diff --git a/.gitignore b/.gitignore index 2d0fadb..37336c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,116 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python,jetbrains+all,git +# Edit at https://www.toptal.com/developers/gitignore?templates=python,jetbrains+all,git + +output +dbs +logs +twistd.pid + + +### Git ### +# Created by git for backups. To disable backups in Git: +# $ git config --global mergetool.keepBackup false +*.orig + +# Created by git when using merge tools for conflicts +*.BACKUP.* +*.BASE.* +*.LOCAL.* +*.REMOTE.* +*_BACKUP_*.txt +*_BASE_*.txt +*_LOCAL_*.txt +*_REMOTE_*.txt + +### JetBrains+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### JetBrains+all Patch ### +# Ignore everything but code style settings and run configurations +# that are supposed to be shared within teams. + +.idea/* + +!.idea/codeStyles +!.idea/runConfigurations + +### Python ### # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -20,7 +133,6 @@ parts/ sdist/ var/ wheels/ -pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg @@ -50,6 +162,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +cover/ # Translations *.mo @@ -72,6 +185,7 @@ instance/ docs/_build/ # PyBuilder +.pybuilder/ target/ # Jupyter Notebook @@ -82,7 +196,9 @@ profile_default/ ipython_config.py # pyenv -.python-version +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. @@ -91,7 +207,22 @@ ipython_config.py # install all needed dependencies. #Pipfile.lock -# PEP 582; used by e.g. github.com/David-OConnor/pyflow +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ # Celery stuff @@ -128,5 +259,27 @@ dmypy.json # Pyre type checker .pyre/ -# templates -.github/templates/* +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +# End of https://www.toptal.com/developers/gitignore/api/python,jetbrains+all,git diff --git a/ABOUT_THIS_TEMPLATE.md b/ABOUT_THIS_TEMPLATE.md deleted file mode 100644 index ed02b66..0000000 --- a/ABOUT_THIS_TEMPLATE.md +++ /dev/null @@ -1,247 +0,0 @@ -# About this template - -Hi, I created this template to help you get started with a new project. - -I have created and maintained a number of python libraries, applications and -frameworks and during those years I have learned a lot about how to create a -project structure and how to structure a project to be as modular and simple -as possible. - -Some decisions I have made while creating this template are: - - - Create a project structure that is as modular as possible. - - Keep it simple and easy to maintain. - - Allow for a lot of flexibility and customizability. - - Low dependency (this template doesn't add dependencies) - -## Structure - -Lets take a look at the structure of this template: - -```text -├── Containerfile # The file to build a container using buildah or docker -├── CONTRIBUTING.md # Onboarding instructions for new contributors -├── docs # Documentation site (add more .md files here) -│   └── index.md # The index page for the docs site -├── .github # Github metadata for repository -│   ├── release_message.sh # A script to generate a release message -│   └── workflows # The CI pipeline for Github Actions -├── .gitignore # A list of files to ignore when pushing to Github -├── HISTORY.md # Auto generated list of changes to the project -├── LICENSE # The license for the project -├── Makefile # A collection of utilities to manage the project -├── MANIFEST.in # A list of files to include in a package -├── mkdocs.yml # Configuration for documentation site -├── neptun_webscraper # The main python package for the project -│   ├── base.py # The base module for the project -│   ├── __init__.py # This tells Python that this is a package -│   ├── __main__.py # The entry point for the project -│   └── VERSION # The version for the project is kept in a static file -├── README.md # The main readme for the project -├── setup.py # The setup.py file for installing and packaging the project -├── requirements.txt # An empty file to hold the requirements for the project -├── requirements-test.txt # List of requirements for testing and devlopment -├── setup.py # The setup.py file for installing and packaging the project -└── tests # Unit tests for the project (add mote tests files here) - ├── conftest.py # Configuration, hooks and fixtures for pytest - ├── __init__.py # This tells Python that this is a test package - └── test_base.py # The base test case for the project -``` - -## FAQ - -Frequent asked questions. - -### Why this template is not using [Poetry](https://python-poetry.org/) ? - -I really like Poetry and I think it is a great tool to manage your python projects, -if you want to switch to poetry, you can run `make switch-to-poetry`. - -But for this template I wanted to keep it simple. - -Setuptools is the most simple and well supported way of packaging a Python project, -it doesn't require extra dependencies and is the easiest way to install the project. - -Also, poetry doesn't have a good support for installing projects in development mode yet. - -### Why the `requirements.txt` is empty ? - -This template is a low dependency project, so it doesn't have any extra dependencies. -You can add new dependencies as you will or you can use the `make init` command to -generate a `requirements.txt` file based on the template you choose `flask, fastapi, click etc`. - -### Why there is a `requirements-test.txt` file ? - -This file lists all the requirements for testing and development, -I think the development environment and testing environment should be as similar as possible. - -Except those tools that are up to the developer choice (like ipython, ipdb etc). - -### Why the template doesn't have a `pyproject.toml` file ? - -It is possible to run `pip install https://github.com/name/repo/tarball/main` and -have pip to download the package direcly from Git repo. - -For that to work you need to have a `setup.py` file, and `pyproject.toml` is not -supported for that kind of installation. - -I think it is easier for example you want to install specific branch or tag you can -do `pip install https://github.com/name/repo/tarball/{TAG|REVISON|COMMIT}` - -People automating CI for your project will be grateful for having a setup.py file - -### Why isn't this template made as a cookiecutter template? - -I really like [cookiecutter](https://github.com/cookiecutter/cookiecutter) and it is a great way to create new projects, -but for this template I wanted to use the Github `Use this template` button, -to use this template doesn't require to install extra tooling such as cookiecutter. - -Just click on [Use this template](https://github.com/rochacbruno/python-project-template/generate) and you are good to go. - -The substituions are done using github actions and a simple sed script. - -### Why `VERSION` is kept in a static plain text file? - -I used to have my version inside my main module in a `__version__` variable, then -I had to do some tricks to read that version variable inside the setuptools -`setup.py` file because that would be available only after the installation. - -I decided to keep the version in a static file because it is easier to read from -wherever I want without the need to install the package. - -e.g: `cat neptun_webscraper/VERSION` will get the project version without harming -with module imports or anything else, it is useful for CI, logs and debugging. - -### Why to include `tests`, `history` and `Containerfile` as part of the release? - -The `MANIFEST.in` file is used to include the files in the release, once the -project is released to PyPI all the files listed on MANIFEST.in will be included -even if the files are static or not related to Python. - -Some build systems such as RPM, DEB, AUR for some Linux distributions, and also -internal repackaging systems tends to run the tests before the packaging is performed. - -The Containerfile can be useful to provide a safer execution environment for -the project when running on a testing environment. - -I added those files to make it easier for packaging in different formats. - -### Why conftest includes a go_to_tmpdir fixture? - -When your project deals with file system operations, it is a good idea to use -a fixture to create a temporary directory and then remove it after the test. - -Before executing each test pytest will create a temporary directory and will -change the working directory to that path and run the test. - -So the test can create temporary artifacts isolated from other tests. - -After the execution Pytest will remove the temporary directory. - -### Why this template is not using [pre-commit](https://pre-commit.com/) ? - -pre-commit is an excellent tool to automate checks and formatting on your code. - -However I figured out that pre-commit adds extra dependency and it an entry barrier -for new contributors. - -Having the linting, checks and formatting as simple commands on the [Makefile](Makefile) -makes it easier to undestand and change. - -Once the project is bigger and complex, having pre-commit as a dependency can be a good idea. - -### Why the CLI is not using click? - -I wanted to provide a simple template for a CLI application on the project main entry point -click and typer are great alternatives but are external dependencies and this template -doesn't add dependencies besides those used for development. - -### Why this doesn't provide a full example of application using Flask or Django? - -as I said before, I want it to be simple and multipurpose, so I decided to not include -external dependencies and programming design decisions. - -It is up to you to decide if you want to use Flask or Django and to create your application -the way you think is best. - -This template provides utilities in the Makefile to make it easier to you can run: - -```bash -$ make init -Which template do you want to apply? [flask, fastapi, click, typer]? > flask -Generating a new project with Flask ... -``` - -Then the above will download the Flask template and apply it to the project. - -## The Makefile - -All the utilities for the template and project are on the Makefile - -```bash -❯ make -Usage: make - -Targets: -help: ## Show the help. -install: ## Install the project in dev mode. -fmt: ## Format code using black & isort. -lint: ## Run pep8, black, mypy linters. -test: lint ## Run tests and generate coverage report. -watch: ## Run tests on every change. -clean: ## Clean unused files. -virtualenv: ## Create a virtual environment. -release: ## Create a new tag for release. -docs: ## Build the documentation. -switch-to-poetry: ## Switch to poetry package manager. -init: ## Initialize the project based on an application template. -``` - -## README - Python Project Template - -A low dependency and really simple to start project template for Python Projects. - -See also -- [Flask-Project-Template](https://github.com/rochacbruno/flask-project-template/) for a full feature Flask project including database, API, admin interface, etc. -- [FastAPI-Project-Template](https://github.com/rochacbruno/fastapi-project-template/) The base to start an openapi project featuring: SQLModel, Typer, FastAPI, JWT Token Auth, Interactive Shell, Management Commands. - -### HOW TO USE THIS TEMPLATE - -> **DO NOT FORK** this is meant to be used from **[Use this template](https://github.com/rochacbruno/python-project-template/generate)** feature. - -1. Click on **[Use this template](https://github.com/rochacbruno/python-project-template/generate)** -3. Give a name to your project - (e.g. `my_awesome_project` recommendation is to use all lowercase and underscores separation for repo names.) -3. Wait until the first run of CI finishes - (Github Actions will process the template and commit to your new repo) -4. If you want [codecov](https://about.codecov.io/sign-up/) Reports and Automatic Release to [PyPI](https://pypi.org) - On the new repository `settings->secrets` add your `PYPI_API_TOKEN` and `CODECOV_TOKEN` (get the tokens on respective websites) -4. Read the file [CONTRIBUTING.md](CONTRIBUTING.md) -5. Then clone your new project and happy coding! - -> **NOTE**: **WAIT** until first CI run on github actions before cloning your new project. - -### What is included on this template? - -- 🖼️ Templates for starting multiple application types: - * **Basic low dependency** Python program (default) [use this template](https://github.com/rochacbruno/python-project-template/generate) - * **Flask** with database, admin interface, restapi and authentication [use this template](https://github.com/rochacbruno/flask-project-template/generate). - **or Run `make init` after cloning to generate a new project based on a template.** -- 📦 A basic [setup.py](setup.py) file to provide installation, packaging and distribution for your project. - Template uses setuptools because it's the de-facto standard for Python packages, you can run `make switch-to-poetry` later if you want. -- 🤖 A [Makefile](Makefile) with the most useful commands to install, test, lint, format and release your project. -- 📃 Documentation structure using [mkdocs](http://www.mkdocs.org) -- 💬 Auto generation of change log using **gitchangelog** to keep a HISTORY.md file automatically based on your commit history on every release. -- 🐋 A simple [Containerfile](Containerfile) to build a container image for your project. - `Containerfile` is a more open standard for building container images than Dockerfile, you can use buildah or docker with this file. -- 🧪 Testing structure using [pytest](https://docs.pytest.org/en/latest/) -- ✅ Code linting using [flake8](https://flake8.pycqa.org/en/latest/) -- 📊 Code coverage reports using [codecov](https://about.codecov.io/sign-up/) -- 🛳️ Automatic release to [PyPI](https://pypi.org) using [twine](https://twine.readthedocs.io/en/latest/) and github actions. -- 🎯 Entry points to execute your program using `python -m ` or `$ neptun_webscraper` with basic CLI argument parsing. -- 🔄 Continuous integration using [Github Actions](.github/workflows/) with jobs to lint, test and release your project on Linux, Mac and Windows environments. - -> Curious about architectural decisions on this template? read [ABOUT_THIS_TEMPLATE.md](ABOUT_THIS_TEMPLATE.md) -> If you want to contribute to this template please open an [issue](https://github.com/rochacbruno/python-project-template/issues) or fork and send a PULL REQUEST. - -[❤️ Sponsor this project](https://github.com/sponsors/rochacbruno/) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 174f70d..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,129 +0,0 @@ -# How to develop on this project - -neptun_webscraper welcomes contributions from the community. - -**You need PYTHON3!** - -This instructions are for linux base systems. (Linux, MacOS, BSD, etc.) -## Setting up your own fork of this repo. - -- On github interface click on `Fork` button. -- Clone your fork of this repo. `git clone git@github.com:YOUR_GIT_USERNAME/tech-stack-ai-configuration-data-scraper.git` -- Enter the directory `cd tech-stack-ai-configuration-data-scraper` -- Add upstream repo `git remote add upstream https://github.com/jonasfroeller/tech-stack-ai-configuration-data-scraper` - -## Installing Dependencies besides Python - -> Needed for JS support, if websites load content after page load client side. - -### Install Playwright - -```bash -npm i playwright@1.44.1 --global -``` - -### Install the browser binaries - -```bash -playwright install -``` - -## Setting up your own virtual environment - -Run `make virtualenv` to create a virtual environment. -then activate it with `source .venv/bin/activate`. - -## Install the project in develop mode - -Run `make install` to install the project in develop mode. - -## Run the tests to ensure everything is working - -Run `make test` to run the tests. - -## Create a new branch to work on your contribution - -Run `git checkout -b my_contribution` - -## Make your changes - -Edit the files using your preferred editor. (we recommend VIM or VSCode) - -## Format the code - -Run `make fmt` to format the code. - -## Run the linter - -Run `make lint` to run the linter. - -## Test your changes - -Run `make test` to run the tests. - -Ensure code coverage report shows `100%` coverage, add tests to your PR. - -## Build the docs locally - -Run `make docs` to build the docs. - -Ensure your new changes are documented. - -## Commit your changes - -This project uses [conventional git commit messages](https://www.conventionalcommits.org/en/v1.0.0/). - -Example: `fix(package): update setup.py arguments 🎉` (emojis are fine too) - -## Push your changes to your fork - -Run `git push origin my_contribution` - -## Submit a pull request - -On github interface, click on `Pull Request` button. - -Wait CI to run and one of the developers will review your PR. -## Makefile utilities - -This project comes with a `Makefile` that contains a number of useful utility. - -```bash -❯ make -Usage: make - -Targets: -help: ## Show the help. -install: ## Install the project in dev mode. -fmt: ## Format code using black & isort. -lint: ## Run pep8, black, mypy linters. -test: lint ## Run tests and generate coverage report. -watch: ## Run tests on every change. -clean: ## Clean unused files. -virtualenv: ## Create a virtual environment. -release: ## Create a new tag for release. -docs: ## Build the documentation. -switch-to-poetry: ## Switch to poetry package manager. -init: ## Initialize the project based on an application template. -``` - -## Making a new release - -This project uses [semantic versioning](https://semver.org/) and tags releases with `X.Y.Z` -Every time a new tag is created and pushed to the remote repo, github actions will -automatically create a new release on github and trigger a release on PyPI. - -For this to work you need to setup a secret called `PIPY_API_TOKEN` on the project settings>secrets, -this token can be generated on [pypi.org](https://pypi.org/account/). - -To trigger a new release all you need to do is. - -1. If you have changes to add to the repo - * Make your changes following the steps described above. - * Commit your changes following the [conventional git commit messages](https://www.conventionalcommits.org/en/v1.0.0/). -2. Run the tests to ensure everything is working. -4. Run `make release` to create a new tag and push it to the remote repo. - -the `make release` will ask you the version number to create the tag, ex: type `0.1.1` when you are asked. - -> **CAUTION**: The make release will change local changelog files and commit all the unstaged changes you have. diff --git a/Containerfile b/Containerfile deleted file mode 100644 index ff605a9..0000000 --- a/Containerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM python:3.7-slim -COPY . /app -WORKDIR /app -RUN pip install . -CMD ["neptun_webscraper"] diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6ada578 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM vimagick/scrapyd:latest + +WORKDIR /code + +COPY . . + +RUN pip install logparser + +RUN logparser & + +RUN pip install scrapy-playwright && \ + playwright install && \ + playwright install-deps + +EXPOSE 6800 + +CMD ["scrapyd"] diff --git a/HISTORY.md b/HISTORY.md deleted file mode 100644 index ac58268..0000000 --- a/HISTORY.md +++ /dev/null @@ -1,7 +0,0 @@ -Changelog -========= - - -0.1.0 (yyyy-mm-dd) ------------------- -- commit message [Author] diff --git a/LICENSE b/LICENSE deleted file mode 100644 index fdddb29..0000000 --- a/LICENSE +++ /dev/null @@ -1,24 +0,0 @@ -This is free and unencumbered software released into the public domain. - -Anyone is free to copy, modify, publish, use, compile, sell, or -distribute this software, either in source code form or as a compiled -binary, for any purpose, commercial or non-commercial, and by any -means. - -In jurisdictions that recognize copyright laws, the author or authors -of this software dedicate any and all copyright interest in the -software to the public domain. We make this dedication for the benefit -of the public at large and to the detriment of our heirs and -successors. We intend this dedication to be an overt act of -relinquishment in perpetuity of all present and future rights to this -software under copyright law. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - -For more information, please refer to diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index d893429..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include LICENSE -include HISTORY.md -include Containerfile -graft tests -graft neptun_webscraper diff --git a/Makefile b/Makefile deleted file mode 100644 index 18282fe..0000000 --- a/Makefile +++ /dev/null @@ -1,122 +0,0 @@ -.ONESHELL: -ENV_PREFIX=$(shell python -c "if __import__('pathlib').Path('.venv/bin/pip').exists(): print('.venv/bin/')") -USING_POETRY=$(shell grep "tool.poetry" pyproject.toml && echo "yes") - -.PHONY: help -help: ## Show the help. - @echo "Usage: make " - @echo "" - @echo "Targets:" - @fgrep "##" Makefile | fgrep -v fgrep - - -.PHONY: show -show: ## Show the current environment. - @echo "Current environment:" - @if [ "$(USING_POETRY)" ]; then poetry env info && exit; fi - @echo "Running using $(ENV_PREFIX)" - @$(ENV_PREFIX)python -V - @$(ENV_PREFIX)python -m site - -.PHONY: install -install: ## Install the project in dev mode. - @if [ "$(USING_POETRY)" ]; then poetry install && exit; fi - @echo "Don't forget to run 'make virtualenv' if you got errors." - $(ENV_PREFIX)pip install -e .[test] - -.PHONY: fmt -fmt: ## Format code using black & isort. - $(ENV_PREFIX)isort neptun_webscraper/ - $(ENV_PREFIX)black -l 79 neptun_webscraper/ - $(ENV_PREFIX)black -l 79 tests/ - -.PHONY: lint -lint: ## Run pep8, black, mypy linters. - $(ENV_PREFIX)flake8 neptun_webscraper/ - $(ENV_PREFIX)black -l 79 --check neptun_webscraper/ - $(ENV_PREFIX)black -l 79 --check tests/ - $(ENV_PREFIX)mypy --ignore-missing-imports neptun_webscraper/ - -.PHONY: test -test: lint ## Run tests and generate coverage report. - $(ENV_PREFIX)pytest -v --cov-config .coveragerc --cov=neptun_webscraper -l --tb=short --maxfail=1 tests/ - $(ENV_PREFIX)coverage xml - $(ENV_PREFIX)coverage html - -.PHONY: watch -watch: ## Run tests on every change. - ls **/**.py | entr $(ENV_PREFIX)pytest -s -vvv -l --tb=long --maxfail=1 tests/ - -.PHONY: clean -clean: ## Clean unused files. - @find ./ -name '*.pyc' -exec rm -f {} \; - @find ./ -name '__pycache__' -exec rm -rf {} \; - @find ./ -name 'Thumbs.db' -exec rm -f {} \; - @find ./ -name '*~' -exec rm -f {} \; - @rm -rf .cache - @rm -rf .pytest_cache - @rm -rf .mypy_cache - @rm -rf build - @rm -rf dist - @rm -rf *.egg-info - @rm -rf htmlcov - @rm -rf .tox/ - @rm -rf docs/_build - -.PHONY: virtualenv -virtualenv: ## Create a virtual environment. - @if [ "$(USING_POETRY)" ]; then poetry install && exit; fi - @echo "creating virtualenv ..." - @rm -rf .venv - @python3 -m venv .venv - @./.venv/bin/pip install -U pip - @./.venv/bin/pip install -e .[test] - @echo - @echo "!!! Please run 'source .venv/bin/activate' to enable the environment !!!" - -.PHONY: release -release: ## Create a new tag for release. - @echo "WARNING: This operation will create s version tag and push to github" - @read -p "Version? (provide the next x.y.z semver) : " TAG - @echo "$${TAG}" > neptun_webscraper/VERSION - @$(ENV_PREFIX)gitchangelog > HISTORY.md - @git add neptun_webscraper/VERSION HISTORY.md - @git commit -m "release: version $${TAG} 🚀" - @echo "creating git tag : $${TAG}" - @git tag $${TAG} - @git push -u origin HEAD --tags - @echo "Github Actions will detect the new tag and release the new version." - -.PHONY: docs -docs: ## Build the documentation. - @echo "building documentation ..." - @$(ENV_PREFIX)mkdocs build - URL="site/index.html"; xdg-open $$URL || sensible-browser $$URL || x-www-browser $$URL || gnome-open $$URL || open $$URL - -.PHONY: switch-to-poetry -switch-to-poetry: ## Switch to poetry package manager. - @echo "Switching to poetry ..." - @if ! poetry --version > /dev/null; then echo 'poetry is required, install from https://python-poetry.org/'; exit 1; fi - @rm -rf .venv - @poetry init --no-interaction --name=a_flask_test --author=rochacbruno - @echo "" >> pyproject.toml - @echo "[tool.poetry.scripts]" >> pyproject.toml - @echo "neptun_webscraper = 'neptun_webscraper.__main__:main'" >> pyproject.toml - @cat requirements.txt | while read in; do poetry add --no-interaction "$${in}"; done - @cat requirements-test.txt | while read in; do poetry add --no-interaction "$${in}" --dev; done - @poetry install --no-interaction - @mkdir -p .github/backup - @mv requirements* .github/backup - @mv setup.py .github/backup - @echo "You have switched to https://python-poetry.org/ package manager." - @echo "Please run 'poetry shell' or 'poetry run neptun_webscraper'" - -.PHONY: init -init: ## Initialize the project based on an application template. - @./.github/init.sh - - -# This project has been generated from rochacbruno/python-project-template -# __author__ = 'rochacbruno' -# __repo__ = https://github.com/rochacbruno/python-project-template -# __sponsor__ = https://github.com/sponsors/rochacbruno/ diff --git a/README.md b/README.md index 6ded390..c19c63b 100644 --- a/README.md +++ b/README.md @@ -1,67 +1,35 @@ -# Neptun Webscraper +# NEPTUN-SCRAPER -[![codecov](https://codecov.io/gh/jonasfroeller/tech-stack-ai-configuration-data-scraper/branch/main/graph/badge.svg?token=tech-stack-ai-configuration-data-scraper_token_here)](https://codecov.io/gh/jonasfroeller/tech-stack-ai-configuration-data-scraper) -[![CI](https://github.com/jonasfroeller/tech-stack-ai-configuration-data-scraper/actions/workflows/main.yml/badge.svg)](https://github.com/jonasfroeller/tech-stack-ai-configuration-data-scraper/actions/workflows/main.yml) +## RUN -## Install it from PyPI +* Install dependencies -```bash -pip install neptun_webscraper +```shell +poetry install ``` -## Usage +* Run Neptun-Scraper -```py -from neptun_webscraper import BaseClass -from neptun_webscraper import base_function - -BaseClass().base_method() -base_function() +```shell +scrapy crawl dockerhubDockerRegistrySpider -a query="python" ``` -```bash -python -m neptun_webscraper +* Run Neptun-Bot (WebUI, Rest-Interface, Scrapy-Daemon) +```shell +docker compose up --build ``` -```bash -neptun_webscraper +* Access the WebUI: +```shell +http://localhost:8000 ``` -## Docker Hub Scraper - -```bash -python -m neptun_webscraper dockerhub --query=python -``` - -## Quay IO Scraper - -```bash -python -m neptun_webscraper quay --query=python -``` - -## Development - -## Create a virtualenv - -```bash -make virtualenv -source .venv/bin/activ +* Access the DAEMON-Interface: +```shell +http://localhost:6802 ``` -## Format the code - -```bash -make fmt -``` - -## Lint the code - -```bash -make lint -``` - -## Contributing - -More commands are in the [Makefile](Makefile). - -Read the [CONTRIBUTING.md](CONTRIBUTING.md) file. +* Create a Scrapyd-Job +```shell +curl http://localhost:6802/schedule.json -d project=neptunscraper -d spider=dockerhubDockerRegistrySpider +``` \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..a7a813f --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,57 @@ +version: "3.8" + +services: + scrapyd: + build: + context: . + dockerfile: Dockerfile + container_name: scrapyd + volumes: + - .:/code + working_dir: /code + ports: + - "6802:6800" + restart: unless-stopped + networks: + scrapyd_network: + ipv4_address: 172.20.0.10 + + scrapydweb: + build: + context: ./scrapydweb + container_name: scrapydweb + ports: + - "8000:8000" + environment: + - SCRAPYD_ADMIN=admin + - SCRAPYD_PASSWD=secret + - SCRAPYD_SERVERS=scrapyd:6800 + - DATABASE_URL=sqlite:///db.sqlite3 + depends_on: + - scrapyd + networks: + scrapyd_network: + ipv4_address: 172.20.0.11 + + scrapyrt: + container_name: scrapyrt + image: vimagick/scrapyd + command: scrapyrt -i 0.0.0.0 -p 9080 + ports: + - "9080:9080" + volumes: + - .:/code + working_dir: /code + restart: unless-stopped + networks: + scrapyd_network: + ipv4_address: 172.20.0.12 + depends_on: + - scrapyd + +networks: + scrapyd_network: + driver: bridge + ipam: + config: + - subnet: 172.20.0.0/16 diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 000ea34..0000000 --- a/docs/index.md +++ /dev/null @@ -1,17 +0,0 @@ -# Welcome to MkDocs - -For full documentation visit [mkdocs.org](https://www.mkdocs.org). - -## Commands - -* `mkdocs new [dir-name]` - Create a new project. -* `mkdocs serve` - Start the live-reloading docs server. -* `mkdocs build` - Build the documentation site. -* `mkdocs -h` - Print help message and exit. - -## Project layout - - mkdocs.yml # The configuration file. - docs/ - index.md # The documentation homepage. - ... # Other markdown pages, images and other files. diff --git a/mkdocs.yml b/mkdocs.yml deleted file mode 100644 index 4dac54e..0000000 --- a/mkdocs.yml +++ /dev/null @@ -1,2 +0,0 @@ -site_name: neptun_webscraper -theme: readthedocs diff --git a/neptun_webscraper/VERSION b/neptun_webscraper/VERSION deleted file mode 100644 index 6e8bf73..0000000 --- a/neptun_webscraper/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.1.0 diff --git a/neptun_webscraper/__init__.py b/neptun_webscraper/__init__.py deleted file mode 100644 index cdf2c71..0000000 --- a/neptun_webscraper/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .spiders.dockerhub import DockerhubDockerRegistrySpider -from .spiders.quay import QuayDockerRegistrySpider - -__all__ = ["DockerhubDockerRegistrySpider", "QuayDockerRegistrySpider"] diff --git a/neptun_webscraper/__main__.py b/neptun_webscraper/__main__.py deleted file mode 100644 index f3483c3..0000000 --- a/neptun_webscraper/__main__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Entry point for neptun_webscraper.""" - -from neptun_webscraper.cli import main # pragma: no cover - -if __name__ == "__main__": # pragma: no cover - main() diff --git a/neptun_webscraper/base.py b/neptun_webscraper/base.py deleted file mode 100644 index 5f128c8..0000000 --- a/neptun_webscraper/base.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -neptun_webscraper base module. - -This is the principal module of the neptun_webscraper project. -Main classes and objects live here. -""" - -NAME = "neptun_webscraper" diff --git a/neptun_webscraper/cli.py b/neptun_webscraper/cli.py deleted file mode 100644 index 1d80a62..0000000 --- a/neptun_webscraper/cli.py +++ /dev/null @@ -1,41 +0,0 @@ -import argparse - -from scrapy.crawler import CrawlerProcess - -from neptun_webscraper.spiders.dockerhub import DockerhubDockerRegistrySpider -from neptun_webscraper.spiders.quay import QuayDockerRegistrySpider - - -def main(): # pragma: no cover - """ - CLI interface for neptun_webscraper project. - This is the program's entry point. The main function executes on commands: - `python -m neptun_webscraper` and `$ neptun_webscraper `. - """ - - parser = argparse.ArgumentParser(description="Neptune WebScraper CLI") - parser.add_argument( - "spider", - choices=["dockerhub", "quay"], - default="dockerhub", - help="Choose a spider to run", - ) - parser.add_argument( - "--query", default="", help="Search query for the registry" - ) - args = parser.parse_args() - - process = CrawlerProcess() - - if args.spider == "dockerhub": - spider = DockerhubDockerRegistrySpider - start_urls = [ - f"https://hub.docker.com/search?q={args.query}&page={i}" - for i in range(1, 11) - ] - elif args.spider == "quay": - spider = QuayDockerRegistrySpider - start_urls = [f"https://quay.io/search?q={args.query}"] - - process.crawl(spider, start_urls=start_urls) - process.start() diff --git a/neptun_webscraper/spiders/logs/20240702000854.json b/neptun_webscraper/spiders/logs/20240702000854.json deleted file mode 100644 index 5ef1e1e..0000000 --- a/neptun_webscraper/spiders/logs/20240702000854.json +++ /dev/null @@ -1,374 +0,0 @@ -[ -{ - "name": "python", - "is_official_image": true, - "is_verified_publisher": false, - "last_update": "2024-06-28", - "description": "Python is an interpreted, interactive, object-oriented, open-source programming language.", - "chips": [ - "Languages & Frameworks" - ], - "downloads": "1B+", - "pulls_last_week": "7201638", - "stars": "9.7K" -} -, -{ - "name": "pypy", - "is_official_image": true, - "is_verified_publisher": false, - "last_update": "2024-06-22", - "description": "PyPy is a fast, compliant alternative implementation of the Python language.", - "chips": [ - "Languages & Frameworks" - ], - "downloads": "10M+", - "pulls_last_week": "17171", - "stars": "9.7K" -} -, -{ - "name": "hylang", - "is_official_image": true, - "is_verified_publisher": false, - "last_update": "2024-06-28", - "description": "Hy is a Lisp dialect that translates expressions into Python's abstract syntax tree.", - "chips": [ - "Languages & Frameworks" - ], - "downloads": "10M+", - "pulls_last_week": "22302", - "stars": "9.7K" -} -, -{ - "name": "circleci/python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2021-07-02", - "description": "Python is an interpreted, interactive, object-oriented, open-source programming language.", - "chips": [], - "downloads": "100M+", - "pulls_last_week": "161684", - "stars": "9.7K" -} -, -{ - "name": "cimg/python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-04-02", - "description": null, - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Security" - ], - "downloads": "100M+", - "pulls_last_week": "1121955", - "stars": "9.7K" -} -, -{ - "name": "bitnami/python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-30", - "description": "Bitnami container image for Python", - "chips": [], - "downloads": "1M+", - "pulls_last_week": "15146", - "stars": "9.7K" -} -, -{ - "name": "amazon/aws-lambda-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-28", - "description": "AWS Lambda base images for Python", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Machine Learning & AI" - ], - "downloads": "1M+", - "pulls_last_week": "15574", - "stars": "9.7K" -} -, -{ - "name": "okteto/python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": null, - "chips": [ - "Languages & Frameworks", - "Integration & Delivery" - ], - "downloads": "100K+", - "pulls_last_week": "26", - "stars": "9.7K" -} -, -{ - "name": "google/guestbook-python-redis", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2015-07-02", - "description": "A simple guestbook example written in Python. It works against a redis backend.", - "chips": [ - "Databases & Storage", - "Languages & Frameworks", - "Integration & Delivery" - ], - "downloads": "1M+", - "pulls_last_week": "6", - "stars": "9.7K" -} -, -{ - "name": "ubuntu/python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-19", - "description": "A chiselled Ubuntu rock with the Python runtime", - "chips": [ - "Languages & Frameworks" - ], - "downloads": "10K+", - "pulls_last_week": "561", - "stars": "9.7K" -} -, -{ - "name": "balenalib/artik533s-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "500K+", - "pulls_last_week": "1", - "stars": "9.7K" -} -, -{ - "name": "jetbrains/qodana-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-02", - "description": "Qodana for Python projects", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Security" - ], - "downloads": "100K+", - "pulls_last_week": "860", - "stars": "9.7K" -} -, -{ - "name": "chainguard/python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-30", - "description": "Minimal Python image based on Wolfi.", - "chips": [], - "downloads": "10K+", - "pulls_last_week": "543", - "stars": "9.7K" -} -, -{ - "name": "intel/python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": null, - "chips": [], - "downloads": "10K+", - "pulls_last_week": "18", - "stars": "9.7K" -} -, -{ - "name": "balenalib/amd64-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "500K+", - "pulls_last_week": "15", - "stars": "9.7K" -} -, -{ - "name": "balenalib/amd64-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": "242", - "stars": "9.7K" -} -, -{ - "name": "amazon/aws-sam-cli-build-image-python3.8", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": null, - "chips": [ - "Languages & Frameworks", - "Integration & Delivery" - ], - "downloads": "1M+", - "pulls_last_week": "626", - "stars": "9.7K" -} -, -{ - "name": "balenalib/artik5-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "500K+", - "pulls_last_week": "1", - "stars": "9.7K" -} -, -{ - "name": "balenalib/amd64-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Data Science", - "Languages & Frameworks", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": "23", - "stars": "9.7K" -} -, -{ - "name": "balenalib/artik10-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things" - ], - "downloads": "100K+", - "pulls_last_week": "1", - "stars": "9.7K" -} -, -{ - "name": "balenalib/apalis-imx6q-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": "1", - "stars": "9.7K" -} -, -{ - "name": "balenalib/fincm3-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "500K+", - "pulls_last_week": "58", - "stars": "9.7K" -} -, -{ - "name": "balenalib/raspberry-pi-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": "161", - "stars": "9.7K" -} -, -{ - "name": "balenalib/artik710-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": "1", - "stars": "9.7K" -} -, -{ - "name": "balenalib/apalis-imx6q-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "500K+", - "pulls_last_week": "1", - "stars": "9.7K" -} -] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/20240702194006/data_page_1.json b/neptun_webscraper/spiders/logs/20240702194006/data_page_1.json deleted file mode 100644 index f40ab67..0000000 --- a/neptun_webscraper/spiders/logs/20240702194006/data_page_1.json +++ /dev/null @@ -1,351 +0,0 @@ -[ - { - "name": "python", - "is_official_image": true, - "is_verified_publisher": false, - "last_update": "2024-06-27", - "description": "Python is an interpreted, interactive, object-oriented, open-source programming language.", - "chips": [ - "Languages & Frameworks" - ], - "downloads": "1B+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "pypy", - "is_official_image": true, - "is_verified_publisher": false, - "last_update": "2024-06-21", - "description": "PyPy is a fast, compliant alternative implementation of the Python language.", - "chips": [ - "Languages & Frameworks" - ], - "downloads": "10M+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "hylang", - "is_official_image": true, - "is_verified_publisher": false, - "last_update": "2024-06-27", - "description": "Hy is a Lisp dialect that translates expressions into Python's abstract syntax tree.", - "chips": [ - "Languages & Frameworks" - ], - "downloads": "10M+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "circleci/python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2021-07-02", - "description": "Python is an interpreted, interactive, object-oriented, open-source programming language.", - "chips": [], - "downloads": "100M+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "cimg/python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-04-02", - "description": null, - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Security" - ], - "downloads": "100M+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "bitnami/python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-29", - "description": "Bitnami container image for Python", - "chips": [], - "downloads": "1M+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "amazon/aws-lambda-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-27", - "description": "AWS Lambda base images for Python", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Machine Learning & AI" - ], - "downloads": "1M+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "okteto/python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": null, - "chips": [ - "Languages & Frameworks", - "Integration & Delivery" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "google/guestbook-python-redis", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2015-07-02", - "description": "A simple guestbook example written in Python. It works against a redis backend.", - "chips": [ - "Databases & Storage", - "Languages & Frameworks", - "Integration & Delivery" - ], - "downloads": "1M+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "ubuntu/python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-18", - "description": "A chiselled Ubuntu rock with the Python runtime", - "chips": [ - "Languages & Frameworks" - ], - "downloads": "10K+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "jetbrains/qodana-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-02", - "description": "Qodana for Python projects", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Security" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "chainguard/python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-29", - "description": "Minimal Python image based on Wolfi.", - "chips": [], - "downloads": "10K+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "intel/python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": null, - "chips": [], - "downloads": "10K+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "balenalib/amd64-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "500K+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "amazon/aws-sam-cli-build-image-python3.8", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": null, - "chips": [ - "Languages & Frameworks", - "Integration & Delivery" - ], - "downloads": "1M+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "balenalib/artik533s-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "500K+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "balenalib/artik5-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "500K+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "balenalib/fincm3-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "500K+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "balenalib/artik10-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "500K+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "balenalib/raspberrypi3-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "balenalib/apalis-imx6q-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "500K+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "balenalib/artik530-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "500K+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "amazon/aws-sam-cli-build-image-python3.7", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": null, - "chips": [ - "API Management", - "Languages & Frameworks", - "Integration & Delivery" - ], - "downloads": "1M+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "balenalib/armv7hf-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "9.7K" - }, - { - "name": "balenalib/amd64-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "9.7K" - } -] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/20240702194006/data_page_10.json b/neptun_webscraper/spiders/logs/20240702194006/data_page_10.json deleted file mode 100644 index c1e930e..0000000 --- a/neptun_webscraper/spiders/logs/20240702194006/data_page_10.json +++ /dev/null @@ -1,346 +0,0 @@ -[ - { - "name": "appdynamics/python-agent-init", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-02", - "description": "AppDynamics Repository for Python agent installation in container", - "chips": [ - "Integration & Delivery", - "Monitoring & Observability", - "Security" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/ts4900-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/armv7hf-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/odroid-xu4-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberrypi3-64-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/srd3-tx2-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Machine Learning & AI", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-nano-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/imx8mm-var-dart-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/odroid-xu4-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/imx6ul-var-dart-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-tx2-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Internet of Things", - "Machine Learning & AI", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/parallella-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Data Science", - "Languages & Frameworks", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-tx2-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-04-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Internet of Things", - "Machine Learning & AI", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/revpi-core-3-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/generic-armv7ahf-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/beaglebone-green-wifi-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/j120-tx2-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Machine Learning & AI", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberrypi400-64-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/blackboard-tx2-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/surface-pro-6-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/surface-pro-6-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberrypi0-2w-64-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/artik533s-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/i386-nlp-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2021-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/iot-gate-imx8-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": null, - "chips": [], - "pulls_last_week": null, - "stars": "0" - } -] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/20240702194006/data_page_2.json b/neptun_webscraper/spiders/logs/20240702194006/data_page_2.json deleted file mode 100644 index 57e61b5..0000000 --- a/neptun_webscraper/spiders/logs/20240702194006/data_page_2.json +++ /dev/null @@ -1,349 +0,0 @@ -[ - { - "name": "balenalib/aarch64-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/i386-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2021-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/i386-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "amazon/aws-sam-cli-emulation-image-python3.8", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-27", - "description": null, - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Monitoring & Observability" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/amd64-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Data Science", - "Languages & Frameworks", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/artik10-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/apalis-imx6q-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/iot2000-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2021-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/aarch64-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/raspberry-pi-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/artik710-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/intel-nuc-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/amd64-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/armv7hf-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/amd64-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/aarch64-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-04-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/armv7hf-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/artik5-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/cybertan-ze250-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2021-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/raspberrypi3-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/raspberrypi3-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/artik533s-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/iot2000-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2021-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Internet of Things" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/intel-nuc-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - }, - { - "name": "balenalib/artik710-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "1" - } -] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/20240702194006/data_page_3.json b/neptun_webscraper/spiders/logs/20240702194006/data_page_3.json deleted file mode 100644 index 7b1a847..0000000 --- a/neptun_webscraper/spiders/logs/20240702194006/data_page_3.json +++ /dev/null @@ -1,374 +0,0 @@ -[ - { - "name": "balenalib/revpi-core-3-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/parallella-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberrypi3-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/artik533s-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/iot2000-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2021-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Internet of Things" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/intel-nuc-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/beaglebone-pocket-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "amazon/aws-sam-cli-emulation-image-python3.7", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-27", - "description": null, - "chips": [ - "Languages & Frameworks", - "Integration & Delivery" - ], - "downloads": "500K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberrypi4-64-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/intel-nuc-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/am571x-evm-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/nitrogen6x-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/i386-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberry-pi2-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/surface-pro-6-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/asus-tinker-board-s-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/apalis-imx6q-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/orangepi-plus2-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/beaglebone-green-wifi-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-tx1-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/up-board-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/spacely-tx2-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-tx2-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/beaglebone-pocket-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/up-core-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Internet of Things" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - } -] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/20240702194006/data_page_4.json b/neptun_webscraper/spiders/logs/20240702194006/data_page_4.json deleted file mode 100644 index b2a109c..0000000 --- a/neptun_webscraper/spiders/logs/20240702194006/data_page_4.json +++ /dev/null @@ -1,369 +0,0 @@ -[ - { - "name": "balenalib/artik10-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/kitra520-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-tx2-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/amd64-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/up-squared-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/kitra520-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/cybertan-ze250-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2021-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "rapidfort/python-chromedriver", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-29", - "description": null, - "chips": [], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/intel-edison-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberrypi3-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/up-board-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/artik710-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/apalis-imx6q-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/artik710-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/artik5-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Internet of Things" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/aarch64-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/kitra710-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "airbyte/base-airbyte-protocol-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2021-07-02", - "description": null, - "chips": [], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/armv7hf-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/am571x-evm-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/nitrogen6x-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/beaglebone-green-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberry-pi-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/srd3-tx2-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/qemux86-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-06-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - } -] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/20240702194006/data_page_5.json b/neptun_webscraper/spiders/logs/20240702194006/data_page_5.json deleted file mode 100644 index 40c1d16..0000000 --- a/neptun_webscraper/spiders/logs/20240702194006/data_page_5.json +++ /dev/null @@ -1,347 +0,0 @@ -[ - { - "name": "balenalib/orangepi-plus2-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/surface-go-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": null, - "chips": [], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/beaglebone-green-wifi-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/beaglebone-black-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-tx1-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/intel-nuc-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-tx1-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/rpi-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/spacely-tx2-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-tx2-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/beaglebone-pocket-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/kitra520-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/beaglebone-green-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/up-core-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Internet of Things" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/artik10-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/kitra520-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/odroid-c1-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/artik530-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-tx2-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/intel-edison-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/up-squared-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/intel-edison-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/n510-tx2-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Internet of Things", - "Machine Learning & AI", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/fincm3-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Internet of Things" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/beagleboard-xm-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - } -] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/20240702194006/data_page_6.json b/neptun_webscraper/spiders/logs/20240702194006/data_page_6.json deleted file mode 100644 index 22b0e53..0000000 --- a/neptun_webscraper/spiders/logs/20240702194006/data_page_6.json +++ /dev/null @@ -1,351 +0,0 @@ -[ - { - "name": "balenalib/intel-nuc-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/artik10-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/generic-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jn30b-nano-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/beaglebone-pocket-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/rpi-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/asus-tinker-board-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/spacely-tx2-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/ts7700-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2020-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/odroid-c1-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/artik530-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/asus-tinker-board-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/am571x-evm-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/nitrogen6xq2g-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/nitrogen6x-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/beaglebone-pocket-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/surface-pro-6-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberry-pi2-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/fincm3-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Internet of Things" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/asus-tinker-board-s-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/ts4900-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Internet of Things" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/cl-som-imx8-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/imx8m-var-dart-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/srd3-tx2-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/orangepi-plus2-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "pulls_last_week": null, - "stars": "0" - } -] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/20240702194006/data_page_7.json b/neptun_webscraper/spiders/logs/20240702194006/data_page_7.json deleted file mode 100644 index 138f39e..0000000 --- a/neptun_webscraper/spiders/logs/20240702194006/data_page_7.json +++ /dev/null @@ -1,375 +0,0 @@ -[ - { - "name": "balenalib/artik10-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/nitrogen8mm-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/imx6ul-var-dart-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/qemux86-64-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Internet of Things" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/generic-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jn30b-nano-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/srd3-tx2-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Machine Learning & AI", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-tx1-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/asus-tinker-board-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-nano-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/parallella-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Data Science", - "Languages & Frameworks", - "Machine Learning & AI" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/ts7700-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2020-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "10K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/beaglebone-pocket-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/blackboard-tx2-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/odroid-c1-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/surface-pro-6-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/asus-tinker-board-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/spacely-tx2-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/nitrogen6xq2g-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/artik710-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberrypicm4-ioboard-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-tx2-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/cl-som-imx8-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/kitra710-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/imx8m-var-dart-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - } -] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/20240702194006/data_page_8.json b/neptun_webscraper/spiders/logs/20240702194006/data_page_8.json deleted file mode 100644 index 700cb89..0000000 --- a/neptun_webscraper/spiders/logs/20240702194006/data_page_8.json +++ /dev/null @@ -1,365 +0,0 @@ -[ - { - "name": "balenalib/imx7-var-som-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/generic-aarch64-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/nanopi-neo-air-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/skx2-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": null, - "chips": [], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/surface-go-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": null, - "chips": [], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/cl-som-imx8-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberry-pi2-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/nanopi-neo-air-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/imx6ul-var-dart-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/intel-edison-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2021-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberry-pi-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberrypi4-64-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-tx1-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/generic-aarch64-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/intel-nuc-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/j120-tx2-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/colibri-imx6dl-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/imx8m-var-dart-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-tx2-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/odyssey-x86-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/orbitty-tx2-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/generic-aarch64-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/kitra710-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/qemux86-64-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/blackboard-tx2-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - } -] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/20240702194006/data_page_9.json b/neptun_webscraper/spiders/logs/20240702194006/data_page_9.json deleted file mode 100644 index 5756741..0000000 --- a/neptun_webscraper/spiders/logs/20240702194006/data_page_9.json +++ /dev/null @@ -1,373 +0,0 @@ -[ - { - "name": "balenalib/iot-gate-imx8-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Internet of Things" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/colibri-imx6dl-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/nitrogen6x-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberry-pi2-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/beaglebone-green-gateway-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberry-pi-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/orbitty-tx2-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/generic-aarch64-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/intel-nuc-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/srd3-tx2-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/orbitty-tx2-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Internet of Things", - "Machine Learning & AI", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/generic-amd64-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": null, - "chips": [], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/artik533s-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/zynq-xz702-ubuntu-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2022-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/photon-tx2-nx-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/jetson-tx2-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Machine Learning & AI" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/kitra710-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/qemux86-64-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Integration & Delivery", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/nanopi-neo-air-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-05-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/parallella-alpine-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-09-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Data Science", - "Languages & Frameworks", - "Internet of Things" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/up-board-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/surface-go-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2024-04-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "50K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/beaglebone-pocket-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/raspberrypi3-fedora-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - }, - { - "name": "balenalib/fincm3-debian-python", - "is_official_image": false, - "is_verified_publisher": true, - "last_update": "2023-07-02", - "description": "This image is part of the balena.io base image series for IoT devices.", - "chips": [ - "Languages & Frameworks", - "Internet of Things", - "Operating Systems" - ], - "downloads": "100K+", - "pulls_last_week": null, - "stars": "0" - } -] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_1.png b/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_1.png deleted file mode 100644 index 16f097b..0000000 Binary files a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_1.png and /dev/null differ diff --git a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_10.png b/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_10.png deleted file mode 100644 index ae1f6aa..0000000 Binary files a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_10.png and /dev/null differ diff --git a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_2.png b/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_2.png deleted file mode 100644 index 0d7e544..0000000 Binary files a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_2.png and /dev/null differ diff --git a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_3.png b/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_3.png deleted file mode 100644 index f6e0999..0000000 Binary files a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_3.png and /dev/null differ diff --git a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_4.png b/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_4.png deleted file mode 100644 index 3b6b848..0000000 Binary files a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_4.png and /dev/null differ diff --git a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_5.png b/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_5.png deleted file mode 100644 index 1a1ba9a..0000000 Binary files a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_5.png and /dev/null differ diff --git a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_6.png b/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_6.png deleted file mode 100644 index a77ed92..0000000 Binary files a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_6.png and /dev/null differ diff --git a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_7.png b/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_7.png deleted file mode 100644 index e03b9ac..0000000 Binary files a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_7.png and /dev/null differ diff --git a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_8.png b/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_8.png deleted file mode 100644 index 1928f2f..0000000 Binary files a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_8.png and /dev/null differ diff --git a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_9.png b/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_9.png deleted file mode 100644 index 94e4a4a..0000000 Binary files a/neptun_webscraper/spiders/logs/20240702194006/screenshot_page_9.png and /dev/null differ diff --git a/neptun_webscraper/spiders/logs/html/dockerhub_example_response.html b/neptun_webscraper/spiders/logs/html/dockerhub_example_response.html deleted file mode 100644 index 61cfe93..0000000 --- a/neptun_webscraper/spiders/logs/html/dockerhub_example_response.html +++ /dev/null @@ -1,380 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/html/dockerhub_example_response_using_playwright_firefox.html b/neptun_webscraper/spiders/logs/html/dockerhub_example_response_using_playwright_firefox.html deleted file mode 100644 index 0d3b2ef..0000000 --- a/neptun_webscraper/spiders/logs/html/dockerhub_example_response_using_playwright_firefox.html +++ /dev/null @@ -1,141 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - Docker Hub - - - -
- -
- - - - - - - \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/html/dockerhub_example_response_using_splash.html b/neptun_webscraper/spiders/logs/html/dockerhub_example_response_using_splash.html deleted file mode 100644 index 7b2f717..0000000 --- a/neptun_webscraper/spiders/logs/html/dockerhub_example_response_using_splash.html +++ /dev/null @@ -1,5216 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/html/dockerhub_search_result_format.html b/neptun_webscraper/spiders/logs/html/dockerhub_search_result_format.html deleted file mode 100644 index 15a2346..0000000 --- a/neptun_webscraper/spiders/logs/html/dockerhub_search_result_format.html +++ /dev/null @@ -1,249 +0,0 @@ -SEARCH RESULT FORMAT (30.6.2024): - - \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/screenshots/20240702000846.png b/neptun_webscraper/spiders/logs/screenshots/20240702000846.png deleted file mode 100644 index 03cdd07..0000000 Binary files a/neptun_webscraper/spiders/logs/screenshots/20240702000846.png and /dev/null differ diff --git a/neptun_webscraper/spiders/quay.py b/neptun_webscraper/spiders/quay.py deleted file mode 100644 index 6534509..0000000 --- a/neptun_webscraper/spiders/quay.py +++ /dev/null @@ -1,10 +0,0 @@ -import scrapy - - -class QuayDockerRegistrySpider(scrapy.Spider): - name = "quayDockerRegistry" - - def parse(self, response): - # item = {} - - print("TODO: Extract data from Quay.io search results") diff --git a/neptun_webscraper/spiders/__init__.py b/neptunscraper/__init__.py similarity index 100% rename from neptun_webscraper/spiders/__init__.py rename to neptunscraper/__init__.py diff --git a/neptunscraper/items.py b/neptunscraper/items.py new file mode 100644 index 0000000..30ef19d --- /dev/null +++ b/neptunscraper/items.py @@ -0,0 +1,30 @@ +# Define here the models for your scraped items +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/items.html + +import scrapy + + +class DockerhubItem(scrapy.Item): + name: scrapy.Field + is_official_image: scrapy.Field + is_verified_publisher: scrapy.Field + last_update: scrapy.Field + description: scrapy.Field + chips: scrapy.Field + downloads: scrapy.Field + pulls_last_week: scrapy.Field + stars: scrapy.Field + + +class QuayIoItem(scrapy.Item): + name: scrapy.Field + is_official_image: scrapy.Field + is_verified_publisher: scrapy.Field + last_update: scrapy.Field + description: scrapy.Field + chips: scrapy.Field + downloads: scrapy.Field + pulls_last_week: scrapy.Field + stars: scrapy.Field diff --git a/neptunscraper/middlewares.py b/neptunscraper/middlewares.py new file mode 100644 index 0000000..353726b --- /dev/null +++ b/neptunscraper/middlewares.py @@ -0,0 +1,103 @@ +# Define here the models for your spider middleware +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +from scrapy import signals + +# useful for handling different item types with a single interface +from itemadapter import is_item, ItemAdapter + + +class NeptunscraperSpiderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(self, response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(self, response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, or item objects. + for i in result: + yield i + + def process_spider_exception(self, response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Request or item objects. + pass + + def process_start_requests(self, start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info("Spider opened: %s" % spider.name) + + +class NeptunscraperDownloaderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info("Spider opened: %s" % spider.name) diff --git a/neptunscraper/pipelines.py b/neptunscraper/pipelines.py new file mode 100644 index 0000000..58d7f78 --- /dev/null +++ b/neptunscraper/pipelines.py @@ -0,0 +1,13 @@ +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html + + +# useful for handling different item types with a single interface +from itemadapter import ItemAdapter + + +class NeptunscraperPipeline: + def process_item(self, item, spider): + return item diff --git a/neptunscraper/settings.py b/neptunscraper/settings.py new file mode 100644 index 0000000..307d220 --- /dev/null +++ b/neptunscraper/settings.py @@ -0,0 +1,102 @@ +# Scrapy settings for neptunscraper project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# https://docs.scrapy.org/en/latest/topics/settings.html +# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +BOT_NAME = "neptunscraper" + +SPIDER_MODULES = ["neptunscraper.spiders"] +NEWSPIDER_MODULE = "neptunscraper.spiders" + + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +#USER_AGENT = "neptunscraper (+http://www.yourdomain.com)" + +# Obey robots.txt rules +ROBOTSTXT_OBEY = True + +DOWNLOAD_HANDLERS = { + "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", + "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", +} + +REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" +PLAYWRIGHT_BROWSER_TYPE = "chromium" +USER_AGENT = None + +# Configure maximum concurrent requests performed by Scrapy (default: 16) +#CONCURRENT_REQUESTS = 32 + +# Configure a delay for requests for the same website (default: 0) +# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs +#DOWNLOAD_DELAY = 3 +# The download delay setting will honor only one of: +#CONCURRENT_REQUESTS_PER_DOMAIN = 16 +#CONCURRENT_REQUESTS_PER_IP = 16 + +# Disable cookies (enabled by default) +#COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +#TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +#DEFAULT_REQUEST_HEADERS = { +# "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", +# "Accept-Language": "en", +#} + +# Enable or disable spider middlewares +# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html +#SPIDER_MIDDLEWARES = { +# "neptunscraper.middlewares.NeptunscraperSpiderMiddleware": 543, +#} + +# Enable or disable downloader middlewares +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +#DOWNLOADER_MIDDLEWARES = { +# "neptunscraper.middlewares.NeptunscraperDownloaderMiddleware": 543, +#} + +# Enable or disable extensions +# See https://docs.scrapy.org/en/latest/topics/extensions.html +#EXTENSIONS = { +# "scrapy.extensions.telnet.TelnetConsole": None, +#} + +# Configure item pipelines +# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html +#ITEM_PIPELINES = { +# "neptunscraper.pipelines.NeptunscraperPipeline": 300, +#} + +# Enable and configure the AutoThrottle extension (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/autothrottle.html +#AUTOTHROTTLE_ENABLED = True +# The initial download delay +#AUTOTHROTTLE_START_DELAY = 5 +# The maximum download delay to be set in case of high latencies +#AUTOTHROTTLE_MAX_DELAY = 60 +# The average number of requests Scrapy should be sending in parallel to +# each remote server +#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +# Enable showing throttling stats for every response received: +#AUTOTHROTTLE_DEBUG = False + +# Enable and configure HTTP caching (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED = True +#HTTPCACHE_EXPIRATION_SECS = 0 +#HTTPCACHE_DIR = "httpcache" +#HTTPCACHE_IGNORE_HTTP_CODES = [] +#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" + +# Set settings whose default value is deprecated to a future-proof value +REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" +TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" +FEED_EXPORT_ENCODING = "utf-8" diff --git a/neptunscraper/spiders/__init__.py b/neptunscraper/spiders/__init__.py new file mode 100644 index 0000000..ebd689a --- /dev/null +++ b/neptunscraper/spiders/__init__.py @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/neptun_webscraper/spiders/dockerhub.py b/neptunscraper/spiders/dockerhub.py similarity index 68% rename from neptun_webscraper/spiders/dockerhub.py rename to neptunscraper/spiders/dockerhub.py index 4e6fea6..f42ed3a 100644 --- a/neptun_webscraper/spiders/dockerhub.py +++ b/neptunscraper/spiders/dockerhub.py @@ -1,63 +1,22 @@ -import json -import os -import re -from datetime import datetime, timedelta - import scrapy -from dateutil.relativedelta import relativedelta from scrapy.selector import Selector from scrapy_playwright.page import PageMethod - -SCRAPY_SETTINGS = { - "DOWNLOAD_HANDLERS": { - "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", - "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", - }, - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", - "PLAYWRIGHT_BROWSER_TYPE": "chromium", - "USER_AGENT": None, # None => using browsers user agent instead -} - - -def parse_update_string(update_string): - print(f"Parsing update string: '{update_string}'") - match = re.search( - r"Updated (a|\d+) (day|week|month|year)s? ago", update_string - ) - if not match: - print(f"String format not recognized: '{update_string}'") - raise ValueError("String format not recognized") - - value_str, unit = match.group(1), match.group(2) - print(f"Extracted value: {value_str}, unit: {unit}") - - # Handle the case where 'a' is used instead of a number - value = 1 if value_str == "a" else int(value_str) - - current_date = datetime.now() - - if unit == "day": - past_date = current_date - timedelta(days=value) - elif unit == "week": - past_date = current_date - timedelta(weeks=value) - elif unit == "month": - past_date = current_date - relativedelta(months=value) - elif unit == "year": - past_date = current_date - relativedelta(years=value) - - formatted_date = past_date.strftime("%Y-%m-%d") - print(f"Calculated past date: {formatted_date}") - return formatted_date +from datetime import datetime +import os +import json class DockerhubDockerRegistrySpider(scrapy.Spider): name = "dockerhubDockerRegistrySpider" - custom_settings = SCRAPY_SETTINGS - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + def __init__(self, query=None, output_dir="output", *args, **kwargs): + super(DockerhubDockerRegistrySpider, self).__init__(*args, **kwargs) self.timestamp = datetime.now().strftime("%Y%m%d%H%M%S") - self.output_dir = f"./neptun_webscraper/spiders/logs/{self.timestamp}" + self.output_dir = output_dir + self.start_urls = [ + f'https://hub.docker.com/search?q={query}&page={i}' + for i in range(1, 11) + ] os.makedirs(self.output_dir, exist_ok=True) self.items = {} @@ -74,14 +33,9 @@ def start_requests(self): playwright_include_page=True, playwright_page_methods=[ PageMethod("wait_for_selector", "div#searchResults"), - # timeouts, if more than one page to crawl - # PageMethod( - # "click", - # selector="button#onetrust-reject-all-handler", - # ), PageMethod( "screenshot", - path=f"{self.output_dir}/screenshot_page_{index}.png", # noqa: E501 + path=f"{self.output_dir}/screenshot_page_{index}.png", full_page=True, ), ], @@ -146,9 +100,9 @@ def parse_result(self, result): # Extract last update and description update_elem = result.css('span:contains("Updated")::text').get() if update_elem: - item["last_update"] = parse_update_string(update_elem.strip()) + item["last_update"] = self.parse_update_string(update_elem.strip()) desc_elem = result.xpath( - './/span[contains(text(), "Updated")]/ancestor::div[1]/following-sibling::p[1]/text()' # noqa: E501 + './/span[contains(text(), "Updated")]/ancestor::div[1]/following-sibling::p[1]/text()' ).get() item["description"] = desc_elem.strip() if desc_elem else None else: @@ -179,12 +133,17 @@ def parse_result(self, result): # Extract stars stars_elem = result.xpath( - '//svg[@data-testid="StarOutlineIcon"]/following-sibling::span/strong/text()' # noqa: E501 + '//svg[@data-testid="StarOutlineIcon"]/following-sibling::span/strong/text()' ).get() item["stars"] = stars_elem.strip() if stars_elem else None return item + @staticmethod + def parse_update_string(update_string): + # Implement the logic to parse the update string into a desired format + return update_string + # https://docs.scrapy.org/en/latest/topics/feed-exports.html?highlight=close#close def close(self, reason): for page_number, items in self.items.items(): @@ -192,5 +151,5 @@ def close(self, reason): with open(filename, "w") as f: json.dump(items, f, indent=2) self.logger.info( - f"Items from page {page_number} have been written to {filename}" # noqa: E501 + f"Items from page {page_number} have been written to {filename}" ) diff --git a/tests/__init__.py b/neptunscraper/spiders/quay.py similarity index 100% rename from tests/__init__.py rename to neptunscraper/spiders/quay.py diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..2043213 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,1183 @@ +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "automat" +version = "22.10.0" +description = "Self-service finite-state machines for the programmer on the go." +optional = false +python-versions = "*" +files = [ + {file = "Automat-22.10.0-py2.py3-none-any.whl", hash = "sha256:c3164f8742b9dc440f3682482d32aaff7bb53f71740dd018533f9de286b64180"}, + {file = "Automat-22.10.0.tar.gz", hash = "sha256:e56beb84edad19dcc11d30e8d9b895f75deeb5ef5e96b84a467066b3b84bb04e"}, +] + +[package.dependencies] +attrs = ">=19.2.0" +six = "*" + +[package.extras] +visualize = ["Twisted (>=16.1.1)", "graphviz (>0.5.1)"] + +[[package]] +name = "certifi" +version = "2024.7.4" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "constantly" +version = "23.10.4" +description = "Symbolic constants in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "constantly-23.10.4-py3-none-any.whl", hash = "sha256:3fd9b4d1c3dc1ec9757f3c52aef7e53ad9323dbe39f51dfd4c43853b68dfa3f9"}, + {file = "constantly-23.10.4.tar.gz", hash = "sha256:aa92b70a33e2ac0bb33cd745eb61776594dc48764b06c35e0efd050b7f1c7cbd"}, +] + +[[package]] +name = "cryptography" +version = "42.0.8" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"}, + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"}, + {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"}, + {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"}, + {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, + {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, + {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"}, + {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] +nox = ["nox"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + +[[package]] +name = "cssselect" +version = "1.2.0" +description = "cssselect parses CSS3 Selectors and translates them to XPath 1.0" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cssselect-1.2.0-py2.py3-none-any.whl", hash = "sha256:da1885f0c10b60c03ed5eccbb6b68d6eff248d91976fcde348f395d54c9fd35e"}, + {file = "cssselect-1.2.0.tar.gz", hash = "sha256:666b19839cfaddb9ce9d36bfe4c969132c647b92fc9088c4e23f786b30f1b3dc"}, +] + +[[package]] +name = "defusedxml" +version = "0.7.1" +description = "XML bomb protection for Python stdlib modules" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, + {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, +] + +[[package]] +name = "filelock" +version = "3.15.4" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "filelock-3.15.4-py3-none-any.whl", hash = "sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7"}, + {file = "filelock-3.15.4.tar.gz", hash = "sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-asyncio (>=0.21)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)", "virtualenv (>=20.26.2)"] +typing = ["typing-extensions (>=4.8)"] + +[[package]] +name = "greenlet" +version = "3.0.3" +description = "Lightweight in-process concurrent programming" +optional = false +python-versions = ">=3.7" +files = [ + {file = "greenlet-3.0.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb"}, + {file = "greenlet-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9"}, + {file = "greenlet-3.0.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d"}, + {file = "greenlet-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728"}, + {file = "greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6"}, + {file = "greenlet-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2"}, + {file = "greenlet-3.0.3-cp37-cp37m-macosx_11_0_universal2.whl", hash = "sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6"}, + {file = "greenlet-3.0.3-cp37-cp37m-win32.whl", hash = "sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d"}, + {file = "greenlet-3.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67"}, + {file = "greenlet-3.0.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da"}, + {file = "greenlet-3.0.3-cp38-cp38-win32.whl", hash = "sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3"}, + {file = "greenlet-3.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf"}, + {file = "greenlet-3.0.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113"}, + {file = "greenlet-3.0.3-cp39-cp39-win32.whl", hash = "sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e"}, + {file = "greenlet-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067"}, + {file = "greenlet-3.0.3.tar.gz", hash = "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491"}, +] + +[package.extras] +docs = ["Sphinx", "furo"] +test = ["objgraph", "psutil"] + +[[package]] +name = "hyperlink" +version = "21.0.0" +description = "A featureful, immutable, and correct URL for Python." +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "hyperlink-21.0.0-py2.py3-none-any.whl", hash = "sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4"}, + {file = "hyperlink-21.0.0.tar.gz", hash = "sha256:427af957daa58bc909471c6c40f74c5450fa123dd093fc53efd2e91d2705a56b"}, +] + +[package.dependencies] +idna = ">=2.5" + +[[package]] +name = "idna" +version = "3.7" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, +] + +[[package]] +name = "incremental" +version = "22.10.0" +description = "\"A small library that versions your Python projects.\"" +optional = false +python-versions = "*" +files = [ + {file = "incremental-22.10.0-py2.py3-none-any.whl", hash = "sha256:b864a1f30885ee72c5ac2835a761b8fe8aa9c28b9395cacf27286602688d3e51"}, + {file = "incremental-22.10.0.tar.gz", hash = "sha256:912feeb5e0f7e0188e6f42241d2f450002e11bbc0937c65865045854c24c0bd0"}, +] + +[package.extras] +mypy = ["click (>=6.0)", "mypy (==0.812)", "twisted (>=16.4.0)"] +scripts = ["click (>=6.0)", "twisted (>=16.4.0)"] + +[[package]] +name = "itemadapter" +version = "0.9.0" +description = "Common interface for data container classes" +optional = false +python-versions = ">=3.8" +files = [ + {file = "itemadapter-0.9.0-py3-none-any.whl", hash = "sha256:cfd108c9d5205d056fcac402ec8f8e9d799ce9066911eec1cd521ea442f87af1"}, + {file = "itemadapter-0.9.0.tar.gz", hash = "sha256:e4f958a6b6b6f5831fa207373010031a0bd7ed0429ddd09b51979c011475cafd"}, +] + +[[package]] +name = "itemloaders" +version = "1.3.1" +description = "Base library for scrapy's ItemLoader" +optional = false +python-versions = ">=3.8" +files = [ + {file = "itemloaders-1.3.1-py3-none-any.whl", hash = "sha256:70be155cd050b8c532e1054f0241dcc4711bd15e62c0a0174963d1c110d9f0fa"}, + {file = "itemloaders-1.3.1.tar.gz", hash = "sha256:81571c941cc189bb55e211f0cd3476fde7511239d3bf7ff91eb6ed68a1b0ec10"}, +] + +[package.dependencies] +itemadapter = ">=0.1.0" +jmespath = ">=0.9.5" +parsel = ">=1.5.0" +w3lib = ">=1.17.0" + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + +[[package]] +name = "lxml" +version = "5.2.2" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +optional = false +python-versions = ">=3.6" +files = [ + {file = "lxml-5.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:364d03207f3e603922d0d3932ef363d55bbf48e3647395765f9bfcbdf6d23632"}, + {file = "lxml-5.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:50127c186f191b8917ea2fb8b206fbebe87fd414a6084d15568c27d0a21d60db"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:74e4f025ef3db1c6da4460dd27c118d8cd136d0391da4e387a15e48e5c975147"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:981a06a3076997adf7c743dcd0d7a0415582661e2517c7d961493572e909aa1d"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aef5474d913d3b05e613906ba4090433c515e13ea49c837aca18bde190853dff"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1e275ea572389e41e8b039ac076a46cb87ee6b8542df3fff26f5baab43713bca"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5b65529bb2f21ac7861a0e94fdbf5dc0daab41497d18223b46ee8515e5ad297"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:bcc98f911f10278d1daf14b87d65325851a1d29153caaf146877ec37031d5f36"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:b47633251727c8fe279f34025844b3b3a3e40cd1b198356d003aa146258d13a2"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:fbc9d316552f9ef7bba39f4edfad4a734d3d6f93341232a9dddadec4f15d425f"}, + {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:13e69be35391ce72712184f69000cda04fc89689429179bc4c0ae5f0b7a8c21b"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3b6a30a9ab040b3f545b697cb3adbf3696c05a3a68aad172e3fd7ca73ab3c835"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a233bb68625a85126ac9f1fc66d24337d6e8a0f9207b688eec2e7c880f012ec0"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:dfa7c241073d8f2b8e8dbc7803c434f57dbb83ae2a3d7892dd068d99e96efe2c"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1a7aca7964ac4bb07680d5c9d63b9d7028cace3e2d43175cb50bba8c5ad33316"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ae4073a60ab98529ab8a72ebf429f2a8cc612619a8c04e08bed27450d52103c0"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ffb2be176fed4457e445fe540617f0252a72a8bc56208fd65a690fdb1f57660b"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e290d79a4107d7d794634ce3e985b9ae4f920380a813717adf61804904dc4393"}, + {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:96e85aa09274955bb6bd483eaf5b12abadade01010478154b0ec70284c1b1526"}, + {file = "lxml-5.2.2-cp310-cp310-win32.whl", hash = "sha256:f956196ef61369f1685d14dad80611488d8dc1ef00be57c0c5a03064005b0f30"}, + {file = "lxml-5.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:875a3f90d7eb5c5d77e529080d95140eacb3c6d13ad5b616ee8095447b1d22e7"}, + {file = "lxml-5.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:45f9494613160d0405682f9eee781c7e6d1bf45f819654eb249f8f46a2c22545"}, + {file = "lxml-5.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b0b3f2df149efb242cee2ffdeb6674b7f30d23c9a7af26595099afaf46ef4e88"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d28cb356f119a437cc58a13f8135ab8a4c8ece18159eb9194b0d269ec4e28083"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:657a972f46bbefdbba2d4f14413c0d079f9ae243bd68193cb5061b9732fa54c1"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b74b9ea10063efb77a965a8d5f4182806fbf59ed068b3c3fd6f30d2ac7bee734"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07542787f86112d46d07d4f3c4e7c760282011b354d012dc4141cc12a68cef5f"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:303f540ad2dddd35b92415b74b900c749ec2010e703ab3bfd6660979d01fd4ed"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:2eb2227ce1ff998faf0cd7fe85bbf086aa41dfc5af3b1d80867ecfe75fb68df3"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:1d8a701774dfc42a2f0b8ccdfe7dbc140500d1049e0632a611985d943fcf12df"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:56793b7a1a091a7c286b5f4aa1fe4ae5d1446fe742d00cdf2ffb1077865db10d"}, + {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:eb00b549b13bd6d884c863554566095bf6fa9c3cecb2e7b399c4bc7904cb33b5"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1a2569a1f15ae6c8c64108a2cd2b4a858fc1e13d25846be0666fc144715e32ab"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:8cf85a6e40ff1f37fe0f25719aadf443686b1ac7652593dc53c7ef9b8492b115"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:d237ba6664b8e60fd90b8549a149a74fcc675272e0e95539a00522e4ca688b04"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0b3f5016e00ae7630a4b83d0868fca1e3d494c78a75b1c7252606a3a1c5fc2ad"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:23441e2b5339bc54dc949e9e675fa35efe858108404ef9aa92f0456929ef6fe8"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2fb0ba3e8566548d6c8e7dd82a8229ff47bd8fb8c2da237607ac8e5a1b8312e5"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:79d1fb9252e7e2cfe4de6e9a6610c7cbb99b9708e2c3e29057f487de5a9eaefa"}, + {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6dcc3d17eac1df7859ae01202e9bb11ffa8c98949dcbeb1069c8b9a75917e01b"}, + {file = "lxml-5.2.2-cp311-cp311-win32.whl", hash = "sha256:4c30a2f83677876465f44c018830f608fa3c6a8a466eb223535035fbc16f3438"}, + {file = "lxml-5.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:49095a38eb333aaf44c06052fd2ec3b8f23e19747ca7ec6f6c954ffea6dbf7be"}, + {file = "lxml-5.2.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:7429e7faa1a60cad26ae4227f4dd0459efde239e494c7312624ce228e04f6391"}, + {file = "lxml-5.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:50ccb5d355961c0f12f6cf24b7187dbabd5433f29e15147a67995474f27d1776"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc911208b18842a3a57266d8e51fc3cfaccee90a5351b92079beed912a7914c2"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33ce9e786753743159799fdf8e92a5da351158c4bfb6f2db0bf31e7892a1feb5"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ec87c44f619380878bd49ca109669c9f221d9ae6883a5bcb3616785fa8f94c97"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08ea0f606808354eb8f2dfaac095963cb25d9d28e27edcc375d7b30ab01abbf6"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75a9632f1d4f698b2e6e2e1ada40e71f369b15d69baddb8968dcc8e683839b18"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:74da9f97daec6928567b48c90ea2c82a106b2d500f397eeb8941e47d30b1ca85"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:0969e92af09c5687d769731e3f39ed62427cc72176cebb54b7a9d52cc4fa3b73"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:9164361769b6ca7769079f4d426a41df6164879f7f3568be9086e15baca61466"}, + {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d26a618ae1766279f2660aca0081b2220aca6bd1aa06b2cf73f07383faf48927"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab67ed772c584b7ef2379797bf14b82df9aa5f7438c5b9a09624dd834c1c1aaf"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:3d1e35572a56941b32c239774d7e9ad724074d37f90c7a7d499ab98761bd80cf"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:8268cbcd48c5375f46e000adb1390572c98879eb4f77910c6053d25cc3ac2c67"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e282aedd63c639c07c3857097fc0e236f984ceb4089a8b284da1c526491e3f3d"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfdc2bfe69e9adf0df4915949c22a25b39d175d599bf98e7ddf620a13678585"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4aefd911793b5d2d7a921233a54c90329bf3d4a6817dc465f12ffdfe4fc7b8fe"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:8b8df03a9e995b6211dafa63b32f9d405881518ff1ddd775db4e7b98fb545e1c"}, + {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f11ae142f3a322d44513de1018b50f474f8f736bc3cd91d969f464b5bfef8836"}, + {file = "lxml-5.2.2-cp312-cp312-win32.whl", hash = "sha256:16a8326e51fcdffc886294c1e70b11ddccec836516a343f9ed0f82aac043c24a"}, + {file = "lxml-5.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:bbc4b80af581e18568ff07f6395c02114d05f4865c2812a1f02f2eaecf0bfd48"}, + {file = "lxml-5.2.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e3d9d13603410b72787579769469af730c38f2f25505573a5888a94b62b920f8"}, + {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38b67afb0a06b8575948641c1d6d68e41b83a3abeae2ca9eed2ac59892b36706"}, + {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c689d0d5381f56de7bd6966a4541bff6e08bf8d3871bbd89a0c6ab18aa699573"}, + {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:cf2a978c795b54c539f47964ec05e35c05bd045db5ca1e8366988c7f2fe6b3ce"}, + {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:739e36ef7412b2bd940f75b278749106e6d025e40027c0b94a17ef7968d55d56"}, + {file = "lxml-5.2.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d8bbcd21769594dbba9c37d3c819e2d5847656ca99c747ddb31ac1701d0c0ed9"}, + {file = "lxml-5.2.2-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:2304d3c93f2258ccf2cf7a6ba8c761d76ef84948d87bf9664e14d203da2cd264"}, + {file = "lxml-5.2.2-cp36-cp36m-win32.whl", hash = "sha256:02437fb7308386867c8b7b0e5bc4cd4b04548b1c5d089ffb8e7b31009b961dc3"}, + {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"}, + {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"}, + {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"}, + {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"}, + {file = "lxml-5.2.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7ed07b3062b055d7a7f9d6557a251cc655eed0b3152b76de619516621c56f5d3"}, + {file = "lxml-5.2.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f60fdd125d85bf9c279ffb8e94c78c51b3b6a37711464e1f5f31078b45002421"}, + {file = "lxml-5.2.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a7e24cb69ee5f32e003f50e016d5fde438010c1022c96738b04fc2423e61706"}, + {file = "lxml-5.2.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23cfafd56887eaed93d07bc4547abd5e09d837a002b791e9767765492a75883f"}, + {file = "lxml-5.2.2-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:19b4e485cd07b7d83e3fe3b72132e7df70bfac22b14fe4bf7a23822c3a35bff5"}, + {file = "lxml-5.2.2-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:7ce7ad8abebe737ad6143d9d3bf94b88b93365ea30a5b81f6877ec9c0dee0a48"}, + {file = "lxml-5.2.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e49b052b768bb74f58c7dda4e0bdf7b79d43a9204ca584ffe1fb48a6f3c84c66"}, + {file = "lxml-5.2.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d14a0d029a4e176795cef99c056d58067c06195e0c7e2dbb293bf95c08f772a3"}, + {file = "lxml-5.2.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:be49ad33819d7dcc28a309b86d4ed98e1a65f3075c6acd3cd4fe32103235222b"}, + {file = "lxml-5.2.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:a6d17e0370d2516d5bb9062c7b4cb731cff921fc875644c3d751ad857ba9c5b1"}, + {file = "lxml-5.2.2-cp38-cp38-win32.whl", hash = "sha256:5b8c041b6265e08eac8a724b74b655404070b636a8dd6d7a13c3adc07882ef30"}, + {file = "lxml-5.2.2-cp38-cp38-win_amd64.whl", hash = "sha256:f61efaf4bed1cc0860e567d2ecb2363974d414f7f1f124b1df368bbf183453a6"}, + {file = "lxml-5.2.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:fb91819461b1b56d06fa4bcf86617fac795f6a99d12239fb0c68dbeba41a0a30"}, + {file = "lxml-5.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d4ed0c7cbecde7194cd3228c044e86bf73e30a23505af852857c09c24e77ec5d"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54401c77a63cc7d6dc4b4e173bb484f28a5607f3df71484709fe037c92d4f0ed"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:625e3ef310e7fa3a761d48ca7ea1f9d8718a32b1542e727d584d82f4453d5eeb"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:519895c99c815a1a24a926d5b60627ce5ea48e9f639a5cd328bda0515ea0f10c"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c7079d5eb1c1315a858bbf180000757db8ad904a89476653232db835c3114001"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:343ab62e9ca78094f2306aefed67dcfad61c4683f87eee48ff2fd74902447726"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:cd9e78285da6c9ba2d5c769628f43ef66d96ac3085e59b10ad4f3707980710d3"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_ppc64le.whl", hash = "sha256:546cf886f6242dff9ec206331209db9c8e1643ae642dea5fdbecae2453cb50fd"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_s390x.whl", hash = "sha256:02f6a8eb6512fdc2fd4ca10a49c341c4e109aa6e9448cc4859af5b949622715a"}, + {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:339ee4a4704bc724757cd5dd9dc8cf4d00980f5d3e6e06d5847c1b594ace68ab"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0a028b61a2e357ace98b1615fc03f76eb517cc028993964fe08ad514b1e8892d"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:f90e552ecbad426eab352e7b2933091f2be77115bb16f09f78404861c8322981"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d83e2d94b69bf31ead2fa45f0acdef0757fa0458a129734f59f67f3d2eb7ef32"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a02d3c48f9bb1e10c7788d92c0c7db6f2002d024ab6e74d6f45ae33e3d0288a3"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6d68ce8e7b2075390e8ac1e1d3a99e8b6372c694bbe612632606d1d546794207"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:453d037e09a5176d92ec0fd282e934ed26d806331a8b70ab431a81e2fbabf56d"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:3b019d4ee84b683342af793b56bb35034bd749e4cbdd3d33f7d1107790f8c472"}, + {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:cb3942960f0beb9f46e2a71a3aca220d1ca32feb5a398656be934320804c0df9"}, + {file = "lxml-5.2.2-cp39-cp39-win32.whl", hash = "sha256:ac6540c9fff6e3813d29d0403ee7a81897f1d8ecc09a8ff84d2eea70ede1cdbf"}, + {file = "lxml-5.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:610b5c77428a50269f38a534057444c249976433f40f53e3b47e68349cca1425"}, + {file = "lxml-5.2.2-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b537bd04d7ccd7c6350cdaaaad911f6312cbd61e6e6045542f781c7f8b2e99d2"}, + {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4820c02195d6dfb7b8508ff276752f6b2ff8b64ae5d13ebe02e7667e035000b9"}, + {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a09f6184f17a80897172863a655467da2b11151ec98ba8d7af89f17bf63dae"}, + {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:76acba4c66c47d27c8365e7c10b3d8016a7da83d3191d053a58382311a8bf4e1"}, + {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b128092c927eaf485928cec0c28f6b8bead277e28acf56800e972aa2c2abd7a2"}, + {file = "lxml-5.2.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ae791f6bd43305aade8c0e22f816b34f3b72b6c820477aab4d18473a37e8090b"}, + {file = "lxml-5.2.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a2f6a1bc2460e643785a2cde17293bd7a8f990884b822f7bca47bee0a82fc66b"}, + {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e8d351ff44c1638cb6e980623d517abd9f580d2e53bfcd18d8941c052a5a009"}, + {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bec4bd9133420c5c52d562469c754f27c5c9e36ee06abc169612c959bd7dbb07"}, + {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:55ce6b6d803890bd3cc89975fca9de1dff39729b43b73cb15ddd933b8bc20484"}, + {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8ab6a358d1286498d80fe67bd3d69fcbc7d1359b45b41e74c4a26964ca99c3f8"}, + {file = "lxml-5.2.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:06668e39e1f3c065349c51ac27ae430719d7806c026fec462e5693b08b95696b"}, + {file = "lxml-5.2.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9cd5323344d8ebb9fb5e96da5de5ad4ebab993bbf51674259dbe9d7a18049525"}, + {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89feb82ca055af0fe797a2323ec9043b26bc371365847dbe83c7fd2e2f181c34"}, + {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e481bba1e11ba585fb06db666bfc23dbe181dbafc7b25776156120bf12e0d5a6"}, + {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d6c6ea6a11ca0ff9cd0390b885984ed31157c168565702959c25e2191674a14"}, + {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3d98de734abee23e61f6b8c2e08a88453ada7d6486dc7cdc82922a03968928db"}, + {file = "lxml-5.2.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:69ab77a1373f1e7563e0fb5a29a8440367dec051da6c7405333699d07444f511"}, + {file = "lxml-5.2.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:34e17913c431f5ae01d8658dbf792fdc457073dcdfbb31dc0cc6ab256e664a8d"}, + {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05f8757b03208c3f50097761be2dea0aba02e94f0dc7023ed73a7bb14ff11eb0"}, + {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a520b4f9974b0a0a6ed73c2154de57cdfd0c8800f4f15ab2b73238ffed0b36e"}, + {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5e097646944b66207023bc3c634827de858aebc226d5d4d6d16f0b77566ea182"}, + {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b5e4ef22ff25bfd4ede5f8fb30f7b24446345f3e79d9b7455aef2836437bc38a"}, + {file = "lxml-5.2.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:ff69a9a0b4b17d78170c73abe2ab12084bdf1691550c5629ad1fe7849433f324"}, + {file = "lxml-5.2.2.tar.gz", hash = "sha256:bb2dc4898180bea79863d5487e5f9c7c34297414bad54bcd0f0852aee9cfdb87"}, +] + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html-clean = ["lxml-html-clean"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] +source = ["Cython (>=3.0.10)"] + +[[package]] +name = "packaging" +version = "24.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, +] + +[[package]] +name = "parsel" +version = "1.9.1" +description = "Parsel is a library to extract data from HTML and XML using XPath and CSS selectors" +optional = false +python-versions = ">=3.8" +files = [ + {file = "parsel-1.9.1-py2.py3-none-any.whl", hash = "sha256:c4a777ee6c3ff5e39652b58e351c5cf02c12ff420d05b07a7966aebb68ab1700"}, + {file = "parsel-1.9.1.tar.gz", hash = "sha256:14e00dc07731c9030db620c195fcae884b5b4848e9f9c523c6119f708ccfa9ac"}, +] + +[package.dependencies] +cssselect = ">=1.2.0" +jmespath = "*" +lxml = "*" +packaging = "*" +w3lib = ">=1.19.0" + +[[package]] +name = "playwright" +version = "1.45.0" +description = "A high-level API to automate web browsers" +optional = false +python-versions = ">=3.8" +files = [ + {file = "playwright-1.45.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:7d49aee5907d8e72060f04bc299cb6851c2dc44cb227540ade89d7aa529e907a"}, + {file = "playwright-1.45.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:210c9f848820f58b5b5ed48047748620b780ca3acc3e2b7560dafb2bfdd6d90a"}, + {file = "playwright-1.45.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:13b5398831f5499580e819ddc996633446a93bf88029e89451e51da188e16ae3"}, + {file = "playwright-1.45.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:0ba5a39f25fb9b9cf1bd48678f44536a29f6d83376329de2dee1567dac220afe"}, + {file = "playwright-1.45.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b09fa76614ba2926d45a4c0581f710c13652d5e32290ba6a1490fbafff7f0be8"}, + {file = "playwright-1.45.0-py3-none-win32.whl", hash = "sha256:97a7d53af89af54208b69c051046b462675fcf5b93f7fbfb7c0fa7f813424ee2"}, + {file = "playwright-1.45.0-py3-none-win_amd64.whl", hash = "sha256:701db496928429aec103739e48e3110806bd5cf49456cc95b89f28e1abda71da"}, +] + +[package.dependencies] +greenlet = "3.0.3" +pyee = "11.1.0" + +[[package]] +name = "protego" +version = "0.3.1" +description = "Pure-Python robots.txt parser with support for modern conventions" +optional = false +python-versions = ">=3.8" +files = [ + {file = "Protego-0.3.1-py2.py3-none-any.whl", hash = "sha256:2fbe8e9b7a7dbc5016a932b14c98d236aad4c29290bbe457b8d2779666ef7a41"}, + {file = "Protego-0.3.1.tar.gz", hash = "sha256:e94430d0d25cbbf239bc849d86c5e544fbde531fcccfa059953c7da344a1712c"}, +] + +[[package]] +name = "pyasn1" +version = "0.6.0" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyasn1-0.6.0-py2.py3-none-any.whl", hash = "sha256:cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473"}, + {file = "pyasn1-0.6.0.tar.gz", hash = "sha256:3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.0" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyasn1_modules-0.4.0-py3-none-any.whl", hash = "sha256:be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b"}, + {file = "pyasn1_modules-0.4.0.tar.gz", hash = "sha256:831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.7.0" + +[[package]] +name = "pycparser" +version = "2.22" +description = "C parser in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, +] + +[[package]] +name = "pydispatcher" +version = "2.0.7" +description = "Multi-producer multi-consumer in-memory signal dispatch system" +optional = false +python-versions = "*" +files = [ + {file = "PyDispatcher-2.0.7-py3-none-any.whl", hash = "sha256:96543bea04115ffde08f851e1d45cacbfd1ee866ac42127d9b476dc5aefa7de0"}, + {file = "PyDispatcher-2.0.7.tar.gz", hash = "sha256:b777c6ad080dc1bad74a4c29d6a46914fa6701ac70f94b0d66fbcfde62f5be31"}, +] + +[package.extras] +dev = ["tox"] + +[[package]] +name = "pyee" +version = "11.1.0" +description = "A rough port of Node.js's EventEmitter to Python with a few tricks of its own" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyee-11.1.0-py3-none-any.whl", hash = "sha256:5d346a7d0f861a4b2e6c47960295bd895f816725b27d656181947346be98d7c1"}, + {file = "pyee-11.1.0.tar.gz", hash = "sha256:b53af98f6990c810edd9b56b87791021a8f54fd13db4edd1142438d44ba2263f"}, +] + +[package.dependencies] +typing-extensions = "*" + +[package.extras] +dev = ["black", "build", "flake8", "flake8-black", "isort", "jupyter-console", "mkdocs", "mkdocs-include-markdown-plugin", "mkdocstrings[python]", "pytest", "pytest-asyncio", "pytest-trio", "sphinx", "toml", "tox", "trio", "trio", "trio-typing", "twine", "twisted", "validate-pyproject[all]"] + +[[package]] +name = "pyopenssl" +version = "24.1.0" +description = "Python wrapper module around the OpenSSL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyOpenSSL-24.1.0-py3-none-any.whl", hash = "sha256:17ed5be5936449c5418d1cd269a1a9e9081bc54c17aed272b45856a3d3dc86ad"}, + {file = "pyOpenSSL-24.1.0.tar.gz", hash = "sha256:cabed4bfaa5df9f1a16c0ef64a0cb65318b5cd077a7eda7d6970131ca2f41a6f"}, +] + +[package.dependencies] +cryptography = ">=41.0.5,<43" + +[package.extras] +docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx-rtd-theme"] +test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"] + +[[package]] +name = "pypydispatcher" +version = "2.1.2" +description = "Multi-producer-multi-consumer signal dispatching mechanism" +optional = false +python-versions = "*" +files = [ + {file = "PyPyDispatcher-2.1.2.tar.gz", hash = "sha256:b6bec5dfcff9d2535bca2b23c80eae367b1ac250a645106948d315fcfa9130f2"}, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "queuelib" +version = "1.7.0" +description = "Collection of persistent (disk-based) and non-persistent (memory-based) queues" +optional = false +python-versions = ">=3.8" +files = [ + {file = "queuelib-1.7.0-py2.py3-none-any.whl", hash = "sha256:b07aaa2410caac3a0021ee4f4026acdac992b0fb9a2cbeb34a918617df3c12a7"}, + {file = "queuelib-1.7.0.tar.gz", hash = "sha256:2855162096cf0230510890b354379ea1c0ff19d105d3147d349d2433bb222b08"}, +] + +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-file" +version = "2.1.0" +description = "File transport adapter for Requests" +optional = false +python-versions = "*" +files = [ + {file = "requests_file-2.1.0-py2.py3-none-any.whl", hash = "sha256:cf270de5a4c5874e84599fc5778303d496c10ae5e870bfa378818f35d21bda5c"}, + {file = "requests_file-2.1.0.tar.gz", hash = "sha256:0f549a3f3b0699415ac04d167e9cb39bccfb730cb832b4d20be3d9867356e658"}, +] + +[package.dependencies] +requests = ">=1.0.0" + +[[package]] +name = "scrapy" +version = "2.11.2" +description = "A high-level Web Crawling and Web Scraping framework" +optional = false +python-versions = ">=3.8" +files = [ + {file = "Scrapy-2.11.2-py2.py3-none-any.whl", hash = "sha256:4be353d6abbb942a9f7e7614ca8b5f3d9037381176ac8d8859c8cac676e74fa0"}, + {file = "scrapy-2.11.2.tar.gz", hash = "sha256:dfbd565384fc3fffeba121f5a3a2d0899ac1f756d41432ca0879933fbfb3401d"}, +] + +[package.dependencies] +cryptography = ">=36.0.0" +cssselect = ">=0.9.1" +defusedxml = ">=0.7.1" +itemadapter = ">=0.1.0" +itemloaders = ">=1.0.1" +lxml = ">=4.4.1" +packaging = "*" +parsel = ">=1.5.0" +protego = ">=0.1.15" +PyDispatcher = {version = ">=2.0.5", markers = "platform_python_implementation == \"CPython\""} +pyOpenSSL = ">=21.0.0" +PyPyDispatcher = {version = ">=2.1.0", markers = "platform_python_implementation == \"PyPy\""} +queuelib = ">=1.4.2" +service-identity = ">=18.1.0" +setuptools = "*" +tldextract = "*" +Twisted = ">=18.9.0" +w3lib = ">=1.17.0" +"zope.interface" = ">=5.1.0" + +[[package]] +name = "scrapy-playwright" +version = "0.0.38" +description = "Playwright integration for Scrapy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "scrapy_playwright-0.0.38-py3-none-any.whl", hash = "sha256:7ffb3db8dbd2b240de7f8a4f7b6f8806a0997c5d2d150abe940951a9afcd2135"}, + {file = "scrapy_playwright-0.0.38.tar.gz", hash = "sha256:7df0440fd46423122b2adb7cc67ec449d62cb5d7ee304bba7a20a8e326fa05d0"}, +] + +[package.dependencies] +playwright = ">=1.15" +scrapy = ">=2.0,<2.4.0 || >2.4.0" + +[[package]] +name = "scrapyd" +version = "1.4.3" +description = "A service for running Scrapy spiders, with an HTTP API" +optional = false +python-versions = "*" +files = [ + {file = "scrapyd-1.4.3-py2.py3-none-any.whl", hash = "sha256:4ad70b3043e76b125316ce71d1ed1cc8e17e8c3f75e0fa95208fadcb69d9407e"}, + {file = "scrapyd-1.4.3.tar.gz", hash = "sha256:ad2cfb47cdfb03e30d0401db6ff95864988ad5cce84081ad6b55f2e2b52b50bd"}, +] + +[package.dependencies] +packaging = "*" +scrapy = ">=2.0.0" +setuptools = "*" +twisted = ">=17.9" +w3lib = "*" +"zope.interface" = "*" + +[package.extras] +docs = ["furo", "sphinx", "sphinx-autobuild"] +test = ["pytest", "pytest-cov", "requests"] + +[[package]] +name = "scrapyd-client" +version = "1.2.3" +description = "A client for Scrapyd" +optional = false +python-versions = ">=3.6" +files = [ + {file = "scrapyd-client-1.2.3.tar.gz", hash = "sha256:fef27cc0d8bf54cbd03467250fb0705635733de81a577b0ba00c739c2d96c894"}, + {file = "scrapyd_client-1.2.3-py3-none-any.whl", hash = "sha256:beeab9310f1ff942978ca472e4ccdc88648cfde3c33734a11337a93514a43859"}, +] + +[package.dependencies] +requests = "*" +scrapy = ">=0.17" +uberegg = ">=0.1.1" +urllib3 = "*" +w3lib = "*" + +[package.extras] +test = ["pytest", "pytest-console-scripts", "pytest-cov", "pytest-mock"] + +[[package]] +name = "service-identity" +version = "24.1.0" +description = "Service identity verification for pyOpenSSL & cryptography." +optional = false +python-versions = ">=3.8" +files = [ + {file = "service_identity-24.1.0-py3-none-any.whl", hash = "sha256:a28caf8130c8a5c1c7a6f5293faaf239bbfb7751e4862436920ee6f2616f568a"}, + {file = "service_identity-24.1.0.tar.gz", hash = "sha256:6829c9d62fb832c2e1c435629b0a8c476e1929881f28bee4d20bc24161009221"}, +] + +[package.dependencies] +attrs = ">=19.1.0" +cryptography = "*" +pyasn1 = "*" +pyasn1-modules = "*" + +[package.extras] +dev = ["pyopenssl", "service-identity[idna,mypy,tests]"] +docs = ["furo", "myst-parser", "pyopenssl", "sphinx", "sphinx-notfound-page"] +idna = ["idna"] +mypy = ["idna", "mypy", "types-pyopenssl"] +tests = ["coverage[toml] (>=5.0.2)", "pytest"] + +[[package]] +name = "setuptools" +version = "70.2.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-70.2.0-py3-none-any.whl", hash = "sha256:b8b8060bb426838fbe942479c90296ce976249451118ef566a5a0b7d8b78fb05"}, + {file = "setuptools-70.2.0.tar.gz", hash = "sha256:bd63e505105011b25c3c11f753f7e3b8465ea739efddaccef8f0efac2137bac1"}, +] + +[package.extras] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "tldextract" +version = "5.1.2" +description = "Accurately separates a URL's subdomain, domain, and public suffix, using the Public Suffix List (PSL). By default, this includes the public ICANN TLDs and their exceptions. You can optionally support the Public Suffix List's private domains as well." +optional = false +python-versions = ">=3.8" +files = [ + {file = "tldextract-5.1.2-py3-none-any.whl", hash = "sha256:4dfc4c277b6b97fa053899fcdb892d2dc27295851ab5fac4e07797b6a21b2e46"}, + {file = "tldextract-5.1.2.tar.gz", hash = "sha256:c9e17f756f05afb5abac04fe8f766e7e70f9fe387adb1859f0f52408ee060200"}, +] + +[package.dependencies] +filelock = ">=3.0.8" +idna = "*" +requests = ">=2.1.0" +requests-file = ">=1.4" + +[package.extras] +release = ["build", "twine"] +testing = ["black", "mypy", "pytest", "pytest-gitignore", "pytest-mock", "responses", "ruff", "syrupy", "tox", "types-filelock", "types-requests"] + +[[package]] +name = "twisted" +version = "24.3.0" +description = "An asynchronous networking framework written in Python" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "twisted-24.3.0-py3-none-any.whl", hash = "sha256:039f2e6a49ab5108abd94de187fa92377abe5985c7a72d68d0ad266ba19eae63"}, + {file = "twisted-24.3.0.tar.gz", hash = "sha256:6b38b6ece7296b5e122c9eb17da2eeab3d98a198f50ca9efd00fb03e5b4fd4ae"}, +] + +[package.dependencies] +attrs = ">=21.3.0" +automat = ">=0.8.0" +constantly = ">=15.1" +hyperlink = ">=17.1.1" +incremental = ">=22.10.0" +twisted-iocpsupport = {version = ">=1.0.2,<2", markers = "platform_system == \"Windows\""} +typing-extensions = ">=4.2.0" +zope-interface = ">=5" + +[package.extras] +all-non-platform = ["twisted[conch,http2,serial,test,tls]", "twisted[conch,http2,serial,test,tls]"] +conch = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)"] +dev = ["coverage (>=6b1,<7)", "pyflakes (>=2.2,<3.0)", "python-subunit (>=1.4,<2.0)", "twisted[dev-release]", "twistedchecker (>=0.7,<1.0)"] +dev-release = ["pydoctor (>=23.9.0,<23.10.0)", "pydoctor (>=23.9.0,<23.10.0)", "sphinx (>=6,<7)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "towncrier (>=23.6,<24.0)"] +gtk-platform = ["pygobject", "pygobject", "twisted[all-non-platform]", "twisted[all-non-platform]"] +http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"] +macos-platform = ["pyobjc-core", "pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyobjc-framework-cocoa", "twisted[all-non-platform]", "twisted[all-non-platform]"] +mypy = ["mypy (>=1.8,<2.0)", "mypy-zope (>=1.0.3,<1.1.0)", "twisted[all-non-platform,dev]", "types-pyopenssl", "types-setuptools"] +osx-platform = ["twisted[macos-platform]", "twisted[macos-platform]"] +serial = ["pyserial (>=3.0)", "pywin32 (!=226)"] +test = ["cython-test-exception-raiser (>=1.0.2,<2)", "hypothesis (>=6.56)", "pyhamcrest (>=2)"] +tls = ["idna (>=2.4)", "pyopenssl (>=21.0.0)", "service-identity (>=18.1.0)"] +windows-platform = ["pywin32 (!=226)", "pywin32 (!=226)", "twisted[all-non-platform]", "twisted[all-non-platform]"] + +[[package]] +name = "twisted-iocpsupport" +version = "1.0.4" +description = "An extension for use in the twisted I/O Completion Ports reactor." +optional = false +python-versions = "*" +files = [ + {file = "twisted-iocpsupport-1.0.4.tar.gz", hash = "sha256:858096c0d15e33f15ac157f455d8f86f2f2cdd223963e58c0f682a3af8362d89"}, + {file = "twisted_iocpsupport-1.0.4-cp310-cp310-win32.whl", hash = "sha256:afa2b630797f9ed2f27f3d9f55e3f72b4244911e45a8c82756f44babbf0b243e"}, + {file = "twisted_iocpsupport-1.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:0058c963c8957bcd3deda62122e89953c9de1e867a274facc9b15dde1a9f31e8"}, + {file = "twisted_iocpsupport-1.0.4-cp311-cp311-win32.whl", hash = "sha256:196f7c7ccad4ba4d1783b1c4e1d1b22d93c04275cd780bf7498d16c77319ad6e"}, + {file = "twisted_iocpsupport-1.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:4e5f97bcbabdd79cbaa969b63439b89801ea560f11d42b0a387634275c633623"}, + {file = "twisted_iocpsupport-1.0.4-cp312-cp312-win32.whl", hash = "sha256:6081bd7c2f4fcf9b383dcdb3b3385d75a26a7c9d2be25b6950c3d8ea652d2d2d"}, + {file = "twisted_iocpsupport-1.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:76f7e67cec1f1d097d1f4ed7de41be3d74546e1a4ede0c7d56e775c4dce5dfb0"}, + {file = "twisted_iocpsupport-1.0.4-cp36-cp36m-win32.whl", hash = "sha256:3d306fc4d88a6bcf61ce9d572c738b918578121bfd72891625fab314549024b5"}, + {file = "twisted_iocpsupport-1.0.4-cp36-cp36m-win_amd64.whl", hash = "sha256:391ac4d6002a80e15f35adc4ad6056f4fe1c17ceb0d1f98ba01b0f4f917adfd7"}, + {file = "twisted_iocpsupport-1.0.4-cp37-cp37m-win32.whl", hash = "sha256:0c1b5cf37f0b2d96cc3c9bc86fff16613b9f5d0ca565c96cf1f1fb8cfca4b81c"}, + {file = "twisted_iocpsupport-1.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:3c5dc11d72519e55f727320e3cee535feedfaee09c0f0765ed1ca7badff1ab3c"}, + {file = "twisted_iocpsupport-1.0.4-cp38-cp38-win32.whl", hash = "sha256:cc86c2ef598c15d824a243c2541c29459881c67fc3c0adb6efe2242f8f0ec3af"}, + {file = "twisted_iocpsupport-1.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:c27985e949b9b1a1fb4c20c71d315c10ea0f93fdf3ccdd4a8c158b5926edd8c8"}, + {file = "twisted_iocpsupport-1.0.4-cp39-cp39-win32.whl", hash = "sha256:e311dfcb470696e3c077249615893cada598e62fa7c4e4ca090167bd2b7d331f"}, + {file = "twisted_iocpsupport-1.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:4574eef1f3bb81501fb02f911298af3c02fe8179c31a33b361dd49180c3e644d"}, + {file = "twisted_iocpsupport-1.0.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:872747a3b64e2909aee59c803ccd0bceb9b75bf27915520ebd32d69687040fa2"}, + {file = "twisted_iocpsupport-1.0.4-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:c2712b778bacf1db434e3e065adfed3db300754186a29aecac1efae9ef4bcaff"}, + {file = "twisted_iocpsupport-1.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:7c66fa0aa4236b27b3c61cb488662d85dae746a6d1c7b0d91cf7aae118445adf"}, + {file = "twisted_iocpsupport-1.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:300437af17396a945a58dcfffd77863303a8b6d9e65c6e81f1d2eed55b50d444"}, +] + +[[package]] +name = "types-python-dateutil" +version = "2.9.0.20240316" +description = "Typing stubs for python-dateutil" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-python-dateutil-2.9.0.20240316.tar.gz", hash = "sha256:5d2f2e240b86905e40944dd787db6da9263f0deabef1076ddaed797351ec0202"}, + {file = "types_python_dateutil-2.9.0.20240316-py3-none-any.whl", hash = "sha256:6b8cb66d960771ce5ff974e9dd45e38facb81718cc1e208b10b1baccbfdbee3b"}, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[[package]] +name = "uberegg" +version = "0.1.1" +description = "Uber-egg (egg with dependencies) builder" +optional = false +python-versions = "*" +files = [ + {file = "uberegg-0.1.1-py3-none-any.whl", hash = "sha256:d1372b906cdbf897333b8eec9611939b907e18c1eaef2687c69c39fccf9aeef5"}, + {file = "uberegg-0.1.1.tar.gz", hash = "sha256:30052db3346e75afac1724a5953b9d7fa35fc6c37f861f980cb2a2faed26f5e5"}, +] + +[[package]] +name = "urllib3" +version = "2.2.2" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "w3lib" +version = "2.2.1" +description = "Library of web-related functions" +optional = false +python-versions = ">=3.8" +files = [ + {file = "w3lib-2.2.1-py3-none-any.whl", hash = "sha256:e56d81c6a6bf507d7039e0c95745ab80abd24b465eb0f248af81e3eaa46eb510"}, + {file = "w3lib-2.2.1.tar.gz", hash = "sha256:756ff2d94c64e41c8d7c0c59fea12a5d0bc55e33a531c7988b4a163deb9b07dd"}, +] + +[[package]] +name = "zope-interface" +version = "6.4.post2" +description = "Interfaces for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "zope.interface-6.4.post2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2eccd5bef45883802848f821d940367c1d0ad588de71e5cabe3813175444202c"}, + {file = "zope.interface-6.4.post2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:762e616199f6319bb98e7f4f27d254c84c5fb1c25c908c2a9d0f92b92fb27530"}, + {file = "zope.interface-6.4.post2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ef8356f16b1a83609f7a992a6e33d792bb5eff2370712c9eaae0d02e1924341"}, + {file = "zope.interface-6.4.post2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e4fa5d34d7973e6b0efa46fe4405090f3b406f64b6290facbb19dcbf642ad6b"}, + {file = "zope.interface-6.4.post2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d22fce0b0f5715cdac082e35a9e735a1752dc8585f005d045abb1a7c20e197f9"}, + {file = "zope.interface-6.4.post2-cp310-cp310-win_amd64.whl", hash = "sha256:97e615eab34bd8477c3f34197a17ce08c648d38467489359cb9eb7394f1083f7"}, + {file = "zope.interface-6.4.post2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:599f3b07bde2627e163ce484d5497a54a0a8437779362395c6b25e68c6590ede"}, + {file = "zope.interface-6.4.post2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:136cacdde1a2c5e5bc3d0b2a1beed733f97e2dad8c2ad3c2e17116f6590a3827"}, + {file = "zope.interface-6.4.post2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47937cf2e7ed4e0e37f7851c76edeb8543ec9b0eae149b36ecd26176ff1ca874"}, + {file = "zope.interface-6.4.post2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f0a6be264afb094975b5ef55c911379d6989caa87c4e558814ec4f5125cfa2e"}, + {file = "zope.interface-6.4.post2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47654177e675bafdf4e4738ce58cdc5c6d6ee2157ac0a78a3fa460942b9d64a8"}, + {file = "zope.interface-6.4.post2-cp311-cp311-win_amd64.whl", hash = "sha256:e2fb8e8158306567a3a9a41670c1ff99d0567d7fc96fa93b7abf8b519a46b250"}, + {file = "zope.interface-6.4.post2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b912750b13d76af8aac45ddf4679535def304b2a48a07989ec736508d0bbfbde"}, + {file = "zope.interface-6.4.post2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4ac46298e0143d91e4644a27a769d1388d5d89e82ee0cf37bf2b0b001b9712a4"}, + {file = "zope.interface-6.4.post2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:86a94af4a88110ed4bb8961f5ac72edf782958e665d5bfceaab6bf388420a78b"}, + {file = "zope.interface-6.4.post2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:73f9752cf3596771c7726f7eea5b9e634ad47c6d863043589a1c3bb31325c7eb"}, + {file = "zope.interface-6.4.post2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00b5c3e9744dcdc9e84c24ed6646d5cf0cf66551347b310b3ffd70f056535854"}, + {file = "zope.interface-6.4.post2-cp312-cp312-win_amd64.whl", hash = "sha256:551db2fe892fcbefb38f6f81ffa62de11090c8119fd4e66a60f3adff70751ec7"}, + {file = "zope.interface-6.4.post2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96ac6b3169940a8cd57b4f2b8edcad8f5213b60efcd197d59fbe52f0accd66e"}, + {file = "zope.interface-6.4.post2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cebff2fe5dc82cb22122e4e1225e00a4a506b1a16fafa911142ee124febf2c9e"}, + {file = "zope.interface-6.4.post2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33ee982237cffaf946db365c3a6ebaa37855d8e3ca5800f6f48890209c1cfefc"}, + {file = "zope.interface-6.4.post2-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:fbf649bc77510ef2521cf797700b96167bb77838c40780da7ea3edd8b78044d1"}, + {file = "zope.interface-6.4.post2-cp37-cp37m-win_amd64.whl", hash = "sha256:4c0b208a5d6c81434bdfa0f06d9b667e5de15af84d8cae5723c3a33ba6611b82"}, + {file = "zope.interface-6.4.post2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d3fe667935e9562407c2511570dca14604a654988a13d8725667e95161d92e9b"}, + {file = "zope.interface-6.4.post2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a96e6d4074db29b152222c34d7eec2e2db2f92638d2b2b2c704f9e8db3ae0edc"}, + {file = "zope.interface-6.4.post2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:866a0f583be79f0def667a5d2c60b7b4cc68f0c0a470f227e1122691b443c934"}, + {file = "zope.interface-6.4.post2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5fe919027f29b12f7a2562ba0daf3e045cb388f844e022552a5674fcdf5d21f1"}, + {file = "zope.interface-6.4.post2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e0343a6e06d94f6b6ac52fbc75269b41dd3c57066541a6c76517f69fe67cb43"}, + {file = "zope.interface-6.4.post2-cp38-cp38-win_amd64.whl", hash = "sha256:dabb70a6e3d9c22df50e08dc55b14ca2a99da95a2d941954255ac76fd6982bc5"}, + {file = "zope.interface-6.4.post2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:706efc19f9679a1b425d6fa2b4bc770d976d0984335eaea0869bd32f627591d2"}, + {file = "zope.interface-6.4.post2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3d136e5b8821073e1a09dde3eb076ea9988e7010c54ffe4d39701adf0c303438"}, + {file = "zope.interface-6.4.post2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1730c93a38b5a18d24549bc81613223962a19d457cfda9bdc66e542f475a36f4"}, + {file = "zope.interface-6.4.post2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bc2676312cc3468a25aac001ec727168994ea3b69b48914944a44c6a0b251e79"}, + {file = "zope.interface-6.4.post2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a62fd6cd518693568e23e02f41816adedfca637f26716837681c90b36af3671"}, + {file = "zope.interface-6.4.post2-cp39-cp39-win_amd64.whl", hash = "sha256:d3f7e001328bd6466b3414215f66dde3c7c13d8025a9c160a75d7b2687090d15"}, + {file = "zope.interface-6.4.post2.tar.gz", hash = "sha256:1c207e6f6dfd5749a26f5a5fd966602d6b824ec00d2df84a7e9a924e8933654e"}, +] + +[package.dependencies] +setuptools = "*" + +[package.extras] +docs = ["Sphinx", "repoze.sphinx.autointerface", "sphinx-rtd-theme"] +test = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] +testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.12" +content-hash = "56191ba931dd1fa541c0a1ef9d0f309fc8753b420e57358a75c984cb67e6baba" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..00af8cb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,21 @@ +[tool.poetry] +name = "neptunscraper" +version = "0.0.1" +description = "This webscraper is used to scrap data from dockerhub and quayiio" +authors = ["stevan06v", "jonasfroeller"] +license = "MIT License" +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.11" +scrapy = "^2.11.2" +scrapyd = "^1.4.3" +scrapy-playwright = "^0.0.38" +python-dateutil = "^2.9.0.post0" +types-python-dateutil = "^2.9.0.20240316" +scrapyd-client = "^1.2.3" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/requirements-test.txt b/requirements-test.txt deleted file mode 100644 index 45bc4a8..0000000 --- a/requirements-test.txt +++ /dev/null @@ -1,11 +0,0 @@ -# These requirements are for development and testing only, not for production. - -pytest -coverage -flake8 -black -isort -pytest-cov -mypy -gitchangelog -mkdocs diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index a214948..0000000 --- a/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -# `make init` => automatically based on the template -# run `make switch-to-poetry` to use the poetry package manager - -Scrapy==2.11.1 -scrapy-playwright==v0.0.36 -python-dateutil -types-python-dateutil diff --git a/scrapy.cfg b/scrapy.cfg new file mode 100644 index 0000000..6124443 --- /dev/null +++ b/scrapy.cfg @@ -0,0 +1,7 @@ +[settings] +default = neptunscraper.settings +neptunscraper = neptunscraper.settings + +[deploy] +url = http://localhost:6800/ +project = neptunbot diff --git a/scrapydweb/.DS_Store b/scrapydweb/.DS_Store new file mode 100644 index 0000000..5008ddf Binary files /dev/null and b/scrapydweb/.DS_Store differ diff --git a/scrapydweb/Dockerfile b/scrapydweb/Dockerfile new file mode 100644 index 0000000..70ee115 --- /dev/null +++ b/scrapydweb/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.6-alpine +LABEL maintainer="Zentek Servicios Tecnologicos" +LABEL description="Web app for Scrapyd cluster management, Scrapy log analysis & visualization, Auto packaging, Timer tasks, Monitor & Alert, and Mobile UI." + +ENV PYTHONUNBUFFERED 1 + +RUN set -ex && apk --no-cache --virtual .build-deps add build-base g++ bash curl gcc libgcc tzdata psutils linux-headers openssl-dev postgresql-dev libffi-dev libxml2-dev libxslt-dev + +RUN ln -sf /usr/share/zoneinfo/America/Mexico_City /etc/localtime +RUN echo "America/Mexico_City" > /etc/timezone + +RUN pip install pip==20.2.4 +RUN pip install scrapydweb +RUN pip install psycopg2 + +RUN mkdir /scrapydweb +COPY scrapydweb_settings_v10.py /scrapydweb/ +WORKDIR /scrapydweb/ + +EXPOSE 8000 + +ENTRYPOINT ["scrapydweb"] \ No newline at end of file diff --git a/scrapydweb/scrapydweb_settings_v10.py b/scrapydweb/scrapydweb_settings_v10.py new file mode 100644 index 0000000..3ded469 --- /dev/null +++ b/scrapydweb/scrapydweb_settings_v10.py @@ -0,0 +1,354 @@ +# coding: utf-8 +""" +How ScrapydWeb works: +BROWSER <<<>>> SCRAPYDWEB_BIND:SCRAPYDWEB_PORT <<<>>> your SCRAPYD_SERVERS + +GitHub: https://github.com/my8100/scrapydweb +DOCS: https://github.com/my8100/files/blob/master/scrapydweb/README.md +文档：https://github.com/my8100/files/blob/master/scrapydweb/README_CN.md +""" +import os + + +############################## QUICK SETUP start ############################## +############################## 快速设置 开始 ################################### +# Setting SCRAPYDWEB_BIND to '0.0.0.0' or IP-OF-THE-CURRENT-HOST would make +# ScrapydWeb server visible externally; Otherwise, set it to '127.0.0.1'. +# The default is '0.0.0.0'. +SCRAPYDWEB_BIND = '0.0.0.0' +# Accept connections on the specified port, the default is 5000. +SCRAPYDWEB_PORT = 8000 + +# The default is False, set it to True to enable basic auth for the web UI. +ENABLE_AUTH = True +# In order to enable basic auth, both USERNAME and PASSWORD should be non-empty strings. +USERNAME = os.environ.get('SCRAPYD_ADMIN', 'admin') +PASSWORD = os.environ.get('SCRAPYD_PASSWD', 'secret') + + +# Make sure that [Scrapyd](https://github.com/scrapy/scrapyd) has been installed +# and started on all of your hosts. +# Note that for remote access, you have to manually set 'bind_address = 0.0.0.0' +# in the configuration file of Scrapyd and restart Scrapyd to make it visible externally. +# Check out 'https://scrapyd.readthedocs.io/en/latest/config.html#example-configuration-file' for more info. +# ------------------------------ Chinese -------------------------------------- +# 请先确保所有主机都已经安装和启动 [Scrapyd](https://github.com/scrapy/scrapyd)。 +# 如需远程访问 Scrapyd，则需在 Scrapyd 配置文件中设置 'bind_address = 0.0.0.0'，然后重启 Scrapyd。 +# 详见 https://scrapyd.readthedocs.io/en/latest/config.html#example-configuration-file + +# - the string format: username:password@ip:port#group +# - The default port would be 6800 if not provided, +# - Both basic auth and group are optional. +# - e.g. '127.0.0.1:6800' or 'username:password@localhost:6801#group' +# - the tuple format: (username, password, ip, port, group) +# - When the username, password, or group is too complicated (e.g. contains ':@#'), +# - or if ScrapydWeb fails to parse the string format passed in, +# - it's recommended to pass in a tuple of 5 elements. +# - e.g. ('', '', '127.0.0.1', '6800', '') or ('username', 'password', 'localhost', '6801', 'group') +SCRAPYD_SERVERS = os.environ.get("SCRAPYD_SERVERS").split(',') if os.environ.get("SCRAPYD_SERVERS") else [] + + +# It's recommended to update the three options below +# if both ScrapydWeb and one of your Scrapyd servers run on the same machine. +# ------------------------------ Chinese -------------------------------------- +# 假如 ScrapydWeb 和某个 Scrapyd 运行于同一台主机，建议更新如下三个设置项。 + +# If both ScrapydWeb and one of your Scrapyd servers run on the same machine, +# ScrapydWeb would try to directly read Scrapy logfiles from disk, instead of making a request +# to the Scrapyd server. +# e.g. '127.0.0.1:6800' or 'localhost:6801', do not forget the port number. +LOCAL_SCRAPYD_SERVER = '' + +# Enter the directory when you run Scrapyd, run the command below +# to find out where the Scrapy logs are stored: +# python -c "from os.path import abspath, isdir; from scrapyd.config import Config; path = abspath(Config().get('logs_dir')); print(path); print(isdir(path))" +# Check out https://scrapyd.readthedocs.io/en/stable/config.html#logs-dir for more info. +# e.g. 'C:/Users/username/logs' or '/home/username/logs' +LOCAL_SCRAPYD_LOGS_DIR = '' + +# The default is False, set it to True to automatically run LogParser as a subprocess at startup. +# Note that you can run the LogParser service separately via command 'logparser' as you like. +# Run 'logparser -h' to find out the config file of LogParser for more advanced settings. +# Visit https://github.com/my8100/logparser for more info. +ENABLE_LOGPARSER = False +############################## QUICK SETUP end ################################ +############################## 快速设置 结束 ################################### + + +############################## ScrapydWeb ##################################### +# The default is False, set it to True and add both CERTIFICATE_FILEPATH and PRIVATEKEY_FILEPATH +# to run ScrapydWeb in HTTPS mode. +# Note that this feature is not fully tested, please leave your comment here if ScrapydWeb +# raises any excepion at startup: https://github.com/my8100/scrapydweb/issues/18 +ENABLE_HTTPS = False +# e.g. '/home/username/cert.pem' +CERTIFICATE_FILEPATH = '' +# e.g. '/home/username/cert.key' +PRIVATEKEY_FILEPATH = '' + + +############################## Scrapy ######################################### +# ScrapydWeb is able to locate projects in the SCRAPY_PROJECTS_DIR, +# so that you can simply select a project to deploy, instead of packaging it in advance. +# e.g. 'C:/Users/username/myprojects' or '/home/username/myprojects' +SCRAPY_PROJECTS_DIR = '' + + +############################## Scrapyd ######################################## +# ScrapydWeb would try every extension in sequence to locate the Scrapy logfile. +# The default is ['.log', '.log.gz', '.txt']. +SCRAPYD_LOG_EXTENSIONS = ['.log', '.log.gz', '.txt'] + + +############################## LogParser ###################################### +# Whether to backup the stats json files locally after you visit the Stats page of a job +# so that it is still accessible even if the original logfile has been deleted. +# The default is True, set it to False to disable this behaviour. +BACKUP_STATS_JSON_FILE = True + + +############################## Timer Tasks #################################### +# Run ScrapydWeb with argument '-sw' or '--switch_scheduler_state', or click the ENABLED|DISABLED button +# on the Timer Tasks page to turn on/off the scheduler for the timer tasks and the snapshot mechanism below. + +# The default is 300, which means ScrapydWeb would automatically create a snapshot of the Jobs page +# and save the jobs info in the database in the background every 300 seconds. +# Note that this behaviour would be paused if the scheduler for timer tasks is disabled. +# Set it to 0 to disable this behaviour. +JOBS_SNAPSHOT_INTERVAL = 300 + + +############################## Run Spider ##################################### +# The default is False, set it to True to automatically +# expand the 'settings & arguments' section in the Run Spider page. +SCHEDULE_EXPAND_SETTINGS_ARGUMENTS = False + +# The default is 'Mozilla/5.0', set it a non-empty string to customize the default value of `custom` +# in the drop-down list of `USER_AGENT`. +SCHEDULE_CUSTOM_USER_AGENT = 'Mozilla/5.0' + +# The default is None, set it to any value of ['custom', 'Chrome', 'iPhone', 'iPad', 'Android'] +# to customize the default value of `USER_AGENT`. +SCHEDULE_USER_AGENT = None + +# The default is None, set it to True or False to customize the default value of `ROBOTSTXT_OBEY`. +SCHEDULE_ROBOTSTXT_OBEY = None + +# The default is None, set it to True or False to customize the default value of `COOKIES_ENABLED`. +SCHEDULE_COOKIES_ENABLED = None + +# The default is None, set it to a non-negative integer to customize the default value of `CONCURRENT_REQUESTS`. +SCHEDULE_CONCURRENT_REQUESTS = None + +# The default is None, set it to a non-negative number to customize the default value of `DOWNLOAD_DELAY`. +SCHEDULE_DOWNLOAD_DELAY = None + +# The default is "-d setting=CLOSESPIDER_TIMEOUT=60\r\n-d setting=CLOSESPIDER_PAGECOUNT=10\r\n-d arg1=val1", +# set it to '' or any non-empty string to customize the default value of `additional`. +# Use '\r\n' as the line separator. +SCHEDULE_ADDITIONAL = "-d setting=CLOSESPIDER_TIMEOUT=60\r\n-d setting=CLOSESPIDER_PAGECOUNT=10\r\n-d arg1=val1" + + +############################## Page Display ################################### +# The default is True, set it to False to hide the Items page, as well as +# the Items column in the Jobs page. +SHOW_SCRAPYD_ITEMS = True + +# The default is True, set it to False to hide the Job column in the Jobs page with non-database view. +SHOW_JOBS_JOB_COLUMN = True + +# The default is 0, which means unlimited, set it to a positive integer so that +# only the latest N finished jobs would be shown in the Jobs page with non-database view. +JOBS_FINISHED_JOBS_LIMIT = 0 + +# If your browser stays on the Jobs page, it would be reloaded automatically every N seconds. +# The default is 300, set it to 0 to disable auto-reloading. +JOBS_RELOAD_INTERVAL = 300 + +# The load status of the current Scrapyd server is checked every N seconds, +# which is displayed in the top right corner of the page. +# The default is 10, set it to 0 to disable auto-refreshing. +DAEMONSTATUS_REFRESH_INTERVAL = 10 + + +############################## Send Text ###################################### +########## usage in scrapy projects ########## +# See the "Send Text" page + +########## slack ########## +# How to create a slack app: +# 1. Visit https://api.slack.com/apps and press the "Create New App" button. +# 2. Enter your App Name (e.g. myapp)and select one of your Slack Workspaces, the press "Create App". +# 3. Click the "OAuth & Permissions" menu in the sidebar on the left side of the page. +# 4. Scroll down the page and find out "Select Permission Scopes" in the "Scopes" section +# 5. Enter "send" and select "Send messages as ", then press "Save Changes" +# 6. Scroll up the page and press "Install App to Workspace", then press "Install" +# 7. Copy the "OAuth Access Token", e.g. xoxp-123-456-789-abcde +# See https://api.slack.com/apps for more info + +# See step 1~7 above, e.g. 'xoxp-123-456-789-abcde' +SLACK_TOKEN = os.environ.get('SLACK_TOKEN', '') +# The default channel to use when sending text via slack, e.g. 'general' +SLACK_CHANNEL = os.environ.get("SLACK_CHANNEL", "general") + +########## telegram ########## +# How to create a telegram bot: +# 1. Visit https://telegram.me/botfather to start a conversation with Telegram's bot that creates other bots. +# 2. Send the /newbot command to create a new bot in a chat with BotFather. +# 3. Follow the instructions to set up name and username (e.g. my_bot) for your bot. +# 4. You would get a token (e.g. 123:abcde) after step 3. +# 5. Visit telegram.me/ (e.g. telegram.me/my_bot) and say hi to your bot to initiate a conversation. +# 6. Visit https://api.telegram.org/bot/getUpdates to get the chat_id. +# (e.g. Visit https://api.telegram.org/bot123:abcde/getUpdates +# and you can find the chat_id in "chat":{"id":123456789,...) +# See https://core.telegram.org/bots#6-botfather for more info + +# See step 1~4 above, e.g. '123:abcde' +TELEGRAM_TOKEN = os.environ.get('TELEGRAM_TOKEN', '') +# See step 5~6 above, e.g. 123456789 +TELEGRAM_CHAT_ID = int(os.environ.get('TELEGRAM_CHAT_ID', 0)) + +########## email ########## +# The default subject to use when sending text via email. +EMAIL_SUBJECT = os.environ.get("EMAIL_SUBJECT","Email from #scrapydweb") + +########## email sender & recipients ########## +# Leave this option as '' to default to the EMAIL_SENDER option below; Otherwise, set it up +# if your email service provider requires an username which is different from the EMAIL_SENDER option below to login. +# e.g. 'username' +EMAIL_USERNAME = os.environ.get('EMAIL_USERNAME', '') +# As for different email service provider, you might have to get an APP password (like Gmail) +# or an authorization code (like QQ mail) and set it as the EMAIL_PASSWORD. +# Check out links below to get more help: +# https://stackoverflow.com/a/27515833/10517783 How to send an email with Gmail as the provider using Python? +# https://stackoverflow.com/a/26053352/10517783 Python smtplib proxy support +# e.g. 'password4gmail' +EMAIL_PASSWORD = os.environ.get('EMAIL_PASSWORD', '') + +# e.g. 'username@gmail.com' +EMAIL_SENDER = os.environ.get('EMAIL_SENDER', '') +# e.g. ['username@gmail.com', ] +EMAIL_RECIPIENTS = [EMAIL_SENDER] + +########## email smtp settings ########## +# Check out this link if you are using ECS of Alibaba Cloud and your SMTP server provides TCP port 25 only: +# https://www.alibabacloud.com/help/doc-detail/56130.htm +# Config for https://mail.google.com using SSL: ('smtp.gmail.com', 465, True) +# Config for https://mail.google.com: ('smtp.gmail.com', 587, False) +# Config for https://mail.qq.com using SSL: ('smtp.qq.com', 465, True) +# Config for http://mail.10086.cn: ('smtp.139.com', 25, False) +SMTP_SERVER = os.environ.get('SMTP_SERVER', '') +SMTP_PORT = os.environ.get('SMTP_PORT', 587) +SMTP_OVER_SSL = os.environ.get("SMTP_OVER_SSL", 'False').lower() in ['true', '1'] +# The timeout in seconds for the connection attempt, the default is 30. +SMTP_CONNECTION_TIMEOUT = 30 + + +############################## Monitor & Alert ################################ +# The default is False, set it to True to launch the poll subprocess to monitor your crawling jobs. +ENABLE_MONITOR = os.environ.get("ENABLE_MONITOR", 'False').lower() in ['true', '1'] + +########## poll interval ########## +# Tip: In order to be notified (and stop or forcestop a job when triggered) in time, +# you can reduce the value of POLL_ROUND_INTERVAL and POLL_REQUEST_INTERVAL, +# at the cost of burdening both CPU and bandwidth of your servers. + +# Sleep N seconds before starting next round of poll, the default is 300. +POLL_ROUND_INTERVAL = 300 +# Sleep N seconds between each request to the Scrapyd server while polling, the default is 10. +POLL_REQUEST_INTERVAL = 10 + +########## alert switcher ########## +# Tip: Set the SCRAPYDWEB_BIND option the in "QUICK SETUP" section to the actual IP of your host, +# then you can visit ScrapydWeb via the links attached in the alert. + +# The default is False, set it to True to enable alert via Slack, Telegram, or Email. +# You have to set up your accounts in the "Send text" section above first. +ENABLE_SLACK_ALERT = os.environ.get("ENABLE_SLACK_ALERT", 'False').lower() in ['true', '1'] +ENABLE_TELEGRAM_ALERT = os.environ.get("ENABLE_TELEGRAM_ALERT", 'False').lower() in ['true', '1'] +ENABLE_EMAIL_ALERT = os.environ.get("ENABLE_EMAIL_ALERT", 'False').lower() in ['true', '1'] + +########## alert working time ########## +# Monday is 1 and Sunday is 7. +# e.g, [1, 2, 3, 4, 5, 6, 7] +ALERT_WORKING_DAYS = os.environ.get("ALERT_WORKING_DAYS").split(',') if os.environ.get("ALERT_WORKING_DAYS") else [] + +# From 0 to 23. +# e.g. [9] + list(range(15, 18)) >>> [9, 15, 16, 17], or range(24) for 24 hours +ALERT_WORKING_HOURS = os.environ.get("ALERT_WORKING_HOURS").split(',') if os.environ.get("ALERT_WORKING_HOURS") else [] + +########## basic triggers ########## +# Trigger alert every N seconds for each running job. +# The default is 0, set it to a positive integer to enable this trigger. +ON_JOB_RUNNING_INTERVAL = 0 + +# Trigger alert when a job is finished. +# The default is False, set it to True to enable this trigger. +ON_JOB_FINISHED = os.environ.get("ON_JOB_FINISHED", 'False').lower() in ['true', '1'] + +########## advanced triggers ########## +# - LOG_XXX_THRESHOLD: +# - Trigger alert the first time reaching the threshold for a specific kind of log. +# - The default is 0, set it to a positive integer to enable this trigger. +# - LOG_XXX_TRIGGER_STOP (optional): +# - The default is False, set it to True to stop current job automatically when reaching the LOG_XXX_THRESHOLD. +# - The SIGTERM signal would be sent only one time to shut down the crawler gracefully. +# - In order to avoid an UNCLEAN shutdown, the 'STOP' action would be executed one time at most +# - if none of the 'FORCESTOP' triggers is enabled, no matter how many 'STOP' triggers are enabled. +# - LOG_XXX_TRIGGER_FORCESTOP (optional): +# - The default is False, set it to True to FORCESTOP current job automatically when reaching the LOG_XXX_THRESHOLD. +# - The SIGTERM signal would be sent twice resulting in an UNCLEAN shutdown, without the Scrapy stats dumped! +# - The 'FORCESTOP' action would be executed if both of the 'STOP' and 'FORCESTOP' triggers are enabled. + +# Note that the 'STOP' action and the 'FORCESTOP' action would still be executed even when the current time +# is NOT within the ALERT_WORKING_DAYS and the ALERT_WORKING_HOURS, though no alert would be sent. + +LOG_CRITICAL_THRESHOLD = 0 +LOG_CRITICAL_TRIGGER_STOP = False +LOG_CRITICAL_TRIGGER_FORCESTOP = False + +LOG_ERROR_THRESHOLD = 0 +LOG_ERROR_TRIGGER_STOP = False +LOG_ERROR_TRIGGER_FORCESTOP = False + +LOG_WARNING_THRESHOLD = 0 +LOG_WARNING_TRIGGER_STOP = False +LOG_WARNING_TRIGGER_FORCESTOP = False + +LOG_REDIRECT_THRESHOLD = 0 +LOG_REDIRECT_TRIGGER_STOP = False +LOG_REDIRECT_TRIGGER_FORCESTOP = False + +LOG_RETRY_THRESHOLD = 0 +LOG_RETRY_TRIGGER_STOP = False +LOG_RETRY_TRIGGER_FORCESTOP = False + +LOG_IGNORE_THRESHOLD = 0 +LOG_IGNORE_TRIGGER_STOP = False +LOG_IGNORE_TRIGGER_FORCESTOP = False + + +############################## System ######################################### +# The default is False, set it to True to enable debug mode and the interactive debugger +# would be shown in the browser instead of the "500 Internal Server Error" page. +# Note that use_reloader is set to False in run.py +DEBUG = False + +# The default is False, set it to True to change the logging level from INFO to DEBUG +# for getting more information about how ScrapydWeb works, especially while debugging. +VERBOSE = False + +# The default is '', which means saving all program data in the Python directory. +# e.g. 'C:/Users/username/scrapydweb_data' or '/home/username/scrapydweb_data' +DATA_PATH = os.environ.get('DATA_PATH', '') + +# The default is '', which means saving data of Jobs and Timer Tasks in DATA_PATH using SQLite. +# The data could be also saved in MySQL or PostgreSQL backend in order to improve concurrency. +# To use MySQL backend, run command: pip install --upgrade pymysql +# To use PostgreSQL backend, run command: pip install --upgrade psycopg2 +# e.g. +# 'mysql://username:password@127.0.0.1:3306' +# 'postgres://username:password@127.0.0.1:5432' +# 'sqlite:///C:/Users/username' +# 'sqlite:////home/username' +DATABASE_URL = os.environ.get('DATABASE_URL', '') \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 9edb6f2..0000000 --- a/setup.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Python setup.py for neptun_webscraper package""" -import io -import os -from setuptools import find_packages, setup - - -def read(*paths, **kwargs): - """Read the contents of a text file safely. - >>> read("neptun_webscraper", "VERSION") - '0.1.0' - >>> read("README.md") - ... - """ - - content = "" - with io.open( - os.path.join(os.path.dirname(__file__), *paths), - encoding=kwargs.get("encoding", "utf8"), - ) as open_file: - content = open_file.read().strip() - return content - - -def read_requirements(path): - return [ - line.strip() - for line in read(path).split("\n") - if not line.startswith(('"', "#", "-", "git+")) - ] - - -setup( - name="neptun_webscraper", - version=read("neptun_webscraper", "VERSION"), - description="Awesome neptun_webscraper created by jonasfroeller", - url="https://github.com/jonasfroeller/tech-stack-ai-configuration-data-scraper/", - long_description=read("README.md"), - long_description_content_type="text/markdown", - author="jonasfroeller", - packages=find_packages(exclude=["tests", ".github"]), - install_requires=read_requirements("requirements.txt"), - entry_points={ - "console_scripts": ["neptun_webscraper = neptun_webscraper.__main__:main"] - }, - extras_require={"test": read_requirements("requirements-test.txt")}, -) diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 1cbb7b1..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,14 +0,0 @@ -import sys -import pytest - - -# each test runs on cwd to its temp dir -@pytest.fixture(autouse=True) -def go_to_tmpdir(request): - # Get the fixture dynamically by its name. - tmpdir = request.getfixturevalue("tmpdir") - # ensure local test created packages can be imported - sys.path.insert(0, str(tmpdir)) - # Chdir only for the duration of the test. - with tmpdir.as_cwd(): - yield diff --git a/tests/test_base.py b/tests/test_base.py deleted file mode 100644 index a4d7818..0000000 --- a/tests/test_base.py +++ /dev/null @@ -1,5 +0,0 @@ -from neptun_webscraper.base import NAME - - -def test_base(): - assert NAME == "neptun_webscraper"