diff --git a/neptun_webscraper/spiders/dockerhub.py b/neptun_webscraper/spiders/dockerhub.py index 300a9f0..d561cc8 100644 --- a/neptun_webscraper/spiders/dockerhub.py +++ b/neptun_webscraper/spiders/dockerhub.py @@ -49,6 +49,7 @@ def parse(self, response): search_result_items = search_result.xpath('//a[@data-testid="imageSearchResult"]') for result in search_result_items: + # print("current result html: ", result.extract()) item = self.parse_result(result) timestamp = datetime.now().strftime('%Y%m%d%H%M%S') @@ -100,14 +101,14 @@ def parse_result(self, result): update_elem = result.css('span:contains("Updated")::text').get() if update_elem: item['last_update'] = update_elem.strip() - desc_elem = result.xpath('following-sibling::p/text()').get() + desc_elem = result.xpath('.//span[contains(text(), "Updated")]/ancestor::div[1]/following-sibling::p[1]/text()').get() item['description'] = desc_elem.strip() if desc_elem else None else: item['last_update'] = None item['description'] = None # Extract chips (categories) - chips = result.css('[data-testid="productChip"]::text').getall() + chips = result.css('[data-testid="productChip"] span::text').getall() item['chips'] = chips # Extract downloads (total pulls) @@ -117,11 +118,11 @@ def parse_result(self, result): item['downloads'] = downloads.strip() if downloads else None # Extract pulls last week - pulls_elem = result.css('span:contains("Pulls:")').xpath('following-sibling::p/text()').get() + pulls_elem = result.css('p:contains("Pulls:")').xpath('following-sibling::p/text()').get() item['pulls_last_week'] = pulls_elem.replace(',', '') if pulls_elem else None # Extract stars - stars_elem = result.css('[data-testid="StarOutlineIcon"] + span::text').get() + stars_elem = result.xpath('//svg[@data-testid="StarOutlineIcon"]/following-sibling::span/strong/text()').get() item['stars'] = stars_elem.strip() if stars_elem else None return item diff --git a/neptun_webscraper/spiders/logs/20240701210421.json b/neptun_webscraper/spiders/logs/20240701210421.json new file mode 100644 index 0000000..b0d9e51 --- /dev/null +++ b/neptun_webscraper/spiders/logs/20240701210421.json @@ -0,0 +1,374 @@ +[ +{ + "name": "python", + "is_official_image": true, + "is_verified_publisher": false, + "last_update": "Updated 4 days ago", + "description": "Python is an interpreted, interactive, object-oriented, open-source programming language.", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "1B+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "pypy", + "is_official_image": true, + "is_verified_publisher": false, + "last_update": "Updated 10 days ago", + "description": "PyPy is a fast, compliant alternative implementation of the Python language.", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "10M+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "hylang", + "is_official_image": true, + "is_verified_publisher": false, + "last_update": "Updated 4 days ago", + "description": "Hy is a Lisp dialect that translates expressions into Python's abstract syntax tree.", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "10M+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "circleci/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 3 years ago", + "description": "Python is an interpreted, interactive, object-oriented, open-source programming language.", + "chips": [], + "downloads": "100M+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "cimg/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 3 months ago", + "description": null, + "chips": [ + "Languages & Frameworks", + "Integration & Delivery", + "Security" + ], + "downloads": "100M+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "bitnami/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 days ago", + "description": "Bitnami container image for Python", + "chips": [], + "downloads": "1M+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "amazon/aws-lambda-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 4 days ago", + "description": "AWS Lambda base images for Python", + "chips": [ + "Languages & Frameworks", + "Integration & Delivery", + "Machine Learning & AI" + ], + "downloads": "1M+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "okteto/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": null, + "chips": [ + "Languages & Frameworks", + "Integration & Delivery" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "google/guestbook-python-redis", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 9 years ago", + "description": "A simple guestbook example written in Python. It works against a redis backend.", + "chips": [ + "Databases & Storage", + "Languages & Frameworks", + "Integration & Delivery" + ], + "downloads": "1M+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "ubuntu/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 13 days ago", + "description": "A chiselled Ubuntu rock with the Python runtime", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "10K+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "intel/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": null, + "chips": [], + "downloads": "10K+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "balenalib/artik533s-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "jetbrains/qodana-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated a month ago", + "description": "Qodana for Python projects", + "chips": [ + "Languages & Frameworks", + "Integration & Delivery", + "Security" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "chainguard/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 days ago", + "description": "Minimal Python image based on Wolfi.", + "chips": [], + "downloads": "10K+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "balenalib/fincm3-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated a year ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "balenalib/amd64-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Machine Learning & AI" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "balenalib/amd64-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "amazon/aws-sam-cli-build-image-python3.8", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 10 months ago", + "description": null, + "chips": [ + "Languages & Frameworks", + "Integration & Delivery" + ], + "downloads": "1M+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "balenalib/artik5-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "balenalib/armv7hf-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "balenalib/amd64-ubuntu-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Data Science", + "Languages & Frameworks", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "balenalib/artik10-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "balenalib/apalis-imx6q-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "balenalib/aarch64-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Machine Learning & AI" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": null +} +, +{ + "name": "balenalib/raspberry-pi-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": null +} +] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/20240701210511.json b/neptun_webscraper/spiders/logs/20240701210511.json new file mode 100644 index 0000000..5f5393a --- /dev/null +++ b/neptun_webscraper/spiders/logs/20240701210511.json @@ -0,0 +1,350 @@ +[ +{ + "name": "python", + "is_official_image": true, + "is_verified_publisher": false, + "last_update": "Updated 4 days ago", + "description": "Python is an interpreted, interactive, object-oriented, open-source programming language.", + "chips": [ + "Languages & Frameworks" + ], + "pulls_last_week": "6904987", + "stars": null +} +, +{ + "name": "pypy", + "is_official_image": true, + "is_verified_publisher": false, + "last_update": "Updated 10 days ago", + "description": "PyPy is a fast, compliant alternative implementation of the Python language.", + "chips": [ + "Languages & Frameworks" + ], + "pulls_last_week": "17171", + "stars": null +} +, +{ + "name": "hylang", + "is_official_image": true, + "is_verified_publisher": false, + "last_update": "Updated 4 days ago", + "description": "Hy is a Lisp dialect that translates expressions into Python's abstract syntax tree.", + "chips": [ + "Languages & Frameworks" + ], + "pulls_last_week": "22302", + "stars": null +} +, +{ + "name": "circleci/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 3 years ago", + "description": "Python is an interpreted, interactive, object-oriented, open-source programming language.", + "chips": [], + "pulls_last_week": "161684", + "stars": null +} +, +{ + "name": "cimg/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 3 months ago", + "description": null, + "chips": [ + "Languages & Frameworks", + "Integration & Delivery", + "Security" + ], + "pulls_last_week": "1121955", + "stars": null +} +, +{ + "name": "bitnami/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 days ago", + "description": "Bitnami container image for Python", + "chips": [], + "pulls_last_week": "15146", + "stars": null +} +, +{ + "name": "amazon/aws-lambda-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 4 days ago", + "description": "AWS Lambda base images for Python", + "chips": [ + "Languages & Frameworks", + "Integration & Delivery", + "Machine Learning & AI" + ], + "pulls_last_week": "15574", + "stars": null +} +, +{ + "name": "okteto/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": null, + "chips": [ + "Languages & Frameworks", + "Integration & Delivery" + ], + "pulls_last_week": "26", + "stars": null +} +, +{ + "name": "google/guestbook-python-redis", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 9 years ago", + "description": "A simple guestbook example written in Python. It works against a redis backend.", + "chips": [ + "Databases & Storage", + "Languages & Frameworks", + "Integration & Delivery" + ], + "pulls_last_week": "6", + "stars": null +} +, +{ + "name": "intel/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": null, + "chips": [], + "pulls_last_week": "18", + "stars": null +} +, +{ + "name": "ubuntu/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 13 days ago", + "description": "A chiselled Ubuntu rock with the Python runtime", + "chips": [ + "Languages & Frameworks" + ], + "pulls_last_week": "561", + "stars": null +} +, +{ + "name": "jetbrains/qodana-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated a month ago", + "description": "Qodana for Python projects", + "chips": [ + "Languages & Frameworks", + "Integration & Delivery", + "Security" + ], + "pulls_last_week": "860", + "stars": null +} +, +{ + "name": "chainguard/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 days ago", + "description": "Minimal Python image based on Wolfi.", + "chips": [], + "pulls_last_week": "543", + "stars": null +} +, +{ + "name": "balenalib/fincm3-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated a year ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "pulls_last_week": "58", + "stars": null +} +, +{ + "name": "balenalib/amd64-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "pulls_last_week": "15", + "stars": null +} +, +{ + "name": "amazon/aws-sam-cli-build-image-python3.8", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 10 months ago", + "description": null, + "chips": [ + "Languages & Frameworks", + "Integration & Delivery" + ], + "pulls_last_week": "626", + "stars": null +} +, +{ + "name": "balenalib/artik5-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "pulls_last_week": "1", + "stars": null +} +, +{ + "name": "balenalib/artik533s-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "pulls_last_week": "1", + "stars": null +} +, +{ + "name": "balenalib/armv7hf-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "pulls_last_week": "6", + "stars": null +} +, +{ + "name": "balenalib/aarch64-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Machine Learning & AI" + ], + "pulls_last_week": "216", + "stars": null +} +, +{ + "name": "balenalib/i386-ubuntu-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 3 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "pulls_last_week": "0", + "stars": null +} +, +{ + "name": "balenalib/artik10-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "pulls_last_week": "1", + "stars": null +} +, +{ + "name": "balenalib/raspberrypi3-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "pulls_last_week": "833", + "stars": null +} +, +{ + "name": "balenalib/apalis-imx6q-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "pulls_last_week": "1", + "stars": null +} +, +{ + "name": "balenalib/artik530-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "pulls_last_week": "58", + "stars": null +} +] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/20240701210721.json b/neptun_webscraper/spiders/logs/20240701210721.json new file mode 100644 index 0000000..a16f84c --- /dev/null +++ b/neptun_webscraper/spiders/logs/20240701210721.json @@ -0,0 +1,374 @@ +[ +{ + "name": "python", + "is_official_image": true, + "is_verified_publisher": false, + "last_update": "Updated 4 days ago", + "description": "Python is an interpreted, interactive, object-oriented, open-source programming language.", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "1B+", + "pulls_last_week": null, + "stars": "9.7K" +} +, +{ + "name": "pypy", + "is_official_image": true, + "is_verified_publisher": false, + "last_update": "Updated 10 days ago", + "description": "PyPy is a fast, compliant alternative implementation of the Python language.", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "10M+", + "pulls_last_week": null, + "stars": "390" +} +, +{ + "name": "hylang", + "is_official_image": true, + "is_verified_publisher": false, + "last_update": "Updated 4 days ago", + "description": "Hy is a Lisp dialect that translates expressions into Python's abstract syntax tree.", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "10M+", + "pulls_last_week": null, + "stars": "60" +} +, +{ + "name": "circleci/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 3 years ago", + "description": "Python is an interpreted, interactive, object-oriented, open-source programming language.", + "chips": [], + "downloads": "100M+", + "pulls_last_week": null, + "stars": "90" +} +, +{ + "name": "cimg/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 3 months ago", + "description": null, + "chips": [ + "Languages & Frameworks", + "Integration & Delivery", + "Security" + ], + "downloads": "100M+", + "pulls_last_week": null, + "stars": "18" +} +, +{ + "name": "bitnami/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 days ago", + "description": "Bitnami container image for Python", + "chips": [], + "downloads": "1M+", + "pulls_last_week": null, + "stars": "27" +} +, +{ + "name": "amazon/aws-lambda-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 4 days ago", + "description": "AWS Lambda base images for Python", + "chips": [ + "Languages & Frameworks", + "Integration & Delivery", + "Machine Learning & AI" + ], + "downloads": "1M+", + "pulls_last_week": null, + "stars": "93" +} +, +{ + "name": "okteto/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": null, + "chips": [ + "Languages & Frameworks", + "Integration & Delivery" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": "0" +} +, +{ + "name": "google/guestbook-python-redis", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 9 years ago", + "description": "A simple guestbook example written in Python. It works against a redis backend.", + "chips": [ + "Databases & Storage", + "Languages & Frameworks", + "Integration & Delivery" + ], + "downloads": "1M+", + "pulls_last_week": null, + "stars": "5" +} +, +{ + "name": "ubuntu/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 13 days ago", + "description": "A chiselled Ubuntu rock with the Python runtime", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "10K+", + "pulls_last_week": null, + "stars": "6" +} +, +{ + "name": "intel/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": null, + "chips": [], + "downloads": "10K+", + "pulls_last_week": null, + "stars": "0" +} +, +{ + "name": "balenalib/artik533s-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": null, + "stars": "0" +} +, +{ + "name": "jetbrains/qodana-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated a month ago", + "description": "Qodana for Python projects", + "chips": [ + "Languages & Frameworks", + "Integration & Delivery", + "Security" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": "0" +} +, +{ + "name": "chainguard/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 days ago", + "description": "Minimal Python image based on Wolfi.", + "chips": [], + "downloads": "10K+", + "pulls_last_week": null, + "stars": "2" +} +, +{ + "name": "balenalib/fincm3-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated a year ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": null, + "stars": "1" +} +, +{ + "name": "balenalib/amd64-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Machine Learning & AI" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": "0" +} +, +{ + "name": "balenalib/amd64-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": null, + "stars": "0" +} +, +{ + "name": "amazon/aws-sam-cli-build-image-python3.8", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 10 months ago", + "description": null, + "chips": [ + "Languages & Frameworks", + "Integration & Delivery" + ], + "downloads": "1M+", + "pulls_last_week": null, + "stars": "11" +} +, +{ + "name": "balenalib/artik5-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": null, + "stars": "0" +} +, +{ + "name": "balenalib/armv7hf-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": "1" +} +, +{ + "name": "balenalib/amd64-ubuntu-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Data Science", + "Languages & Frameworks", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": "0" +} +, +{ + "name": "balenalib/artik10-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": "0" +} +, +{ + "name": "balenalib/apalis-imx6q-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": "0" +} +, +{ + "name": "balenalib/aarch64-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Machine Learning & AI" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": "1" +} +, +{ + "name": "balenalib/raspberry-pi-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": null, + "stars": "11" +} +] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/20240701210818.json b/neptun_webscraper/spiders/logs/20240701210818.json new file mode 100644 index 0000000..667aa25 --- /dev/null +++ b/neptun_webscraper/spiders/logs/20240701210818.json @@ -0,0 +1,374 @@ +[ +{ + "name": "python", + "is_official_image": true, + "is_verified_publisher": false, + "last_update": "Updated 4 days ago", + "description": "Python is an interpreted, interactive, object-oriented, open-source programming language.", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "1B+", + "pulls_last_week": "6904987", + "stars": null +} +, +{ + "name": "pypy", + "is_official_image": true, + "is_verified_publisher": false, + "last_update": "Updated 10 days ago", + "description": "PyPy is a fast, compliant alternative implementation of the Python language.", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "10M+", + "pulls_last_week": "17171", + "stars": null +} +, +{ + "name": "hylang", + "is_official_image": true, + "is_verified_publisher": false, + "last_update": "Updated 4 days ago", + "description": "Hy is a Lisp dialect that translates expressions into Python's abstract syntax tree.", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "10M+", + "pulls_last_week": "22302", + "stars": null +} +, +{ + "name": "circleci/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 3 years ago", + "description": "Python is an interpreted, interactive, object-oriented, open-source programming language.", + "chips": [], + "downloads": "100M+", + "pulls_last_week": "161684", + "stars": null +} +, +{ + "name": "cimg/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 3 months ago", + "description": null, + "chips": [ + "Languages & Frameworks", + "Integration & Delivery", + "Security" + ], + "downloads": "100M+", + "pulls_last_week": "1121955", + "stars": null +} +, +{ + "name": "bitnami/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 days ago", + "description": "Bitnami container image for Python", + "chips": [], + "downloads": "1M+", + "pulls_last_week": "15146", + "stars": null +} +, +{ + "name": "amazon/aws-lambda-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 4 days ago", + "description": "AWS Lambda base images for Python", + "chips": [ + "Languages & Frameworks", + "Integration & Delivery", + "Machine Learning & AI" + ], + "downloads": "1M+", + "pulls_last_week": "15574", + "stars": null +} +, +{ + "name": "okteto/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": null, + "chips": [ + "Languages & Frameworks", + "Integration & Delivery" + ], + "downloads": "100K+", + "pulls_last_week": "26", + "stars": null +} +, +{ + "name": "google/guestbook-python-redis", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 9 years ago", + "description": "A simple guestbook example written in Python. It works against a redis backend.", + "chips": [ + "Databases & Storage", + "Languages & Frameworks", + "Integration & Delivery" + ], + "downloads": "1M+", + "pulls_last_week": "6", + "stars": null +} +, +{ + "name": "ubuntu/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 13 days ago", + "description": "A chiselled Ubuntu rock with the Python runtime", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "10K+", + "pulls_last_week": "561", + "stars": null +} +, +{ + "name": "intel/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": null, + "chips": [], + "downloads": "10K+", + "pulls_last_week": "18", + "stars": null +} +, +{ + "name": "balenalib/artik533s-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": "1", + "stars": null +} +, +{ + "name": "jetbrains/qodana-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated a month ago", + "description": "Qodana for Python projects", + "chips": [ + "Languages & Frameworks", + "Integration & Delivery", + "Security" + ], + "downloads": "100K+", + "pulls_last_week": "860", + "stars": null +} +, +{ + "name": "chainguard/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 days ago", + "description": "Minimal Python image based on Wolfi.", + "chips": [], + "downloads": "10K+", + "pulls_last_week": "543", + "stars": null +} +, +{ + "name": "balenalib/fincm3-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated a year ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": "58", + "stars": null +} +, +{ + "name": "balenalib/amd64-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Machine Learning & AI" + ], + "downloads": "100K+", + "pulls_last_week": "242", + "stars": null +} +, +{ + "name": "balenalib/amd64-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": "15", + "stars": null +} +, +{ + "name": "amazon/aws-sam-cli-build-image-python3.8", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 10 months ago", + "description": null, + "chips": [ + "Languages & Frameworks", + "Integration & Delivery" + ], + "downloads": "1M+", + "pulls_last_week": "626", + "stars": null +} +, +{ + "name": "balenalib/artik5-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": "1", + "stars": null +} +, +{ + "name": "balenalib/armv7hf-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": "6", + "stars": null +} +, +{ + "name": "balenalib/amd64-ubuntu-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Data Science", + "Languages & Frameworks", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": "23", + "stars": null +} +, +{ + "name": "balenalib/artik10-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things" + ], + "downloads": "100K+", + "pulls_last_week": "1", + "stars": null +} +, +{ + "name": "balenalib/apalis-imx6q-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": "1", + "stars": null +} +, +{ + "name": "balenalib/aarch64-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Machine Learning & AI" + ], + "downloads": "100K+", + "pulls_last_week": "216", + "stars": null +} +, +{ + "name": "balenalib/raspberry-pi-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": "161", + "stars": null +} +] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/20240701210856.json b/neptun_webscraper/spiders/logs/20240701210856.json new file mode 100644 index 0000000..7a238cf --- /dev/null +++ b/neptun_webscraper/spiders/logs/20240701210856.json @@ -0,0 +1,374 @@ +[ +{ + "name": "python", + "is_official_image": true, + "is_verified_publisher": false, + "last_update": "Updated 4 days ago", + "description": "Python is an interpreted, interactive, object-oriented, open-source programming language.", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "1B+", + "pulls_last_week": "6904987", + "stars": "9.7K" +} +, +{ + "name": "pypy", + "is_official_image": true, + "is_verified_publisher": false, + "last_update": "Updated 10 days ago", + "description": "PyPy is a fast, compliant alternative implementation of the Python language.", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "10M+", + "pulls_last_week": "17171", + "stars": "9.7K" +} +, +{ + "name": "hylang", + "is_official_image": true, + "is_verified_publisher": false, + "last_update": "Updated 4 days ago", + "description": "Hy is a Lisp dialect that translates expressions into Python's abstract syntax tree.", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "10M+", + "pulls_last_week": "22302", + "stars": "9.7K" +} +, +{ + "name": "circleci/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 3 years ago", + "description": "Python is an interpreted, interactive, object-oriented, open-source programming language.", + "chips": [], + "downloads": "100M+", + "pulls_last_week": "161684", + "stars": "9.7K" +} +, +{ + "name": "cimg/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 3 months ago", + "description": null, + "chips": [ + "Languages & Frameworks", + "Integration & Delivery", + "Security" + ], + "downloads": "100M+", + "pulls_last_week": "1121955", + "stars": "9.7K" +} +, +{ + "name": "bitnami/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 days ago", + "description": "Bitnami container image for Python", + "chips": [], + "downloads": "1M+", + "pulls_last_week": "15146", + "stars": "9.7K" +} +, +{ + "name": "amazon/aws-lambda-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 4 days ago", + "description": "AWS Lambda base images for Python", + "chips": [ + "Languages & Frameworks", + "Integration & Delivery", + "Machine Learning & AI" + ], + "downloads": "1M+", + "pulls_last_week": "15574", + "stars": "9.7K" +} +, +{ + "name": "okteto/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": null, + "chips": [ + "Languages & Frameworks", + "Integration & Delivery" + ], + "downloads": "100K+", + "pulls_last_week": "26", + "stars": "9.7K" +} +, +{ + "name": "google/guestbook-python-redis", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 9 years ago", + "description": "A simple guestbook example written in Python. It works against a redis backend.", + "chips": [ + "Databases & Storage", + "Languages & Frameworks", + "Integration & Delivery" + ], + "downloads": "1M+", + "pulls_last_week": "6", + "stars": "9.7K" +} +, +{ + "name": "ubuntu/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 13 days ago", + "description": "A chiselled Ubuntu rock with the Python runtime", + "chips": [ + "Languages & Frameworks" + ], + "downloads": "10K+", + "pulls_last_week": "561", + "stars": "9.7K" +} +, +{ + "name": "balenalib/artik533s-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": "1", + "stars": "9.7K" +} +, +{ + "name": "jetbrains/qodana-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated a month ago", + "description": "Qodana for Python projects", + "chips": [ + "Languages & Frameworks", + "Integration & Delivery", + "Security" + ], + "downloads": "100K+", + "pulls_last_week": "860", + "stars": "9.7K" +} +, +{ + "name": "chainguard/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 days ago", + "description": "Minimal Python image based on Wolfi.", + "chips": [], + "downloads": "10K+", + "pulls_last_week": "543", + "stars": "9.7K" +} +, +{ + "name": "intel/python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": null, + "chips": [], + "downloads": "10K+", + "pulls_last_week": "18", + "stars": "9.7K" +} +, +{ + "name": "balenalib/amd64-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": "15", + "stars": "9.7K" +} +, +{ + "name": "balenalib/amd64-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Machine Learning & AI" + ], + "downloads": "100K+", + "pulls_last_week": "242", + "stars": "9.7K" +} +, +{ + "name": "amazon/aws-sam-cli-build-image-python3.8", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 10 months ago", + "description": null, + "chips": [ + "Languages & Frameworks", + "Integration & Delivery" + ], + "downloads": "1M+", + "pulls_last_week": "626", + "stars": "9.7K" +} +, +{ + "name": "balenalib/artik5-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": "1", + "stars": "9.7K" +} +, +{ + "name": "balenalib/amd64-ubuntu-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Data Science", + "Languages & Frameworks", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": "23", + "stars": "9.7K" +} +, +{ + "name": "balenalib/artik10-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things" + ], + "downloads": "100K+", + "pulls_last_week": "1", + "stars": "9.7K" +} +, +{ + "name": "balenalib/apalis-imx6q-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": "1", + "stars": "9.7K" +} +, +{ + "name": "balenalib/fincm3-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated a year ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": "58", + "stars": "9.7K" +} +, +{ + "name": "balenalib/raspberry-pi-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": "161", + "stars": "9.7K" +} +, +{ + "name": "balenalib/artik710-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 10 months ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "100K+", + "pulls_last_week": "1", + "stars": "9.7K" +} +, +{ + "name": "balenalib/apalis-imx6q-alpine-python", + "is_official_image": false, + "is_verified_publisher": true, + "last_update": "Updated 2 years ago", + "description": "This image is part of the balena.io base image series for IoT devices.", + "chips": [ + "Languages & Frameworks", + "Internet of Things", + "Operating Systems" + ], + "downloads": "500K+", + "pulls_last_week": "1", + "stars": "9.7K" +} +] \ No newline at end of file diff --git a/neptun_webscraper/spiders/logs/screenshots/20240701210415.png b/neptun_webscraper/spiders/logs/screenshots/20240701210415.png new file mode 100644 index 0000000..0b13383 Binary files /dev/null and b/neptun_webscraper/spiders/logs/screenshots/20240701210415.png differ diff --git a/neptun_webscraper/spiders/logs/screenshots/20240701210505.png b/neptun_webscraper/spiders/logs/screenshots/20240701210505.png new file mode 100644 index 0000000..0c0b3d5 Binary files /dev/null and b/neptun_webscraper/spiders/logs/screenshots/20240701210505.png differ diff --git a/neptun_webscraper/spiders/logs/screenshots/20240701210715.png b/neptun_webscraper/spiders/logs/screenshots/20240701210715.png new file mode 100644 index 0000000..9fec3e1 Binary files /dev/null and b/neptun_webscraper/spiders/logs/screenshots/20240701210715.png differ diff --git a/neptun_webscraper/spiders/logs/screenshots/20240701210813.png b/neptun_webscraper/spiders/logs/screenshots/20240701210813.png new file mode 100644 index 0000000..8bdfa9c Binary files /dev/null and b/neptun_webscraper/spiders/logs/screenshots/20240701210813.png differ diff --git a/neptun_webscraper/spiders/logs/screenshots/20240701210849.png b/neptun_webscraper/spiders/logs/screenshots/20240701210849.png new file mode 100644 index 0000000..ddaeb49 Binary files /dev/null and b/neptun_webscraper/spiders/logs/screenshots/20240701210849.png differ