Use pip-tools to help manage dependencies.

ao3scrape/README.md

@@ -6,3 +6,7 @@ To run locally:
 4. Run `scrapy crawl ao3`.
 
 Markdown files will be created in `../backup/content/posts`.
+
+To upgrade requirements:
+1. Make sure pip-tools is installed: `pip install pip-tools`.
+2. Run `pip-compile requirements.in > requirements.txt`

ao3scrape/requirements.in

@@ -0,0 +1,2 @@
+markdownify
+Scrapy

ao3scrape/requirements.txt

@@ -1,40 +1,121 @@
-attrs==22.2.0
-Automat==22.10.0
-beautifulsoup4==4.11.2
+#
+# This file is autogenerated by pip-compile with Python 3.12
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+attrs==24.2.0
+    # via
+    #   automat
+    #   service-identity
+    #   twisted
+automat==22.10.0
+    # via twisted
+beautifulsoup4==4.12.3
+    # via markdownify
 certifi==2024.7.4
-cffi==1.15.1
-charset-normalizer==3.1.0
-constantly==15.1.0
-cryptography==42.0.4
+    # via requests
+cffi==1.17.0
+    # via cryptography
+charset-normalizer==3.3.2
+    # via requests
+constantly==23.10.4
+    # via twisted
+cryptography==43.0.0
+    # via
+    #   pyopenssl
+    #   scrapy
+    #   service-identity
 cssselect==1.2.0
-filelock==3.10.0
+    # via
+    #   parsel
+    #   scrapy
+defusedxml==0.7.1
+    # via scrapy
+filelock==3.15.4
+    # via tldextract
 hyperlink==21.0.0
+    # via twisted
 idna==3.7
-incremental==22.10.0
-itemadapter==0.7.0
-itemloaders==1.0.6
+    # via
+    #   hyperlink
+    #   requests
+    #   tldextract
+incremental==24.7.2
+    # via twisted
+itemadapter==0.9.0
+    # via
+    #   itemloaders
+    #   scrapy
+itemloaders==1.3.1
+    # via scrapy
 jmespath==1.0.1
-lxml==4.9.2
-markdownify==0.11.6
-packaging==23.0
-parsel==1.7.0
-Protego==0.2.1
-pyasn1==0.4.8
-pyasn1-modules==0.2.8
-pycparser==2.21
-PyDispatcher==2.0.7
-pyOpenSSL==24.0.0
-PyYAML==6.0
-queuelib==1.6.2
-requests==2.32.0
-requests-file==1.5.1
-Scrapy==2.11.2
-service-identity==21.1.0
+    # via
+    #   itemloaders
+    #   parsel
+lxml==5.3.0
+    # via
+    #   parsel
+    #   scrapy
+markdownify==0.13.1
+    # via -r requirements.in
+packaging==24.1
+    # via
+    #   parsel
+    #   scrapy
+parsel==1.9.1
+    # via
+    #   itemloaders
+    #   scrapy
+protego==0.3.1
+    # via scrapy
+pyasn1==0.6.0
+    # via
+    #   pyasn1-modules
+    #   service-identity
+pyasn1-modules==0.4.0
+    # via service-identity
+pycparser==2.22
+    # via cffi
+pydispatcher==2.0.7
+    # via scrapy
+pyopenssl==24.2.1
+    # via scrapy
+queuelib==1.7.0
+    # via scrapy
+requests==2.32.3
+    # via
+    #   requests-file
+    #   tldextract
+requests-file==2.1.0
+    # via tldextract
+scrapy==2.11.2
+    # via -r requirements.in
+service-identity==24.1.0
+    # via scrapy
 six==1.16.0
-soupsieve==2.4
-tldextract==3.4.0
-Twisted==24.7.0
-typing_extensions==4.5.0
-urllib3==1.26.19
-w3lib==2.1.1
-zope.interface==5.5.2
+    # via
+    #   automat
+    #   markdownify
+soupsieve==2.6
+    # via beautifulsoup4
+tldextract==5.1.2
+    # via scrapy
+twisted==24.7.0
+    # via scrapy
+typing-extensions==4.12.2
+    # via twisted
+urllib3==2.2.2
+    # via requests
+w3lib==2.2.1
+    # via
+    #   itemloaders
+    #   parsel
+    #   scrapy
+zope-interface==7.0.1
+    # via
+    #   scrapy
+    #   twisted
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools