bumps python version

.github/deploy_failure.md

@@ -8,4 +8,4 @@ Job Failed: {{ env.GITHUB_JOB }}
 Last Commit: {{ env.LAST_COMMIT }}
 Number of times run: {{ env.GITHUB_ATTEMPTS }}
 Last run by: {{ env.LAST_RUN_BY }}
-Github Action Run: https://github.com/{{ env.REPO }}/actions/runs/{{ env.RUN_ID }}
+Github Action Run: https://github.com/{{ env.REPO }}/actions/runs/{{ env.RUN_ID }}

.github/workflows/load_test.yml

@@ -1,7 +1,7 @@
 name: Load Test
 on:
   workflow_dispatch:
-    
+
 jobs:
   load-test:
     name: H2.0 Load Test

.python-version

@@ -1 +1 @@
-3.10.0
+3.10.14

.yamllint

@@ -6,4 +6,3 @@ rules:
     max: 220
     level: warning
   truthy: disable
-

Dockerfile

@@ -6,7 +6,7 @@ COPY . /app
 
 RUN pip install --no-cache-dir -r requirements.txt
 
-ARG DEV 
+ARG DEV
 
 RUN if [ $DEV ]; \
     then pip install --no-cache-dir -r requirements-dev.txt; \
@@ -17,4 +17,4 @@ EXPOSE 8080
 ENV FLASK_APP=run.py
 
 # Run run.py when the container launches
-CMD ["flask", "run", "--host=0.0.0.0", "--port=8080"]
+CMD ["flask", "run", "--host=0.0.0.0", "--port=8080"]

LICENSE.md

@@ -2,7 +2,7 @@
 
 This project constitutes a work of the United States Government and is not subject to domestic copyright protection under 17 USC § 105. Additionally, we waive copyright and related rights in the work worldwide through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/).
 
-All contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest. See [CONTRIBUTING](CONTRIBUTING.md) for more information. 
+All contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest. See [CONTRIBUTING](CONTRIBUTING.md) for more information.
 
 ## GNU General Public License
 

Makefile

@@ -45,6 +45,9 @@ test-ci: ## Runs all tests using only db and required test resources. NOT compat
 	make test-functional
 	make down
 
+test-pre-commit: ## Runs all pre-commit hooks manually
+	pre-commit run --all-files
+
 up: ## Sets up local flask and harvest runner docker environments. harvest runner gets DATABASE_PORT from .env
 	DATABASE_PORT=5433 docker compose up -d
 	docker compose -p harvest-app up db -d

app/templates/get_data_sources.html

@@ -8,7 +8,7 @@
 <h2>Harvest Sources</h2>
 <div class="form-group">
     <label for="source_id">Get Harvest Source by id:</label>
-    <select name="source_id" id="source_id" class="form-control" 
+    <select name="source_id" id="source_id" class="form-control"
             onchange="updateSourceLink()">
         <option value="">Select a source</option>
         {% for source in sources %}
@@ -22,7 +22,7 @@ <h2>Harvest Sources</h2>
 
 <div class="form-group">
     <label for="organization_id">Get Harvest Source by Organization:</label>
-    <select name="organization_id" id="organization_id" class="form-control" 
+    <select name="organization_id" id="organization_id" class="form-control"
         onchange="updateOrgLink()">
         <option value="">Select an organization</option>
         {% for org in organizations %}
@@ -39,7 +39,7 @@ <h2>Harvest Sources</h2>
 <h2>Organizations</h2>
 <div class="form-group">
     <label for="organization_id_2">Get Organizations:</label>
-    <select name="organization_id" id="organization_id_2" class="form-control" 
+    <select name="organization_id" id="organization_id_2" class="form-control"
         onchange="updateOrgEditLink()">
         <option value="">Select an organization</option>
         {% for org in organizations %}
@@ -60,7 +60,7 @@ <h2>Organizations</h2>
         var link = document.getElementById('source_link');
         link.href = selectedId ? `/harvest_source/${selectedId}` : "#";
         var exampleLink = document.getElementById('example_source_link');
-        exampleLink.textContent = 
+        exampleLink.textContent =
             selectedId ? `/harvest_source/${selectedId}` : '/harvest_source/';
 
     }
@@ -70,7 +70,7 @@ <h2>Organizations</h2>
         var link = document.getElementById('org_link');
         var exampleLink = document.getElementById('example_org_link');
         link.href = selectedId ? `/harvest_source?organization_id=${selectedId}` : "#";
-        exampleLink.textContent = 
+        exampleLink.textContent =
             selectedId ? `/harvest_source?organization_id
                 =${selectedId}` : '/harvest_source/?organization_id=';
     }
@@ -80,12 +80,11 @@ <h2>Organizations</h2>
         var link = document.getElementById('org_link_2');
         var exampleLink = document.getElementById('example_org_link_2');
         link.href = selectedId ? `/organization/${selectedId}` : "#";
-        exampleLink.textContent = 
+        exampleLink.textContent =
             selectedId ? `/organization/${selectedId}` : '/organization';
     }
 
 
 
 </script>
 {% endblock %}
-

create_cloudgov_service.sh

@@ -1,8 +1,8 @@
 #!/bin/sh
 
-set -e 
+set -e
 
-# If an argument was provided, use it as the service name prefix. 
+# If an argument was provided, use it as the service name prefix.
 # Otherwise default to "harvesting-logic".
 app_name=${1:-harvesting-logic}
 

docker-compose_debug.yml

@@ -3,4 +3,4 @@ services:
     ports:
       - "8080:8080"
       - "5678:5678"
-    command: python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m flask run --host=0.0.0.0 --port=8080
+    command: python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m flask run --host=0.0.0.0 --port=8080

docs/diagrams/mermaid/src/dcat.md

@@ -137,7 +137,7 @@ flowchart LR
     update ==> update_object_reference
     update_object_reference ==> ie
   end
-  
+
 
   %% Data
   error[\Error/]

docs/diagrams/mermaid/src/etl_pipeline.md

@@ -32,7 +32,7 @@ sequenceDiagram
         DHL->>+MD: MDTransform(dataset)
         MD-->>-DHL: Transformed Item
     end
-    DHL-->>HDB: Log failures as harvest_error with type: transform<br>update harvest_record status: error_transform    
+    DHL-->>HDB: Log failures as harvest_error with type: transform<br>update harvest_record status: error_transform
     note over DHL: PUT TO S3
     DHL->>S3: write source_metadata (plus transform artifact) to S3<br>S3://{BUCKET_PREFIX}/{HARVEST_SOURCE_ID}/{UNIQUE_IDENTIFIER}
     note over DHL: DELETE

docs/diagrams/mermaid/src/h20_compare_dcat.md

@@ -19,15 +19,15 @@ end
   updateDataset([Update existing dataset])
 
 
-%% flow  
+%% flow
   getHarvest -- 1-to-N --> extractHarvest
   extractHarvest --> hashDataset
   hashDataset --> compareHash
-  
+
   queryCKAN -- 1-to-N --> extractHash
   extractHash --> compareHash
 
-  
+
   compareHash -- ID found; Hash not same --> updateDataset
   compareHash -- ID not found in Catalog --> createDataset
   compareHash -- ID not found in Harvest Source --> deleteDataset

docs/diagrams/uml/src/erd.plantuml

@@ -1,4 +1,4 @@
-@startuml harvester2.0 ERD 
+@startuml harvester2.0 ERD
 
 skinparam linetype ortho
 
@@ -12,7 +12,7 @@ card "Entity Relationship Diagram" as entities {
     organization_name: txt
     frequency: text
     config: json
-    urls: array( txt ) 
+    urls: array( txt )
     schema_validation_type: text
   }
 
@@ -30,7 +30,7 @@ card "Entity Relationship Diagram" as entities {
     --
     source_id: uuid (FK)
     status: text
-    date_created 
+    date_created
     date_finished: datetime
     extract_started: datetime
     extract_finished: datetime
@@ -59,7 +59,7 @@ card "Entity Relationship Diagram" as entities {
 
 ' lookup tables
 card "Lookup Tables" as lookup {
-  
+
   entity "data_format" as data_format {
     name: text (PK)
     --
@@ -91,7 +91,7 @@ card "Enumerators" as enumerators {
     ACTIVE
     INVALID
   }
-} 
+}
 
 ' relationships
 source ||--|{ job

example_data/iso_2_waf/invalid_47662.xml

@@ -511,4 +511,4 @@ Wells NERR, ME</gco:CharacterString>
       </gmd:transferOptions>
     </gmd:MD_Distribution>
   </gmd:distributionInfo>
-</gmi:MI_Metadata>
+</gmi:MI_Metadata>

example_data/iso_2_waf/valid_47598.xml

@@ -680,4 +680,4 @@ Email: [email protected]</gco:CharacterString>
       </gmd:transferOptions>
     </gmd:MD_Distribution>
   </gmd:distributionInfo>
-</gmi:MI_Metadata>
+</gmi:MI_Metadata>

example_data/waf/USGSImageryOnly.xml

@@ -136,7 +136,7 @@
                 <digtinfo>
                     <formname>ImageryOnly Base Map Service (ArcGIS)</formname>
                     <formvern>10.21</formvern>
-                    <formspec>https://www.geoplatform.gov/spec/</formspec>               
+                    <formspec>https://www.geoplatform.gov/spec/</formspec>
                 </digtinfo>
                 <digtopt>
                     <onlinopt>
@@ -152,7 +152,7 @@
                 <digtinfo>
                     <formname>ImageryOnly Base Map Service (WMS)</formname>
                     <formvern>1.3.0</formvern>
-                    <formspec>https://www.opengeospatial.org/standards/wms</formspec>              
+                    <formspec>https://www.opengeospatial.org/standards/wms</formspec>
                 </digtinfo>
                 <digtopt>
                     <onlinopt>

example_data/waf/USGSImageryTopo.xml

@@ -140,7 +140,7 @@
             <digtinfo>
                <formname>ImageryTopo Base Map Service (ArcGIS)</formname>
                <formvern>10.21</formvern>
-               <formspec>https://www.geoplatform.gov/spec/</formspec>               
+               <formspec>https://www.geoplatform.gov/spec/</formspec>
             </digtinfo>
             <digtopt>
                <onlinopt>
@@ -156,7 +156,7 @@
             <digtinfo>
                <formname>ImageryTopo Base Map Service (WMS)</formname>
                <formvern>1.3.0</formvern>
-               <formspec>https://www.opengeospatial.org/standards/wms</formspec>              
+               <formspec>https://www.opengeospatial.org/standards/wms</formspec>
             </digtinfo>
             <digtopt>
                <onlinopt>

example_data/waf/USGSShadedReliefOnly.xml

@@ -52,7 +52,7 @@
             <themekey>National Geospatial Data Asset</themekey>
             <themekey>NGDA</themekey>
             <themekey>Elevation Theme</themekey>
-         </theme>         
+         </theme>
          <theme>
             <themekt>The National Map Theme Thesaurus</themekt>
             <themekey>Elevation</themekey>
@@ -135,7 +135,7 @@
             <digtinfo>
                <formname>ShadedReliefOnly Base Map Service (ArcGIS)</formname>
                <formvern>10.21</formvern>
-               <formspec>https://www.geoplatform.gov/spec/</formspec>               
+               <formspec>https://www.geoplatform.gov/spec/</formspec>
             </digtinfo>
             <digtopt>
                <onlinopt>
@@ -151,7 +151,7 @@
             <digtinfo>
                <formname>ShadedReliefOnly Base Map Service (WMS)</formname>
                <formvern>1.3.0</formvern>
-               <formspec>https://www.opengeospatial.org/standards/wms</formspec>              
+               <formspec>https://www.opengeospatial.org/standards/wms</formspec>
             </digtinfo>
             <digtopt>
                <onlinopt>

example_data/waf/browse/more/USGSShadedReliefOnly.xml

@@ -52,7 +52,7 @@
             <themekey>National Geospatial Data Asset</themekey>
             <themekey>NGDA</themekey>
             <themekey>Elevation Theme</themekey>
-         </theme>         
+         </theme>
          <theme>
             <themekt>The National Map Theme Thesaurus</themekt>
             <themekey>Elevation</themekey>
@@ -135,7 +135,7 @@
             <digtinfo>
                <formname>ShadedReliefOnly Base Map Service (ArcGIS)</formname>
                <formvern>10.21</formvern>
-               <formspec>https://www.geoplatform.gov/spec/</formspec>               
+               <formspec>https://www.geoplatform.gov/spec/</formspec>
             </digtinfo>
             <digtopt>
                <onlinopt>
@@ -151,7 +151,7 @@
             <digtinfo>
                <formname>ShadedReliefOnly Base Map Service (WMS)</formname>
                <formvern>1.3.0</formvern>
-               <formspec>https://www.opengeospatial.org/standards/wms</formspec>              
+               <formspec>https://www.opengeospatial.org/standards/wms</formspec>
             </digtinfo>
             <digtopt>
                <onlinopt>

example_data/waf/other/USGSImageryOnly.xml

@@ -136,7 +136,7 @@
                 <digtinfo>
                     <formname>ImageryOnly Base Map Service (ArcGIS)</formname>
                     <formvern>10.21</formvern>
-                    <formspec>https://www.geoplatform.gov/spec/</formspec>               
+                    <formspec>https://www.geoplatform.gov/spec/</formspec>
                 </digtinfo>
                 <digtopt>
                     <onlinopt>
@@ -152,7 +152,7 @@
                 <digtinfo>
                     <formname>ImageryOnly Base Map Service (WMS)</formname>
                     <formvern>1.3.0</formvern>
-                    <formspec>https://www.opengeospatial.org/standards/wms</formspec>              
+                    <formspec>https://www.opengeospatial.org/standards/wms</formspec>
                 </digtinfo>
                 <digtopt>
                     <onlinopt>

nginx.conf

@@ -9,8 +9,8 @@ server {
     location / {
         root   /usr/share/nginx/html;
         index  index.html index.htm;
-        
-        # Only customization from default; 
+
+        # Only customization from default;
         # Allows to create a WAF harvest source
         autoindex on;
     }
@@ -23,4 +23,4 @@ server {
     location = /50x.html {
         root   /usr/share/nginx/html;
     }
-}
+}

scripts/monitor-cf-logs.sh

@@ -8,7 +8,7 @@ app_to_monitor=$1
 task_to_monitor=$2
 
 # install a better version of grep which supports --line-buffered
-apk add grep 
+apk add grep
 
 while read -r line ; do
   echo "$line"

tests/integration/harvest/test_harvest_full_flow.py

@@ -199,4 +199,3 @@ def test_send_notification_emails(
         mock_smtp.side_effect = Exception("SMTP failed")
         harvest_source.send_notification_emails(results)
         assert "Failed to send notification email: SMTP failed" in caplog.text
-