Merge pull request #36 from AgPipeline/develop

Merging develop to main branch - no review

.github/workflows/check_plotclip_details.py

@@ -5,6 +5,8 @@
 import argparse
 import os
 import subprocess
+import PIL.Image
+import numpy as np
 
 # The default file extension to look for
 DEFAULT_FILE_EXT = '.tif'
@@ -34,6 +36,30 @@ def get_arguments() -> argparse.Namespace:
     return args
 
 
+def check_images(first_path: str, second_path: str) -> None:
+    """Compares the two image files and throws an exception if they don't match
+    Arguments:
+        first_path: the path to the first file to compare
+        second_path: the path to the second file to compare
+    """
+    first_pixels = PIL.Image.open(first_path)
+    first_array = np.array(first_pixels)
+    del first_pixels
+
+    second_pixels = PIL.Image.open(second_path)
+    second_array = np.array(second_pixels)
+    del second_pixels
+
+    if first_array.shape != second_array.shape:
+        raise RuntimeError("Image dimensions are different: %s vs %s" % (str(first_array.shape), str(second_array.shape)))
+
+    for i in range(0, first_array.shape[0]):
+        for j in range(0, first_array.shape[1]):
+            for k in range(0, first_array.shape[2]):
+                if first_array[i][j][k] - second_array[i][j][k] != 0:
+                    raise RuntimeError("Image pixels are different. First difference at %s %s %s" % (str(i), str(j), str(k)))
+
+
 def compare_file_contents(source_file: str, compare_file: str) -> bool:
     """Compares the source file against the comparison file for differences
     Arguments:
@@ -42,6 +68,10 @@ def compare_file_contents(source_file: str, compare_file: str) -> bool:
     Returns:
         Returns True if the files are considered the same, and False if they are not
     """
+    if (os.path.splitext(source_file)[1]).lower() in ['.tif', '.tiff']:
+        check_images(source_file, compare_file)
+        return True
+
     cmd = ['diff', '--brief', source_file, compare_file]
     res = subprocess.run(cmd, capture_output=True, check=True)
     return len(res.stdout) == 0
@@ -75,6 +105,7 @@ def find_compare_files(truth_folder: str, compare_folder: str, file_ext: str, ch
         # Check truth folder for matching files and sub-folders
         for one_file in os.listdir(cur_truth):
             if os.path.splitext(one_file)[1].lower() == check_ext:
+                print("Checking file:", os.path.join(cur_truth, one_file))
                 if not os.path.exists(os.path.join(cur_compare, one_file)):
                     raise RuntimeError('Unable to find expected file "%s"' % os.path.join(cur_subpath, one_file))
                 if not compare_file_contents(os.path.join(cur_truth, one_file), os.path.join(cur_compare, one_file)):

.github/workflows/testing_checks.yaml

@@ -17,7 +17,7 @@ on:
 
 jobs:
   testing:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     name: Running testing
     strategy:
       matrix:
@@ -33,10 +33,10 @@ jobs:
     steps:
       - name: Current python version
         run: python3 --version || echo python3 not installed
-      - name: Install Python 3.7
+      - name: Install Python 3.8
         uses: actions/setup-python@v2
         with:
-          python-version: '3.7'
+          python-version: '3.8'
       - name: Updated python version
         run: python3 --version
       - name: PYTHONPATH environment variable
@@ -53,13 +53,13 @@ jobs:
         run: find . -type f -name "*.py" > action_test_files.txt
       - name: Install system requirements
         shell: bash
-        run: 'sudo apt-get install -y python3-gdal gdal-bin libgdal-dev gcc g++ python3.7-dev'
+        run: 'sudo apt-get install -y python3-gdal gdal-bin libgdal-dev gcc g++ python3.8-dev'
       - name: Install Python numpy
         shell: bash
         run: 'python3 -m pip install --upgrade --no-cache-dir numpy wheel'
       - name: Install Python pygdal
         shell: bash
-        run: 'python3 -m pip install --no-cache-dir pygdal==2.2.3.5'
+        run: 'python3 -m pip install --no-cache-dir pygdal==3.0.4.*'
       - name: Install system requirements from source
         shell: bash
         run: '[ -s "packages.txt" ] && (cat packages.txt | sudo xargs apt-get install -y --no-install-recommends) || (echo "Failed to install additional packages" && exit 1)'

.github/workflows/testing_docker.yaml

@@ -20,10 +20,16 @@ jobs:
     runs-on: ubuntu-latest
     name: Running Docker testing
     steps:
-      - name: Install Python 3.7
+      - name: Install Python 3.8
         uses: actions/setup-python@v2
         with:
-          python-version: '3.7'
+          python-version: '3.8'
+      - name: Updated python version
+        run: python3 --version
+      - name: Install python dependencies
+        run: |
+             python3 -m pip install -U pip
+             python3 -m pip install -U numpy Pillow
       - name: Fetch source code
         uses: actions/checkout@v2
       - name: Build docker image

Dockerfile

@@ -1,5 +1,6 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 LABEL maintainer="Chris Schnaufer <[email protected]>"
+ENV DEBIAN_FRONTEND noninteractive
 
 # Add user
 RUN useradd -u 49044 extractor \
@@ -11,11 +12,11 @@ RUN chown -R extractor /home/extractor \
 RUN apt-get update && \
     apt-get upgrade -y && \
     apt-get install -y --no-install-recommends \
-        python3.7 \
+        python3.8 \
         python3-pip && \
-    ln -sfn /usr/bin/python3.7 /usr/bin/python && \
-    ln -sfn /usr/bin/python3.7 /usr/bin/python3 && \
-    ln -sfn /usr/bin/python3.7m /usr/bin/python3m && \
+    ln -sfn /usr/bin/python3.8 /usr/bin/python && \
+    ln -sfn /usr/bin/python3.8 /usr/bin/python3 && \
+    ln -sfn /usr/bin/python3.8m /usr/bin/python3m && \
     apt-get autoremove -y && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
@@ -32,13 +33,13 @@ RUN apt-get update && \
         libgdal-dev  \
         gcc \
         g++ \
-        python3.7-dev && \
+        python3.8-dev && \
     python3 -m pip install --upgrade --no-cache-dir \
         wheel && \
     python3 -m pip install --upgrade --no-cache-dir \
         numpy && \
     python3 -m pip install --upgrade --no-cache-dir \
-        pygdal==2.2.3.* && \
+        pygdal==3.0.4.* && \
     apt-get remove -y \
         libgdal-dev \
         gcc \

packages.txt

@@ -1,2 +1 @@
 pdal
-liblas-bin

plotclip.py

@@ -3,7 +3,6 @@
 """
 
 import argparse
-import copy
 import datetime
 import json
 import logging
@@ -14,6 +13,7 @@
 import numpy as np
 from agpypeline import algorithm, entrypoint, geometries, geoimage, lasfile
 from agpypeline.environment import Environment
+from agpypeline.checkmd import CheckMD
 from osgeo import gdal, ogr, osr
 
 from configuration import ConfigurationPlotclip
@@ -299,7 +299,7 @@ def calculate_overlap_percent(check_bounds: ogr.Geometry, other_bounds: ogr.Geom
         return 0.0
 
     @staticmethod
-    def cleanup_request_md(source_md: dict) -> dict:
+    def cleanup_request_md(source_md: CheckMD) -> dict:
         """Makes a copy of the source metadata and cleans it up for use as plot-level information
         Arguments:
             source_md: the source metadata to clone and clean up
@@ -309,10 +309,14 @@ def cleanup_request_md(source_md: dict) -> dict:
         if not source_md:
             return {}
 
-        new_md = copy.deepcopy(source_md)
-        new_md.pop('list_files', None)
-        new_md.pop('context_md', None)
-        new_md.pop('working_folder', None)
+        new_md = {
+            'timestamp': source_md.timestamp,
+            'season': source_md.season,
+            'experiment': source_md.experiment,
+            'container_name': source_md.container_name,
+            'target_container_name': source_md.target_container_name,
+            'trigger_name': source_md.trigger_name
+        }
 
         return new_md
 
@@ -517,7 +521,7 @@ def add_parameters(self, parser: argparse.ArgumentParser) -> None:
                             help='the name of the column in the plot geometry file containing plot names')
         parser.add_argument('plot_file', type=str, help='the path of the GeoJSON file to use for plot boundaries')
 
-    def perform_process(self, environment: Environment, check_md: dict, transformer_md: dict, full_md: list) -> dict:
+    def perform_process(self, environment: Environment, check_md: CheckMD, transformer_md: dict, full_md: list) -> dict:
         """Performs the processing of the data
         Arguments:
             environment: instance of environment class
@@ -532,7 +536,7 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_
         processed_files = 0
         processed_plots = 0
         start_timestamp = datetime.datetime.now()
-        file_list = check_md['list_files']()
+        file_list = check_md.get_list_files()
         files_to_process = __internal__.get_files_to_process(file_list, environment.args.epsg)
         logging.info("Found %s files to process", str(len(files_to_process)))
 
@@ -567,7 +571,7 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_
 
                     if filename.endswith('.tif'):
                         # If file is a geoTIFF, simply clip it
-                        out_path = os.path.join(check_md['working_folder'], plot_name)
+                        out_path = os.path.join(check_md.working_folder, plot_name)
                         out_file = os.path.join(out_path, filename)
                         __internal__.clip_tiff(file_path, file_bounds, plot_bounds, out_file, environment.args.full_plot_fill)
 
@@ -579,7 +583,7 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_
 
                     elif filename.endswith('.las'):
                         tuples = geometries.geometry_to_tuples(plot_bounds)
-                        out_path = os.path.join(check_md['working_folder'], plot_name)
+                        out_path = os.path.join(check_md.working_folder, plot_name)
                         out_file = os.path.join(out_path, filename)
                         __internal__.clip_las(file_path, tuples, out_file)
 

requirements.txt

@@ -1,3 +1,2 @@
-liblas
 dbfread
 agpypeline