[tests] Update tests to allow for tolerances on ARM platforms.

.github/workflows/build.yml

@@ -30,8 +30,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        # TODO(v1.6.1): Add macos-14 builder.
-        os: [macos-14]
+        os: [macos-13, macos-14, ubuntu-20.04, ubuntu-latest, windows-latest]
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
@@ -45,19 +44,40 @@ jobs:
 
       - name: Install Dependencies
         run: |
-          python -m pip install --upgrade pip build wheel virtualenv
-          pip install opencv-python-headless opencv-contrib-python-headless --only-binary :all:
-          pip install -r requirements_headless.txt
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade build wheel virtualenv
+          python -m pip install opencv-python-headless opencv-contrib-python-headless --only-binary :all:
+          python -m pip install -r requirements_headless.txt
 
-      - name: Create Mask File
+      - name: Unit Test
         run: |
-          python -m dvr_scan -i tests/resources/traffic_camera.mp4 -mo mask.avi -fm -so --add-region 631 532 841 532 841 659 631 659 --min-event-length 4 --time-before-event 0 -k 0
+          python -m pytest tests/
 
-      - name: Upload Artifact
+      - name: Build Package
+        shell: bash
+        run: |
+          python -m build
+          echo "dvr_scan_version=`python -c \"import dvr_scan; print(dvr_scan.__version__.replace('-', '.'))\"`" >> "$GITHUB_ENV"
+
+      - name: Smoke Test (Source)
+        run: |
+          python -m pip install dist/dvr_scan-${{ env.dvr_scan_version }}.tar.gz
+          python -m dvr_scan --version
+          python -m dvr_scan -i tests/resources/simple_movement.mp4 -so -df 4 -et 100
+          python -m pip uninstall -y dvr-scan
+
+      - name: Smoke Test (Wheel)
+        run: |
+          python -m pip install dist/dvr_scan-${{ env.dvr_scan_version }}-py3-none-any.whl
+          python -m dvr_scan --version
+          python -m dvr_scan -i tests/resources/simple_movement.mp4 -so -df 4 -et 100
+          python -m pip uninstall -y dvr-scan
+
+      - name: Upload Package
+        if: ${{ matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest' }}
         uses: actions/upload-artifact@v3
         with:
-          name: macos-14-test-file
-          path: mask.avi
-      #- name: Unit Test
-      #  run: |
-      #    python -m pytest tests/
+          name: dvr-scan-dist
+          path: |
+            dist/*.tar.gz
+            dist/*.whl

.github/workflows/check-code-format.yml

@@ -23,7 +23,7 @@ jobs:
       - name: Set up Python 3.x
         uses: actions/setup-python@v3
         with:
-          python-version: '3.x'
+          python-version: '3.12'
 
       - name: Update pip
         run: python -m pip install --upgrade pip

.github/workflows/update-site.yml

@@ -1,4 +1,5 @@
 # Update www.dvr-scan.com
+# TODO: Add versioning for documentation. Currently only the latest docs are shown.
 name: Update Website
 
 on:
@@ -17,10 +18,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: '3.12'
           cache: 'pip'
 
       - name: Install Dependencies

tests/test_cli.py

@@ -15,6 +15,7 @@
 """
 
 import os
+import platform
 import subprocess
 from typing import List
 
@@ -26,6 +27,8 @@
 
 from dvr_scan.subtractor import SubtractorCNT, SubtractorCudaMOG2
 
+MACHINE_ARCH = platform.machine().upper()
+
 # TODO: Open extracted motion events and validate the actual frames.
 
 DVR_SCAN_COMMAND: List[str] = 'python -m dvr_scan'.split(' ')
@@ -40,12 +43,15 @@
     '4',
     '--time-before-event',
     '0',
+    '--threshold',
+    '0.2',
 ]
 BASE_COMMAND_NUM_EVENTS = 3
 
 TEST_CONFIG_FILE = """
 min-event-length = 4
 time-before-event = 0
+threshold = 0.2
 """
 
 # TODO: Need to generate goldens for CNT/MOG2_CUDA, as their output can differ slightly.
@@ -59,8 +65,12 @@
 -------------------------------------------------------------
 """[1:]
 
+# On some ARM chips (e.g. Apple M1), results are slightly different, so we allow a 1 frame
+# delta on the events for those platforms.
 BASE_COMMAND_TIMECODE_LIST_GOLDEN = """
-00:00:00.360,00:00:05.960,00:00:14.320,00:00:19.640,00:00:21.680,00:00:23.040
+00:00:00.400,00:00:05.960,00:00:14.320,00:00:19.640,00:00:21.680,00:00:23.040
+"""[1:] if not ('ARM' in MACHINE_ARCH or 'AARCH' in MACHINE_ARCH) else """
+00:00:00.400,00:00:06.000,00:00:14.320,00:00:19.640,00:00:21.680,00:00:23.040
 """[1:]
 
 

tests/test_scan_context.py

@@ -14,13 +14,24 @@
 Validates functionality of the motion scanning context using various parameters.
 """
 
+import platform
+import typing as ty
+
 import pytest
 
-from dvr_scan.detector import Rectangle
 from dvr_scan.scanner import DetectorType, MotionScanner
 from dvr_scan.subtractor import SubtractorCNT, SubtractorCudaMOG2
 from dvr_scan.region import Point
 
+MACHINE_ARCH = platform.machine().upper()
+
+# On some ARM chips (e.g. Apple M1), results are slightly different, so we allow a 1 frame
+# delta on the events for those platforms.
+EVENT_FRAME_TOLERANCE = 1 if ('ARM' in MACHINE_ARCH or 'AARCH' in MACHINE_ARCH) else 0
+
+# Similar to ARM, the CUDA version gives slightly different results.
+CUDA_EVENT_TOLERANCE = 1
+
 # ROI within the frame used for the test case (see traffic_camera.txt for details).
 TRAFFIC_CAMERA_ROI = [
     Point(631, 532),
@@ -35,9 +46,6 @@
     (542, 576),
 ]
 
-# Allow up to 1 frame difference in ground truth due to different floating point handling.
-CUDA_EVENT_TOLERANCE = 1
-
 TRAFFIC_CAMERA_EVENTS_TIME_PRE_5 = [
     (3, 149),
     (352, 491),
@@ -69,6 +77,20 @@
 ]
 
 
+def compare_event_lists(a: ty.List[ty.Tuple[int, int]],
+                        b: ty.List[ty.Tuple[int, int]],
+                        tolerance: int = 0):
+    if tolerance == 0:
+        assert a == b
+        return
+    for i, (start, end) in enumerate(a):
+        start_matches = abs(start - b[i][0]) <= tolerance
+        end_matches = abs(end - b[i][1]) <= tolerance
+        assert start_matches and end_matches, (
+            f"Event mismatch at index {i} with tolerance {tolerance}.\n"
+            f"Actual = {a[i]}, Expected = {b[i]}")
+
+
 def test_scan_context(traffic_camera_video):
     """Test functionality of MotionScanner with default parameters (DetectorType.MOG2)."""
     scanner = MotionScanner([traffic_camera_video])
@@ -77,7 +99,7 @@ def test_scan_context(traffic_camera_video):
     scanner.set_event_params(min_event_len=4, time_pre_event=0)
     event_list = scanner.scan().event_list
     event_list = [(event.start.frame_num, event.end.frame_num) for event in event_list]
-    assert event_list == TRAFFIC_CAMERA_EVENTS
+    compare_event_lists(event_list, TRAFFIC_CAMERA_EVENTS, EVENT_FRAME_TOLERANCE)
 
 
 @pytest.mark.skipif(not SubtractorCudaMOG2.is_available(), reason="CUDA module not available.")
@@ -90,12 +112,7 @@ def test_scan_context_cuda(traffic_camera_video):
     event_list = scanner.scan().event_list
     assert len(event_list) == len(TRAFFIC_CAMERA_EVENTS)
     event_list = [(event.start.frame_num, event.end.frame_num) for event in event_list]
-    for i, event in enumerate(event_list):
-        start_matches = abs(event.start - TRAFFIC_CAMERA_EVENTS[i][0]) <= CUDA_EVENT_TOLERANCE
-        end_matches = abs(event.start - TRAFFIC_CAMERA_EVENTS[i][0]) <= CUDA_EVENT_TOLERANCE
-        assert start_matches and end_matches, (
-            "Event mismatch at index %d with tolerance %d:\n Actual:   %s\n Expected: %s" %
-            (i, CUDA_EVENT_TOLERANCE, str(event), str(TRAFFIC_CAMERA_EVENTS[i])))
+    compare_event_lists(event_list, TRAFFIC_CAMERA_EVENTS, CUDA_EVENT_TOLERANCE)
 
 
 @pytest.mark.skipif(not SubtractorCNT.is_available(), reason="CNT algorithm not available.")
@@ -107,7 +124,7 @@ def test_scan_context_cnt(traffic_camera_video):
     scanner.set_event_params(min_event_len=3, time_pre_event=0)
     event_list = scanner.scan().event_list
     event_list = [(event.start.frame_num, event.end.frame_num) for event in event_list]
-    assert event_list == TRAFFIC_CAMERA_EVENTS_CNT
+    compare_event_lists(event_list, TRAFFIC_CAMERA_EVENTS_CNT, EVENT_FRAME_TOLERANCE)
 
 
 def test_pre_event_shift(traffic_camera_video):
@@ -117,7 +134,7 @@ def test_pre_event_shift(traffic_camera_video):
     scanner.set_event_params(min_event_len=4, time_pre_event=6)
     event_list = scanner.scan().event_list
     event_list = [(event.start.frame_num, event.end.frame_num) for event in event_list]
-    assert event_list == TRAFFIC_CAMERA_EVENTS_TIME_PRE_5
+    compare_event_lists(event_list, TRAFFIC_CAMERA_EVENTS_TIME_PRE_5, EVENT_FRAME_TOLERANCE)
 
 
 def test_pre_event_shift_with_frame_skip(traffic_camera_video):
@@ -148,7 +165,7 @@ def test_post_event_shift(traffic_camera_video):
     event_list = scanner.scan().event_list
     assert len(event_list) == len(TRAFFIC_CAMERA_EVENTS_TIME_POST_40)
     event_list = [(event.start.frame_num, event.end.frame_num) for event in event_list]
-    assert all([x == y for x, y in zip(event_list, TRAFFIC_CAMERA_EVENTS_TIME_POST_40)])
+    compare_event_lists(event_list, TRAFFIC_CAMERA_EVENTS_TIME_POST_40, EVENT_FRAME_TOLERANCE)
 
 
 def test_post_event_shift_with_frame_skip(traffic_camera_video):
@@ -180,7 +197,7 @@ def test_decode_corrupt_video(corrupt_video):
     scanner.set_regions(regions=[CORRUPT_VIDEO_ROI])
     event_list = scanner.scan().event_list
     event_list = [(event.start.frame_num, event.end.frame_num) for event in event_list]
-    assert event_list == CORRUPT_VIDEO_EVENTS
+    compare_event_lists(event_list, CORRUPT_VIDEO_EVENTS, EVENT_FRAME_TOLERANCE)
 
 
 def test_start_end_time(traffic_camera_video):
@@ -192,7 +209,7 @@ def test_start_end_time(traffic_camera_video):
     event_list = scanner.scan().event_list
     event_list = [(event.start.frame_num, event.end.frame_num) for event in event_list]
     # The set duration should only cover the middle event.
-    assert event_list == TRAFFIC_CAMERA_EVENTS[1:2]
+    compare_event_lists(event_list, TRAFFIC_CAMERA_EVENTS[1:2], EVENT_FRAME_TOLERANCE)
 
 
 def test_start_duration(traffic_camera_video):
@@ -204,4 +221,4 @@ def test_start_duration(traffic_camera_video):
     event_list = scanner.scan().event_list
     event_list = [(event.start.frame_num, event.end.frame_num) for event in event_list]
     # The set duration should only cover the middle event.
-    assert event_list == TRAFFIC_CAMERA_EVENTS[1:2]
+    compare_event_lists(event_list, TRAFFIC_CAMERA_EVENTS[1:2], EVENT_FRAME_TOLERANCE)