fixed docker build (#256)

Apra-Labs · Jul 8, 2023 · 79edaf2 · 79edaf2
1 parent c0838bf
commit 79edaf2
Show file tree

Hide file tree

Showing 6 changed files with 68 additions and 73 deletions.
diff --git a/.github/workflows/CI-Linux-CUDA-Docker.yml b/.github/workflows/CI-Linux-CUDA-Docker.yml
@@ -10,6 +10,18 @@ env:
   NOTE_TO_SELF: "environments can not be passed from here to reused workflows!"
 
 jobs:
+  # linux-cuda-build-prep:
+  # this is in caching, not using it for now
+  #   uses: ./.github/workflows/build-test-lin-container.yml
+  #   with:
+  #     runner: 'ubuntu-20.04'
+  #     flav: 'Linux-Cuda'
+  #     is-selfhosted: false
+  #     cuda: 'ON'
+  #     prep-cmd: 'echo skipping builder prep as I can not sudo'
+  #     is-prep-phase: true
+  #     cache-path: '/github/home/.cache/vcpkg/archives'      
+  #     nProc: 3
   linux-cuda-build-no-test:
     uses: ./.github/workflows/build-test-lin-container.yml
     with:

diff --git a/.github/workflows/build-test-lin-container.yml b/.github/workflows/build-test-lin-container.yml
@@ -73,7 +73,7 @@ jobs:
     env:
       TEST_EXE: build/aprapipesut
       CMAKE_TC_FILE: '../vcpkg/scripts/buildsystems/vcpkg.cmake' # Note: naming this variable as CMAKE_TOOLCHAIN_FILE can cause havoc!!!
-    container: ghcr.io/kumaakh/aprapipes-build-x86-ubutu18.04-cuda:latest
+    container: ghcr.io/kumaakh/aprapipes-build-x86-ubutu18.04-cuda:last-good
     defaults:
       run:
         shell: bash  
@@ -104,12 +104,14 @@ jobs:
         submodules: 'recursive' 
         lfs: true
         fetch-depth: 0
-    - name: Checkout submodule
+
+    - name: List Submodules
       run: |
+        git config --global --add safe.directory "*"
         git submodule status > submodule_ver.txt
         cat submodule_ver.txt
         git rev-list --all --count 
-    
+
     - name: Run VCPKG bootstrap
       run: ${{ inputs.bootstrap-cmd }}
 
@@ -125,12 +127,12 @@ jobs:
       run: .\fix-vcpkg-json.ps1 -removeOpenCV
       shell: pwsh
 
-    - name: cache init on container build
-      if: ${{ job.container != null }} # we are running on a cloud hosted container... move the cache "initially"
-      run: |
-        cp -R /root/.cache /github/home/ || true
-        ls ${{ inputs.cache-path }} || true
-      continue-on-error: true
+    # - name: cache init on container build
+    #   if: ${{ job.container != null }} # we are running on a cloud hosted container... move the cache "initially"
+    #   run: |
+    #     cp -R /root/.cache /github/home/ || true
+    #     ls ${{ inputs.cache-path }} || true
+    #   continue-on-error: true
 
     - name: Cache dependencies for fast cloud build
       id: cache-all
@@ -157,17 +159,13 @@ jobs:
 
     - name: List Test cases
       if: ${{!inputs.is-prep-phase}}
-      env:
-        LD_LIBRARY_PATH: ${{env.LD_LIB_PATH}}
       run: |
-        ldd ${{env.TEST_EXE}} | grep 'not found' || true
-        ${{env.TEST_EXE}} --list_content > tests.txt
+        ldd ${{env.TEST_EXE}} | tee >(grep 'not found') || true
+        ${{env.TEST_EXE}} --list_content > tests.txt || true
       timeout-minutes: 1
 
     - name: Run Tests
       if: ${{!inputs.is-prep-phase && !inputs.skip-test}}
-      env:
-        LD_LIBRARY_PATH: ${{env.LD_LIB_PATH}}
       run: |
         ${{env.TEST_EXE}} --log_format=JUNIT --log_sink=CI_test_result_${{inputs.flav}}.xml -p -l all || echo 'test execution returned error'
       timeout-minutes: ${{ inputs.nTestTimeoutMins}}

diff --git a/.github/workflows/build-test-lin-wsl.yml b/.github/workflows/build-test-lin-wsl.yml
@@ -106,12 +106,14 @@ jobs:
         submodules: 'recursive' 
         lfs: true
         fetch-depth: 0
-    - name: Checkout submodule
+
+    - name: List Submodules
       run: |
+        git config --global --add safe.directory "*"
         git submodule status > submodule_ver.txt
         cat submodule_ver.txt
         git rev-list --all --count 
-    
+
     - name: Run VCPKG bootstrap
       run: ${{ inputs.bootstrap-cmd }}
       shell: wsl-bash {0}
@@ -128,12 +130,12 @@ jobs:
       run: .\fix-vcpkg-json.ps1 -removeOpenCV
       shell: pwsh
 
-    - name: cache init on container build
-      if: ${{ job.container != null }} # we are running on a cloud hosted container... move the cache "initially"
-      run: |
-        cp -R /root/.cache /github/home/ || true
-        ls ${{ inputs.cache-path }} || true
-      continue-on-error: true
+    # - name: cache init on container build
+    #   if: ${{ job.container != null }} # we are running on a cloud hosted container... move the cache "initially"
+    #   run: |
+    #     cp -R /root/.cache /github/home/ || true
+    #     ls ${{ inputs.cache-path }} || true
+    #   continue-on-error: true
 
     - name: Cache dependencies for fast cloud build
       id: cache-all
@@ -168,16 +170,14 @@ jobs:
     - name: List Test cases
       if: ${{!inputs.is-prep-phase}}
       run: |
-        export LD_LIBRARY_PATH=${{env.LD_LIB_PATH}}
-        ldd ${{env.TEST_EXE}} | grep 'not found' || true
-        ${{env.TEST_EXE}} --list_content > tests.txt
+        ldd ${{env.TEST_EXE}} | tee >(grep 'not found') || true
+        ${{env.TEST_EXE}} --list_content > tests.txt || true
       timeout-minutes: 1
       shell: wsl-bash {0}
 
     - name: Run Tests
       if: ${{!inputs.is-prep-phase && !inputs.skip-test}}
       run: |
-        export LD_LIBRARY_PATH=${{env.LD_LIB_PATH}}
         ${{env.TEST_EXE}} --log_format=JUNIT --log_sink=CI_test_result_${{inputs.flav}}.xml -p -l all || echo 'test execution returned error'
       timeout-minutes: ${{ inputs.nTestTimeoutMins}}
       shell: wsl-bash {0}

diff --git a/.github/workflows/build-test-lin.yml b/.github/workflows/build-test-lin.yml
@@ -100,12 +100,14 @@ jobs:
         submodules: 'recursive' 
         lfs: true
         fetch-depth: 0
-    - name: Checkout submodule
+
+    - name: List Submodules
       run: |
+        git config --global --add safe.directory "*"
         git submodule status > submodule_ver.txt
         cat submodule_ver.txt
         git rev-list --all --count 
-    
+
     - name: Run VCPKG bootstrap
       run: ${{ inputs.bootstrap-cmd }}
 
@@ -121,12 +123,12 @@ jobs:
       run: .\fix-vcpkg-json.ps1 -removeOpenCV
       shell: pwsh
 
-    - name: cache init on container build
-      if: ${{ job.container != null }} # we are running on a cloud hosted container... move the cache "initially"
-      run: |
-        cp -R /root/.cache /github/home/ || true
-        ls ${{ inputs.cache-path }} || true
-      continue-on-error: true
+    # - name: cache init on container build
+    #   if: ${{ job.container != null }} # we are running on a cloud hosted container... move the cache "initially"
+    #   run: |
+    #     cp -R /root/.cache /github/home/ || true
+    #     ls ${{ inputs.cache-path }} || true
+    #   continue-on-error: true
 
     - name: Cache dependencies for fast cloud build
       id: cache-all
@@ -153,17 +155,13 @@ jobs:
 
     - name: List Test cases
       if: ${{!inputs.is-prep-phase}}
-      env:
-        LD_LIBRARY_PATH: ${{env.LD_LIB_PATH}}
       run: |
-        ldd ${{env.TEST_EXE}} | grep 'not found' || true
-        ${{env.TEST_EXE}} --list_content > tests.txt
+        ldd ${{env.TEST_EXE}} | tee >(grep 'not found') || true
+        ${{env.TEST_EXE}} --list_content > tests.txt || true
       timeout-minutes: 1
 
     - name: Run Tests
       if: ${{!inputs.is-prep-phase && !inputs.skip-test}}
-      env:
-        LD_LIBRARY_PATH: ${{env.LD_LIB_PATH}}
       run: |
         ${{env.TEST_EXE}} --log_format=JUNIT --log_sink=CI_test_result_${{inputs.flav}}.xml -p -l all || echo 'test execution returned error'
       timeout-minutes: ${{ inputs.nTestTimeoutMins}}

diff --git a/.github/workflows/build-test-win.yml b/.github/workflows/build-test-win.yml
@@ -37,7 +37,7 @@ on:
         description: 'command to check if the builder is ready'
         default: 'cmake --version ; ninja --version ; git --version; pwsh --version'
         #pwsh is required on self hosted runners: see here: https://learn.microsoft.com/en-us/powershell/scripting/install/install-other-linux?view=powershell-7.2#installation-using-a-binary-archive-file     
-        required: false                
+        required: false
       bootstrap-cmd:
         type: string
         description: 'commands required to boot strap builder after code checkout'
@@ -72,7 +72,6 @@ jobs:
   build:
     env:
       TEST_EXE: build/${{inputs.buildConf}}/aprapipesut
-      LD_LIB_PATH: ''
       CMAKE_TC_FILE: '${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake'
     runs-on: ${{ inputs.runner }}
     steps:
@@ -103,12 +102,14 @@ jobs:
         submodules: 'recursive' 
         lfs: true
         fetch-depth: 0
-    - name: Checkout submodule
+
+    - name: List Submodules
       run: |
+        git config --global --add safe.directory "*"
         git submodule status > submodule_ver.txt
         cat submodule_ver.txt
         git rev-list --all --count 
-    
+
     - name: Run VCPKG bootstrap
       run: ${{ inputs.bootstrap-cmd }}
 
@@ -124,22 +125,22 @@ jobs:
       run: .\fix-vcpkg-json.ps1 -onlyOpenCV
       shell: pwsh
 
-    - name: cache init on container build
-      if: ${{ job.container != null }} # we are running on a cloud hosted container... move the cache "initially"
-      run: |
-        cp -R /root/.cache /github/home/ || true
-        ls ${{ inputs.cache-path }} || true
-      continue-on-error: true
+    # - name: cache init on container build
+    #   if: ${{ job.container != null }} # we are running on a cloud hosted container... move the cache "initially"
+    #   run: |
+    #     cp -R /root/.cache /github/home/ || true
+    #     ls ${{ inputs.cache-path }} || true
+    #   continue-on-error: true
 
     - name: Cache dependencies for fast cloud build
       id: cache-all
       uses: actions/cache@v3
       with:
           path: |
             ${{ inputs.cache-path }}
-          key:  ${{ inputs.flav }}-3-${{ hashFiles( 'base/vcpkg.json', 'vcpkg/baseline.json', 'submodule_ver.txt') }}
+          key:  ${{ inputs.flav }}-4-${{ hashFiles( 'base/vcpkg.json', 'vcpkg/baseline.json', 'submodule_ver.txt') }}
           restore-keys: ${{ inputs.flav }}-
-
+    
     - name: Make build folder
       run: mkdir -p build
       continue-on-error: true
@@ -160,7 +161,7 @@ jobs:
       if: ${{!inputs.is-prep-phase}}
       working-directory: ${{github.workspace}}/build
       run: cmake --build . --config ${{inputs.buildConf}} -j ${{ inputs.nProc }}
-    
+
     - name: List Test cases
       if: ${{!inputs.is-prep-phase}}
       run: |
@@ -169,9 +170,8 @@ jobs:
 
     - name: Run Tests
       if: ${{!inputs.is-prep-phase && !inputs.skip-test}}
-      env:
-        LD_LIBRARY_PATH: ${{env.LD_LIB_PATH}}
-      run: ${{env.TEST_EXE}} --log_format=JUNIT --log_sink=CI_test_result_${{inputs.flav}}.xml -p -l all || echo 'test execution returned error'
+      run: |
+        ${{env.TEST_EXE}} --log_format=JUNIT --log_sink=CI_test_result_${{inputs.flav}}.xml -p -l all || echo 'test execution returned error'
       timeout-minutes: ${{ inputs.nTestTimeoutMins}}
 
     - name: Upload Test Results

diff --git a/README.md b/README.md
@@ -70,12 +70,6 @@ Automatically built and tested on Ubuntu 18.04, Jetson Boards and Windows 11 x64
   ```
   git clone --recursive https://github.com/Apra-Labs/ApraPipes.git
   ```
-* Build libmp4
-  ```
-  cd thirdparty\libmp4
-  .\build.cmd
-  ```
-* __Note__ As of this revision, there is no need to build thirdparty\gstreamer for windows as we leverage vcpkg for the same.
 
 ### Build for windows
 
@@ -124,13 +118,6 @@ build_windows_cuda.bat
   ```
   git clone --recursive https://github.com/Apra-Labs/ApraPipes.git
   ```
-* build gstreamer
-   * ``` cd thirdparty && sh ./build_gstreamer.sh && cd -```
-   * update .bashrc and append following line at the end of it. Adjust the path based on your environment.
-     ```
-     export LD_LIBRARY_PATH=~/ApraPipes/thirdparty/gst-build/gst-build-1.16/outInstall/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
-     ```
-   * load symbols from .bashrc ```source ~/.bashrc```. 
 
 
 ### Build for linux
@@ -162,15 +149,15 @@ Build can take ~2 hours depending on the machine configuration.
 
 * Use this [docker image](https://github.com/users/kumaakh/packages/container/package/aprapipes-build-x86-ubutu18.04-cuda) with all the software setup.
   ```
-  docker pull ghcr.io/kumaakh/aprapipes-build-x86-ubutu18.04-cuda:latest
+  docker pull ghcr.io/kumaakh/aprapipes-build-x86-ubutu18.04-cuda:last-good
   ```
 * Mount an external volume as a build area, and then use the Windows command line to create a Docker container using the above image with the following command:  
   ```
-  docker run -dit --gpus all -v "</path/to/external_volume>":"/mnt/b/" --name <give-container-name> 02ed8b575e94802e19b8eb6424e0a0d52d260ea51173ce8dae0eac229acac725
+  docker run -dit --gpus all -v "</path/to/external_volume>":"/mnt/b/" --name <give-container-name> a799cc26f4b7
   ```
   ..your command should look like this [where D:\ws\docker-pipes->local_folder_path , pipes->container_name ]
   ```
-  docker run -dit --gpus all -v "D:\ws\docker-pipes":"/mnt/b/" --name pipes 02ed8b575e94802e19b8eb6424e0a0d52d260ea51173ce8dae0eac229acac725
+  docker run -dit --gpus all -v "D:\ws\docker-pipes":"/mnt/b/" --name pipes a799cc26f4b7
   ```
 * After creating the container, execute the following command to access its command line interface
   ```