Merge remote-tracking branch 'origin/candidate-9.8.x'

Signed-off-by: Gavin Halliday <[email protected]> # Conflicts: # helm/hpcc/Chart.yaml # helm/hpcc/templates/_helpers.tpl # version.cmake
stuartort · Oct 9, 2024 · b08c180 · b08c180
2 parents e2074a3 + 9f55380
commit b08c180
Show file tree

Hide file tree

Showing 14 changed files with 476 additions and 39 deletions.
diff --git a/.github/workflows/build-vcpkg.yml b/.github/workflows/build-vcpkg.yml
@@ -123,6 +123,36 @@ jobs:
       asset-name: 'docker-ubuntu-22_04'
       generate-zap: ""
     secrets: inherit
+
+  check-documentation-changes:
+    if: ${{ contains('pull_request,push', github.event_name) }}
+    runs-on: ubuntu-22.04
+    outputs:
+      documentation_contents_changed: ${{ steps.variables.outputs.documentation_contents_changed }}
+    steps:
+      - name: Check for Documentation Changes
+        id: changed
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            src:
+              - 'docs/**'
+              - '.github/workflows/test-documentation.yml'
+      - name: Set Output
+        id: variables
+        run: |
+          echo "documentation_contents_changed=${{ steps.changed.outputs.src }}" >> $GITHUB_OUTPUT
+      - name: Print Variables
+        run: |
+          echo "${{ toJSON(steps.variables.outputs)}}"
+
+  test-documentation-ubuntu-22_04:
+    needs: check-documentation-changes
+    if: ${{ contains('pull_request,push', github.event_name) && needs.check-documentation-changes.outputs.documentation_contents_changed == 'true' }}
+    uses: ./.github/workflows/test-documentation.yml
+    with:
+      os: 'ubuntu-22.04'
+      asset-name: 'Documentation'
 
   build-docker-ubuntu-20_04:
     if: ${{ contains('schedule,push', github.event_name) }}

diff --git a/.github/workflows/test-documentation.yml b/.github/workflows/test-documentation.yml
@@ -0,0 +1,108 @@
+name: Build Documentation
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        type: string
+        description: 'Operating System'
+        required: false
+        default: 'ubuntu-22.04'
+      asset-name:
+        type: string
+        description: 'Asset Name'
+        required: false
+        default: 'Documentation'
+
+  workflow_dispatch:
+    inputs:
+      os:
+        type: string
+        description: 'Operating System'
+        required: false
+        default: 'ubuntu-22.04'
+      asset-name:
+        type: string
+        description: 'Asset Name'
+        required: false
+        default: 'Documentation'
+
+jobs:
+  build-documentation:
+    name: Build Documentation
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Checkout HPCC-Platform
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+          submodules: recursive
+          path: ${{ github.workspace }}/HPCC-Platform
+
+      - name: Calculate vars
+        id: vars
+        working-directory: ${{ github.workspace }}/HPCC-Platform/vcpkg
+        run: |
+          vcpkg_sha_short=$(git rev-parse --short=8 HEAD)
+          echo "vcpkg_sha_short=$vcpkg_sha_short" >> $GITHUB_OUTPUT
+          docker_build_label=hpccsystems/platform-build-${{ inputs.os }}
+          echo "docker_build_label=$docker_build_label" >> $GITHUB_OUTPUT
+          echo "docker_tag=$docker_build_label:$vcpkg_sha_short" >> $GITHUB_OUTPUT
+          community_base_ref=${{ github.event.base_ref || github.ref }}
+          candidate_branch=$(echo $community_base_ref | cut -d'/' -f3)
+          candidate_base_branch=$(echo $candidate_branch | awk -F'.' -v OFS='.' '{ $3="x"; print }')
+          echo "docker_tag_candidate_base=$docker_build_label:$candidate_base_branch" >> $GITHUB_OUTPUT
+          community_ref=${{ github.ref }}
+          community_tag=$(echo $community_ref | cut -d'/' -f3)
+          echo "community_tag=$community_tag" >> $GITHUB_OUTPUT
+
+  
+      - name: Print vars
+        run: |
+          echo "${{ toJSON(steps.vars.outputs) }})"
+  
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Create Build Image
+        uses: docker/build-push-action@v5
+        with:
+          builder: ${{ steps.buildx.outputs.name }}
+          file: ${{ github.workspace }}/HPCC-Platform/dockerfiles/vcpkg/${{ inputs.os }}.dockerfile
+          context: ${{ github.workspace }}/HPCC-Platform/dockerfiles/vcpkg
+          push: false
+          load: true
+          build-args: |
+            VCPKG_REF=${{ steps.vars.outputs.vcpkg_sha_short }}
+          tags: |
+            ${{ steps.vars.outputs.docker_tag_candidate_base }}
+          cache-from: |
+            type=registry,ref=${{ steps.vars.outputs.docker_tag_candidate_base }}
+            type=registry,ref=${{ steps.vars.outputs.docker_tag }}
+          cache-to: type=inline
+
+      - name: CMake documentation
+        run: |
+          mkdir -p {${{ github.workspace }}/build,EN_US,PT_BR}
+          docker run --rm --mount source="${{ github.workspace }}/HPCC-Platform",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached --mount source="${{ github.workspace }}/build",target=/hpcc-dev/build,type=bind,consistency=cached ${{ steps.vars.outputs.docker_tag_candidate_base }} "\
+            cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build -DVCPKG_FILES_DIR=/hpcc-dev -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \
+            cmake --build /hpcc-dev/build --parallel $(nproc) --target all"
+          docker run --rm --mount source="${{ github.workspace }}/HPCC-Platform",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached --mount source="${{ github.workspace }}/build",target=/hpcc-dev/build,type=bind,consistency=cached ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/EN_US && zip ALL_HPCC_DOCS_EN_US-$(echo '${{ steps.vars.outputs.community_tag }}' | sed 's/community_//' ).zip *.pdf"
+          docker run --rm --mount source="${{ github.workspace }}/HPCC-Platform",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached --mount source="${{ github.workspace }}/build",target=/hpcc-dev/build,type=bind,consistency=cached ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/PT_BR && zip ALL_HPCC_DOCS_PT_BR-$(echo '${{ steps.vars.outputs.community_tag }}' | sed 's/community_//' ).zip *.pdf"
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ inputs.asset-name }}
+          path: |
+            ${{ github.workspace }}/build/Release/docs/*.zip
+            ${{ github.workspace }}/build/Release/docs/EN_US/*.zip
+            ${{ github.workspace }}/build/Release/docs/PT_BR/*.zip
+            ${{ github.workspace }}/build/docs/EN_US/EclipseHelp/*.zip
+            ${{ github.workspace }}/build/docs/EN_US/HTMLHelp/*.zip
+            ${{ github.workspace }}/build/docs/PT_BR/HTMLHelp/*.zip
+          compression-level: 0
+
diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp
@@ -2466,9 +2466,15 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
                     checkTimeLimitExceeded(&remainingMS);
                     Url &connUrl = master->proxyUrlArray.empty() ? url : master->proxyUrlArray.item(0);
 
+                    CCycleTimer dnsTimer;
+
                     // TODO: for DNS, do we use timeoutMS or remainingMS or remainingMS / maxRetries+1 or ?
                     ep.set(connUrl.host.get(), connUrl.port, master->timeoutMS);
 
+                    unsigned __int64 dnsNs = dnsTimer.elapsedNs();
+                    master->logctx.noteStatistic(StTimeSoapcallDNS, dnsNs);
+                    master->activitySpanScope->setSpanAttribute("SoapcallDNSTimeNs", dnsNs);
+
                     if (ep.isNull())
                         throw MakeStringException(-1, "Failed to resolve host '%s'", nullText(connUrl.host.get()));
 
@@ -2489,6 +2495,8 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
                         isReused = false;
                         keepAlive = true;
 
+                        CCycleTimer connTimer;
+
                         // TODO: for each connect attempt, do we use timeoutMS or remainingMS or remainingMS / maxRetries or ?
                         socket.setown(blacklist->connect(ep, master->logctx, (unsigned)master->maxRetries, master->timeoutMS, master->roxieAbortMonitor, master->rowProvider));
 
@@ -2518,11 +2526,17 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
                             throw makeStringException(0, err.str());
 #endif
                         }
+
+                        unsigned __int64 connNs = connTimer.elapsedNs();
+                        master->logctx.noteStatistic(StTimeSoapcallConnect, connNs);
+                        master->activitySpanScope->setSpanAttribute("SoapcallConnectTimeNs", connNs);
                     }
                     break;
                 }
                 catch (IException *e)
                 {
+                    master->logctx.noteStatistic(StNumSoapcallConnectFailures, 1);
+
                     if (master->timeLimitExceeded)
                     {
                         master->activitySpanScope->recordError(SpanError("Time Limit Exceeded", e->errorCode(), true, true));

diff --git a/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV1Blob.yaml b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV1Blob.yaml
@@ -0,0 +1,103 @@
+# Configures HPCC logAccess to target Azure Log Analytics Workspace
+global:
+  logAccess:
+    name: "Azure LogAnalytics LogAccess"
+    type: "AzureLogAnalyticsCurl"
+    blobMode: "true"
+    #connection:
+      #All connection attributes are optional.
+      #It is preferable to provide connection values as secret values category 'esp', secret name 'azure_logaccess'
+      # NOTE: secret 'azure_logaccess' must include 'aad-client-secret' and it cannot be provided in configuration
+      #
+      #workspaceID: "XYZ"      #ID of the Azure LogAnalytics workspace to query logs from
+                               # Secret value equivalent: 'ala-workspace-id'
+      #clientID: "DEF"         #ID of Azure Active Directory registered application with api.loganalytics.io access - format: 00000000-0000-0000-0000-000000000000
+                               # Secret value equivalent: 'aad-client-id'
+      #tenantID: "ABC"         #The Azure Active Directory Tenant ID, required for KQL API access
+                               # Secret value equivalent: 'aad-tenant-id'
+    logMaps:
+    - type: "global"
+      storeName: "ContainerLog"
+      searchColumn: "LogEntry"
+      columnType: "string"
+      columnMode: "MIN"
+      timeStampColumn: "TimeGenerated"
+    - type: "workunits"
+      searchColumn: "LogEntry"
+      columnMode: "DEFAULT"
+      columnType: "string"
+    - type: "components"
+      storeName: "ContainerInventory"
+      searchColumn: "Name"
+      keyColumn: "ContainerID"
+      columnMode: "MIN"
+      columnType: "string"
+      timeStampColumn: "TimeGenerated"
+      disableJoins: false #Potentially expensive join operations needed to fetch a certain column can be disabled
+    - type: "audience"
+      searchColumn: "LogEntry"
+      enumValues:
+      - code: OPR
+      - code: USR
+      - code: PRO
+      - code: ADT
+      - code: MON
+      columnMode: "DEFAULT"
+      columnType: "enum"
+    - type: "class"
+      searchColumn: "LogEntry"
+      enumValues:
+      - code: DIS
+      - code: ERR
+      - code: WRN
+      - code: INF
+      - code: PRO
+      - code: MET
+      - code: EVT
+      columnMode: "DEFAULT"
+      columnType: "enum"
+    - type: "instance"
+      columnMode: "DEFAULT"
+      searchColumn: "Computer"
+      columnMode: "ALL"
+      columnType: "string"
+    - type: "message"
+      searchColumn: "LogEntry"
+      columnMode: "MIN"
+      columnType: "string"
+    - type: "logid"
+      searchColumn: "LogEntry"
+      columnMode: "DEFAULT"
+      columnType: "string"
+    - type: "processid"
+      searchColumn: "LogEntry"
+      columnMode: "ALL"
+      columnType: "string"
+    - type: "threadid"
+      searchColumn: "LogEntry"
+      columnMode: "DEFAULT"
+      columnType: "string"
+    - type: "timestamp"
+      searchColumn: "TimeGenerated"
+      columnMode: "MIN"
+      columnType: "Timestamp"
+    #- type: "pod"
+    #  searchColumn: "PodName"
+    #  columnMode: "DEFAULT"
+    #  columnType: "string"
+    - type: "spanid"
+      searchColumn: "LogEntry"
+      columnMode: "DEFAULT"
+      columnType: "string"
+    - type: "traceid"
+      searchColumn: "LogEntry"
+      columnMode: "DEFAULT"
+      columnType: "string"
+secrets:
+  esp:
+    azure-logaccess: "azure-logaccess"
+vaults:
+  esp:
+  - name: my-azure-logaccess-vault
+    url: http://${env.VAULT_SERVICE_HOST}:${env.VAULT_SERVICE_PORT}/v1/secret/data/esp/${secret}
+    kind: kv-v2
diff --git a/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2Blob.yaml b/helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2Blob.yaml
@@ -0,0 +1,101 @@
+# Configures HPCC logAccess to target Azure Log Analytics Workspace
+global:
+  logAccess:
+    name: "Azure LogAnalytics LogAccess"
+    type: "AzureLogAnalyticsCurl"
+    blobMode: "true"
+    #connection:
+      #All connection attributes are optional.
+      #It is preferable to provide connection values as secret values category 'esp', secret name 'azure_logaccess'
+      # NOTE: secret 'azure_logaccess' must include 'aad-client-secret' and it cannot be provided in configuration
+      #
+      #workspaceID: "XYZ"      #ID of the Azure LogAnalytics workspace to query logs from
+                               # Secret value equivalent: 'ala-workspace-id'
+      #clientID: "DEF"         #ID of Azure Active Directory registered application with api.loganalytics.io access - format: 00000000-0000-0000-0000-000000000000
+                               # Secret value equivalent: 'aad-client-id'
+      #tenantID: "ABC"         #The Azure Active Directory Tenant ID, required for KQL API access
+                               # Secret value equivalent: 'aad-tenant-id'
+    logMaps:
+    - type: "global"
+      storeName: "ContainerLogV2"
+      searchColumn: "LogMessage"
+      columnType: "dynamic"
+      columnMode: "MIN"
+      timeStampColumn: "TimeGenerated"
+    - type: "workunits"
+      searchColumn: "LogMessage"
+      columnMode: "DEFAULT"
+      columnType: "string"
+    - type: "components"
+      storeName: "ContainerLogV2"
+      searchColumn: "ContainerName" # Container name happens to coincide with component name
+      keyColumn: "ContainerName"
+      columnMode: "DEFAULT"
+      columnType: "string"
+    - type: "audience"
+      searchColumn: "LogMessage"
+      enumValues:
+      - code: OPR
+      - code: USR
+      - code: PRO
+      - code: ADT
+      - code: MON
+      columnMode: "DEFAULT"
+      columnType: "enum"
+    - type: "class"
+      searchColumn: "LogMessage"
+      enumValues:
+      - code: DIS
+      - code: ERR
+      - code: WRN
+      - code: INF
+      - code: PRO
+      - code: MET
+      - code: EVT
+      columnMode: "DEFAULT"
+      columnType: "enum"
+    - type: "instance"
+      columnMode: "DEFAULT"
+      searchColumn: "Computer"
+      columnMode: "ALL"
+      columnType: "string"
+    - type: "message"
+      searchColumn: "LogMessage"
+      columnMode: "MIN"
+      columnType: "string"
+    - type: "logid"
+      searchColumn: "LogMessage"
+      columnMode: "DEFAULT"
+      columnType: "string"
+    - type: "processid"
+      searchColumn: "LogMessage"
+      columnMode: "ALL"
+      columnType: "string"
+    - type: "threadid"
+      searchColumn: "LogMessage"
+      columnMode: "DEFAULT"
+      columnType: "string"
+    - type: "timestamp"
+      searchColumn: "LogMessage"
+      columnMode: "MIN"
+      columnType: "string"
+    - type: "pod"
+      searchColumn: "PodName"
+      columnMode: "DEFAULT"
+      columnType: "string"
+    - type: "spanid"
+      searchColumn: "LogMessage"
+      columnMode: "DEFAULT"
+      columnType: "string"
+    - type: "traceid"
+      searchColumn: "LogMessage"
+      columnMode: "DEFAULT"
+      columnType: "string"
+secrets:
+  esp:
+    azure-logaccess: "azure-logaccess"
+vaults:
+  esp:
+  - name: my-azure-logaccess-vault
+    url: http://${env.VAULT_SERVICE_HOST}:${env.VAULT_SERVICE_PORT}/v1/secret/data/esp/${secret}
+    kind: kv-v2