Stable 24 2

.github/CODEOWNERS

@@ -1,13 +1 @@
-/ydb/core/fq/ @ydb-platform/fq
-/ydb/core/public_http/ @ydb-platform/fq
-
-/ydb/docs/ @ydb-platform/docs
-
-/ydb/library/yql/ @ydb-platform/yql
-/ydb/library/yql/providers/generic @ydb-platform/fq
-/ydb/library/yql/providers/pq @ydb-platform/fq
-/ydb/library/yql/providers/s3 @ydb-platform/fq
-/ydb/library/yql/providers/solomon @ydb-platform/fq
-
-/ydb/services/fq/ @ydb-platform/fq
-
+* @ydb-platform/ReleaseApprovers

.github/actions/build/action.yml

@@ -11,6 +11,9 @@ inputs:
     required: false
     default: ""
     description: "extra compile flags will be added to the end of C_FLAGS and CXX_FLAGS"
+  ninja_target:
+    required: false
+    type: string
 
 runs:
   using: "composite"
@@ -59,7 +62,7 @@ runs:
       export CCACHE_SLOPPINESS=locale
       export CCACHE_MAXSIZE=50G
       cd ../build
-      ninja
+      ninja ${{ inputs.ninja_target }}
       ccache -s
       df -h
   - name: report Build failed

.github/actions/build_and_test_ya/action.yml

@@ -0,0 +1,138 @@
+name: Ya-Build-and-Test
+inputs:
+  build_target:
+    type: string
+    default: "ydb/"
+    description: "limit build and test to specific target"
+  build_preset:
+    type: string
+  run_build:
+    type: boolean
+    default: true
+    description: "run build"
+  run_tests:
+    type: boolean
+    default: true
+    description: "run tests"
+  run_tests_if_build_fails:
+    default: "true"
+    description: "run tests if build fails"
+  test_threads:
+    type: string
+    default: 28
+    description: "Test threads count"
+  link_threads:
+    type: string
+    default: 8
+    description: "link threads count"
+  test_size:
+    type: string
+    default: "small,medium,large"
+  test_type:
+    type: string
+    default: "unittest,py3test,py2test,pytest"
+  increment:
+    type: boolean
+    required: true
+    description: If true, compares build graphs between the current and previous commits to find a list of test suites to run. Otherwise, runs all tests.
+  folder_prefix:
+    type: string
+    default: "ya-"
+  put_build_results_to_cache:
+    type: boolean
+    default: true
+  secs:
+    type: string
+    default: ""
+  vars:
+    type: string
+    default: ""
+defaults:
+  run:
+    shell: bash
+runs:
+  using: "composite"
+  steps:
+    - name: comment-build-start
+      if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
+      shell: bash
+      env:
+        BUILD_PRESET: ${{ inputs.build_preset }}
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        jobs_url="https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs"
+        # tricky: we are searching job with name that contains build_preset
+        check_url=$(curl -s $jobs_url | jq --arg n "$BUILD_PRESET" -r '.jobs[] | select(.name | contains($n)) | .html_url')
+
+        echo "Pre-commit [check]($check_url) for $(git rev-parse HEAD) has started." | .github/scripts/tests/comment-pr.py --rewrite
+
+    - name: Prepare s3cmd
+      uses: ./.github/actions/s3cmd
+      with:
+        s3_bucket: ${{ fromJSON( inputs.vars ).AWS_BUCKET }}
+        s3_endpoint: ${{ fromJSON( inputs.vars ).AWS_ENDPOINT }}
+        s3_key_id: ${{ fromJSON( inputs.secs ).AWS_KEY_ID }}
+        s3_key_secret: ${{ fromJSON( inputs.secs ).AWS_KEY_VALUE }}
+        folder_prefix: ya-
+        build_preset: ${{ inputs.build_preset }}
+
+    - name: Build
+      uses: ./.github/actions/build_ya
+      id: build
+      if: ${{ inputs.run_build == 'true' }}
+      with:
+        build_target: ${{ inputs.build_target }}
+        build_preset: ${{ inputs.build_preset }}
+        bazel_remote_uri: ${{  fromJSON( inputs.vars ).REMOTE_CACHE_URL || '' }}
+        bazel_remote_username: ${{ inputs.put_build_results_to_cache &&  fromJSON( inputs.secs ).REMOTE_CACHE_USERNAME || '' }}
+        bazel_remote_password: ${{ inputs.put_build_results_to_cache &&  fromJSON( inputs.secs ).REMOTE_CACHE_PASSWORD || '' }}
+        link_threads: ${{ inputs.link_threads }}
+
+    - name: Generate ya.make with affected test suites list
+      if: inputs.run_tests == 'true' && inputs.increment == 'true'
+      uses: ./.github/actions/graph_compare
+
+    - name: Check if there's a list of tests to run
+      id: test_run_choice
+      shell: bash
+      run: |
+        if [ -f ya.make ];then
+          echo "target='.'" >> $GITHUB_OUTPUT
+          echo "Listed test targets: "
+          cat ya.make
+        else
+          echo "target=${{ inputs.build_target }}" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Run tests
+      uses: ./.github/actions/test_ya
+      if: ${{ inputs.run_tests == 'true' &&  (steps.build.outputs.success == 'true' || inputs.run_tests_if_build_fails == 'true') }}
+      with:
+        build_target: ${{ steps.test_run_choice.outputs.target }}
+        build_preset: ${{ inputs.build_preset }}
+        test_size: ${{ inputs.test_size }}
+        testman_token: ${{ fromJSON( inputs.secs ).TESTMO_TOKEN }}
+        testman_url: ${{  fromJSON( inputs.vars ).TESTMO_URL }}
+        testman_project_id: ${{  fromJSON( inputs.vars ).TESTMO_PROJECT_ID }}
+        link_threads: ${{ inputs.link_threads }}
+        test_threads: ${{ inputs.test_threads }}
+
+    - name: Notify about failed build
+      if: ${{ steps.build.outputs.success != 'true' && inputs.run_tests == 'true' && inputs.run_tests_if_build_fails == 'false' }}
+      shell: bash
+      run: |
+        echo 'Build failed. See the [build log](${{ steps.build.outputs.log_url }}).' >> $GITHUB_STEP_SUMMARY
+
+        if [[ "$GITHUB_EVENT_NAME" =~ ^pull_request ]]; then
+          echo "Tests run skipped." | .github/scripts/tests/comment-pr.py --fail
+        fi
+
+        exit 1
+
+    - name: comment-if-cancel
+      shell: bash
+      if: cancelled() && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target')
+      env:
+        BUILD_PRESET: ${{ inputs.build_preset }}
+        GITHUB_TOKEN: ${{ github.token }}
+      run:  echo "Check cancelled" | .github/scripts/tests/comment-pr.py --color black

.github/actions/build_ya/action.yml

@@ -7,7 +7,7 @@ inputs:
   build_preset:
     required: true
     default: "relwithdebinfo"
-    description: "relwithdebinfo, release-asan, release-tsan"
+    description: "debug, relwithdebinfo, release-asan, release-tsan, release, release-cmake14"
   bazel_remote_uri:
     required: false
     description: "bazel-remote endpoint"
@@ -21,7 +21,13 @@ inputs:
     required: false
     default: "8"
     description: "link threads count"
-
+outputs:
+  success:
+    value: ${{ steps.build.outputs.status }}
+    description: "build success"
+  log_url:
+    value: ${{ steps.init.outputs.log_url }}
+    description: "build log url"
 runs:
   using: "composite"
   steps:
@@ -34,10 +40,15 @@ runs:
         echo "SHELLOPTS=xtrace" >> $GITHUB_ENV
         export TMP_DIR=$(pwd)/tmp_build
         echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV
+
+        export log_url="$S3_URL_PREFIX/build_logs/ya_make.log"
+
         rm -rf $TMP_DIR && mkdir $TMP_DIR
 
         echo "BUILD_PRESET=$build_preset" >> $GITHUB_ENV
         echo "GITHUB_TOKEN=${{ github.token }}" >> $GITHUB_ENV
+        echo "LOG_URL=$log_url" >> $GITHUB_ENV
+        echo "log_url=$log_url" >> $GITHUB_OUTPUT
 
     - name: build
       id: build
@@ -67,6 +78,14 @@ runs:
           relwithdebinfo)
             build_type=relwithdebinfo
             ;;
+          release)
+            build_type=release
+            ;;
+          release-clang14)
+            build_type=release
+            extra_params+=(--target-platform="CLANG14-LINUX-X86_64")
+            extra_params+=(-DLLD_VERSION=16)
+            ;;
           release-asan)
             build_type=release
             extra_params+=(--sanitize="address")
@@ -99,7 +118,7 @@ runs:
         ./ya make -k --build "${build_type}" --force-build-depends -D'BUILD_LANGUAGES=CPP PY3 PY2 GO' -T --stat -DCONSISTENT_DEBUG \
           --log-file "$TMP_DIR/ya_log.txt" --evlog-file "$TMP_DIR/ya_evlog.jsonl" \
           --cache-size 512G --link-threads "${{ inputs.link_threads }}" \
-          "${extra_params[@]}" |& tee $TMP_DIR/ya_make.log || (
+          "${extra_params[@]}" |& tee $TMP_DIR/ya_make.log && echo "status=true" >> $GITHUB_OUTPUT || (
               RC=$?
               echo "::debug::ya make RC=$RC"
               echo "status=failed" >> $GITHUB_OUTPUT
@@ -118,10 +137,8 @@ runs:
       if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
       shell: bash
       run: |
-        log_url="$S3_URL_PREFIX/build_logs/ya_make.log"
-
         if [ "${{ steps.build.outputs.status }}" == "failed" ]; then
-          echo "Build failed. see the [build logs]($log_url)." | .github/scripts/tests/comment-pr.py --fail
+          echo "Build failed. see the [build logs]($LOG_URL)." | .github/scripts/tests/comment-pr.py --fail
         else
           echo "Build successful." | .github/scripts/tests/comment-pr.py --ok
         fi

.github/actions/graph_compare/action.yml

@@ -0,0 +1,18 @@
+name: graph_compare
+description: Compare graphs between current and previous commits (merge commit base in case of a merge commit), and list affected tests in ya.make
+runs:
+  using: "composite"
+  steps:
+    - name: original_ref
+      id: oref
+      shell: bash
+      run: |
+        echo "value=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+    - name: generate_ya_make
+      shell: bash
+      run: |
+        ./.github/scripts/graph_compare.sh ${{ steps.oref.outputs.value }}~1 ${{ steps.oref.outputs.value }}
+    - name: restore_ref
+      shell: bash
+      run: |
+        git checkout ${{ steps.oref.outputs.value }}

.github/actions/s3cmd/action.yml

@@ -54,9 +54,10 @@ runs:
             exit 1
             ;;
         esac
-
         echo "S3_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
         echo "S3_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
+        echo "S3_TEST_ARTIFACTS_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
+        echo "S3_TEST_ARTIFACTS_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
       env:
         s3_key_id: ${{ inputs.s3_key_id }}
         s3_secret_access_key: ${{ inputs.s3_key_secret }}