Skip to content

Commit

Permalink
testmo retry wip
Browse files Browse the repository at this point in the history
  • Loading branch information
nikitka committed Sep 24, 2024
1 parent 5424e5a commit 2e3dc3b
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 7 deletions.
12 changes: 7 additions & 5 deletions .github/actions/test_ya/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,9 @@ runs:
shell: bash
run: |
set -x
echo "$(pwd)/ydb/ci/scripts" >> $GITHUB_PATH
export TMP_DIR=$(pwd)/tmp
rm -rf $TMP_DIR
mkdir -p $TMP_DIR
Expand Down Expand Up @@ -345,7 +347,7 @@ runs:
TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:resources:add-link --name build --url "$TESTMO_RUN_URL" --resources $CURRENT_PUBLIC_DIR/testmo.json
TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:resources:add-field --name git-sha --type string --value "${GITHUB_SHA:0:7}" --resources $CURRENT_PUBLIC_DIR/testmo.json
TESTMO_RUN_ID=$(
TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:create --instance "https://$TESTMO_PROXY_ADDR" --project-id ${{ inputs.testman_project_id }} \
TESTMO_TOKEN=${{ inputs.testman_token }} retry.sh -- testmo automation:run:create --instance "https://$TESTMO_PROXY_ADDR" --project-id ${{ inputs.testman_project_id }} \
--name "$TESTMO_RUN_NAME" --source "$TESTMO_SOURCE" --resources $CURRENT_PUBLIC_DIR/testmo.json \
--tags "$TESTMO_BRANCH_TAG" --tags "$TESTMO_EXTRA_TAG"
)
Expand Down Expand Up @@ -459,11 +461,11 @@ runs:
# archive unitest reports (transformed)
tar -C $TESTMO_JUNIT_REPORT_PARTS/.. -czf $PUBLIC_DIR/junit_parts.xml.tar.gz $(basename $TESTMO_JUNIT_REPORT_PARTS)

TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:submit-thread \
TESTMO_TOKEN=${{ inputs.testman_token }} retry.sh -- testmo automation:run:submit-thread \
--instance "https://$TESTMO_PROXY_ADDR" --run-id "$TESTMO_RUN_ID" \
--results "$TESTMO_JUNIT_REPORT_PARTS/*.xml"

TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id $TESTMO_RUN_ID || true
TESTMO_TOKEN=${{ inputs.testman_token }} retry.sh -- testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id $TESTMO_RUN_ID || true
echo "runid=" >> $GITHUB_OUTPUT
fi

Expand Down Expand Up @@ -500,7 +502,7 @@ runs:
shell: bash
run: |
if [ ${{ steps.build.outputs.runid }} ]; then
TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id ${{ steps.build.outputs.runid }} || true
TESTMO_TOKEN=${{ inputs.testman_token }} retry.sh -- testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id ${{ steps.build.outputs.runid }} || true
fi
if [ ${{ inputs.testman_token }} ]; then
kill $TESTMO_PROXY_PID
Expand Down
72 changes: 72 additions & 0 deletions ydb/ci/scripts/retry.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
#!/usr/bin/env bash

usage() {
echo "$0 -r [max_attempts] -t [timeout] -s [sleep] -- cmd"
}

if [ $# -lt 2 ]; then
usage
exit 1
fi

max_attempts=0
timeout=1800
sleep_time=30

while getopts 'r:t:s:' opt; do
case "$opt" in
r)
max_attempts=$OPTARG
;;
t)
timeout=$OPTARG
;;
s)
sleep_time=$OPTARG
;;
:)
usage
exit 1
;;
esac
done


shift $((OPTIND - 2))

if [[ "$1" == "--" ]]; then
shift # Shift past the double dash
else
echo "Error: Missing -- before command"
exit 1
fi

cmd=$@

attempt_num=1
start_time=$(date +%s)

while true; do
elapsed=$(($(date +%s) - $start_time))

if [ "$max_attempts" -ne 0 ] && [ "$attempt_num" -ge "$max_attempts" ]; then
echo "maximum attempts reached, exit" >&2
exit 10
fi

if [ "$timeout" -ne 0 ] && [ $elapsed -ge "$timeout" ]; then
echo "timeout reached, exit" >&2
exit 11
fi

$cmd

if [ $? -eq 0 ]; then
exit
else
attempt_num=$(( attempt_num + 1 ))
if [ $sleep_time != "0" ]; then
sleep $sleep_time
fi
fi
done
4 changes: 2 additions & 2 deletions ydb/ci/testmo-proxy/testmo-proxy.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ class Handler(http.server.BaseHTTPRequestHandler):
error_message_format = '{"status":"error", "code":"%(code)d", "message": "%(message)s"}\n'

# noinspection PyMissingConstructor
def __init__(self, target_url: str, timeout: Tuple[int, int], max_reqest_time: int):
def __init__(self, target_url: str, timeout: Tuple[int, int], max_request_time: int):
self._target_url = target_url
self._timeout = timeout
self._max_request_time = max_reqest_time
self._max_request_time = max_request_time

def __call__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
Expand Down

0 comments on commit 2e3dc3b

Please sign in to comment.