From 3b6b6aff07cbd805986a1ba71afe788e2f332f7b Mon Sep 17 00:00:00 2001 From: Wenxin Zhang Date: Tue, 14 May 2024 09:41:25 +0800 Subject: [PATCH] update Signed-off-by: Wenxin Zhang --- .github/workflows/model_test.yml | 16 +++++++++----- .../workflows/scripts/models/collect_log.sh | 5 +++-- .../workflows/scripts/models/model_test.sh | 22 ++++++++++++------- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/.github/workflows/model_test.yml b/.github/workflows/model_test.yml index 4f18c630..6dd62f9a 100644 --- a/.github/workflows/model_test.yml +++ b/.github/workflows/model_test.yml @@ -36,7 +36,7 @@ jobs: strategy: matrix: include: - - modelName: "facebook/opt-125m" + - modelName: "opt-125m" datasets: "piqa" device: "cpu" tasks: "text-generation" @@ -45,6 +45,11 @@ jobs: steps: - name: Clean Up Working Directory run: sudo rm -rf ${{github.workspace}}/* + + - name: Load environment variables + if: ${{ matrix.device }} == "cpu" + run: + cat ~/actions-runner4/.env >> $GITHUB_ENV - name: Checkout out Repo uses: actions/checkout@v4 @@ -54,7 +59,7 @@ jobs: # We need this because GitHub needs to clone the branch to pipeline - name: Docker Build run: | - docker build -f ${{ github.workspace }}/.github/workflows/docker/common.dockerfile -t ${{ env.DOCKER_NAME }}:${{ env.DOCKER_TAG }} . + docker build -f ${{ github.workspace }}/.github/workflows/docker/common.dockerfile --build-arg http_proxy="${{ env.HTTP_PROXY_IMAGE_BUILD }}" --build-arg https_proxy="${{ env.HTTPS_PROXY_IMAGE_BUILD }}" -t ${{ env.DOCKER_NAME }}:${{ env.DOCKER_TAG }} . - name: Docker Run run: | @@ -64,6 +69,7 @@ jobs: fi docker run -dit --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} -v /dev/shm:/dev/shm \ -v ${{ github.workspace }}:/GenAIEval \ + -e http_proxy="${{ env.HTTP_PROXY_IMAGE_BUILD }}" -e https_proxy="${{ env.HTTPS_PROXY_IMAGE_BUILD }}" \ ${{ env.DOCKER_NAME }}:${{ env.DOCKER_TAG }} - name: Binary build @@ -102,7 +108,7 @@ jobs: && bash -x collect_log.sh --model=${{ matrix.modelName }} \ --device=${{ matrix.device }} \ --datasets=${{ matrix.datasets }} \ - --tasks=${{ matrix.tasks }} + --tasks=${{ matrix.tasks }}" - name: Publish pipeline artifact uses: actions/upload-artifact@v4 @@ -110,7 +116,7 @@ jobs: with: name: ${{ matrix.device }}-${{ matrix.modelName }} path: | - ${{ github.workspace }}/${{ matrix.device }}/${{ matrix.modelName }} - ${{ github.workspace }}/.summary.log + ${{ github.workspace }}/${{ matrix.device }}-${{ matrix.tasks }}-${{ matrix.modelName }}-${{ matrix.datasets }}.log + ${{ github.workspace }}/summary.log if-no-files-found: ignore # 'warn' or 'ignore' are also available, defaults to `warn` retention-days: 60 # 1 <= retention-days <= 90 diff --git a/.github/workflows/scripts/models/collect_log.sh b/.github/workflows/scripts/models/collect_log.sh index a197393a..0cc6db4a 100644 --- a/.github/workflows/scripts/models/collect_log.sh +++ b/.github/workflows/scripts/models/collect_log.sh @@ -14,6 +14,7 @@ # limitations under the License. set -eo pipefail +set -x source /GenAIEval/.github/workflows/scripts/change_color WORKSPACE="/GenAIEval" # get parameters @@ -34,7 +35,7 @@ for i in "$@"; do esac done -log_file="/GenAIEval/${device}/${model}/${device}-${model}-${tasks}-${datasets}.log" +log_file="/log/${device}-${tasks}-${model}-${datasets}.log" $BOLD_YELLOW && echo "-------- Collect logs --------" && $RESET echo "working in" @@ -42,6 +43,6 @@ pwd if [[ ! -f ${log_file} ]]; then echo "${device};${model};${tasks};${datasets};;${logfile}" >> ${WORKSPACE}/summary.log else - acc=$(grep -Po "Accuracy .* is:\\s+(\\d+(\\.\\d+)?)" ${log_file} | head -n 1 | sed 's/.*://;s/[^0-9.]//g') + acc=$(grep -Po "acc .*(\d+(\.\d+)?)" ${log_file} | awk -F "|" '{print $2}' | head -n 1 | sed 's/.*://;s/[^0-9.]//g') echo "${device};${model};${tasks};${datasets};${acc};${logfile}" >> ${WORKSPACE}/summary.log fi diff --git a/.github/workflows/scripts/models/model_test.sh b/.github/workflows/scripts/models/model_test.sh index fb420086..438c43b8 100644 --- a/.github/workflows/scripts/models/model_test.sh +++ b/.github/workflows/scripts/models/model_test.sh @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -eo pipefail +set -o pipefail +set -x source /GenAIEval/.github/workflows/scripts/change_color - +git config --global --add safe.directory /GenAIEval # get parameters PATTERN='[-a-zA-Z0-9_]*=' PERF_STABLE_CHECK=true @@ -34,11 +35,7 @@ for i in "$@"; do esac done -log_dir="/GenAIEval/${device}/${model}" -mkdir -p ${log_dir} working_dir="" -$BOLD_YELLOW && echo "-------- evaluation start --------" && $RESET - main() { case ${tasks} in "text-generation") @@ -48,7 +45,16 @@ main() { *) echo "Not suppotted task"; exit 1;; esac + if [[ ${model} == *"opt"* ]]; then + pretrained="facebook/${model}" + else + pretrained="${model}" + fi + log_dir="/log/${device}/${model}" + mkdir -p ${log_dir} + $BOLD_YELLOW && echo "-------- evaluation start --------" && $RESET run_benchmark + cp ${log_dir}/${device}-${tasks}-${model}-${datasets}.log /GenAIEval/ } function prepare() { @@ -69,10 +75,10 @@ function prepare() { function run_benchmark() { cd ${working_dir} - overall_log="${log_dir}/${device}-${model}-${tasks}-${datasets}.log" + overall_log="${log_dir}/${device}-${tasks}-${model}-${datasets}.log" python main.py \ --model hf \ - --model_args pretrained=${model} \ + --model_args pretrained=${pretrained} \ --tasks ${datasets} \ --device ${device} \ --batch_size 112