CI testing #5330
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI testing | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - "*.md" | |
| - "*.cff" | |
| - "**/*.md" | |
| - "LICENSE*" | |
| schedule: | |
| # every 3 hours | |
| - cron: "0 */3 * * *" | |
| env: | |
| CONTAINER_NAME: onediff-test | |
| REGION_ID: cn-beijing | |
| ACR_ORG: registry.cn-beijing.aliyuncs.com/oneflow | |
| COMFYUI_SRC_DIR: ComfyUI | |
| WEBUI_SRC_DIR: stable-diffusion-webui | |
| WEBUI_DEPENDENCIES_SUBDIR: repos | |
| SDXL_BASE: /share_nfs/hf_models/sd_xl_base_1.0.safetensors | |
| UNET_INT8: /share_nfs/hf_models/unet_int8 | |
| CONTROL_LORA_OPENPOSEXL2_RANK256: /share_nfs/hf_models/controlnet/control-lora-openposeXL2-rank256.safetensors | |
| SELENIUM_CONTAINER_NAME: selenium-test | |
| SELENIUM_IMAGE: standalone-chrome:119.0-chromedriver-119.0-grid-4.15.0-20231129 | |
| # For git repos required by webui | |
| ASSETS_COMMIT_HASH: 6f7db241d2f8ba7457bac5ca9753331f0c266917 | |
| STABLE_DIFFUSION_COMMIT_HASH: cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf | |
| STABLE_DIFFUSION_XL_COMMIT_HASH: 45c443b316737a4ab6e40413d7794a7f5657c19f | |
| K_DIFFUSION_COMMIT_HASH: ab527a9a6d347f364e3d185ba6d714e22d80cb3c | |
| BLIP_COMMIT_HASH: 48211a1594f1321b00f14c9f7a5b4813144b2fb9 | |
| OSS_AK_ID_BASE64: TFRBSTV0RmROVlFTQ2VQdG1wdkhQYlQx | |
| OSS_AK_SC_BASE64: MTlrSlFRT1M0WUZvQ05SV1IyTUxDUDBpTlM0UURS | |
| OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }} | |
| OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }} | |
| concurrency: | |
| group: sd-examples-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| pre-commit: | |
| runs-on: [ubuntu-latest] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: '3.10' | |
| - uses: pre-commit/[email protected] | |
| upload_src: | |
| runs-on: [ubuntu-latest] | |
| outputs: | |
| onediff_src_url: ${{ steps.set_env.outputs.onediff_src_url }} | |
| comfy_src_url: ${{ steps.set_env.outputs.comfy_src_url }} | |
| webui_src_url: ${{ steps.set_env.outputs.webui_src_url }} | |
| steps: | |
| - name: Setup ossutil | |
| if: github.event.pull_request.head.repo.full_name != github.repository | |
| run: | | |
| OSS_ACCESS_KEY_ID=$(echo -n "${{ env.OSS_AK_ID_BASE64 }}" | base64 -d) | |
| OSS_ACCESS_KEY_SECRET=$(echo -n "${{ env.OSS_AK_SC_BASE64 }}" | base64 -d) | |
| echo "OSS_ACCESS_KEY_ID=${OSS_ACCESS_KEY_ID}" >> ${GITHUB_ENV} | |
| echo "OSS_ACCESS_KEY_SECRET=${OSS_ACCESS_KEY_SECRET}" >> ${GITHUB_ENV} | |
| - name: Setup ossutil | |
| run: | | |
| curl http://gosspublic.alicdn.com/ossutil/1.7.15/ossutil64 -o ossutil64 | |
| sudo chmod +x ossutil64 | |
| ./ossutil64 config -e oss-cn-beijing.aliyuncs.com -i ${{ env.OSS_ACCESS_KEY_ID }} -k ${{ env.OSS_ACCESS_KEY_SECRET }} -L EN -c $HOME/.ossutilconfig | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| path: onediff | |
| - name: Checkout ComfyUI | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: comfyanonymous/ComfyUI | |
| path: ComfyUI | |
| - name: Checkout Stable Diffusion WebUI | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: AUTOMATIC1111/stable-diffusion-webui | |
| path: ${{ env.WEBUI_SRC_DIR }} | |
| # -------- The following are the dependencies required by webui -------- | |
| - name: Checkout CLIP (dependency of webui) | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: openai/CLIP | |
| path: ${{ env.WEBUI_SRC_DIR }}/${{ env.WEBUI_DEPENDENCIES_SUBDIR }}/CLIP | |
| ref: d50d76daa670286dd6cacf3bcd80b5e4823fc8e1 | |
| - name: Checkout open clip (dependency of webui) | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: mlfoundations/open_clip | |
| path: ${{ env.WEBUI_SRC_DIR }}/${{ env.WEBUI_DEPENDENCIES_SUBDIR }}/open_clip | |
| ref: bb6e834e9c70d9c27d0dc3ecedeebeaeb1ffad6b | |
| - name: Checkout ${{ env.WEBUI_SRC_DIR }}-assets (dependency of webui) | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: AUTOMATIC1111/stable-diffusion-webui-assets | |
| path: ${{ env.WEBUI_SRC_DIR }}/${{ env.WEBUI_DEPENDENCIES_SUBDIR }}/stable-diffusion-webui-assets | |
| ref: ${{ env.ASSETS_COMMIT_HASH }} | |
| - name: Checkout stablediffusion (dependency of webui) | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Stability-AI/stablediffusion | |
| path: ${{ env.WEBUI_SRC_DIR }}/${{ env.WEBUI_DEPENDENCIES_SUBDIR }}/stable-diffusion-stability-ai | |
| ref: ${{ env.STABLE_DIFFUSION_COMMIT_HASH }} | |
| - name: Checkout generative-models (dependency of webui) | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Stability-AI/generative-models | |
| path: ${{ env.WEBUI_SRC_DIR }}/${{ env.WEBUI_DEPENDENCIES_SUBDIR }}/generative-models | |
| ref: ${{ env.STABLE_DIFFUSION_XL_COMMIT_HASH }} | |
| - name: Checkout k-diffusion (dependency of webui) | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: crowsonkb/k-diffusion | |
| path: ${{ env.WEBUI_SRC_DIR }}/${{ env.WEBUI_DEPENDENCIES_SUBDIR }}/k-diffusion | |
| ref: ${{ env.K_DIFFUSION_COMMIT_HASH }} | |
| - name: Checkout BLIP (dependency of webui) | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: salesforce/BLIP | |
| path: ${{ env.WEBUI_SRC_DIR }}/${{ env.WEBUI_DEPENDENCIES_SUBDIR }}/BLIP | |
| ref: ${{ env.BLIP_COMMIT_HASH }} | |
| # -------- The above are the dependencies required by webui -------- | |
| - name: Pack src | |
| working-directory: onediff | |
| run: | | |
| git reset --hard | |
| git clean -f | |
| git archive --format zip HEAD > onediff-src.zip | |
| - name: Pack comfy | |
| working-directory: ComfyUI | |
| run: | | |
| git reset --hard | |
| git clean -f | |
| git archive --prefix ${{ env.COMFYUI_SRC_DIR }}/ --format zip HEAD > comfyui-src.zip | |
| - name: Pack webui | |
| working-directory: ${{ env.WEBUI_SRC_DIR }} | |
| run: | | |
| git reset --hard | |
| git clean -f | |
| zip -r webui-src.zip . | |
| - name: Set src env | |
| id: set_env | |
| run: | | |
| ONEDIFF_DST="oss://gh-src-cache/onediff/${{ github.sha }}/onediff-src.zip" | |
| COMFY_DST="oss://gh-src-cache/onediff/${{ github.sha }}/comfyui-src.zip" | |
| WEBUI_DST="oss://gh-src-cache/onediff/${{ github.sha }}/webui-src.zip" | |
| echo "onediff_src_url=${ONEDIFF_DST}" >> $GITHUB_OUTPUT | |
| echo "comfy_src_url=${COMFY_DST}" >> $GITHUB_OUTPUT | |
| echo "webui_src_url=${WEBUI_DST}" >> $GITHUB_OUTPUT | |
| - name: Upload src | |
| id: upload_to_oss | |
| run: | | |
| ./ossutil64 cp --disable-ignore-error --update onediff/onediff-src.zip ${{ steps.set_env.outputs.onediff_src_url }} | |
| ./ossutil64 cp --disable-ignore-error --update ComfyUI/comfyui-src.zip ${{ steps.set_env.outputs.comfy_src_url }} | |
| ./ossutil64 cp --disable-ignore-error --update ${{ env.WEBUI_SRC_DIR }}/webui-src.zip ${{ steps.set_env.outputs.webui_src_url }} | |
| run-examples: | |
| name: " ${{ matrix.test-suite }} ${{ matrix.image }}" | |
| runs-on: [self-hosted, cuda] | |
| needs: [upload_src] | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 5 | |
| matrix: | |
| should-run-pro: | |
| - ${{ github.event.pull_request.head.repo.full_name == github.repository }} | |
| image: | |
| - onediff:cu118 | |
| - onediff-pro:cu121 | |
| test-suite: | |
| - diffusers_examples | |
| - comfy | |
| - webui | |
| exclude: | |
| - should-run-pro: false | |
| image: onediff-pro:cu121 | |
| steps: | |
| - name: Login to ACR with the AccessKey pair | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| uses: aliyun/acr-login@v1 | |
| with: | |
| login-server: https://registry.${{env.REGION_ID}}.aliyuncs.com | |
| username: "${{ secrets.ACR_USERNAME }}" | |
| password: "${{ secrets.ACR_PASSWORD }}" | |
| - name: Setup ossutil | |
| if: github.event.pull_request.head.repo.full_name != github.repository | |
| run: | | |
| OSS_ACCESS_KEY_ID=$(echo -n "${{ env.OSS_AK_ID_BASE64 }}" | base64 -d) | |
| OSS_ACCESS_KEY_SECRET=$(echo -n "${{ env.OSS_AK_SC_BASE64 }}" | base64 -d) | |
| echo "OSS_ACCESS_KEY_ID=${OSS_ACCESS_KEY_ID}" >> ${GITHUB_ENV} | |
| echo "OSS_ACCESS_KEY_SECRET=${OSS_ACCESS_KEY_SECRET}" >> ${GITHUB_ENV} | |
| - name: Fix permissions | |
| run: | | |
| set -x | |
| docker run --rm -v $PWD:$PWD -w $PWD busybox rm -rf .[^.]* * | |
| - name: Setup ossutil | |
| run: | | |
| if [ ! -f "$HOME/ossutil64" ]; then | |
| curl http://gosspublic.alicdn.com/ossutil/1.7.15/ossutil64 -o $HOME/ossutil64 | |
| chmod +x $HOME/ossutil64 | |
| fi | |
| $HOME/ossutil64 config -e oss-cn-beijing.aliyuncs.com -i ${{ env.OSS_ACCESS_KEY_ID }} -k ${{ env.OSS_ACCESS_KEY_SECRET }} -L EN -c $HOME/.ossutilconfig | |
| - name: Checkout vis OSS | |
| run: | | |
| $HOME/ossutil64 cp ${{ needs.upload_src.outputs.onediff_src_url }} . | |
| unzip -o $(basename ${{ needs.upload_src.outputs.onediff_src_url }}) | |
| - name: Checkout Comfy vis OSS | |
| if: matrix.test-suite == 'comfy' | |
| run: | | |
| $HOME/ossutil64 cp ${{ needs.upload_src.outputs.comfy_src_url }} . | |
| unzip -o $(basename ${{ needs.upload_src.outputs.comfy_src_url }}) | |
| - name: Checkout WebUI vis OSS | |
| if: matrix.test-suite == 'webui' | |
| run: | | |
| $HOME/ossutil64 cp ${{ needs.upload_src.outputs.webui_src_url }} . | |
| unzip -o $(basename ${{ needs.upload_src.outputs.webui_src_url }}) -d ${{env.WEBUI_SRC_DIR}} | |
| - name: Clean docker containers | |
| run: | | |
| docker rm -f ${{ env.CONTAINER_NAME }} || true | |
| docker rm -f ${{ env.SELENIUM_CONTAINER_NAME }} || true | |
| - run: docker image prune -f | |
| - run: | | |
| docker pull ${{ env.ACR_ORG }}/${{ matrix.image }} | |
| - if: matrix.test-suite == 'comfy' | |
| run: | | |
| docker pull ${{ env.ACR_ORG }}/${{ env.SELENIUM_IMAGE }} | |
| - run: docker image prune -f | |
| - name: Setup docker for ComfyUI Test | |
| if: matrix.test-suite == 'comfy' | |
| run: | | |
| env | |
| docker compose -f tests/comfy-docker-compose.yml up -d | |
| env: | |
| CONTAINER_NAME: ${{ env.CONTAINER_NAME }} | |
| ACR_ORG: ${{ env.ACR_ORG }} | |
| MATRIX_IMAGE: ${{ matrix.image }} | |
| COMFYUI_SRC_DIR: ${{ env.COMFYUI_SRC_DIR }} | |
| SDXL_BASE: ${{ env.SDXL_BASE }} | |
| UNET_INT8: ${{ env.UNET_INT8 }} | |
| SELENIUM_IMAGE: ${{ env.SELENIUM_IMAGE }} | |
| SELENIUM_CONTAINER_NAME: ${{ env.SELENIUM_CONTAINER_NAME }} | |
| SILICON_ONEDIFF_LICENSE_KEY: ${{ secrets.SILICON_ONEDIFF_LICENSE_KEY }} | |
| - name: Setup docker for diffusers examples | |
| if: matrix.test-suite == 'diffusers_examples' | |
| run: | | |
| env | |
| docker compose -f tests/diffusers-docker-compose.yml up -d | |
| env: | |
| CONTAINER_NAME: ${{ env.CONTAINER_NAME }} | |
| ACR_ORG: ${{ env.ACR_ORG }} | |
| MATRIX_IMAGE: ${{ matrix.image }} | |
| SDXL_BASE: ${{ env.SDXL_BASE }} | |
| UNET_INT8: ${{ env.UNET_INT8 }} | |
| SILICON_ONEDIFF_LICENSE_KEY: ${{ secrets.SILICON_ONEDIFF_LICENSE_KEY }} | |
| - name: Setup docker for WebUI Test | |
| if: matrix.test-suite == 'webui' | |
| run: | | |
| env | |
| docker compose -f tests/webui-docker-compose.yml up -d | |
| env: | |
| CONTAINER_NAME: ${{ env.CONTAINER_NAME }} | |
| MATRIX_IMAGE: ${{ matrix.image }} | |
| WEBUI_SRC_DIR: ${{ env.WEBUI_SRC_DIR }} | |
| WEBUI_DEPENDENCIES_SUBDIR: ${{ env.WEBUI_DEPENDENCIES_SUBDIR }} | |
| SELENIUM_IMAGE: ${{ env.SELENIUM_IMAGE }} | |
| SELENIUM_CONTAINER_NAME: ${{ env.SELENIUM_CONTAINER_NAME }} | |
| SILICON_ONEDIFF_LICENSE_KEY: ${{ secrets.SILICON_ONEDIFF_LICENSE_KEY }} | |
| ASSETS_COMMIT_HASH: ${{ env.ASSETS_COMMIT_HASH }} | |
| STABLE_DIFFUSION_COMMIT_HASH: ${{ env.STABLE_DIFFUSION_COMMIT_HASH }} | |
| STABLE_DIFFUSION_XL_COMMIT_HASH: ${{ env.STABLE_DIFFUSION_XL_COMMIT_HASH }} | |
| K_DIFFUSION_COMMIT_HASH: ${{ env.K_DIFFUSION_COMMIT_HASH }} | |
| BLIP_COMMIT_HASH: ${{ env.BLIP_COMMIT_HASH }} | |
| - run: nvidia-smi | |
| - run: nvidia-smi -L | |
| - run: docker exec ${{ env.CONTAINER_NAME }} python3 -m oneflow --doctor | |
| - run: docker exec ${{ env.CONTAINER_NAME }} python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple | |
| - name: Install onediff-quant if needed | |
| if: startsWith(matrix.image, 'onediff-pro') | |
| run: | | |
| docker exec ${{ env.CONTAINER_NAME }} python3 -m pip install --pre onediff-quant -f https://oneflow-pro.oss-cn-beijing.aliyuncs.com/onediff-quant/ | |
| - name: Pip Install Requirements for ComfyUI & Test | |
| if: matrix.test-suite == 'comfy' | |
| run: | | |
| docker exec ${{ env.CONTAINER_NAME }} python3 -m pip install -r tests/comfyui/requirements.txt --user | |
| docker exec ${{ env.CONTAINER_NAME }} python3 -m pip install -r ComfyUI/requirements.txt --user | |
| docker exec ${{ env.CONTAINER_NAME }} python3 -m pip uninstall -y transformer-engine | |
| - name: Print pip package versions | |
| run: | | |
| docker exec ${{ env.CONTAINER_NAME }} python3 -m pip list | |
| - name: Start ComfyUI Web Service | |
| if: matrix.test-suite == 'comfy' | |
| run: | | |
| docker exec -w /app/ComfyUI -d ${{ env.CONTAINER_NAME }} sh -c "python3 /app/ComfyUI/main.py --gpu-only --disable-cuda-malloc --port 8188 --extra-model-paths-config /src/onediff/tests/comfyui/extra_model_paths.yaml > /app/ComfyUI/onediff_comfyui.log 2>&1" | |
| sleep 30 | |
| # print to check if comfy is launched successfully | |
| - run: docker exec ${{ env.CONTAINER_NAME }} ps aux | |
| - name: Test ComfyUI | |
| if: matrix.test-suite == 'comfy' | |
| run: | | |
| run_comfy_test() { | |
| local workflow_name="$1" | |
| local timeout=$2 | |
| docker exec -w /src/onediff/tests/comfyui ${{ env.CONTAINER_NAME }} python3 test_by_ui.py --comfy_port 8188 --workflow "$workflow_name" --timeout $timeout || { | |
| echo "test_by_ui.py fails! print the ComfyUI logs..." | |
| docker exec ${{ env.CONTAINER_NAME }} cat /app/ComfyUI/onediff_comfyui.log | |
| false | |
| } | |
| } | |
| # run_comfy_test "/share_nfs/hf_models/comfyui_resources/workflows/deep-cache.json" 600 | |
| # run_comfy_test "/share_nfs/hf_models/comfyui_resources/workflows/deep-cache-with-lora.json" 800 | |
| # run_comfy_test "workflows/text-to-video-speedup.json" 5000 | |
| docker exec -w /src/onediff/onediff_comfy_nodes/benchmarks ${{ env.CONTAINER_NAME }} bash scripts/install_env.sh /app/ComfyUI | |
| docker exec -w /src/onediff/onediff_comfy_nodes/benchmarks ${{ env.CONTAINER_NAME }} bash scripts/run_all_tests.sh || { | |
| echo "Test fails! print the ComfyUI logs..." | |
| docker exec onediff-test cat /app/ComfyUI/onediff_comfyui.log | |
| false | |
| } | |
| - name: Show ComfyUI Log | |
| if: matrix.test-suite == 'comfy' | |
| run: | | |
| docker exec ${{ env.CONTAINER_NAME }} cat /app/ComfyUI/onediff_comfyui.log | |
| - name: Setup OneDiff diffusers extension | |
| if: matrix.test-suite == 'diffusers_examples' | |
| run: | | |
| docker exec -w /src/onediff ${{ env.CONTAINER_NAME }} sh -c 'cd onediff_diffusers_extensions && python3 -m pip install -e .' | |
| docker exec ${{ env.CONTAINER_NAME }} python3 -m pip uninstall -y transformer-engine | |
| - if: matrix.test-suite == 'diffusers_examples' && startsWith(matrix.image, 'onediff-pro') | |
| run: docker exec -w /src/onediff ${{ env.CONTAINER_NAME }} python3 tests/test_quantize_custom_model.py | |
| - if: matrix.test-suite == 'diffusers_examples' && startsWith(matrix.image, 'onediff-pro') | |
| run: docker exec -w /src/onediff/onediff_diffusers_extensions ${{ env.CONTAINER_NAME }} python3 examples/text_to_image_sdxl_enterprise.py --model /share_nfs/hf_models/stable-diffusion-xl-base-1.0-int8 --width 512 --height 512 --saved_image output_enterprise_sdxl.png | |
| - if: matrix.test-suite == 'diffusers_examples' && startsWith(matrix.image, 'onediff-pro') | |
| run: docker exec -w /src/onediff/onediff_diffusers_extensions ${{ env.CONTAINER_NAME }} python3 examples/text_to_image_deep_cache_sd_sdxl_enterprise.py --model /share_nfs/stable-diffusion-xl-base-1.0-int8-deep-cache --model_type sdxl --width 512 --height 512 --saved_image output_enterprise_deepcache_sdxl.png | |
| - if: matrix.test-suite == 'diffusers_examples' && startsWith(matrix.image, 'onediff-pro') | |
| run: docker exec -w /src/onediff/onediff_diffusers_extensions ${{ env.CONTAINER_NAME }} python3 examples/text_to_image_deep_cache_sdxl.py --base /share_nfs/hf_models/stable-diffusion-xl-base-1.0 --width 512 --height 512 --run_multiple_resolutions true --saved_image deepcache_sdxl.png | |
| - if: matrix.test-suite == 'diffusers_examples' | |
| run: docker exec -w /src/onediff ${{ env.CONTAINER_NAME }} python3 tests/test_model_inference.py | |
| - if: matrix.test-suite == 'diffusers_examples' | |
| run: docker exec -w /src/onediff/onediff_diffusers_extensions ${{ env.CONTAINER_NAME }} python3 examples/text_to_image.py --model_id=/share_nfs/hf_models/stable-diffusion-v1-5 | |
| - if: matrix.test-suite == 'diffusers_examples' | |
| run: docker exec -w /src/onediff/onediff_diffusers_extensions ${{ env.CONTAINER_NAME }} python3 examples/image_to_image.py --model_id=/share_nfs/hf_models/stable-diffusion-2-1 | |
| - if: matrix.test-suite == 'diffusers_examples' | |
| run: docker exec -w /src/onediff/onediff_diffusers_extensions ${{ env.CONTAINER_NAME }} python3 examples/text_to_image_sdxl.py --base /share_nfs/hf_models/stable-diffusion-xl-base-1.0 --compiler oneflow --height 512 --width 512 --run_multiple_resolutions True | |
| - if: matrix.test-suite == 'diffusers_examples' | |
| run: docker exec -w /src/onediff/onediff_diffusers_extensions ${{ env.CONTAINER_NAME }} bash examples/save_and_load_pipeline.sh | |
| - if: matrix.test-suite == 'diffusers_examples' | |
| run: docker exec -w /src/onediff/onediff_diffusers_extensions ${{ env.CONTAINER_NAME }} python3 examples/text_to_image_controlnet.py --base=/share_nfs/hf_models/stable-diffusion-v1-5 --controlnet=/share_nfs/hf_models/sd-controlnet-canny --input_image=/share_nfs/hf_models/input_image_vermeer.png | |
| - if: matrix.test-suite == 'diffusers_examples' | |
| run: docker exec -w /src/onediff/onediff_diffusers_extensions ${{ env.CONTAINER_NAME }} bash examples/unet_save_and_load.sh --model_id=/share_nfs/hf_models/stable-diffusion-2-1 | |
| - if: matrix.test-suite == 'diffusers_examples' | |
| run: docker exec -w /src/onediff/onediff_diffusers_extensions ${{ env.CONTAINER_NAME }} python3 examples/text_to_image_sdxl_turbo.py --compile true --base /share_nfs/hf_models/sdxl-turbo | |
| - if: matrix.test-suite == 'diffusers_examples' | |
| run: | | |
| docker exec ${{ env.CONTAINER_NAME }} python3 -m pip install scikit-image | |
| docker exec -e ONEFLOW_MLIR_ENABLE_INFERENCE_OPTIMIZATION=0 ${{ env.CONTAINER_NAME }} python3 -m pytest -v onediff_diffusers_extensions/tests/test_lora.py --disable-warnings | |
| # - if: matrix.test-suite == 'diffusers_examples' | |
| # run: docker exec -w /src/onediff/onediff_diffusers_extensions -e ONEFLOW_MLIR_ENABLE_INFERENCE_OPTIMIZATION=0 ${{ env.CONTAINER_NAME }} python3 examples/text_to_image_sdxl_reuse_pipe.py --base /share_nfs/hf_models/stable-diffusion-xl-base-1.0 --new_base /share_nfs/hf_models/dataautogpt3-OpenDalleV1.1 | |
| - if: matrix.test-suite == 'diffusers_examples' && startsWith(matrix.image, 'onediff-pro') | |
| run: | | |
| docker exec -w /src/onediff ${{ env.CONTAINER_NAME }} python3 onediff_diffusers_extensions/examples/text_to_image_sd_enterprise.py --model /share_nfs/hf_models/stable-diffusion-v1-5-int8 --width 512 --height 512 --saved_image /src/onediff/output_enterprise_sd.png | |
| docker exec -w /src/onediff ${{ env.CONTAINER_NAME }} python3 tests/test_quantitative_quality.py | |
| - name: Install Requirements for WebUI | |
| if: matrix.test-suite == 'webui' | |
| run: | | |
| docker exec ${{ env.CONTAINER_NAME }} python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple | |
| docker exec ${{ env.CONTAINER_NAME }} python3 -m pip config set global.extra-index-url https://pypi.tuna.tsinghua.edu.cn/simple | |
| docker exec ${{ env.CONTAINER_NAME }} python3 -m pip install pytorch-lightning gradio==3.41.2 diskcache gitpython pytorch_lightning==1.9.4 scikit-image jsonmerge pillow-avif-plugin torchdiffeq torchsde clean-fid resize-right lark tomesd blendmodes facexlib opencv-python==4.8.0.74 piexif inflection ftfy regex tqdm pydantic==1.10.13 | |
| docker exec ${{ env.CONTAINER_NAME }} python3 -m pip uninstall -y transformer-engine | |
| - name: Prepare environment for WebUI | |
| if: matrix.test-suite == 'webui' | |
| run: | | |
| # hack code to print error msg for debugging | |
| # docker exec -w /app/${{ env.WEBUI_SRC_DIR }} -d ${{ env.CONTAINER_NAME }} sed -i '/except RuntimeError:/c\ except RuntimeError as e:\n print(f"Error occurred while running git command: {e}")' modules/launch_utils.py | |
| docker exec -d ${{ env.CONTAINER_NAME }} mkdir /app/${{ env.WEBUI_SRC_DIR }}/.git /app/${{ env.WEBUI_SRC_DIR }}/openai | |
| docker exec -d ${{ env.CONTAINER_NAME }} ln -s /share_nfs/onediff_ci/sd-webui/models/clips/clip-vit-large-patch14 /app/${{ env.WEBUI_SRC_DIR }}/openai/clip-vit-large-patch14 | |
| docker exec -w /app/${{ env.WEBUI_SRC_DIR }} -d ${{ env.CONTAINER_NAME }} git config --global --add safe.directory /app/${{ env.WEBUI_SRC_DIR }} | |
| for dir in $SAFE_DIRECTORIES; do | |
| docker exec -w /app/${{ env.WEBUI_SRC_DIR }} -d ${{ env.CONTAINER_NAME }} git config --global --add safe.directory /app/${{ env.WEBUI_SRC_DIR }}/${{ env.WEBUI_DEPENDENCIES_SUBDIR }}/$dir | |
| echo /app/${{ env.WEBUI_SRC_DIR }}/${{ env.WEBUI_DEPENDENCIES_SUBDIR }}/$dir | |
| done | |
| docker exec -w /app/${{ env.WEBUI_SRC_DIR }} -e venv_dir=- ${{ env.CONTAINER_NAME }} sh -c "bash webui.sh -f --exit --api --no-download-sd-model --do-not-download-clip --disable-safe-unpickle --ckpt-dir /share_nfs/onediff_ci/sd-webui/models" | |
| - name: Start WebUI Web Service | |
| if: matrix.test-suite == 'webui' | |
| run: | | |
| docker exec -w /app/${{ env.WEBUI_SRC_DIR }} -d ${{ env.CONTAINER_NAME }} sh -c "python3 webui.py --port 7860 --api --no-download-sd-model --do-not-download-clip --disable-safe-unpickle --ckpt-dir /share_nfs/onediff_ci/sd-webui/models --skip-version-check > /app/${{ env.WEBUI_SRC_DIR }}/onediff_webui.log 2>&1" | |
| sleep 60 | |
| - run: docker exec ${{ env.CONTAINER_NAME }} ps aux | |
| - if: matrix.test-suite == 'webui' | |
| run: docker exec ${{ env.CONTAINER_NAME }} python3 -m pytest -v -s tests/sd-webui/test_api.py | |
| - name: Show WebUI Log | |
| if: matrix.test-suite == 'webui' | |
| run: docker exec ${{ env.CONTAINER_NAME }} cat /app/${{ env.WEBUI_SRC_DIR }}/onediff_webui.log | |
| - name: Shutdown docker for ComfyUI Test | |
| if: matrix.test-suite == 'comfy' | |
| run: | | |
| docker compose -f tests/comfy-docker-compose.yml down | |
| env: | |
| CONTAINER_NAME: ${{ env.CONTAINER_NAME }} | |
| ACR_ORG: ${{ env.ACR_ORG }} | |
| MATRIX_IMAGE: ${{ matrix.image }} | |
| COMFYUI_SRC_DIR: ${{ env.COMFYUI_SRC_DIR }} | |
| SDXL_BASE: ${{ env.SDXL_BASE }} | |
| UNET_INT8: ${{ env.UNET_INT8 }} | |
| SELENIUM_IMAGE: ${{ env.SELENIUM_IMAGE }} | |
| SELENIUM_CONTAINER_NAME: ${{ env.SELENIUM_CONTAINER_NAME }} | |
| SILICON_ONEDIFF_LICENSE_KEY: ${{ secrets.SILICON_ONEDIFF_LICENSE_KEY }} | |
| - name: Shutdown docker for diffusers examples | |
| if: matrix.test-suite == 'diffusers_examples' | |
| run: | | |
| docker compose -f tests/diffusers-docker-compose.yml down | |
| env: | |
| CONTAINER_NAME: ${{ env.CONTAINER_NAME }} | |
| ACR_ORG: ${{ env.ACR_ORG }} | |
| MATRIX_IMAGE: ${{ matrix.image }} | |
| SDXL_BASE: ${{ env.SDXL_BASE }} | |
| UNET_INT8: ${{ env.UNET_INT8 }} | |
| SILICON_ONEDIFF_LICENSE_KEY: ${{ secrets.SILICON_ONEDIFF_LICENSE_KEY }} | |
| - name: Shutdown docker for WebUI Test | |
| if: matrix.test-suite == 'webui' | |
| run: | | |
| docker compose -f tests/webui-docker-compose.yml down | |
| env: | |
| CONTAINER_NAME: ${{ env.CONTAINER_NAME }} | |
| ACR_ORG: ${{ env.ACR_ORG }} | |
| MATRIX_IMAGE: ${{ matrix.image }} | |
| WEBUI_SRC_DIR: ${{ env.WEBUI_SRC_DIR }} | |
| SELENIUM_IMAGE: ${{ env.SELENIUM_IMAGE }} | |
| SELENIUM_CONTAINER_NAME: ${{ env.SELENIUM_CONTAINER_NAME }} | |
| SILICON_ONEDIFF_LICENSE_KEY: ${{ secrets.SILICON_ONEDIFF_LICENSE_KEY }} |