From 939187a12969153dd6e4d6a73add54cdb1713b25 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 23 Dec 2023 14:31:34 -0500 Subject: [PATCH] env(conda): use transformers for vall-e-x (#1481) --- .github/workflows/test-extra.yml | 52 +++++++++---------- .../transformers/transformers-nvidia.yml | 17 ++++++ .../common-env/transformers/transformers.yml | 19 ++++++- backend/python/vall-e-x/Makefile | 4 +- backend/python/vall-e-x/install.sh | 5 +- backend/python/vall-e-x/run.sh | 2 +- backend/python/vall-e-x/test.sh | 2 +- 7 files changed, 67 insertions(+), 34 deletions(-) diff --git a/.github/workflows/test-extra.yml b/.github/workflows/test-extra.yml index b83e6a18afe0..a42cf4526a50 100644 --- a/.github/workflows/test-extra.yml +++ b/.github/workflows/test-extra.yml @@ -222,29 +222,29 @@ jobs: # export PATH=$PATH:/opt/conda/bin # make -C backend/python/vllm # make -C backend/python/vllm test - # tests-vallex: - # runs-on: ubuntu-latest - # steps: - # - name: Clone - # uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Dependencies - # run: | - # sudo apt-get update - # sudo apt-get install build-essential ffmpeg - # curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \ - # sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \ - # gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \ - # sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \ - # sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \ - # sudo apt-get update && \ - # sudo apt-get install -y conda - # sudo apt-get install -y ca-certificates cmake curl patch - # sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2 - # sudo rm -rfv /usr/bin/conda || true - # - name: Test vall-e-x - # run: | - # export PATH=$PATH:/opt/conda/bin - # make -C backend/python/vall-e-x - # make -C backend/python/vall-e-x test + tests-vallex: + runs-on: ubuntu-latest + steps: + - name: Clone + uses: actions/checkout@v4 + with: + submodules: true + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install build-essential ffmpeg + curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \ + sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \ + gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \ + sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \ + sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \ + sudo apt-get update && \ + sudo apt-get install -y conda + sudo apt-get install -y ca-certificates cmake curl patch + sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2 + sudo rm -rfv /usr/bin/conda || true + - name: Test vall-e-x + run: | + export PATH=$PATH:/opt/conda/bin + make -C backend/python/vall-e-x + make -C backend/python/vall-e-x test diff --git a/backend/python/common-env/transformers/transformers-nvidia.yml b/backend/python/common-env/transformers/transformers-nvidia.yml index a3a44d4b66bb..6884ab977ef7 100644 --- a/backend/python/common-env/transformers/transformers-nvidia.yml +++ b/backend/python/common-env/transformers/transformers-nvidia.yml @@ -96,4 +96,21 @@ dependencies: - urllib3==1.26.17 - xxhash==3.4.1 - yarl==1.9.2 + - soundfile + - langid + - wget + - unidecode + - pyopenjtalk-prebuilt + - pypinyin + - inflect + - cn2an + - jieba + - eng_to_ipa + - openai-whisper + - matplotlib + - gradio==3.41.2 + - nltk + - sudachipy + - sudachidict_core + - vocos prefix: /opt/conda/envs/transformers diff --git a/backend/python/common-env/transformers/transformers.yml b/backend/python/common-env/transformers/transformers.yml index 941fec27e6c7..a28ad4436abf 100644 --- a/backend/python/common-env/transformers/transformers.yml +++ b/backend/python/common-env/transformers/transformers.yml @@ -84,4 +84,21 @@ dependencies: - urllib3==1.26.17 - xxhash==3.4.1 - yarl==1.9.2 -prefix: /opt/conda/envs/transformers + - soundfile + - langid + - wget + - unidecode + - pyopenjtalk-prebuilt + - pypinyin + - inflect + - cn2an + - jieba + - eng_to_ipa + - openai-whisper + - matplotlib + - gradio==3.41.2 + - nltk + - sudachipy + - sudachidict_core + - vocos +prefix: /opt/conda/envs/transformers \ No newline at end of file diff --git a/backend/python/vall-e-x/Makefile b/backend/python/vall-e-x/Makefile index fdb0ab7c900f..4804f12ff521 100644 --- a/backend/python/vall-e-x/Makefile +++ b/backend/python/vall-e-x/Makefile @@ -1,8 +1,6 @@ .PHONY: ttsvalle ttsvalle: - @echo "Creating virtual environment..." - @conda env create --name ttsvalle --file ttsvalle.yml - @echo "Virtual environment created." + $(MAKE) -C ../common-env/transformers bash install.sh .PHONY: run diff --git a/backend/python/vall-e-x/install.sh b/backend/python/vall-e-x/install.sh index 71ec3998d5e2..29f23684100a 100644 --- a/backend/python/vall-e-x/install.sh +++ b/backend/python/vall-e-x/install.sh @@ -3,12 +3,13 @@ ## ## A bash script installs the required dependencies of VALL-E-X and prepares the environment export PATH=$PATH:/opt/conda/bin +export SHA=3faaf8ccadb154d63b38070caf518ce9309ea0f4 # Activate conda environment -source activate ttsvalle +source activate transformers echo $CONDA_PREFIX -git clone https://github.com/Plachtaa/VALL-E-X.git $CONDA_PREFIX/vall-e-x && pushd $CONDA_PREFIX/vall-e-x && pip install -r requirements.txt && popd +git clone https://github.com/Plachtaa/VALL-E-X.git $CONDA_PREFIX/vall-e-x && pushd $CONDA_PREFIX/vall-e-x && git checkout -b build $SHA && pip install -r requirements.txt && popd cp -rfv $CONDA_PREFIX/vall-e-x/* ./ \ No newline at end of file diff --git a/backend/python/vall-e-x/run.sh b/backend/python/vall-e-x/run.sh index 41936882172f..3286e951daa1 100755 --- a/backend/python/vall-e-x/run.sh +++ b/backend/python/vall-e-x/run.sh @@ -5,7 +5,7 @@ export PATH=$PATH:/opt/conda/bin # Activate conda environment -source activate ttsvalle +source activate transformers # get the directory where the bash script is located DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" diff --git a/backend/python/vall-e-x/test.sh b/backend/python/vall-e-x/test.sh index df897a0b87e2..6184966d2d07 100644 --- a/backend/python/vall-e-x/test.sh +++ b/backend/python/vall-e-x/test.sh @@ -3,7 +3,7 @@ ## A bash script wrapper that runs the ttsvalle server with conda # Activate conda environment -source activate ttsvalle +source activate transformers # get the directory where the bash script is located DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"