From 874c2e028c3be0b052934d08e2dfa097e60b9513 Mon Sep 17 00:00:00 2001 From: Rob Ballantyne Date: Thu, 4 Jul 2024 13:10:27 +0100 Subject: [PATCH] add newbase provisioning to test venv base --- config/provisioning/newbase.sh | 181 +++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100755 config/provisioning/newbase.sh diff --git a/config/provisioning/newbase.sh b/config/provisioning/newbase.sh new file mode 100755 index 0000000..8707714 --- /dev/null +++ b/config/provisioning/newbase.sh @@ -0,0 +1,181 @@ +#!/bin/bash +# This file will be sourced in init.sh +# Namespace functions with provisioning_ + +# https://raw.githubusercontent.com/ai-dock/stable-diffusion-webui/main/config/provisioning/default.sh + +### Edit the following arrays to suit your workflow - values must be quoted and separated by newlines or spaces. + +DISK_GB_REQUIRED=30 + +PIP_PACKAGES=( + +) + +EXTENSIONS=( + "https://github.com/Mikubill/sd-webui-controlnet" + "https://github.com/d8ahazard/sd_dreambooth_extension" + "https://github.com/deforum-art/sd-webui-deforum" + "https://github.com/adieyal/sd-dynamic-prompts" + "https://github.com/ototadana/sd-face-editor" + "https://github.com/AlUlkesh/stable-diffusion-webui-images-browser" + "https://github.com/hako-mikan/sd-webui-regional-prompter" + "https://github.com/Coyote-A/ultimate-upscale-for-automatic1111" + "https://github.com/Gourieff/sd-webui-reactor" +) + +CHECKPOINT_MODELS=( + "https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt" + #"https://huggingface.co/stabilityai/stable-diffusion-2-1/resolve/main/v2-1_768-ema-pruned.ckpt" + #"https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/sd_xl_base_1.0.safetensors" + #"https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-1.0/resolve/main/sd_xl_refiner_1.0.safetensors" +) + +LORA_MODELS=( + #"https://civitai.com/api/download/models/16576" +) + +VAE_MODELS=( + "https://huggingface.co/stabilityai/sd-vae-ft-ema-original/resolve/main/vae-ft-ema-560000-ema-pruned.safetensors" + "https://huggingface.co/stabilityai/sd-vae-ft-mse-original/resolve/main/vae-ft-mse-840000-ema-pruned.safetensors" + "https://huggingface.co/stabilityai/sdxl-vae/resolve/main/sdxl_vae.safetensors" +) + +ESRGAN_MODELS=( + "https://huggingface.co/ai-forever/Real-ESRGAN/resolve/main/RealESRGAN_x4.pth" + "https://huggingface.co/FacehugmanIII/4x_foolhardy_Remacri/resolve/main/4x_foolhardy_Remacri.pth" + "https://huggingface.co/Akumetsu971/SD_Anime_Futuristic_Armor/resolve/main/4x_NMKD-Siax_200k.pth" +) + +CONTROLNET_MODELS=( + "https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_canny-fp16.safetensors" + #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_depth-fp16.safetensors" + #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_hed-fp16.safetensors" + #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_mlsd-fp16.safetensors" + #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_normal-fp16.safetensors" + "https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_openpose-fp16.safetensors" + #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_scribble-fp16.safetensors" + #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_seg-fp16.safetensors" + "https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_canny-fp16.safetensors" + #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_color-fp16.safetensors" + #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_depth-fp16.safetensors" + #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_keypose-fp16.safetensors" + "https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_openpose-fp16.safetensors" + #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_seg-fp16.safetensors" + #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_sketch-fp16.safetensors" + #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_style-fp16.safetensors" +) + + +### DO NOT EDIT BELOW HERE UNLESS YOU KNOW WHAT YOU ARE DOING ### + +function provisioning_start() { + source /opt/ai-dock/etc/environment.sh + source /opt/ai-dock/bin/venv-set.sh webui + + DISK_GB_AVAILABLE=$(($(df --output=avail -m "${WORKSPACE}" | tail -n1) / 1000)) + DISK_GB_USED=$(($(df --output=used -m "${WORKSPACE}" | tail -n1) / 1000)) + DISK_GB_ALLOCATED=$(($DISK_GB_AVAILABLE + $DISK_GB_USED)) + provisioning_print_header + provisioning_get_pip_packages + provisioning_get_extensions + provisioning_get_models \ + "${WORKSPACE}/storage/stable_diffusion/models/ckpt" \ + "${CHECKPOINT_MODELS[@]}" + provisioning_get_models \ + "${WORKSPACE}/storage/stable_diffusion/models/lora" \ + "${LORA_MODELS[@]}" + provisioning_get_models \ + "${WORKSPACE}/storage/stable_diffusion/models/controlnet" \ + "${CONTROLNET_MODELS[@]}" + provisioning_get_models \ + "${WORKSPACE}/storage/stable_diffusion/models/vae" \ + "${VAE_MODELS[@]}" + provisioning_get_models \ + "${WORKSPACE}/storage/stable_diffusion/models/esrgan" \ + "${ESRGAN_MODELS[@]}" + + PLATFORM_FLAGS="" + if [[ $XPU_TARGET = "CPU" ]]; then + PLATFORM_FLAGS="--use-cpu all --skip-torch-cuda-test --no-half" + fi + PROVISIONING_FLAGS="--skip-python-version-check --no-download-sd-model --do-not-download-clip --port 11404 --exit" + FLAGS_COMBINED="${PLATFORM_FLAGS} $(cat /etc/a1111_webui_flags.conf) ${PROVISIONING_FLAGS}" + + # Start and exit because webui will probably require a restart + cd /opt/stable-diffusion-webui && \ + source "$WEBUI_VENV/bin/activate" + LD_PRELOAD=libtcmalloc.so python launch.py \ + ${FLAGS_COMBINED} + deactivate + provisioning_print_end +} + +function provisioning_get_pip_packages() { + if [[ -n $PIP_PACKAGES ]]; then + "$WEBUI_VENV_PIP" install --no-cache-dir ${PIP_PACKAGES[@]} + fi +} + +function provisioning_get_extensions() { + for repo in "${EXTENSIONS[@]}"; do + dir="${repo##*/}" + path="/opt/stable-diffusion-webui/extensions/${dir}" + requirements="${path}/requirements.txt" + if [[ -d $path ]]; then + # Pull only if AUTO_UPDATE + if [[ ${AUTO_UPDATE,,} == "true" ]]; then + printf "Updating extension: %s...\n" "${repo}" + ( cd "$path" && git pull ) + fi + # Always pip install + if [[ -e $requirements ]]; then + "$WEBUI_VENV_PIP" install --no-cache-dir -r "$requirements" + fi + else + printf "Downloading extension: %s...\n" "${repo}" + git clone "${repo}" "${path}" --recursive + if [[ -e $requirements ]]; then + "$WEBUI_VENV_PIP" install --no-cache-dir -r "${requirements}" + fi + fi + done +} + +function provisioning_get_models() { + if [[ -z $2 ]]; then return 1; fi + dir="$1" + mkdir -p "$dir" + shift + if [[ $DISK_GB_ALLOCATED -ge $DISK_GB_REQUIRED ]]; then + arr=("$@") + else + printf "WARNING: Low disk space allocation - Only the first model will be downloaded!\n" + arr=("$1") + fi + + printf "Downloading %s model(s) to %s...\n" "${#arr[@]}" "$dir" + for url in "${arr[@]}"; do + printf "Downloading: %s\n" "${url}" + provisioning_download "${url}" "${dir}" + printf "\n" + done +} + +function provisioning_print_header() { + printf "\n##############################################\n# #\n# Provisioning container #\n# #\n# This will take some time #\n# #\n# Your container will be ready on completion #\n# #\n##############################################\n\n" + if [[ $DISK_GB_ALLOCATED -lt $DISK_GB_REQUIRED ]]; then + printf "WARNING: Your allocated disk size (%sGB) is below the recommended %sGB - Some models will not be downloaded\n" "$DISK_GB_ALLOCATED" "$DISK_GB_REQUIRED" + fi +} + +function provisioning_print_end() { + printf "\nProvisioning complete: Web UI will start now\n\n" +} + +# Download from $1 URL to $2 file path +function provisioning_download() { + wget -qnc --content-disposition --show-progress -e dotbytes="${3:-4M}" -P "$2" "$1" +} + +provisioning_start \ No newline at end of file