Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Autoconfigure fds for readiness checks in mods #995

Open
wants to merge 11 commits into
base: mod-scripts
Choose a base branch
from
2 changes: 1 addition & 1 deletion .editorconfig
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,6 @@ trim_trailing_whitespace = false
indent_style = space
indent_size = 2

[{**.sh,root/etc/cont-init.d/**,root/etc/services.d/**}]
[{docker-mods.*}]
indent_style = space
indent_size = 4
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

These files are used by Linuxserver build processes to handle mods in our images. Not for end-user consumption.

* **22.12.24:** - Add modcache support.
* **26.06.24:** - Add RO and User handlers.
* **10.06.24:** - Move lsiown to its own file. Remove support for legacy v2 and hybrid mods.
* **13.04.24:** - Let lsiown ignore broken symlinks (requires gnu find).
Expand Down
127 changes: 94 additions & 33 deletions docker-mods.v3
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,16 @@

# Version 3
# 2022-09-25 - Initial Release
MOD_SCRIPT_VER="3.20241207"
MOD_SCRIPT_VER="3.20241223"

# Define custom folder paths
SCRIPTS_DIR="/custom-cont-init.d"
SERVICES_DIR="/custom-services.d"

if [[ ${DOCKER_MODS_DEBUG_CURL,,} = "true" ]]; then
CURL_NOISE_LEVEL="-v"
CURL_NOISE_LEVEL="-v"
else
CURL_NOISE_LEVEL="--silent"
CURL_NOISE_LEVEL="--silent"
fi

tamper_check() {
Expand Down Expand Up @@ -120,7 +120,7 @@ create_with_contenv_alias() {
# Check for curl
curl_check() {
if [[ ! -f /usr/bin/curl ]] || [[ ! -f /usr/bin/jq ]]; then
write_mod_info "Curl/JQ was not found on this system for Docker mods installing"
write_mod_info "Curl/JQ was not found on this system and is required for Docker mods, installing..."
if [[ -f /usr/bin/apt ]]; then
# Ubuntu
export DEBIAN_FRONTEND="noninteractive"
Expand Down Expand Up @@ -149,22 +149,22 @@ curl_check() {

write_mod_info() {
local MSG=$*
echo "[mod-init] $MSG"
echo -e "[mod-init] $MSG"
}

write_mod_error() {
local MSG=$*
echo "[mod-init] (ERROR) $MSG"
echo -e "[mod-init] (ERROR) $MSG"
}

write_mod_debug() {
local MSG=$*
if [[ ${DOCKER_MODS_DEBUG,,} = "true" ]]; then echo "[mod-init] (DEBUG) $MSG"; fi
if [[ ${DOCKER_MODS_DEBUG,,} = "true" ]]; then echo -e "[mod-init] (DEBUG) $MSG"; fi
}

# Use different filtering depending on URL
get_blob_sha() {
MULTIDIGEST=$(curl -f --retry 10 --retry-max-time 60 --retry-connrefused \
MULTIDIGEST=$(curl -f --retry 5 --retry-max-time 30 --retry-connrefused \
${CURL_NOISE_LEVEL} \
--location \
--header "Accept: application/vnd.docker.distribution.manifest.v2+json" \
Expand Down Expand Up @@ -194,7 +194,7 @@ get_blob_sha() {
write_mod_debug "Mod only has a single arch manifest" >&2
MULTIDIGEST=$(jq -r ".manifests[].digest?" <<< "${MULTIDIGEST}")
fi
if DIGEST=$(curl -f --retry 10 --retry-max-time 60 --retry-connrefused \
if DIGEST=$(curl -f --retry 5 --retry-max-time 30 --retry-connrefused \
${CURL_NOISE_LEVEL} \
--location \
--header "Accept: application/vnd.docker.distribution.manifest.v2+json" \
Expand Down Expand Up @@ -254,6 +254,14 @@ run_mods() {
write_mod_info "Running Docker Modification Logic"
write_mod_debug "Running in debug mode"
write_mod_debug "Mod script version ${MOD_SCRIPT_VER}"
mkdir -p /modcache
# Fetch internal fds
FD_MAX=3
while IFS= read -r -d '' FD; do
if [[ $(cat "${FD}") -gt "${FD_MAX}" ]]; then
FD_MAX=$(cat "${FD}")
fi
done < <(find /etc/s6-overlay/s6-rc.d -type f -name 'notification-fd' -print0)
for DOCKER_MOD in $(echo "${DOCKER_MODS}" | tr '|' '\n'); do
# Support alternative endpoints
case "${DOCKER_MOD}" in
Expand Down Expand Up @@ -320,7 +328,7 @@ run_mods() {
if [[ -n "${AUTH_URL}" ]]; then
# Get registry token for api operations
TOKEN="$(
curl -f --retry 10 --retry-max-time 60 --retry-connrefused \
curl -f --retry 5 --retry-max-time 30 --retry-connrefused \
${CURL_NOISE_LEVEL} \
"${AUTH_URL}" |
jq -r '.token'
Expand All @@ -335,7 +343,7 @@ run_mods() {
write_mod_debug "Couldn't fetch manifest from ghcr.io, trying docker.io"
AUTH_URL="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ENDPOINT}:pull"
TOKEN="$(
curl -f --retry 10 --retry-max-time 60 --retry-connrefused \
curl -f --retry 5 --retry-max-time 30 --retry-connrefused \
${CURL_NOISE_LEVEL} \
"${AUTH_URL}" |
jq -r '.token'
Expand All @@ -347,50 +355,98 @@ run_mods() {
fi
ARCH=$(get_arch)
write_mod_debug "Arch detected as ${ARCH}"
# Determine first and only layer of image
SHALAYER=$(get_blob_sha "${TOKEN}" "${MANIFEST_URL}" "${TAG}" "${ARCH:=-amd64}")
if [[ $? -eq 1 ]]; then
write_mod_error "No manifest available for arch ${ARCH:=-amd64}, cannot fetch mod"
continue
elif [[ -z "${SHALAYER}" ]]; then
write_mod_error "${DOCKER_MOD} digest could not be fetched from ${REGISTRY}"
continue
if [[ -z "${TOKEN}" ]]; then
write_mod_error "Couldn't fetch auth token from ${REGISTRY}, switching to offline mode for ${DOCKER_MOD}"
MOD_OFFLINE="true"
else
# Determine first and only layer of image
SHALAYER=$(get_blob_sha "${TOKEN}" "${MANIFEST_URL}" "${TAG}" "${ARCH:--amd64}")
if [[ $? -eq 1 ]]; then
write_mod_error "No manifest available for arch ${ARCH:--amd64}, cannot fetch mod"
continue
elif [[ -z "${SHALAYER}" ]]; then
write_mod_info "${DOCKER_MOD} digest could not be fetched from ${REGISTRY}, checking local cache"
MOD_OFFLINE="true"
fi
write_mod_debug "Mod SHA is ${SHALAYER:-unknown, mod may not exist}"
fi
# Check if we have allready applied this layer
if [[ -f "/${FILENAME}" ]] && [[ "${SHALAYER}" == "$(cat /"${FILENAME}")" ]]; then
write_mod_info "${DOCKER_MOD} at ${SHALAYER} has been previously applied skipping"
continue
elif [[ -f "/modcache/${FILENAME}.tar.xz" ]] && [[ "${SHALAYER}" =~ $(sha256sum "/modcache/${FILENAME}.tar.xz" | cut -f1 -d" ") ]]; then
write_mod_info "${DOCKER_MOD} at ${SHALAYER} found in modcache, applying"
elif [[ -f "/modcache/${FILENAME}.tar.xz" ]] && [[ "${MOD_OFFLINE}" = "true" ]]; then
write_mod_info "OFFLINE: ${DOCKER_MOD} found in modcache, applying"
elif [[ ! -f "/modcache/${FILENAME}.tar.xz" ]] && [[ "${MOD_OFFLINE}" = "true" ]]; then
write_mod_error "OFFLINE: ${DOCKER_MOD} not found in modcache, skipping"
continue
else
write_mod_info "Downloading ${DOCKER_MOD} from ${REGISTRY}"
# Download and extract layer to /
curl -f --retry 10 --retry-max-time 60 --retry-all-errors \
${CURL_NOISE_LEVEL} \
--location \
--header "Authorization: Bearer ${TOKEN}" \
--user-agent "${MOD_UA}" \
"${BLOB_URL}${SHALAYER}" -o \
"/${FILENAME}.tar.xz"
mkdir -p /tmp/mod
if ! tar -tzf "/${FILENAME}.tar.xz" >/dev/null 2>&1; then
write_mod_error "Invalid tarball, could not download ${DOCKER_MOD} from ${REGISTRY}"
if [[ -f "/modcache/${FILENAME}.lock" ]]; then
write_mod_info "${DOCKER_MOD} is already being downloaded by another container, waiting..."
for ((i = 5 ; i < 21 ; i=i*2 )); do
sleep $i
if [[ ! -f "/modcache/${FILENAME}.lock" ]]; then
SKIP_MOD_DOWNLOAD=true
break
elif [[ $i == 20 ]]; then
write_mod_error "${DOCKER_MOD} timed out waiting for lock, skipping\n\tIf no other containers are using this mod you may need to delete /modcache/${FILENAME}.lock"
SKIP_MOD=true
fi
done
fi
if [[ "${SKIP_MOD}" == "true" ]]; then
continue
elif [[ "${SKIP_MOD_DOWNLOAD}" != "true" ]]; then
# Download and extract layer to /
touch "/modcache/${FILENAME}.lock"
curl -f --retry 5 --retry-max-time 30 --retry-all-errors \
${CURL_NOISE_LEVEL} \
--location \
--header "Authorization: Bearer ${TOKEN}" \
--user-agent "${MOD_UA}" \
"${BLOB_URL}${SHALAYER}" -o \
"/modcache/${FILENAME}.tar.xz"
fi
fi
if ! tar -tzf "/modcache/${FILENAME}.tar.xz" >/dev/null 2>&1; then
write_mod_error "Invalid tarball for ${DOCKER_MOD}, skipping"
if [[ -f "/modcache/${FILENAME}.lock" ]]; then
rm "/modcache/${FILENAME}.lock" || write_mod_error "Failed to delete lock file /modcache/${FILENAME}.lock"
fi
continue
fi
if [[ -z "${MODMANAGER_MODONLY}" ]]; then
write_mod_info "Installing ${DOCKER_MOD}"
tar xzf "/${FILENAME}.tar.xz" -C /tmp/mod
mkdir -p /tmp/mod
tar xzf "/modcache/${FILENAME}.tar.xz" -C /tmp/mod
# Remove any v2 mod elements as they're no longer supported
if [[ -d /tmp/mod/etc/cont-init.d ]]; then
rm -rf /tmp/mod/etc/cont-init.d
fi
if [[ -d /tmp/mod/etc/services.d ]]; then
rm -rf /tmp/mod/etc/services.d
fi
# Rationalise fds
if [[ ! -f "/tmp/mod/fd-static" ]]; then
while IFS= read -r -d '' FD; do
(( FD_MAX++ ))
echo $FD_MAX > "${FD}"
done < <(find /tmp/mod/etc/s6-overlay/s6-rc.d -type f -name 'notification-fd' -print0)
fi
shopt -s dotglob
cp -R /tmp/mod/* /
shopt -u dotglob
rm -rf /tmp/mod
rm -rf "/${FILENAME}.tar.xz"
echo "${SHALAYER}" >"/${FILENAME}"
write_mod_info "${DOCKER_MOD} applied to container"
else
write_mod_debug "Modmanager skipping mod application"
fi
if [[ -f "/modcache/${FILENAME}.lock" ]]; then
rm "/modcache/${FILENAME}.lock" || write_mod_error "Failed to delete lock file /modcache/${FILENAME}.lock"
fi
write_mod_info "${DOCKER_MOD} applied to container"
done
}

Expand Down Expand Up @@ -474,6 +530,11 @@ run_branding() {
}

# Main script loop
if [[ -n "${MODMANAGER_MODONLY}" ]]; then
run_mods
exit 0
fi

if grep -qEe ' / \w+ ro' /proc/mounts; then
printf '1' > /run/s6/container_environment/LSIO_READ_ONLY_FS
LSIO_READ_ONLY_FS=1
Expand Down