Merge pull request #105 from jim60105/main

Rewrite Dockerfile

.gitattributes

@@ -0,0 +1 @@
+*.sh eol=lf

.github/workflows/docker_publish.yml

@@ -21,43 +21,51 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           submodules: true
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           images: ryu1845/${{ github.event.repository.name }},ghcr.io/${{ github.repository }}
           flavor: |
             latest=${{ github.ref == 'refs/heads/main' }}
             prefix=
             suffix=
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       # Create a Access Token and save it as as Actions secret
       # https://hub.docker.com/settings/security
       # DOCKERHUB_USERNAME
       # DOCKERHUB_TOKEN
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ryu1845
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       # Create a Access Token with `read:packages` and `write:packages` scopes
       # CR_PAT
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.CR_PAT }}
 
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true
+          target: final
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64

.gitignore

@@ -7,3 +7,4 @@ tmp*
 __pycache__
 .python-version
 cookies.txt
+master_urls.txt

Dockerfile

@@ -1,27 +1,56 @@
-FROM python:alpine as base
+FROM python:3.11-alpine as build
 
-WORKDIR /output
-VOLUME /output
+# RUN mount cache for multi-arch: https://github.com/docker/buildx/issues/549#issuecomment-1788297892
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+# Install build dependencies
+RUN apk add --no-cache build-base libffi-dev
 
-FROM base as builder
+WORKDIR /app
 
-ENV PIP_DEFAULT_TIMEOUT=100 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_NO_CACHE_DIR=1
+# Set up venv
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
 
-RUN apk add --no-cache gcc libffi-dev musl-dev
-RUN pip install poetry
-RUN python -m venv /venv
+# Install poetry
+RUN --mount=type=cache,id=pip-$TARGETARCH$TARGETVARIANT,sharing=locked,target=/root/.cache/pip pip3.11 install poetry
 
+# Install dependencies
 COPY pyproject.toml poetry.lock ./
-RUN poetry export -f requirements.txt | /venv/bin/pip install -r /dev/stdin
+RUN --mount=type=cache,id=pip-$TARGETARCH$TARGETVARIANT,sharing=locked,target=/root/.cache/pip poetry export -f requirements.txt | pip3.11 install -r /dev/stdin
 
+# Build
 COPY . .
-RUN poetry build && /venv/bin/pip install dist/*.whl
+RUN --mount=type=cache,id=pip-$TARGETARCH$TARGETVARIANT,sharing=locked,target=/root/.cache/pip poetry build && pip3.11 install dist/*.whl
+
+# Uninstall them inside venv
+RUN pip3.11 uninstall -y setuptools pip && \
+    pip3.11 uninstall -y setuptools pip
+
+FROM python:3.11-alpine as final
+
+# Uninstall them for security purpose
+RUN pip3.11 uninstall -y setuptools pip && \
+    rm -rf /root/.cache/pip
 
-FROM base as final
+# Copy venv
+COPY --link --from=build /venv /venv
+ENV PATH="/venv/bin:$PATH"
 
-RUN apk add --no-cache dumb-init ffmpeg
-COPY --from=builder /venv /venv
+# Use dumb-init to handle signals
+RUN apk add --no-cache dumb-init
+
+# ffmpeg
+COPY --link --from=mwader/static-ffmpeg:6.0 /ffmpeg /usr/local/bin/
+
+# Create output directory
+RUN mkdir -p /output && chown 1001:1001 /output
+VOLUME [ "/output" ]
+
+# Run as non-root user
+USER 1001
+WORKDIR /output
 
-ENTRYPOINT [ "/usr/bin/dumb-init", "--", "/venv/bin/twspace_dl"]
+STOPSIGNAL SIGINT
+ENTRYPOINT [ "dumb-init", "--", "/venv/bin/twspace_dl" ]

README.md

@@ -173,7 +173,7 @@ Example: `[%(creator_screen_name)s]-%(title)s|%(start_date)s`
 
 `Changing ID3 metadata in HLS audio elementary stream is not implemented....`
 
-This is an error in ffmpeg that does not affect twspace_dl at all as far as I know.
+This is an error in ffmpeg that does not affect twspace_dl at all as far as I know.
 
 ## Service
 
@@ -183,19 +183,13 @@ To run as a systemd service please refer to https://github.com/HoloArchivists/tw
 
 ### Run once
 
-> Use `${pwd}` in powershell, or `$(pwd)` in bash
-
 ```bash
-docker run --rm -v ${pwd}:/output ryu1845/twspace-dl -i space_url
+docker run --rm -v .:/output holoarchivists/twspace-dl -i space_url
 ```
 
 ### Run as monitoring service
 
-Using a cookie can help solve some problem with the twitter api. However, using one is not necessary.
-
-#### Without cookie
-
 1. Download the `docker-compose.yml`, `.env`, `monitor.sh` files and put them in a folder named `twspace-dl`.
 2. Edit `.env` and fill in the Twitter username you want to monitor.
-3. \[Optional] If you want to used a cookies file, put it into the folder and named it `cookies.txt`.
+3. Put a cookies file into the folder and named it `cookies.txt`.
 4. `docker-compose up -d`

docker-compose.yml

@@ -7,10 +7,6 @@ services:
     restart: "no"
     env_file: .env
     volumes:
-      - ./:/output
-    entrypoint: [ "/usr/bin/dumb-init", "--", "sh" ]
+      - .:/output
+    entrypoint: [ "dumb-init", "--", "sh" ]
     command: [ "monitor.sh" ]
-    # logging:
-    #   driver: "gelf"
-    #   options:
-    #     gelf-address: "udp://127.0.0.1:12201"

monitor.sh

@@ -7,22 +7,22 @@ if [ -z "$TWITTER_ID" ]; then
   exit 1
 fi
 
+COOKIES_PATH="./cookies.txt"
+if [ ! -f "$COOKIES_PATH" ]; then
+  echo "The cookies file is now required due to the Twitter API change that prohibited guest user access to Twitter API endpoints on 2023-07-01."
+  exit 1
+fi
+
 INTERVAL=${INTERVAL:-10}
-COOKIES_PATH="/output/cookies.txt"
 
 # Monitor live streams of specific user
 while true; do
-  LOG_PREFIX=$(date +"[%m/%d/%y %H:%M:%S] [tw_space@${TWITTER_ID}] ")
+  LOG_PREFIX=$(date +"[%m/%d/%y %H:%M:%S] [tw_space@${TWITTER_ID}]")
 
   # Start recording
-  if [ ! -f "$COOKIES_PATH" ]; then
-    echo "$LOG_PREFIX [VRB] Start trying..."  
-    /venv/bin/twspace_dl -U "https://twitter.com/${TWITTER_ID}" --write-url "master_urls.txt"
-  else
-    echo "$LOG_PREFIX [VRB] Start trying with cookies..."
-    /venv/bin/twspace_dl -U "https://twitter.com/${TWITTER_ID}" --write-url "master_urls.txt" --input-cookie-file "$COOKIES_PATH" -o "$COOKIES_PATH"
-  fi
+  echo "$LOG_PREFIX Start trying..."
+  /venv/bin/twspace_dl -U "https://twitter.com/${TWITTER_ID}" --write-url "master_urls.txt" --input-cookie-file "$COOKIES_PATH"
 
-  echo "$LOG_PREFIX [VRB] Sleep $INTERVAL sec."
+  echo "$LOG_PREFIX Sleep $INTERVAL sec."
   sleep "$INTERVAL"
 done