From 3a5b1d053c2847835988e7e23d074bf0ea57d57d Mon Sep 17 00:00:00 2001 From: Wayback Archiver <66856220+waybackarchiver@users.noreply.github.com> Date: Tue, 30 Jul 2024 00:31:16 +0000 Subject: [PATCH] Add support for placing ipfs variables --- .github/workflows/builder.yml | 12 ++++++++++++ .github/workflows/docker.yml | 8 ++++++++ .github/workflows/release.yml | 12 ++++++++++++ .github/workflows/snapcraft.yml | 2 ++ Dockerfile | 4 ++++ Makefile | 10 +++++++++- build/aur | 2 +- build/debian/Dockerfile | 2 ++ build/docker/Dockerfile.all | 6 +++++- build/docker/Dockerfile.dev | 6 +++++- build/redhat/Dockerfile | 4 ++++ config/config_test.go | 4 ++-- config/options.go | 24 +++++++++++++++--------- docker-bake.hcl | 10 ++++++++++ 14 files changed, 91 insertions(+), 15 deletions(-) diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml index 43940618..aa6f429f 100644 --- a/.github/workflows/builder.yml +++ b/.github/workflows/builder.yml @@ -101,7 +101,9 @@ jobs: artifact-path: ./build/binary/wayback* egress-policy: audit secrets: + wayback-ipfs-target: ${{ secrets.WAYBACK_IPFS_TARGET }} wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} + wayback-ipfs-secret: ${{ secrets.WAYBACK_IPFS_SECRET }} debpkg: name: Build deb @@ -125,7 +127,9 @@ jobs: artifact-path: build/package/wayback*.deb egress-policy: audit secrets: + wayback-ipfs-target: ${{ secrets.WAYBACK_IPFS_TARGET }} wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} + wayback-ipfs-secret: ${{ secrets.WAYBACK_IPFS_SECRET }} rpmpkg: name: Build RPM @@ -136,7 +140,9 @@ jobs: artifact-path: build/package/wayback*.rpm egress-policy: audit secrets: + wayback-ipfs-target: ${{ secrets.WAYBACK_IPFS_TARGET }} wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} + wayback-ipfs-secret: ${{ secrets.WAYBACK_IPFS_SECRET }} wayback-signing-key: ${{ secrets.GEMFURY_SIGNING_KEY }} wayback-signing-passpharse: ${{ secrets.GEMFURY_SIGNING_PASSPHARSE }} @@ -152,7 +158,9 @@ jobs: build/aur/wayback*.pkg.tar.zst egress-policy: audit secrets: + wayback-ipfs-target: ${{ secrets.WAYBACK_IPFS_TARGET }} wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} + wayback-ipfs-secret: ${{ secrets.WAYBACK_IPFS_SECRET }} snapcraft: name: Build Snap @@ -163,7 +171,9 @@ jobs: publish: ${{ github.repository == 'wabarc/wayback' && github.event_name == 'push' }} egress-policy: audit secrets: + wayback-ipfs-target: ${{ secrets.WAYBACK_IPFS_TARGET }} wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} + wayback-ipfs-secret: ${{ secrets.WAYBACK_IPFS_SECRET }} snapcraft-token: ${{ secrets.SNAPCRAFT_TOKEN }} flatpak: @@ -176,4 +186,6 @@ jobs: artifact-path: org.wabarc.wayback-*.x86_64.flatpak egress-policy: audit secrets: + wayback-ipfs-target: ${{ secrets.WAYBACK_IPFS_TARGET }} wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} + wayback-ipfs-secret: ${{ secrets.WAYBACK_IPFS_SECRET }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1a996844..0789c41a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -134,7 +134,9 @@ jobs: - name: Build artifacts uses: docker/bake-action@6c87dcca988e4e074e3ab1f976a70f63ec9673fb # v2.3.0 env: + WAYBACK_IPFS_TARGET: ${{ secrets.WAYBACK_IPFS_TARGET }} WAYBACK_IPFS_APIKEY: ${{ secrets.WAYBACK_IPFS_APIKEY }} + WAYBACK_IPFS_SECRET: ${{ secrets.WAYBACK_IPFS_SECRET }} with: files: ./docker-bake.hcl targets: artifact-all @@ -145,7 +147,9 @@ jobs: - name: Build and push Docker image uses: docker/bake-action@6c87dcca988e4e074e3ab1f976a70f63ec9673fb # v2.3.0 env: + WAYBACK_IPFS_TARGET: ${{ secrets.WAYBACK_IPFS_TARGET }} WAYBACK_IPFS_APIKEY: ${{ secrets.WAYBACK_IPFS_APIKEY }} + WAYBACK_IPFS_SECRET: ${{ secrets.WAYBACK_IPFS_SECRET }} with: files: | ./docker-bake.hcl @@ -271,7 +275,9 @@ jobs: - name: Build artifacts uses: docker/bake-action@6c87dcca988e4e074e3ab1f976a70f63ec9673fb # v2.3.0 env: + WAYBACK_IPFS_TARGET: ${{ secrets.WAYBACK_IPFS_TARGET }} WAYBACK_IPFS_APIKEY: ${{ secrets.WAYBACK_IPFS_APIKEY }} + WAYBACK_IPFS_SECRET: ${{ secrets.WAYBACK_IPFS_SECRET }} with: files: ./docker-bake.hcl targets: artifact-all @@ -282,7 +288,9 @@ jobs: - name: Build and push Docker image uses: docker/bake-action@6c87dcca988e4e074e3ab1f976a70f63ec9673fb # v2.3.0 env: + WAYBACK_IPFS_TARGET: ${{ secrets.WAYBACK_IPFS_TARGET }} WAYBACK_IPFS_APIKEY: ${{ secrets.WAYBACK_IPFS_APIKEY }} + WAYBACK_IPFS_SECRET: ${{ secrets.WAYBACK_IPFS_SECRET }} with: files: | ./docker-bake.hcl diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e071e0fb..1caac454 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -86,7 +86,9 @@ jobs: artifact-path: build/package/wayback* egress-policy: audit secrets: + wayback-ipfs-target: ${{ secrets.WAYBACK_IPFS_TARGET }} wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} + wayback-ipfs-secret: ${{ secrets.WAYBACK_IPFS_SECRET }} debpkg: name: Build DEB @@ -110,7 +112,9 @@ jobs: artifact-path: build/package/wayback*.deb egress-policy: audit secrets: + wayback-ipfs-target: ${{ secrets.WAYBACK_IPFS_TARGET }} wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} + wayback-ipfs-secret: ${{ secrets.WAYBACK_IPFS_SECRET }} rpmpkg: name: Build RPM @@ -121,7 +125,9 @@ jobs: artifact-path: build/package/wayback*.rpm egress-policy: audit secrets: + wayback-ipfs-target: ${{ secrets.WAYBACK_IPFS_TARGET }} wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} + wayback-ipfs-secret: ${{ secrets.WAYBACK_IPFS_SECRET }} wayback-signing-key: ${{ secrets.GEMFURY_SIGNING_KEY }} wayback-signing-passpharse: ${{ secrets.GEMFURY_SIGNING_PASSPHARSE }} @@ -137,7 +143,9 @@ jobs: build/aur/wayback*.pkg.tar.zst egress-policy: audit secrets: + wayback-ipfs-target: ${{ secrets.WAYBACK_IPFS_TARGET }} wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} + wayback-ipfs-secret: ${{ secrets.WAYBACK_IPFS_SECRET }} snapcraft: name: Build Snap @@ -149,7 +157,9 @@ jobs: publish: true egress-policy: audit secrets: + wayback-ipfs-target: ${{ secrets.WAYBACK_IPFS_TARGET }} wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} + wayback-ipfs-secret: ${{ secrets.WAYBACK_IPFS_SECRET }} snapcraft-token: ${{ secrets.SNAPCRAFT_TOKEN }} flatpak: @@ -161,7 +171,9 @@ jobs: artifact-path: org.wabarc.wayback-*.x86_64.flatpak egress-policy: audit secrets: + wayback-ipfs-target: ${{ secrets.WAYBACK_IPFS_TARGET }} wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} + wayback-ipfs-secret: ${{ secrets.WAYBACK_IPFS_SECRET }} release: name: Create and upload release diff --git a/.github/workflows/snapcraft.yml b/.github/workflows/snapcraft.yml index 66f8a802..bc39e784 100644 --- a/.github/workflows/snapcraft.yml +++ b/.github/workflows/snapcraft.yml @@ -22,5 +22,7 @@ jobs: publish: true release: true secrets: + wayback-ipfs-target: ${{ secrets.WAYBACK_IPFS_TARGET }} wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} + wayback-ipfs-secret: ${{ secrets.WAYBACK_IPFS_SECRET }} snapcraft-token: ${{ secrets.SNAPCRAFT_TOKEN }} diff --git a/Dockerfile b/Dockerfile index 814c3ebf..f329d38e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,11 +11,15 @@ COPY --from=tonistiigi/xx:golang / / RUN apk add --no-cache -U build-base ca-certificates linux-headers musl-dev git tar ARG TARGETPLATFORM +ARG WAYBACK_IPFS_TARGET ARG WAYBACK_IPFS_APIKEY +ARG WAYBACK_IPFS_SECRET WORKDIR /src +ENV WAYBACK_IPFS_TARGET ${WAYBACK_IPFS_TARGET} ENV WAYBACK_IPFS_APIKEY ${WAYBACK_IPFS_APIKEY} +ENV WAYBACK_IPFS_SECRET ${WAYBACK_IPFS_SECRET} COPY . . RUN --mount=type=bind,target=/src,rw \ diff --git a/Makefile b/Makefile index 5735cf50..2b2697d8 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,9 @@ PACKDIR ?= ./build/package LDFLAGS := $(shell echo "-X '${REPO}/version.Version=`git describe --tags --abbrev=0`'") LDFLAGS := $(shell echo "${LDFLAGS} -X '${REPO}/version.Commit=`git rev-parse --short HEAD`'") LDFLAGS := $(shell echo "${LDFLAGS} -X '${REPO}/version.BuildDate=`date +%FT%T%z`'") -LDFLAGS := $(shell echo "${LDFLAGS} -X '${REPO}/config.IPFSToken=$(shell echo ${WAYBACK_IPFS_APIKEY})'") +LDFLAGS := $(shell echo "${LDFLAGS} -X '${REPO}/config.IPFSTarget=$(shell echo ${WAYBACK_IPFS_TARGET})'") +LDFLAGS := $(shell echo "${LDFLAGS} -X '${REPO}/config.IPFSApikey=$(shell echo ${WAYBACK_IPFS_APIKEY})'") +LDFLAGS := $(shell echo "${LDFLAGS} -X '${REPO}/config.IPFSSecret=$(shell echo ${WAYBACK_IPFS_SECRET})'") GOBUILD ?= go build -trimpath --ldflags "-s -w ${LDFLAGS} -buildid=" -v VERSION ?= $(shell git describe --tags `git rev-list --tags --max-count=1` | sed -e 's/v//g') GOFILES ?= $(wildcard ./cmd/wayback/*.go) @@ -152,14 +154,18 @@ profile: ## Test and profile docker-image: ## Build Docker image @echo "-> Building docker image..." @$(DOCKER) build \ + --build-arg WAYBACK_IPFS_TARGET=$(shell echo ${WAYBACK_IPFS_TARGET}) \ --build-arg WAYBACK_IPFS_APIKEY=$(shell echo ${WAYBACK_IPFS_APIKEY}) \ + --build-arg WAYBACK_IPFS_SECRET=$(shell echo ${WAYBACK_IPFS_SECRET}) \ -t $(DOCKER_IMAGE):$(VERSION) \ -f ./build/docker/Dockerfile.dev . rpm: ## Build RPM package @echo "-> Building rpm package..." @$(DOCKER) build \ + --build-arg WAYBACK_IPFS_TARGET=$(shell echo ${WAYBACK_IPFS_TARGET}) \ --build-arg WAYBACK_IPFS_APIKEY=$(shell echo ${WAYBACK_IPFS_APIKEY}) \ + --build-arg WAYBACK_IPFS_SECRET=$(shell echo ${WAYBACK_IPFS_SECRET}) \ -t wayback-rpm-builder \ -f build/redhat/Dockerfile . @$(DOCKER) run --rm \ @@ -174,7 +180,9 @@ debian: ## Build Debian packages @$(DOCKER) buildx build --load \ --platform linux/$(DOCKER_PLATFORM) \ --build-arg PKG_VERSION=$(VERSION) \ + --build-arg WAYBACK_IPFS_TARGET=$(shell echo ${WAYBACK_IPFS_TARGET}) \ --build-arg WAYBACK_IPFS_APIKEY=$(shell echo ${WAYBACK_IPFS_APIKEY}) \ + --build-arg WAYBACK_IPFS_SECRET=$(shell echo ${WAYBACK_IPFS_SECRET}) \ -t wayback-deb-builder \ -f build/debian/Dockerfile \ . diff --git a/build/aur b/build/aur index 757d0fc9..0f510455 160000 --- a/build/aur +++ b/build/aur @@ -1 +1 @@ -Subproject commit 757d0fc9bb60172fbde97dd414bfbed72a23624c +Subproject commit 0f510455e1fe11ffc471fc3f8736d1222507bda8 diff --git a/build/debian/Dockerfile b/build/debian/Dockerfile index 05df7ab8..bf5755f3 100644 --- a/build/debian/Dockerfile +++ b/build/debian/Dockerfile @@ -11,6 +11,8 @@ RUN apt-get update -q && \ COPY . /src +ENV WAYBACK_IPFS_TARGET ${WAYBACK_IPFS_TARGET} ENV WAYBACK_IPFS_APIKEY ${WAYBACK_IPFS_APIKEY} +ENV WAYBACK_IPFS_SECRET ${WAYBACK_IPFS_SECRET} CMD ["/src/build/debian/builder"] diff --git a/build/docker/Dockerfile.all b/build/docker/Dockerfile.all index 49ec3d2d..86ce888e 100644 --- a/build/docker/Dockerfile.all +++ b/build/docker/Dockerfile.all @@ -7,11 +7,15 @@ COPY --from=tonistiigi/xx:golang / / RUN apk add --no-cache -U build-base ca-certificates linux-headers musl-dev git tar ARG TARGETPLATFORM +ARG WAYBACK_IPFS_TARGET ARG WAYBACK_IPFS_APIKEY +ARG WAYBACK_IPFS_SECRET WORKDIR /src -ENV WAYBACK_IPFS_APIKEY $WAYBACK_IPFS_APIKEY +ENV WAYBACK_IPFS_TARGET ${WAYBACK_IPFS_TARGET} +ENV WAYBACK_IPFS_APIKEY ${WAYBACK_IPFS_APIKEY} +ENV WAYBACK_IPFS_SECRET ${WAYBACK_IPFS_SECRET} COPY . . RUN --mount=type=bind,target=/src,rw \ diff --git a/build/docker/Dockerfile.dev b/build/docker/Dockerfile.dev index a852453f..25bf0feb 100644 --- a/build/docker/Dockerfile.dev +++ b/build/docker/Dockerfile.dev @@ -6,11 +6,15 @@ FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS builder RUN apk add --no-cache -U build-base ca-certificates linux-headers musl-dev git tar ARG TARGETPLATFORM +ARG WAYBACK_IPFS_TARGET ARG WAYBACK_IPFS_APIKEY +ARG WAYBACK_IPFS_SECRET WORKDIR /src -ENV WAYBACK_IPFS_APIKEY $WAYBACK_IPFS_APIKEY +ENV WAYBACK_IPFS_TARGET ${WAYBACK_IPFS_TARGET} +ENV WAYBACK_IPFS_APIKEY ${WAYBACK_IPFS_APIKEY} +ENV WAYBACK_IPFS_SECRET ${WAYBACK_IPFS_SECRET} COPY . . RUN set -eu && \ diff --git a/build/redhat/Dockerfile b/build/redhat/Dockerfile index 103f928b..b220eb8c 100644 --- a/build/redhat/Dockerfile +++ b/build/redhat/Dockerfile @@ -4,13 +4,17 @@ # FROM golang:1.22-alpine AS builder +ARG WAYBACK_IPFS_TARGET ARG WAYBACK_IPFS_APIKEY +ARG WAYBACK_IPFS_SECRET RUN apk update && apk add --no-cache build-base ca-certificates git # Required by statically linked binary with OpenSSL # RUN apk add linux-headers +ENV WAYBACK_IPFS_TARGET ${WAYBACK_IPFS_TARGET} ENV WAYBACK_IPFS_APIKEY ${WAYBACK_IPFS_APIKEY} +ENV WAYBACK_IPFS_SECRET ${WAYBACK_IPFS_SECRET} WORKDIR /go/src/app diff --git a/config/config_test.go b/config/config_test.go index b3b27b1b..f74b8fcc 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -281,7 +281,7 @@ func TestIPFSTarget(t *testing.T) { os.Clearenv() os.Setenv("WAYBACK_IPFS_TARGET", test.userTarget) os.Setenv("WAYBACK_IPFS_APIKEY", test.userApikey) - IPFSToken = test.token + IPFSApikey = test.token parser := NewParser() opts, err := parser.ParseEnvironmentVariables() @@ -337,7 +337,7 @@ func TestIPFSApikey(t *testing.T) { os.Clearenv() os.Setenv("WAYBACK_IPFS_TARGET", test.userTarget) os.Setenv("WAYBACK_IPFS_APIKEY", test.userApikey) - IPFSToken = test.token + IPFSApikey = test.token parser := NewParser() opts, err := parser.ParseEnvironmentVariables() diff --git a/config/options.go b/config/options.go index 392d61f3..e9a93b3e 100644 --- a/config/options.go +++ b/config/options.go @@ -108,8 +108,9 @@ const ( ) var ( - IPFSToken = "" - IPFSTarget = "web3storage" + IPFSTarget = "" + IPFSApikey = "" + IPFSSecret = "" defStorageDir = path.Join(os.TempDir(), "reduxer") defOnionRemotePorts = []int{80} @@ -443,25 +444,30 @@ func (o *Options) IPFSMode() string { } // IPFSTarget returns which IPFS pinning service to use. +// It returns a managed IPFS credential if env `WAYBACK_IPFS_TARGET` empty. func (o *Options) IPFSTarget() string { - if IPFSToken != "" { - return IPFSTarget + if o.ipfs.target != "" { + return o.ipfs.target } - return o.ipfs.target + return IPFSTarget } // IPFSApiKey returns the apikey of the IPFS pinning service. // It returns a managed IPFS credential if env `WAYBACK_IPFS_APIKEY` empty. func (o *Options) IPFSApikey() string { - if o.ipfs.apikey == "" { - return IPFSToken + if o.ipfs.apikey != "" { + return o.ipfs.apikey } - return o.ipfs.apikey + return IPFSApikey } // IPFSSecret returns the secret of the IPFS pinning service. +// It returns a managed IPFS credential if env `WAYBACK_IPFS_SECRET` empty. func (o *Options) IPFSSecret() string { - return o.ipfs.secret + if o.ipfs.secret != "" { + return o.ipfs.secret + } + return IPFSSecret } // UseTor returns whether to use the Tor proxy when snapshot webpage. diff --git a/docker-bake.hcl b/docker-bake.hcl index 654a6990..fce50359 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -3,10 +3,18 @@ variable "GO_VERSION" { default = "1.22" } +variable "WAYBACK_IPFS_TARGET" { + default = "" +} + variable "WAYBACK_IPFS_APIKEY" { default = "" } +variable "WAYBACK_IPFS_SECRET" { + default = "" +} + // GitHub reference as defined in GitHub Actions (eg. refs/head/master)) variable "GITHUB_REF" { default = "" @@ -15,7 +23,9 @@ variable "GITHUB_REF" { target "_common" { args = { GO_VERSION = GO_VERSION + WAYBACK_IPFS_TARGET = WAYBACK_IPFS_TARGET WAYBACK_IPFS_APIKEY = WAYBACK_IPFS_APIKEY + WAYBACK_IPFS_SECRET = WAYBACK_IPFS_SECRET } }