Add charset converter to convert non utf-8 char

.github/workflows/publish.yml

@@ -0,0 +1,57 @@
+name: Publish
+
+on:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        db: [fdb, gp, mongo, mysql, pg, redis, sqlserver]
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v1
+        with:
+          go-version: '1.21'
+        id: go
+
+      - name: Prepare git
+        env:
+          GITHUB_USER: 1gtm
+          GITHUB_TOKEN: ${{ secrets.LGTM_GITHUB_TOKEN }}
+        run: |
+          set -x
+          git config --global user.name "${GITHUB_USER}"
+          git config --global user.email "${GITHUB_USER}@appscode.com"
+          git config --global \
+            url."https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com".insteadOf \
+            "https://github.com"
+
+      - name: Print version info
+        id: semver
+        run: |
+          make version -f ac.mk
+
+      - name: Set up QEMU
+        id: qemu
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Publish to GitHub Container Registry
+        env:
+          REGISTRY: ghcr.io/kubedb
+          DOCKER_TOKEN: ${{ secrets.LGTM_GITHUB_TOKEN }}
+          USERNAME: 1gtm
+        run: |
+          docker login ghcr.io --username ${USERNAME} --password ${DOCKER_TOKEN}
+          make release -f ac.mk DATABASES=${{ matrix.db }}

Dockerfile

@@ -0,0 +1,43 @@
+# Copyright AppsCode Inc. and Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang:{GO_VERSION} AS builder
+
+ARG TARGETOS
+ARG TARGETARCH
+ARG DB
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN true
+
+RUN set -x \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends apt-transport-https ca-certificates
+
+WORKDIR /src
+COPY . .
+RUN CGO_ENABLED=0 go build -v -o /wal-g ./main/${DB}/main.go
+
+FROM {ARG_FROM}
+
+ARG TARGETOS
+ARG TARGETARCH
+ARG DB
+
+LABEL org.opencontainers.image.source https://github.com/kubedb/wal-g
+
+COPY --from=0 /wal-g /wal-g
+COPY --from=0 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+
+ENTRYPOINT ["/wal-g"]

Makefile

@@ -285,3 +285,9 @@ unlink_libsodium:
 build_client:
 	cd cmd/daemonclient && \
 	go build -o ../../bin/walg-daemon-client -ldflags "-s -w -X main.buildDate=`date -u +%Y.%m.%d_%H:%M:%S` -X main.gitRevision=`git rev-parse --short HEAD` -X main.version=`git tag -l --points-at HEAD`"
+
+
+update: mongo_build
+	docker build --tag walg:2.0 /home/sayed/go/src/kubedb.dev/wal-g/main/mongo
+	docker tag walg:2.0 sayedppqq/walg:2.0
+	docker push sayedppqq/walg:2.0

ac.mk

@@ -0,0 +1,180 @@
+# Copyright AppsCode Inc. and Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SHELL=/bin/bash -o pipefail
+
+BIN      := wal-g
+
+# Where to push the docker image.
+REGISTRY ?= ghcr.io/kubedb
+SRC_REG  ?=
+
+# This version-strategy uses git tags to set the version string
+git_branch       := $(shell git rev-parse --abbrev-ref HEAD)
+git_tag          := $(shell git describe --exact-match --abbrev=0 2>/dev/null || echo "")
+commit_hash      := $(shell git rev-parse --verify HEAD)
+commit_timestamp := $(shell date --date="@$$(git show -s --format=%ct)" --utc +%FT%T)
+
+VERSION          := $(shell git describe --tags --always --dirty)
+version_strategy := commit_hash
+ifdef git_tag
+	VERSION := $(git_tag)
+	version_strategy := tag
+else
+	ifeq (,$(findstring $(git_branch),master HEAD))
+		ifneq (,$(patsubst release-%,,$(git_branch)))
+			VERSION := $(git_branch)
+			version_strategy := branch
+		endif
+	endif
+endif
+
+DATABASES ?= fdb gp mongo mysql pg redis sqlserver
+DB        ?= pg
+
+###
+### These variables should not need tweaking.
+###
+
+DOCKER_PLATFORMS := linux/amd64 linux/arm64
+BIN_PLATFORMS    := $(DOCKER_PLATFORMS)
+
+# Used internally.  Users should pass GOOS and/or GOARCH.
+OS   := $(if $(GOOS),$(GOOS),$(shell go env GOOS))
+ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH))
+
+# bash required
+BASEIMAGE        ?= debian:bookworm
+
+IMAGE            := $(REGISTRY)/$(BIN)
+VERSION          := $(VERSION)_$(DB)
+TAG              := $(VERSION)_$(OS)_$(ARCH)
+
+GO_VERSION       ?= 1.21
+
+# Directories that we need created to build/test.
+BUILD_DIRS  := bin/$(OS)_$(ARCH)
+
+DOCKERFILE  = Dockerfile
+
+# If you want to build all binaries, see the 'all-build' rule.
+# If you want to build all containers, see the 'all-container' rule.
+# If you want to build AND push all containers, see the 'all-push' rule.
+all: fmt build
+
+# For the following OS/ARCH expansions, we transform OS/ARCH into OS_ARCH
+# because make pattern rules don't match with embedded '/' characters.
+
+build-%:
+	@$(MAKE) build                        \
+		-f ac.mk                          \
+		--no-print-directory              \
+		GOOS=$(firstword $(subst _, ,$*)) \
+		GOARCH=$(lastword $(subst _, ,$*))
+
+container-%:
+	@$(MAKE) container                    \
+		-f ac.mk                          \
+		--no-print-directory              \
+		DB=$(firstword $(subst _, ,$*))   \
+		GOOS=$(word 2, $(subst _, ,$*))   \
+		GOARCH=$(lastword $(subst _, ,$*))
+
+push-%:
+	@$(MAKE) push                         \
+		-f ac.mk                          \
+		--no-print-directory              \
+		DB=$(firstword $(subst _, ,$*))   \
+		GOOS=$(word 2, $(subst _, ,$*))   \
+		GOARCH=$(lastword $(subst _, ,$*))
+
+docker-manifest-%:
+	@$(MAKE) docker-manifest              \
+		-f ac.mk                          \
+		--no-print-directory              \
+		DB=$*
+
+all-container: $(addprefix container-, $(subst /,_, $(foreach X,$(DATABASES),$(foreach Y,$(DOCKER_PLATFORMS),$X/$Y))))
+
+all-push: $(addprefix push-, $(subst /,_, $(foreach X,$(DATABASES),$(foreach Y,$(DOCKER_PLATFORMS),$X/$Y))))
+
+all-docker-manifest: $(addprefix docker-manifest-, $(subst /,_, $(DATABASES)))
+
+version:
+	@echo IMAGE=$(IMAGE)
+	@echo TAG=$(TAG)
+	@echo BIN=$(BIN)
+	@echo version=$(VERSION)
+	@echo version_strategy=$(version_strategy)
+	@echo git_tag=$(git_tag)
+	@echo git_branch=$(git_branch)
+	@echo commit_hash=$(commit_hash)
+	@echo commit_timestamp=$(commit_timestamp)
+
+# Used to track state in hidden files.
+DOTFILE_IMAGE    = $(subst /,_,$(IMAGE))-$(TAG)
+
+container: bin/.container-$(DOTFILE_IMAGE)
+ifeq (,$(SRC_REG))
+bin/.container-$(DOTFILE_IMAGE): $(BUILD_DIRS) $(DOCKERFILE)
+	@echo "container: $(IMAGE):$(TAG)"
+	@sed \
+		-e 's|{ARG_FROM}|$(BASEIMAGE)|g'    \
+		-e 's|{GO_VERSION}|$(GO_VERSION)|g' \
+		$(DOCKERFILE) > bin/.dockerfile-$(DB)-$(OS)_$(ARCH)
+	@docker build --platform $(OS)/$(ARCH) --build-arg="DB=$(DB)" --load --pull -t $(IMAGE):$(TAG) -f bin/.dockerfile-$(DB)-$(OS)_$(ARCH) .
+	@docker images -q $(IMAGE):$(TAG) > $@
+	@echo
+else
+bin/.container-$(DOTFILE_IMAGE):
+	@echo "container: $(IMAGE):$(TAG)"
+	@docker tag $(SRC_REG)/$(BIN):$(TAG) $(IMAGE):$(TAG)
+	@echo
+endif
+
+push: bin/.push-$(DOTFILE_IMAGE)
+bin/.push-$(DOTFILE_IMAGE): bin/.container-$(DOTFILE_IMAGE)
+	@docker push $(IMAGE):$(TAG)
+	@echo "pushed: $(IMAGE):$(TAG)"
+	@echo
+
+.PHONY: docker-manifest
+docker-manifest:
+	docker manifest create -a $(IMAGE):$(VERSION) $(foreach PLATFORM,$(DOCKER_PLATFORMS),$(IMAGE):$(VERSION)_$(subst /,_,$(PLATFORM)))
+	docker manifest push $(IMAGE):$(VERSION)
+
+$(BUILD_DIRS):
+	@mkdir -p $@
+
+.PHONY: dev
+dev: gen fmt push
+
+.PHONY: verify
+verify: verify-modules
+
+.PHONY: verify-modules
+verify-modules:
+	go mod tidy
+	go mod vendor
+	@if !(git diff --exit-code HEAD); then \
+		echo "go module files are out of date"; exit 1; \
+	fi
+
+.PHONY: release
+release:
+	@$(MAKE) all-push all-docker-manifest -f ac.mk --no-print-directory
+
+.PHONY: clean
+clean:
+	rm -rf .go bin

cmd/mongo/oplog_push.go

@@ -2,7 +2,9 @@ package mongo
 
 import (
 	"context"
+	storageapi "kubestash.dev/apimachinery/apis/storage/v1alpha1"
 	"os"
+	"path"
 	"syscall"
 	"time"
 
@@ -18,6 +20,15 @@ import (
 	"github.com/wal-g/wal-g/internal/databases/mongo/stats"
 	"github.com/wal-g/wal-g/internal/webserver"
 	"github.com/wal-g/wal-g/utility"
+	"k8s.io/client-go/tools/clientcmd"
+	"kubedb.dev/apimachinery/pkg/factory"
+	controllerclient "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var (
+	snapshotName      string
+	snapshotNamespace string
+	kubeconfig        string
 )
 
 // oplogPushCmd represents the continuous oplog archiving procedure
@@ -49,6 +60,12 @@ var oplogPushCmd = &cobra.Command{
 
 func init() {
 	cmd.AddCommand(oplogPushCmd)
+	oplogPushCmd.PersistentFlags().StringVarP(
+		&snapshotName, "snapshot-name", "", "", "Name of the snapshot")
+	oplogPushCmd.PersistentFlags().StringVarP(
+		&snapshotNamespace, "snapshot-namespace", "n", "", "Namespace of the snapshot")
+	oplogPushCmd.PersistentFlags().StringVarP(
+		&kubeconfig, "kubeconfig", "", "", "Path of the kubeconfig")
 }
 
 func runOplogPush(ctx context.Context, pushArgs oplogPushRunArgs, statsArgs oplogPushStatsArgs) error {
@@ -58,8 +75,14 @@ func runOplogPush(ctx context.Context, pushArgs oplogPushRunArgs, statsArgs oplo
 	if err != nil {
 		return err
 	}
-	uplProvider.ChangeDirectory(models.OplogArchBasePath)
+	subDir := models.OplogArchBasePath
+	if pushArgs.dbProvider == string(storageapi.ProviderLocal) {
+		subDir = path.Join(pushArgs.dbPath, subDir)
+	}
+	uplProvider.ChangeDirectory(subDir)
 	uploader := archive.NewStorageUploader(uplProvider)
+	uploader.SetKubeClient(pushArgs.kubeClient)
+	uploader.SetSnapshot(snapshotName, snapshotNamespace)
 
 	// set up mongodb client and oplog fetcher
 	mongoClient, err := client.NewMongoClient(ctx, pushArgs.mongodbURL)
@@ -128,6 +151,9 @@ type oplogPushRunArgs struct {
 	primaryWait        bool
 	primaryWaitTimeout time.Duration
 	lwUpdate           time.Duration
+	kubeClient         controllerclient.Client
+	dbProvider         string
+	dbPath             string
 }
 
 func buildOplogPushRunArgs() (args oplogPushRunArgs, err error) {
@@ -146,6 +172,10 @@ func buildOplogPushRunArgs() (args oplogPushRunArgs, err error) {
 		return
 	}
 
+	args.dbProvider = internal.GetNonRequiredSetting(internal.MongoDBProvider)
+
+	args.dbPath = internal.GetNonRequiredSetting(internal.MongoDBPath)
+
 	args.primaryWait, err = internal.GetBoolSettingDefault(internal.OplogPushWaitForBecomePrimary, false)
 	if err != nil {
 		return
@@ -159,7 +189,21 @@ func buildOplogPushRunArgs() (args oplogPushRunArgs, err error) {
 	}
 
 	args.lwUpdate, err = internal.GetDurationSetting(internal.MongoDBLastWriteUpdateInterval)
-	return
+	if err != nil {
+		return
+	}
+
+	clientConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+	if err != nil {
+		return
+	}
+
+	args.kubeClient, err = factory.NewUncachedClient(clientConfig)
+	if err != nil {
+		return
+	}
+
+	return args, err
 }
 
 type oplogPushStatsArgs struct {