-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
d7311b0
commit 53bebd2
Showing
13 changed files
with
1,025 additions
and
22 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
FROM golang:1.19-alpine as builder | ||
|
||
WORKDIR /project | ||
|
||
COPY . . | ||
|
||
WORKDIR /project/example | ||
|
||
RUN go mod download | ||
RUN CGO_ENABLED=0 go build -a -o example main.go | ||
|
||
FROM alpine:3.17.0 | ||
|
||
WORKDIR /app | ||
|
||
RUN apk --no-cache add ca-certificates | ||
|
||
USER nobody | ||
COPY --from=builder --chown=nobody:nobody /project/example/example . | ||
COPY --from=builder --chown=nobody:nobody /project/example/config.yml ./config.yml | ||
|
||
ENTRYPOINT ["./example"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1,133 @@ | ||
# Go Elasticsearch Connect Couchbase [![Go Reference](https://pkg.go.dev/badge/github.com/Trendyol/go-elasticsearch-connect-couchbase.svg)](https://pkg.go.dev/github.com/Trendyol/go-elasticsearch-connect-couchbase) [![Go Report Card](https://goreportcard.com/badge/github.com/Trendyol/go-elasticsearch-connect-couchbase)](https://goreportcard.com/report/github.com/Trendyol/go-elasticsearch-connect-couchbase) | ||
|
||
This repository contains the Go implementation of the Couchbase Elasticsearch Connector. | ||
|
||
### Contents | ||
|
||
--- | ||
|
||
* [What is Couchbase Elasticsearch Connector?](#what-is-couchbase-elasticsearch-connector) | ||
* [Why?](#why) | ||
* [Features](#features) | ||
* [Usage](#usage) | ||
* [Configuration](#configuration) | ||
* [Examples](#examples) | ||
|
||
### What is Couchbase Elasticsearch Connector? | ||
|
||
**Go Elasticsearch Connect Couchbase is a source connector**. So it writes Couchbase mutations to Elasticsearch as | ||
documents. | ||
|
||
--- | ||
|
||
### Why? | ||
|
||
+ Build a Couchbase Elasticsearch Connector by using **Go** to reduce resource usage. | ||
+ Suggesting more **configurations** so users can make changes to less code. | ||
+ By presenting the connector as a **library**, ensuring that users do not clone the code they don't need. | ||
|
||
--- | ||
|
||
### Example | ||
|
||
```go | ||
package main | ||
|
||
import ( | ||
goelasticsearchconnectcouchbase "github.com/Trendyol/go-elasticsearch-connect-couchbase" | ||
"github.com/Trendyol/go-elasticsearch-connect-couchbase/couchbase" | ||
"github.com/Trendyol/go-elasticsearch-connect-couchbase/elasticsearch/document" | ||
) | ||
|
||
func mapper(event couchbase.Event) []document.ESActionDocument { | ||
if event.IsMutated { | ||
e := document.NewIndexAction(event.Key, event.Value, nil) | ||
return []document.ESActionDocument{e} | ||
} | ||
e := document.NewDeleteAction(event.Key, nil) | ||
return []document.ESActionDocument{e} | ||
} | ||
|
||
func main() { | ||
connector, err := goelasticsearchconnectcouchbase.NewConnectorBuilder("config.yml"). | ||
SetMapper(mapper). | ||
Build() | ||
if err != nil { | ||
return | ||
} | ||
|
||
defer connector.Close() | ||
connector.Start() | ||
} | ||
|
||
``` | ||
|
||
Custom log structures can be used with the connector | ||
|
||
```go | ||
package main | ||
|
||
import ( | ||
goelasticsearchconnectcouchbase "github.com/Trendyol/go-elasticsearch-connect-couchbase" | ||
"github.com/Trendyol/go-elasticsearch-connect-couchbase/couchbase" | ||
"github.com/Trendyol/go-elasticsearch-connect-couchbase/elasticsearch/document" | ||
"log" | ||
"os" | ||
) | ||
|
||
func mapper(event couchbase.Event) []document.ESActionDocument { | ||
if event.IsMutated { | ||
e := document.NewIndexAction(event.Key, event.Value, nil) | ||
return []document.ESActionDocument{e} | ||
} | ||
e := document.NewDeleteAction(event.Key, nil) | ||
return []document.ESActionDocument{e} | ||
} | ||
|
||
func main() { | ||
logger := log.New(os.Stdout, "cb2elastic: ", log.Ldate|log.Ltime|log.Llongfile) | ||
|
||
connector, err := goelasticsearchconnectcouchbase.NewConnectorBuilder("config.yml"). | ||
SetMapper(mapper). | ||
SetLogger(logger). | ||
SetErrorLogger(logger). | ||
Build() | ||
if err != nil { | ||
return | ||
} | ||
|
||
defer connector.Close() | ||
connector.Start() | ||
} | ||
|
||
``` | ||
|
||
--- | ||
|
||
### Usage | ||
|
||
``` | ||
$ go get github.com/Trendyol/go-elasticsearch-connect-couchbase | ||
``` | ||
|
||
--- | ||
|
||
### Dcp Configuration | ||
|
||
Check out on [go-dcp-client](https://github.com/Trendyol/go-dcp-client#configuration) | ||
|
||
### Elasticsearch Specific Configuration | ||
|
||
| Variable | Type | Required | Default | | ||
|----------------------------------------|-------------------|----------|---------| | ||
| `elasticsearch.collectionIndexMapping` | map[string]string | yes | | | ||
| `elasticsearch.urls` | []string | yes | | | ||
| `elasticsearch.bulkSize` | int | yes | | | ||
| `elasticsearch.bulkTickerDuration` | time.Duration | yes | | | ||
|
||
--- | ||
|
||
### Examples | ||
|
||
- [example](example/main.go) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
FROM golang:1.19-alpine as builder | ||
|
||
WORKDIR /project | ||
|
||
COPY go.mod go.sum ./ | ||
COPY main.go ./ | ||
COPY config.yml ./config.yml | ||
|
||
RUN go mod download | ||
RUN CGO_ENABLED=0 go build -a -o example main.go | ||
|
||
FROM alpine:3.17.0 | ||
|
||
WORKDIR /app | ||
|
||
RUN apk --no-cache add ca-certificates | ||
|
||
USER nobody | ||
COPY --from=builder --chown=nobody:nobody /project/example . | ||
COPY --from=builder --chown=nobody:nobody /project/config.yml ./config.yml | ||
|
||
ENTRYPOINT ["./example"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,68 @@ | ||
module example | ||
|
||
go 1.19 | ||
|
||
replace github.com/Trendyol/go-elasticsearch-connect-couchbase => ../. | ||
|
||
require github.com/Trendyol/go-elasticsearch-connect-couchbase v0.0.0-00010101000000-000000000000 | ||
|
||
require ( | ||
github.com/Trendyol/go-dcp-client v0.0.42 // indirect | ||
github.com/VividCortex/ewma v1.2.0 // indirect | ||
github.com/andybalholm/brotli v1.0.4 // indirect | ||
github.com/ansrivas/fiberprometheus/v2 v2.4.1 // indirect | ||
github.com/avast/retry-go/v4 v4.3.1 // indirect | ||
github.com/beorn7/perks v1.0.1 // indirect | ||
github.com/cespare/xxhash/v2 v2.1.2 // indirect | ||
github.com/couchbase/gocbcore/v10 v10.2.2 // indirect | ||
github.com/davecgh/go-spew v1.1.1 // indirect | ||
github.com/elastic/go-elasticsearch/v7 v7.17.7 // indirect | ||
github.com/go-logr/logr v1.2.2 // indirect | ||
github.com/gofiber/adaptor/v2 v2.1.25 // indirect | ||
github.com/gofiber/fiber/v2 v2.39.0 // indirect | ||
github.com/gogo/protobuf v1.3.2 // indirect | ||
github.com/golang/protobuf v1.5.2 // indirect | ||
github.com/golang/snappy v0.0.4 // indirect | ||
github.com/google/go-cmp v0.5.8 // indirect | ||
github.com/google/gofuzz v1.2.0 // indirect | ||
github.com/google/uuid v1.3.0 // indirect | ||
github.com/googleapis/gnostic v0.5.5 // indirect | ||
github.com/gookit/config/v2 v2.1.8 // indirect | ||
github.com/gookit/goutil v0.5.15 // indirect | ||
github.com/imdario/mergo v0.3.13 // indirect | ||
github.com/json-iterator/go v1.1.12 // indirect | ||
github.com/klauspost/compress v1.15.9 // indirect | ||
github.com/mattn/go-colorable v0.1.13 // indirect | ||
github.com/mattn/go-isatty v0.0.16 // indirect | ||
github.com/mattn/go-runewidth v0.0.14 // indirect | ||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect | ||
github.com/mitchellh/mapstructure v1.5.0 // indirect | ||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect | ||
github.com/modern-go/reflect2 v1.0.2 // indirect | ||
github.com/prometheus/client_golang v1.14.0 // indirect | ||
github.com/prometheus/client_model v0.3.0 // indirect | ||
github.com/prometheus/common v0.37.0 // indirect | ||
github.com/prometheus/procfs v0.8.0 // indirect | ||
github.com/rivo/uniseg v0.2.0 // indirect | ||
github.com/valyala/bytebufferpool v1.0.0 // indirect | ||
github.com/valyala/fasthttp v1.40.0 // indirect | ||
github.com/valyala/tcplisten v1.0.0 // indirect | ||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect | ||
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect | ||
golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect | ||
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect | ||
golang.org/x/text v0.3.8 // indirect | ||
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect | ||
google.golang.org/appengine v1.6.7 // indirect | ||
google.golang.org/protobuf v1.28.1 // indirect | ||
gopkg.in/inf.v0 v0.9.1 // indirect | ||
gopkg.in/yaml.v2 v2.4.0 // indirect | ||
gopkg.in/yaml.v3 v3.0.1 // indirect | ||
k8s.io/api v0.22.5 // indirect | ||
k8s.io/apimachinery v0.22.5 // indirect | ||
k8s.io/client-go v0.22.5 // indirect | ||
k8s.io/klog/v2 v2.30.0 // indirect | ||
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect | ||
sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect | ||
sigs.k8s.io/yaml v1.2.0 // indirect | ||
) |
Oops, something went wrong.