services/ledgerexporter: Move ledgerexporter out of experimental (ste…

…llar#5341)

* services/ledgerexporter: Move ledgerexporter out of experimental

* Add CHANGELOG.md

* Fix git workflow

* Rename Ledgerexporter to Galexie

.github/workflows/ledgerexporter-release.yml → .github/workflows/galexie-release.yml

@@ -1,34 +1,34 @@
-name: Ledger Exporter release
+name: Galexie Release 
 
 on:
   push:
-    tags: ['ledgerexporter-v*']
+    tags: ['galexie-v*']
 
 jobs:
 
   publish-docker:
-    name: Test and push the Ledger Exporter images
+    name: Test and push docker image
     runs-on: ubuntu-latest
     env:
-      LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED: "true"
-      LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
+      GALEXIE_INTEGRATION_TESTS_ENABLED: "true"
+      GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
       # this pins to a version of quickstart:testing that has the same version as STELLAR_CORE_VERSION
       # this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:testing'
-      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:03c6679f838a92b1eda4cd3a9e2bdee4c3586e278a138a0acf36a9bc99a0041f
-      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
+      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:03c6679f838a92b1eda4cd3a9e2bdee4c3586e278a138a0acf36a9bc99a0041f
+      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
       STELLAR_CORE_VERSION: 21.1.0-1921.b3aeb14cc.focal
     steps:
       - name: Set VERSION
         run: |
-          echo "VERSION=${GITHUB_REF_NAME#ledgerexporter-v}" >> $GITHUB_ENV
+          echo "VERSION=${GITHUB_REF_NAME#galexie-v}" >> $GITHUB_ENV
 
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.sha }}
       - name: Pull Quickstart image
         shell: bash
         run: |
-          docker pull "$LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE"
+          docker pull "$GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE"
       - name: Install captive core
         run: |
           # Workaround for https://github.com/actions/virtual-environments/issues/5245,
@@ -42,11 +42,11 @@ jobs:
           sudo apt-get update && sudo apt-get install -y stellar-core="$STELLAR_CORE_VERSION"
           echo "Using stellar core version $(stellar-core version)"
 
-      - name: Run Ledger Exporter test
-        run: go test -v -race -run TestLedgerExporterTestSuite ./exp/services/ledgerexporter/...
+      - name: Run tests 
+        run: go test -v -race -run TestGalexieTestSuite ./exp/services/galexie/...
 
-      - name: Build Ledger Exporter docker
-        run: make -C exp/services/ledgerexporter docker-build
+      - name: Build docker
+        run: make -C exp/services/galexie docker-build
 
       # Push images
       - name: Login to DockerHub
@@ -56,4 +56,4 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Push to DockerHub
-        run: make -C exp/services/ledgerexporter docker-push
+        run: make -C exp/services/galexie docker-push

.github/workflows/ledgerexporter.yml → .github/workflows/galexie.yml

@@ -1,23 +1,22 @@
-name: LedgerExporter
+name: Galexie
 
 on:
   push:
     branches: [master]
   pull_request:
 
 jobs:
-  ledger-exporter:
-    name: Test Ledger Exporter
+  galexie:
+    name: Test
     runs-on: ubuntu-latest
     env:
       CAPTIVE_CORE_DEBIAN_PKG_VERSION: 21.1.0-1921.b3aeb14cc.focal
-      LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED: "true"
-      LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
-      # this pins to a version of quickstart:testing that has the same version of core
-      # as specified on LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN
+      GALEXIE_INTEGRATION_TESTS_ENABLED: "true"
+      GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
+      # this pins to a version of quickstart:testing that has the same version as GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN
       # this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:testing'
-      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:5c8186f53cc98571749054dd782dce33b0aca2d1a622a7610362f7c15b79b1bf
-      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
+      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:5c8186f53cc98571749054dd782dce33b0aca2d1a622a7610362f7c15b79b1bf
+      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
     steps:
       - name: Install captive core
         run: |
@@ -35,12 +34,12 @@ jobs:
       - name: Pull Quickstart image
         shell: bash
         run: |
-          docker pull "$LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE"
+          docker pull "$GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE"
 
       - uses: actions/checkout@v3
         with:
           # For pull requests, build and test the PR head not a merge of the PR with the destination.
           ref:  ${{ github.event.pull_request.head.sha || github.ref }}
 
-      - name: Run Ledger Exporter test
-        run: go test -v -race -run TestLedgerExporterTestSuite ./exp/services/ledgerexporter/... 
+      - name: Run test
+        run: go test -v -race -run TestGalexieTestSuite ./services/galexie/... 

Makefile

@@ -34,8 +34,8 @@ friendbot:
 horizon:
 	$(MAKE) -C services/horizon/ binary-build
 
-ledger-exporter:
-	$(MAKE) -C exp/services/ledgerexporter/ docker-build
+galexie:
+	$(MAKE) -C services/galexie/ docker-build
 
 webauth:
 	$(MAKE) -C exp/services/webauth/ docker-build

exp/services/ledgerexporter/DEVELOPER_GUIDE.md → services/galexie/DEVELOPER_GUIDE.md

@@ -1,61 +1,61 @@
 
-# Ledger Exporter Developer Guide
-The ledger exporter is a tool to export Stellar network transaction data to cloud storage in a way that is easy to access.
+# Galexie Developer Guide
+Galexie is a tool to export Stellar network transaction data to cloud storage in a way that is easy to access.
 
 ## Prerequisites
-This document assumes that you have installed and can run the ledger exporter, and that you have familiarity with its CLI and configuration. If not, please refer to the [Installation Guide](./README.md).
+This document assumes that you have installed and can run Galexie, and that you have familiarity with its CLI and configuration. If not, please refer to the [Installation Guide](./README.md).
 
 ## Goal
-The goal of the ledger exporter is to build an easy-to-use tool to export Stellar network ledger data to a configurable remote data store, such as cloud blob storage.
+The goal of Galexie is to build an easy-to-use tool to export Stellar network ledger data to a configurable remote data store, such as cloud blob storage.
  - Use cloud storage optimally
  - Minimize network usage to export
  - Make it easy and fast to search for a specific ledger or ledger range
 
 ## Architecture
-To achieve its goals, the ledger exporter uses the following architecture, which consists of the 3 main components:
+To achieve its goals, Galexie uses the following architecture, which consists of the 3 main components:
 - Captive-core to extract raw transaction metadata from the Stellar Network.
 - Export manager to bundles and organizes the ledgers to get them ready for export.
 - The cloud storage plugin writes to the cloud storage. This is specific to the type of cloud storage, GCS in this case.
 
 
-![ledgerexporter-architecture](./architecture.png)
+![Architecture](./architecture.png)
 
 
 ## Data Format
-- Ledger exporter uses a compact and efficient data format called [XDR](https://developers.stellar.org/docs/learn/encyclopedia/data-format/xdr) (External Data Representation), which is a compact binary format. A Stellar Captive Core instance emits data in this format and the data structure is referred to as `LedgerCloseMeta`. The exporter bundles multiple `LedgerCloseMeta`'s into a single object using a custom XDR structure called `LedgerCloseMetaBatch` which is defined in [Stellar-exporter.x](https://github.com/stellar/go/blob/master/xdr/Stellar-exporter.x).
+- Galexie uses a compact and efficient data format called [XDR](https://developers.stellar.org/docs/learn/encyclopedia/data-format/xdr) (External Data Representation), which is a compact binary format. A Stellar Captive Core instance emits data in this format and the data structure is referred to as `LedgerCloseMeta`. The exporter bundles multiple `LedgerCloseMeta`'s into a single object using a custom XDR structure called `LedgerCloseMetaBatch` which is defined in [Stellar-exporter.x](https://github.com/stellar/go/blob/master/xdr/Stellar-exporter.x).
 
 - The metadata for the same batch is also stored alongside each exported object. Supported metadata is defined in [metadata.go](https://github.com/stellar/go/blob/master/support/datastore/metadata.go). 
 
 - Objects are compressed before uploading using the [zstd](http://facebook.github.io/zstd/) (zstandard) compression algorithm to optimize network usage and storage needs.
 
 ## Data Storage
 - An example implementation of `DataStore` for GCS, Google Cloud Storage. This plugin is located in the [support](https://github.com/stellar/go/tree/master/support/datastore) package. 
-- The ledger exporter currently implements the interface only for Google Cloud Storage (GCS). The [GCS plugin](https://github.com/stellar/go/blob/master/support/datastore/gcs_datastore.go) uses GCS-specific behaviors like conditional puts, automatic retry, metadata, and CRC checksum.
+- Galexie currently implements the interface only for Google Cloud Storage (GCS). The [GCS plugin](https://github.com/stellar/go/blob/master/support/datastore/gcs_datastore.go) uses GCS-specific behaviors like conditional puts, automatic retry, metadata, and CRC checksum.
 
 ## Build and Run using Docker
-The Dockerfile contains all the necessary dependencies (e.g., Stellar-core) required to run the ledger exporter. 
+The Dockerfile contains all the necessary dependencies (e.g., Stellar-core) required to run Galexie.
 
 - Build: To build the Docker container, use the provided [Makefile](./Makefile). Simply run make `make docker-build` to build a new container after making any changes.
 
 - Run: For instructions on running the Docker container, refer to the [Installation Guide](./README.md).
 
-- Run ledgerexporter with a local, fake GCS backend: Requires `make docker-build` first, then run `make docker-test-fake-gcs`. This will run the ledger exporter against `testnet` and export to the 'fake' GCS instance started in the container.
+- Run Galexie with a local, fake GCS backend: Requires `make docker-build` first, then run `make docker-test-fake-gcs`. This will run it against `testnet` and export to the 'fake' GCS instance started in the container.
 
 ## Running Integration Tests:
-from top directory of stellar/go repo, run go test to launch ledger exporter integration
+from top directory of stellar/go repo, run go test to launch Galexie integration
 tests.
 
-`LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED=true` is required environment variable to allow
+`GALEXIE_INTEGRATION_TESTS_ENABLED=true` is required environment variable to allow
 tests to run.
 
-Optional, tests will try to run `stellar-core` from o/s PATH for captive core, if not resolvable, then set `LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN=/path/to/stellar-core` 
+Optional, tests will try to run `stellar-core` from o/s PATH for captive core, if not resolvable, then set `GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN=/path/to/stellar-core`
 
-Optional, can override the version of quickstart used to run standalone stellar network, `LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE=docker.io/stellar/quickstart:<tag>`. By default it will try to docker pull `stellar/quickstart:testing` image to local host's docker image store. Set `LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL=false` to skip the pull, if you know host has up to date image.
+Optional, can override the version of quickstart used to run standalone stellar network, `GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE=docker.io/stellar/quickstart:<tag>`. By default it will try to docker pull `stellar/quickstart:testing` image to local host's docker image store. Set `GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL=false` to skip the pull, if you know host has up to date image.
 
-Note, the version of stellar core in `LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE` and `LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN` needs to be on the same major rev or the captive core process may not be able to join or parse ledger meta from the `local` network created by `LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE`
+Note, the version of stellar core in `GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE` and `GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN` needs to be on the same major rev or the captive core process may not be able to join or parse ledger meta from the `local` network created by `GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE`
 
 ```
-$ LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED=true go test -v -race -run TestLedgerExporterTestSuite ./exp/services/ledgerexporter/...
+$ GALEXIE_INTEGRATION_TESTS_ENABLED=true go test -v -race -run TestGalexieTestSuite ./services/galexie/...
 ```
 
 ## Adding support for a new storage type
@@ -71,7 +71,7 @@ Support for different data storage types are encapsulated as 'plugins', which ar
 - An emulator such as a GCS emulator [fake-gcs-server](https://github.com/fsouza/fake-gcs-server) can be used for testing without connecting to real cloud storage.
 
 ### Design DOs and DONTs
-- Multiple exporters should be able to run in parallel without the need for explicit locking or synchronization.
+- Multiple Galexie instances should be able to run in parallel without the need for explicit locking or synchronization.
 - Exporters when restarted do not have any memory of prior operation and rely on the already exported data as much as possible to decide where to resume.
 
 ## Using exported data

exp/services/ledgerexporter/Makefile → services/galexie/Makefile

@@ -3,14 +3,14 @@ SUDO := $(shell docker version >/dev/null 2>&1 || echo "sudo")
 # https://github.com/opencontainers/image-spec/blob/master/annotations.md
 BUILD_DATE := $(shell date -u +%FT%TZ)
 VERSION ?= $(shell git rev-parse --short HEAD)
-DOCKER_IMAGE := stellar/ledger-exporter
+DOCKER_IMAGE := stellar/stellar-galexie
 
 docker-build:
-	cd ../../../ && \
+	cd ../../ && \
 	$(SUDO) docker build --platform linux/amd64 --pull --label org.opencontainers.image.created="$(BUILD_DATE)" \
-    --build-arg GOFLAGS="-ldflags=-X=github.com/stellar/go/exp/services/ledgerexporter/internal.version=$(VERSION)" \
+    --build-arg GOFLAGS="-ldflags=-X=github.com/stellar/go/services/galexie/internal.version=$(VERSION)" \
 $(if $(STELLAR_CORE_VERSION), --build-arg STELLAR_CORE_VERSION=$(STELLAR_CORE_VERSION)) \
-	-f exp/services/ledgerexporter/docker/Dockerfile \
+	-f services/galexie/docker/Dockerfile \
 	-t $(DOCKER_IMAGE):$(VERSION) \
 	-t $(DOCKER_IMAGE):latest .
 
@@ -22,7 +22,7 @@ docker-clean:
 
 docker-test-fake-gcs: docker-clean
 	# Create temp storage dir
-	$(SUDO) mkdir -p ${PWD}/storage/exporter-test 
+	$(SUDO) mkdir -p ${PWD}/storage/galexie-test
 
 	# Create test network for docker
 	$(SUDO) docker network create test-network
@@ -31,13 +31,13 @@ docker-test-fake-gcs: docker-clean
 	$(SUDO) docker run -d --name fake-gcs-server -p 4443:4443 \
 		 -v ${PWD}/storage:/data --network test-network fsouza/fake-gcs-server -scheme http
 
-	# Run the ledger-exporter
-	$(SUDO) docker run --platform linux/amd64 -t --network test-network\
-		 -v ${PWD}/exp/services/ledgerexporter/docker/config.test.toml:/config.toml \
-		 -e STORAGE_EMULATOR_HOST=http://fake-gcs-server:4443 \
-		 $(DOCKER_IMAGE):$(VERSION) \
-		 scan-and-fill --start 1000 --end 2000
-		 
+	# Run
+	$(SUDO) docker run --platform linux/amd64 -t --network test-network \
+		-v ${PWD}/exp/services/galexie/docker/config.test.toml:/config.toml \
+		-e STORAGE_EMULATOR_HOST=http://fake-gcs-server:4443 \
+		$(DOCKER_IMAGE):$(VERSION) \
+		scan-and-fill --start 1000 --end 2000
+
 	$(MAKE) docker-clean
 
 docker-push: