Toolchain Upgrade: CMake, Conan, GitHub Actions, and more

.github/workflows/cfpages.yml

@@ -0,0 +1,39 @@
+# This workflow uploads documentation to Cloudflare Pages.
+# @authors: Runzhi He <[email protected]>
+# @date: 2024-01-10
+
+name: Publish Cloudflare Pages
+
+on:
+    push:
+      branches: [ online-docs ]
+    workflow_dispatch: # Manually run workflow
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      deployments: write
+    name: Publish to Cloudflare Pages
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: online-docs
+          fetch-depth: 0
+
+      - name: Copy Assets
+        run: |
+          mkdir -p temp
+          cp -rv $(find ./docs -maxdepth 1 -type f) temp
+          cp -rv ./docs/zh-cn ./docs/help temp
+
+      - name: Publish to Cloudflare Pages
+        uses: cloudflare/pages-action@v1
+        with:
+          apiToken: ${{ secrets.CF_API_TOKEN }}
+          accountId: ${{ secrets.CF_ACCOUNT_ID }}
+          projectName: ${{ vars.CF_PROJECT_NAME }}
+          directory: temp
+          gitHubToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -0,0 +1,77 @@
+# This workflow runs the test suite, measure coverage, and upload the coverage report to Codecov.
+# @authors: Runzhi He <[email protected]>
+# @date: 2024-01-08
+
+name: Build and Test
+
+on:
+    push:
+      # branches: [ master ]  # TODO: have a stable default branch
+    pull_request:
+    workflow_dispatch: # Manually run workflow
+
+jobs:
+    build-and-test:
+      # Only trigger this workflow on the default branch
+      # https://stackoverflow.com/questions/64781462/github-actions-default-branch-variable
+      if: ${{ always() && format('refs/heads/{0}', github.event.repository.default_branch) == github.ref }}
+      # This CI gonna fail if can't build and test on latest Ubuntu LTS
+      runs-on: ubuntu-latest
+      steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      # Test the setup script
+      - name: Install system dependencies
+        run: sudo bash scripts/setup-dev.sh
+
+      - name: Setup Conan
+        run: pip install conan && conan profile detect
+
+      - name: Cache Conan dependencies
+        uses: actions/cache@v3
+        with:
+            path: |
+              ~/.conan2/p/
+            key: ${{ runner.os }}-conan-${{ hashFiles('conanfile.py') }}
+            restore-keys: |
+              ${{ runner.os }}-conan-
+
+      - name: Install conan dependencies
+        run: conan install . -s "&:build_type=Debug" -s "build_type=Release" --build=missing
+
+      - name: Build
+        run: |
+          cmake .. -G Ninja -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Debug -DCOVERAGE=ON
+          ninja prepare
+          ninja install -v
+        working-directory: cmake-build-debug
+
+      - name: Sanity Check
+        run: |
+          bash scripts/init.sh
+          (timeout 10s bin/ghttp &) && sleep 3 && bin/gquery -db small -q data/small/small_q0.sql && killall ghttp
+
+      - name: Test
+        run: ctest -C Debug --output-on-failure --rerun-failed
+        working-directory: cmake-build-debug
+        continue-on-error: ${{ vars.CHEAT_IN_CI == 'true' }}
+
+      - name: Generate coverage report
+        run: ninja coverage
+        working-directory: cmake-build-debug
+
+      - name: Upload coverage report
+        uses: actions/upload-artifact@v2
+        with:
+          name: coverage.tar.gz
+          path: coverage.tar.gz
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: false
+          files: coverage.info
+          verbose: true

.github/workflows/publish-docker.yml

@@ -0,0 +1,59 @@
+# This workflow will build a new Docker image and push it to Docker Hub whenever a new tag is created.
+# @authors: Runzhi He <[email protected]>
+# @date: 2024-01-08
+
+name: Publish Docker
+
+on:
+  push:
+    tags:
+      - 'v*'
+  # This line enables manual triggering
+  workflow_dispatch:  
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ vars.DOCKERHUB_USERNAME }}/${{ vars.DOCKERHUB_REPOSITORY }}
+
+      - name: Creating source tarball
+        run:  |
+          tar -czvf gstore-src.tar.gz "src" "api" "defaults" "tests" "3rdparty" "data" "scripts" "conanfile.py" "CMakeLists.txt" "README.md" "LICENSE"
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./Dockerfile
+          # if triggered by workflow_dispatch, don't push to Docker Hub
+          push: ${{ github.event_name != 'workflow_dispatch' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/publish-release.yml

@@ -0,0 +1,75 @@
+# This workflow will build and draft a new release whenever a new tag is pushed to the repository.
+# @authors: Runzhi He <[email protected]>
+# @date: 2024-01-08
+
+name: Publish Release
+
+on:
+  push:
+    tags:
+      - 'v*'
+  # This line enables manual triggering
+  workflow_dispatch:
+
+jobs:
+  build-and-release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      # Test the setup script
+      - name: Install system dependencies
+        run: sudo bash scripts/setup-dev.sh
+
+      - name: Setup Conan
+        run: pip install conan && conan profile detect
+
+      - name: Install conan dependencies
+        run: conan install . -s "build_type=Release" --build=missing
+
+      - name: Build
+        run: |
+          cmake .. -G Ninja -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release
+          ninja prepare
+          ninja install -v
+        working-directory: cmake-build-release
+
+      - name: Create Tarball
+        run: |
+          ninja -C cmake-build-release tarball
+          ninja -C cmake-build-release package
+
+      - name: Sanity Check
+        run: |
+          bash scripts/init.sh
+          (timeout 10s bin/ghttp &) && sleep 3 && bin/gquery -db small -q data/small/small_q0.sql && killall ghttp
+
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '18'
+
+      - name: Generate Changelog
+        run: |
+            npm install -g conventional-changelog-cli
+            conventional-changelog -p angular -i CHANGELOG.md -s
+
+      - name: Create a new Release
+        uses: ncipollo/release-action@v1
+        id: create_release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          # https://github.com/ncipollo/release-action for the full list of arguments
+          artifacts: gstore-*.tar.gz
+          draft: true  # generate a draft release
+          generateReleaseNotes: false
+          bodyFile: CHANGELOG.md
+          # generate a pre-release if 'pre', 'rc', or 'beta' is in the tag name, or triggered manually
+          prerelease: ${{ github.event_name == 'workflow_dispatch' || contains(github.ref, 'beta') || contains(github.ref, 'rc') || contains(github.ref, 'pre') }}
+          # use the tag name if it's a tag, otherwise use the commit hash
+        continue-on-error: true

.gitignore

@@ -1,3 +1,38 @@
+# Artifacts and Cache
+build/
+include/
+lib/
+bin/
+bin_tests/
+dist/
+cmake-build-*/
+.cmake/
+gstore-*.tar.gz
+CMakeUserPresets.json
+/Testing/
+
+# Runtime directories
+.tmp/
+.debug/
+.obj/
+backup/
+logs/
+upload/
+temp/
+fun/
+
+# config files
+# TODO: add a separate config file folder
+/backup.json
+/conf.ini
+/init.conf
+/ipAllow.config
+/ipDeny.config
+/README.txt
+/slog.properties
+/slog.stdout.properties
+/*.log
+
 # Executables
 *.pyc
 *.exe
@@ -7,19 +42,20 @@
 *.x86_64
 *.hex
 bin/g*
-scripts/update_test
-scripts/dataset_test
+tests/update_test
+tests/dataset_test
 api/cpp/example/example
 
-Parser/Sparql*
+src/Parser/Sparql*
 
 #result logs
 *.txt
 *.log
 logs
+!CMakeLists.txt
 
 #backup files
-backup.json
+defaults/backup.json
 
 #download files
 *.torrent
@@ -90,6 +126,7 @@ tags
 .vs
 *.iml
 .idea
+.vscode
 
 # latex files
 *.aux

3rdparty/.gitignore

@@ -0,0 +1,6 @@
+*
+!.gitignore
+!*.tar*
+!*.t[bgx]z
+!*.zip
+!*.sh

3rdparty/antlr4-cpp-runtime-4.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+mkdir -p ../include/
+rm -rf antlr4-cpp-runtime-4
+tar -xzvf antlr4-cpp-runtime-4.tar.gz;
+cd antlr4-cpp-runtime-4 || echo "./antlr4-cpp-runtime-4 not found"
+cmake .
+
+# if GSTORE_USE_DYNAMIC_LIB is defined, then we need to copy the dynamic library to the lib directory
+# otherwise, we need to copy the static library to the lib directory
+if [ -n "$GSTORE_USE_DYNAMIC_LIB" ]; then
+    make -j2 antlr4_shared
+    cp dist/libantlr4-runtime.so* ../../lib/libantlr4-runtime.so
+    cd ../../lib/ || echo "./lib not found"
+else
+    make -j$(nproc) antlr4_static
+    cp dist/libantlr4-runtime.a ../../lib/
+fi
+cp -r runtime/src/* ../../include/