Kondon

.github/workflows/integration-test.yml

@@ -0,0 +1,34 @@
+name: tests
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "cmd/api/**"
+      - "api/**"
+      - ".github/**"
+      - "**.go"
+  pull_request:
+    paths:
+      - "clients/**"
+
+jobs:
+  build:
+    name: integration
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "1.19"
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+      - name: test
+        run: go test ./... --tags=integration

.github/workflows/test.yml

@@ -4,12 +4,12 @@ on: [pull_request]
 
 jobs:
   build:
-    name: all
+    name: packages
     runs-on: ubuntu-latest
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.19'
+          go-version: "1.19"
       - uses: actions/cache@v3
         with:
           path: |

.gitignore

@@ -1 +1,3 @@
 tmp/
+.env
+*.pprof

Dockerfile

@@ -0,0 +1,18 @@
+FROM golang:1.20-alpine AS build
+WORKDIR /go/src/app
+ENV CGO_ENABLED=0
+
+RUN apk --no-cache add ca-certificates
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+ARG GIT_SHA
+RUN cd cmd/api && go build -ldflags="-X 'main.GitSHA=$GIT_SHA'" main.go && mv main /go/bin/app
+
+FROM scratch
+COPY --from=build /go/bin/app /app
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+ENTRYPOINT ["/app"]
+EXPOSE 8080

README.md

@@ -2,4 +2,4 @@
 
 Create an allow list in seconds that works across web3
 
-[lanyard.build](https://lanyard.build)
+[lanyard.org](https://lanyard.org)

api/api.go

@@ -3,11 +3,16 @@ package api
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
+	"strings"
+	"time"
 
 	"github.com/contextwtf/lanyard/api/tracing"
+	"github.com/ethereum/go-ethereum/common"
 
+	lru "github.com/hashicorp/golang-lru/v2"
 	"github.com/jackc/pgx/v4/pgxpool"
 	"github.com/rs/cors"
 	"github.com/rs/zerolog"
@@ -17,46 +22,93 @@ import (
 )
 
 type Server struct {
-	db *pgxpool.Pool
+	db   *pgxpool.Pool
+	tlru *lru.Cache[common.Hash, cachedTree]
 }
 
 func New(db *pgxpool.Pool) *Server {
+	l, err := lru.New[common.Hash, cachedTree](1000)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed to create lru cache")
+	}
 	return &Server{
-		db: db,
+		db:   db,
+		tlru: l,
 	}
 }
 
 func (s *Server) Handler(env, gitSha string) http.Handler {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/api/v1/tree", s.TreeHandler)
 	mux.HandleFunc("/api/v1/proof", s.GetProof)
+	mux.HandleFunc("/api/v1/root", s.GetRoot)
+	mux.HandleFunc("/api/v1/roots", s.GetRoot)
 	mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprint(w, gitSha)
 	})
 
 	h := http.Handler(mux)
 	h = versionHandler(h, gitSha)
-	h = tracingHandler(os.Getenv("DD_ENV"), os.Getenv("DD_SERVICE"), gitSha, h)
-	h = hlog.NewHandler(log.Logger)(h)
 	h = hlog.UserAgentHandler("user_agent")(h)
 	h = hlog.RefererHandler("referer")(h)
 	h = hlog.RequestIDHandler("req_id", "Request-Id")(h)
 	h = hlog.URLHandler("path")(h)
-	h = hlog.RequestHandler("req")(h)
+	h = hlog.MethodHandler("method")(h)
+	h = tracingHandler(os.Getenv("DD_ENV"), os.Getenv("DD_SERVICE"), gitSha, h)
+	h = RemoteAddrHandler("ip")(h)
+	h = hlog.NewHandler(log.Logger)(h) // needs to be last for log values to correctly be passed to context
 
 	if env == "production" {
 		return h
 	}
 
 	c := cors.New(cors.Options{
-		AllowedOrigins:   []string{"http://localhost:3000"},
-		AllowCredentials: true,
+		AllowedOrigins:     []string{"*"},
+		AllowCredentials:   false,
+		OptionsPassthrough: false,
 	})
+
 	h = c.Handler(h)
 
 	return h
 }
 
+func ipFromRequest(r *http.Request) string {
+	if r.Header.Get("fastly-client-ip") != "" {
+		return r.Header.Get("fastly-client-ip")
+	}
+
+	if r.Header.Get("x-forwarded-for") != "" {
+		group := strings.Split(r.Header.Get("x-forwarded-for"), ", ")
+		if len(group) > 0 {
+			return group[len(group)-1]
+		}
+	}
+
+	host, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err == nil {
+		return host
+	}
+
+	return ""
+}
+
+func RemoteAddrHandler(fieldKey string) func(next http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			ip := ipFromRequest(r)
+			if ip != "" {
+				log := zerolog.Ctx(r.Context())
+				log.UpdateContext(func(c zerolog.Context) zerolog.Context {
+					return c.Str("ip", ip)
+				})
+			}
+
+			next.ServeHTTP(w, r.WithContext(r.Context()))
+		})
+	}
+}
+
 func versionHandler(h http.Handler, sha string) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Add("server-version", sha)
@@ -66,30 +118,40 @@ func versionHandler(h http.Handler, sha string) http.Handler {
 
 func tracingHandler(env, service, sha string, h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/health" {
+			h.ServeHTTP(w, r)
+			return
+		}
+
 		span, ctx := tracing.SpanFromContext(r.Context(), "http.request")
 
 		defer span.Finish()
+		log := zerolog.Ctx(ctx)
 
-		log := zerolog.Ctx(r.Context())
-		log.UpdateContext(func(c zerolog.Context) zerolog.Context {
-			return c.Uint64("dd.trace_id", span.Context().TraceID())
-		})
-		log.UpdateContext(func(c zerolog.Context) zerolog.Context {
-			return c.Str("dd.service", service)
-		})
-		log.UpdateContext(func(c zerolog.Context) zerolog.Context {
-			return c.Str("dd.env", env)
-		})
-		log.UpdateContext(func(c zerolog.Context) zerolog.Context {
-			return c.Str("dd.version", sha)
-		})
+		if env != "" {
+			log.UpdateContext(func(c zerolog.Context) zerolog.Context {
+				return c.Uint64("dd.trace_id", span.Context().TraceID()).
+					Str("dd.service", service).
+					Str("dd.env", env).
+					Str("dd.version", sha)
+			})
+		}
 
 		span.SetTag(ext.ResourceName, r.URL.Path)
 		span.SetTag(ext.SpanType, ext.SpanTypeWeb)
 		span.SetTag(ext.HTTPMethod, r.Method)
 
 		sc := &statusCapture{ResponseWriter: w}
+
+		requestStart := time.Now()
 		h.ServeHTTP(sc, r.WithContext(ctx))
+
+		// log every request
+		log.Info().
+			Int("status", sc.status).
+			Dur("duration", time.Since(requestStart)).
+			Msg("")
+
 		span.SetTag(ext.HTTPCode, sc.status)
 	})
 }
@@ -127,6 +189,11 @@ func (s *Server) sendJSONError(
 	customMessage string,
 ) {
 	w.Header().Set("Content-Type", "application/json")
+	if code == http.StatusNotFound && w.Header().Get("Cache-Control") == "" {
+		w.Header().Set("Cache-Control", "no-cache, max-age=0, must-revalidate")
+	}
+
+	// all headers need to be set before this line
 	w.WriteHeader(code)
 
 	if err != nil {

api/migrations/migrations.go

@@ -1,6 +1,6 @@
 package migrations
 
-import "github.com/contextwtf/lanyard/migrate"
+import "github.com/contextwtf/migrate"
 
 var Migrations = []migrate.Migration{
 	{
@@ -61,4 +61,76 @@ var Migrations = []migrate.Migration{
 			ALTER TABLE merkle_trees RENAME TO trees;
 		`,
 	},
+	{
+		Name: "2022-08-22.0.add-proof-idx.sql",
+		SQL: `
+		    CREATE INDEX on trees USING gin(proofs jsonb_path_ops);
+		`,
+	},
+	{
+		Name: "2022-08-22.1.add-inserted-at.sql",
+		SQL: `
+		ALTER TABLE trees
+		ADD COLUMN "inserted_at" timestamptz NOT NULL DEFAULT now();
+		`,
+	},
+	{
+		Name: "2022-08-30.0.packed-bool.sql",
+		SQL: `
+		UPDATE trees SET packed = true
+		WHERE packed IS NULL;
+
+		ALTER TABLE trees
+		ALTER COLUMN packed
+		SET NOT NULL;
+		`,
+	},
+	{
+		Name: "2022-08-31.0.root-func-index.sql",
+		SQL: `
+		CREATE OR REPLACE FUNCTION proofs_array (data jsonb)
+			RETURNS text[]
+			AS $CODE$
+		BEGIN
+			RETURN ARRAY (
+				SELECT
+					jsonb_array_elements(data) ->> 'proof');
+		END
+		$CODE$
+		LANGUAGE plpgsql
+		IMMUTABLE;
+
+		CREATE INDEX proofs_arr_idx ON trees USING GIN ((proofs_array(proofs)));
+		DROP INDEX "trees_proofs_idx";
+	`,
+	},
+	{
+		Name: "2022-10-07.0.trees-proofs.sql",
+		SQL: `
+		DROP INDEX proofs_arr_idx;
+		CREATE TABLE trees_proofs (
+			root bytea PRIMARY KEY,
+			proofs jsonb,
+			inserted_at timestamp with time zone NOT NULL DEFAULT now()
+		);
+		CREATE INDEX proofs_arr_idx ON trees_proofs USING GIN ((proofs_array(proofs)));
+		`,
+	},
+	{
+		Name: "2023-06-26.0.proofs_hashes.sql",
+		SQL: `
+		CREATE TABLE IF NOT EXISTS proofs_hashes (
+			hash bytea,
+			root bytea
+		);
+		CREATE INDEX IF NOT EXISTS proofs_hashes_hash_idx ON proofs_hashes (hash);
+		`,
+	},
+	{
+		Name: "2023-06-26.1.drop_data.sql",
+		SQL: `
+		ALTER TABLE "trees" DROP COLUMN "proofs";
+		DROP TABLE "trees_proofs";
+		`,
+	},
 }