Refactoring: change ORM, testing and migrate to timescale (#995)

* Change ORM to Bun

* Add storage tests

* Refactoring: rollback

* Refactoring: saving block and migrations

* Remove Save function from model interface

* Refactoring: protocol migrations

* Tests: protocol migration postgres functions

* Fix: rollback

* Fix: rollback protocol

* Refactoring: saving accounts

* Refactoring: Contract.tx_count -> Account.operations_count

* Timescale

* Feature: add stats model

* Extend stats

* Refactoring: remove unused functions

* Refactoring: operations API

* Account statistics

* Feature: add ticket model

* Feature: add ticket balances

* Fix: create extension once

* Fix: change varchar to text

* Fix: register models

* Fix: register m2m models

* Fix: smart rollup count

* Fix: save big map states

* Feature: add ticket balance endpoint

* Fix: content type for ticket balance

* Fix: response in ticket balances

* Fix: rollback test

* Remove concurrent index creation

* Fix: one ticket to all updates

* Fix: ticket hash

* Fix: tests

* Fix: index creation

* Fix and optimize babylon migration

* Fix and optimize babylon migration

* Remove bmd repo from babylon migration

* Change test data for babylon migration test

* Fix: tx_rollup_origination

* Fix: recently called contracts endpoint

* Fix: operations count for account

* Fix: parsing node error

* Fix: smart rollup accounts counting

* Fix: failed originations

* Fix: saving origination accounts

* Fix: same API

* Disable oxfordnet

* Upgrade to go1.21

* Add fromatter tests for new primitives

* Cache scripts

* Create contract indexes

* Add codeql

* Add codeql

* Remove codeql

* Linter update and some API improvements

* Fix: linter job

* Refactoring: logger

* Fix: recently called contracts

* Fix: global constants per network API

* Fix: add gin gonic timeout

* Fix: post-creating index concurrently

* Fix: increase timeout

* Add log

* Fix: gin timeout

* Fix: add GetContractTickets endpoint

* Optimization of GetLastAction query

* Fix: global constants query

* Fix: last action in same API

* Optimize last query

* Fix: sanbox tagging

* Fix: errors scan

* Fix: errors

* Add filters to ticket updates endpoints

* Add ticket_id to ticket updates response

.github/workflows/release.yml

@@ -46,7 +46,7 @@ jobs:
           context: .
           file: build/sandbox/Dockerfile
           build-args: |
-            TAG=4.6.2
+            TAG=4.6.3
           push: true
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/tests_master.yml

@@ -11,12 +11,12 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.51.2
+          version: v1.54.2
           args: --timeout=3m
   test:
     name: Run tests

.github/workflows/tests_pr.yml

@@ -11,12 +11,12 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.51.2
+          version: v1.54.2
           args: --timeout=3m
   test:
     name: Run tests

.golangci.yml

@@ -11,7 +11,11 @@ linters:
     - ineffassign
     - containedctx
     - tenv
+    - musttag
+    - mirror
+    - tagalign
+    - zerologlint
 
 run:
-  go: '1.19'
+  go: '1.21'
   timeout: 3m

Makefile

@@ -8,7 +8,7 @@ BACKUP?=dump_latest.gz
 
 api:
 	docker-compose up -d db
-	cd cmd/api && go run .
+	cd cmd/api && go run -tags=jsoniter .
 
 indexer:
 	docker-compose up -d db
@@ -79,10 +79,10 @@ ps:
 	docker ps --format "table {{.Names}}\t{{.RunningFor}}\t{{.Status}}\t{{.Ports}}"
 
 sandbox-pull:
-	TAG=4.4.0 docker-compose -f docker-compose.flextesa.yml pull
+	TAG_FRONT=4.6.2 TAG_BACK=4.7.2 docker-compose -f docker-compose.flextesa.yml pull
 
 flextesa-sandbox:
-	COMPOSE_PROJECT_NAME=bcdbox TAG=4.6.2 docker-compose -f docker-compose.flextesa.yml up -d
+	COMPOSE_PROJECT_NAME=bcdbox TAG_FRONT=4.6.2 TAG_BACK=4.7.2 docker-compose -f docker-compose.flextesa.yml up -d
 
 sandbox-down:
 	COMPOSE_PROJECT_NAME=bcdbox docker-compose -f docker-compose.flextesa.yml down

build/api/Dockerfile

@@ -1,7 +1,7 @@
 # ---------------------------------------------------------------------
 #  The first stage container, for building the application
 # ---------------------------------------------------------------------
-FROM golang:1.20-alpine as builder
+FROM golang:1.21-alpine as builder
 
 ENV CGO_ENABLED=0
 ENV GO111MODULE=on
@@ -20,7 +20,7 @@ COPY cmd/api cmd/api
 COPY internal internal
 
 WORKDIR $GOPATH/src/github.com/baking-bad/bcdhub/cmd/api/
-RUN go build -a -installsuffix cgo -o /go/bin/api .
+RUN go build -tags=jsoniter -a -installsuffix cgo -o /go/bin/api .
 
 WORKDIR $GOPATH/src/github.com/baking-bad/bcdhub
 COPY scripts scripts

build/indexer/Dockerfile

@@ -1,7 +1,7 @@
 # ---------------------------------------------------------------------
 #  The first stage container, for building the application
 # ---------------------------------------------------------------------
-FROM golang:1.20-alpine as builder
+FROM golang:1.21-alpine as builder
 
 ENV CGO_ENABLED=0
 ENV GO111MODULE=on

cmd/api/handlers/account.go

@@ -5,8 +5,8 @@ import (
 
 	"github.com/baking-bad/bcdhub/internal/bcd"
 	"github.com/baking-bad/bcdhub/internal/config"
-	"github.com/baking-bad/bcdhub/internal/logger"
 	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog/log"
 )
 
 // GetInfo godoc
@@ -33,32 +33,31 @@ func GetInfo() gin.HandlerFunc {
 			return
 		}
 
-		acc, err := ctx.Accounts.Get(req.Address)
-		if handleError(c, ctx.Storage, err, 0) {
-			return
-		}
-		stats, err := ctx.Operations.ContractStats(acc.Address)
+		acc, err := ctx.Accounts.Get(c.Request.Context(), req.Address)
 		if handleError(c, ctx.Storage, err, 0) {
 			return
 		}
+
 		var balance int64
 		if !(bcd.IsRollupAddressLazy(acc.Address) || bcd.IsSmartRollupAddressLazy(acc.Address)) {
-			block, err := ctx.Blocks.Last()
+			block, err := ctx.Blocks.Last(c.Request.Context())
 			if handleError(c, ctx.Storage, err, 0) {
 				return
 			}
 			balance, err = ctx.Cache.TezosBalance(c, acc.Address, block.Level)
 			if err != nil {
-				logger.Err(err)
+				log.Err(err).Msg("receiving tezos balance")
 			}
 		}
 		c.SecureJSON(http.StatusOK, AccountInfo{
-			Address:     acc.Address,
-			Alias:       acc.Alias,
-			TxCount:     stats.Count,
-			Balance:     balance,
-			LastAction:  stats.LastAction.UTC(),
-			AccountType: acc.Type.String(),
+			Address:            acc.Address,
+			OperationsCount:    acc.OperationsCount,
+			EventsCount:        acc.EventsCount,
+			MigrationsCount:    acc.MigrationsCount,
+			TicketUpdatesCount: acc.TicketUpdatesCount,
+			Balance:            balance,
+			LastAction:         acc.LastAction.UTC(),
+			AccountType:        acc.Type.String(),
 		})
 	}
 

cmd/api/handlers/bigmap.go

@@ -1,6 +1,7 @@
 package handlers
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 
@@ -38,7 +39,7 @@ func GetBigMap() gin.HandlerFunc {
 			return
 		}
 
-		stats, err := ctx.BigMapDiffs.GetStats(req.Ptr)
+		stats, err := ctx.BigMapDiffs.GetStats(c.Request.Context(), req.Ptr)
 		if handleError(c, ctx.Storage, err, 0) {
 			return
 		}
@@ -52,29 +53,29 @@ func GetBigMap() gin.HandlerFunc {
 		}
 
 		if stats.Total == 0 {
-			actions, err := ctx.BigMapActions.Get(req.Ptr, 1, 0)
+			actions, err := ctx.BigMapActions.Get(c.Request.Context(), req.Ptr, 1, 0)
 			if handleError(c, ctx.Storage, err, 0) {
 				return
 			}
 			if len(actions) > 0 {
 				res.Address = actions[0].Address
 			}
 		} else {
-			destination, err := ctx.Accounts.Get(res.Address)
+			destination, err := ctx.Accounts.Get(c.Request.Context(), res.Address)
 			if handleError(c, ctx.Storage, err, 0) {
 				return
 			}
-			res.ContractAlias = destination.Alias
 
 			operation, err := ctx.Operations.Last(
+				c.Request.Context(),
 				map[string]interface{}{
 					"status":         types.OperationStatusApplied,
 					"destination_id": destination.ID,
 				}, 0)
 			if handleError(c, ctx.Storage, err, 0) {
 				return
 			}
-			proto, err := ctx.Cache.ProtocolByID(operation.ProtocolID)
+			proto, err := ctx.Cache.ProtocolByID(c.Request.Context(), operation.ProtocolID)
 			if handleError(c, ctx.Storage, err, 0) {
 				return
 			}
@@ -94,7 +95,7 @@ func GetBigMap() gin.HandlerFunc {
 				return
 			}
 
-			script, err := ctx.Contracts.ScriptPart(res.Address, proto.SymLink, consts.STORAGE)
+			script, err := ctx.Contracts.ScriptPart(c.Request.Context(), res.Address, proto.SymLink, consts.STORAGE)
 			if handleError(c, ctx.Storage, err, 0) {
 				return
 			}
@@ -146,7 +147,7 @@ func GetBigMapHistory() gin.HandlerFunc {
 			return
 		}
 
-		bm, err := ctx.BigMapActions.Get(req.Ptr, 0, 0)
+		bm, err := ctx.BigMapActions.Get(c.Request.Context(), req.Ptr, 0, 0)
 		if handleError(c, ctx.Storage, err, 0) {
 			return
 		}
@@ -189,7 +190,7 @@ func GetBigMapKeys() gin.HandlerFunc {
 			return
 		}
 
-		keys, err := ctx.BigMapDiffs.Keys(bigmapdiff.GetContext{
+		keys, err := ctx.BigMapDiffs.Keys(c.Request.Context(), bigmapdiff.GetContext{
 			Ptr:      &req.Ptr,
 			Size:     pageReq.Size,
 			Offset:   pageReq.Offset,
@@ -201,12 +202,12 @@ func GetBigMapKeys() gin.HandlerFunc {
 			return
 		}
 
-		symLink, err := getCurrentSymLink(ctx.Blocks)
+		symLink, err := getCurrentSymLink(c.Request.Context(), ctx.Blocks)
 		if handleError(c, ctx.Storage, err, 0) {
 			return
 		}
 
-		response, err := prepareBigMapKeys(ctx, keys, symLink)
+		response, err := prepareBigMapKeys(c.Request.Context(), ctx, keys, symLink)
 		if handleError(c, ctx.Storage, err, 0) {
 			return
 		}
@@ -246,7 +247,7 @@ func GetBigMapByKeyHash() gin.HandlerFunc {
 			return
 		}
 
-		bm, total, err := ctx.BigMapDiffs.GetByPtrAndKeyHash(req.Ptr, req.KeyHash, pageReq.Size, pageReq.Offset)
+		bm, total, err := ctx.BigMapDiffs.GetByPtrAndKeyHash(c.Request.Context(), req.Ptr, req.KeyHash, pageReq.Size, pageReq.Offset)
 		if handleError(c, ctx.Storage, err, 0) {
 			return
 		}
@@ -255,12 +256,12 @@ func GetBigMapByKeyHash() gin.HandlerFunc {
 			c.SecureJSON(http.StatusNoContent, gin.H{})
 			return
 		}
-		symLink, err := getCurrentSymLink(ctx.Blocks)
+		symLink, err := getCurrentSymLink(c.Request.Context(), ctx.Blocks)
 		if handleError(c, ctx.Storage, err, 0) {
 			return
 		}
 
-		response, err := prepareBigMapItem(ctx, bm, req.KeyHash, symLink)
+		response, err := prepareBigMapItem(c.Request.Context(), ctx, bm, req.KeyHash, symLink)
 		if handleError(c, ctx.Storage, err, 0) {
 			return
 		}
@@ -294,17 +295,17 @@ func GetCurrentBigMapKeyHash() gin.HandlerFunc {
 			return
 		}
 
-		state, err := ctx.BigMapDiffs.Current(req.KeyHash, req.Ptr)
+		state, err := ctx.BigMapDiffs.Current(c.Request.Context(), req.KeyHash, req.Ptr)
 		if handleError(c, ctx.Storage, err, 0) {
 			return
 		}
 
-		symLink, err := getCurrentSymLink(ctx.Blocks)
+		symLink, err := getCurrentSymLink(c.Request.Context(), ctx.Blocks)
 		if handleError(c, ctx.Storage, err, 0) {
 			return
 		}
 
-		bigMapType, err := getBigMapType(ctx, state.Contract, state.Ptr, symLink)
+		bigMapType, err := getBigMapType(c.Request.Context(), ctx, state.Contract, state.Ptr, symLink)
 		if handleError(c, ctx.Storage, err, 0) {
 			return
 		}
@@ -348,7 +349,7 @@ func GetBigMapDiffCount() gin.HandlerFunc {
 			return
 		}
 
-		count, err := ctx.BigMapDiffs.Count(req.Ptr)
+		count, err := ctx.BigMapDiffs.Count(c.Request.Context(), req.Ptr)
 		if err != nil {
 			if ctx.Storage.IsRecordNotFound(err) {
 				c.SecureJSON(http.StatusOK, CountResponse{})
@@ -357,16 +358,16 @@ func GetBigMapDiffCount() gin.HandlerFunc {
 			handleError(c, ctx.Storage, err, 0)
 			return
 		}
-		c.SecureJSON(http.StatusOK, CountResponse{count})
+		c.SecureJSON(http.StatusOK, CountResponse{int64(count)})
 	}
 }
 
-func prepareBigMapKeys(ctx *config.Context, data []bigmapdiff.BigMapState, symLink string) ([]BigMapResponseItem, error) {
+func prepareBigMapKeys(c context.Context, ctx *config.Context, data []bigmapdiff.BigMapState, symLink string) ([]BigMapResponseItem, error) {
 	if len(data) == 0 {
 		return []BigMapResponseItem{}, nil
 	}
 
-	bigMapType, err := getBigMapType(ctx, data[0].Contract, data[0].Ptr, symLink)
+	bigMapType, err := getBigMapType(c, ctx, data[0].Contract, data[0].Ptr, symLink)
 	if err != nil {
 		return nil, err
 	}
@@ -393,12 +394,12 @@ func prepareBigMapKeys(ctx *config.Context, data []bigmapdiff.BigMapState, symLi
 	return res, nil
 }
 
-func prepareBigMapItem(ctx *config.Context, data []bigmapdiff.BigMapDiff, keyHash, symLink string) (res BigMapDiffByKeyResponse, err error) {
+func prepareBigMapItem(c context.Context, ctx *config.Context, data []bigmapdiff.BigMapDiff, keyHash, symLink string) (res BigMapDiffByKeyResponse, err error) {
 	if len(data) == 0 {
 		return
 	}
 
-	bigMapType, err := getBigMapType(ctx, data[0].Contract, data[0].Ptr, symLink)
+	bigMapType, err := getBigMapType(c, ctx, data[0].Contract, data[0].Ptr, symLink)
 	if err != nil {
 		return
 	}
@@ -424,13 +425,13 @@ func prepareBigMapItem(ctx *config.Context, data []bigmapdiff.BigMapDiff, keyHas
 	return
 }
 
-func getBigMapType(ctx *config.Context, contract string, ptr int64, symLink string) (*ast.BigMap, error) {
-	storageType, err := getStorageType(ctx.Contracts, contract, symLink)
+func getBigMapType(c context.Context, ctx *config.Context, contract string, ptr int64, symLink string) (*ast.BigMap, error) {
+	storageType, err := getStorageType(c, ctx.Contracts, contract, symLink)
 	if err != nil {
 		return nil, err
 	}
 
-	actions, err := ctx.BigMapActions.Get(ptr, 2, 0)
+	actions, err := ctx.BigMapActions.Get(c, ptr, 2, 0)
 	if err != nil {
 		return nil, err
 	}
@@ -443,18 +444,18 @@ func getBigMapType(ctx *config.Context, contract string, ptr int64, symLink stri
 				operationID = actions[1].OperationID
 			}
 		}
-		operation, err := ctx.Operations.GetByID(operationID)
+		operation, err := ctx.Operations.GetByID(c, operationID)
 		if err != nil {
 			return nil, err
 		}
 		deffatedStorage = operation.DeffatedStorage
 	} else {
-		contract, err := ctx.Accounts.Get(contract)
+		contract, err := ctx.Accounts.Get(c, contract)
 		if err != nil {
 			return nil, err
 		}
 
-		operation, err := ctx.Operations.Last(map[string]interface{}{
+		operation, err := ctx.Operations.Last(c, map[string]interface{}{
 			"destination_id": contract.ID,
 			"status":         types.OperationStatusApplied,
 		}, 0)