skip migration on new db

go.mod

@@ -7,10 +7,10 @@ go 1.20
 
 require (
 	github.com/Masterminds/semver v1.5.0
-	github.com/aserto-dev/azm v0.0.15
+	github.com/aserto-dev/azm v0.0.16
 	github.com/aserto-dev/errors v0.0.6
 	github.com/aserto-dev/go-directory v0.30.4
-	github.com/bufbuild/protovalidate-go v0.4.0
+	github.com/bufbuild/protovalidate-go v0.4.1
 	github.com/gonvenience/ytbx v1.4.4
 	github.com/google/uuid v1.4.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
@@ -27,9 +27,9 @@ require (
 )
 
 require (
-	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.31.0-20231030212536-12f9cba37c9d.2 // indirect
+	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.31.0-20231106192134-1baebb0a1518.2 // indirect
 	github.com/BurntSushi/toml v1.3.2 // indirect
-	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
@@ -38,7 +38,7 @@ require (
 	github.com/gonvenience/term v1.0.2 // indirect
 	github.com/gonvenience/text v1.0.7 // indirect
 	github.com/gonvenience/wrap v1.1.2 // indirect
-	github.com/google/cel-go v0.18.1 // indirect
+	github.com/google/cel-go v0.18.2 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect

go.sum

@@ -1,22 +1,22 @@
-buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.31.0-20231030212536-12f9cba37c9d.2 h1:m8rKyv88R8ZIR1549RMXckZ4FZJGxrq/7aRYl6U3WHc=
-buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.31.0-20231030212536-12f9cba37c9d.2/go.mod h1:xafc+XIsTxTy76GJQ1TKgvJWsSugFBqMaN27WhUblew=
+buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.31.0-20231106192134-1baebb0a1518.2 h1:iRWpWLm1nrsCHBVhibqPJQB3iIf3FRsAXioJVU8m6w0=
+buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.31.0-20231106192134-1baebb0a1518.2/go.mod h1:xafc+XIsTxTy76GJQ1TKgvJWsSugFBqMaN27WhUblew=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
 github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
-github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 h1:goHVqTbFX3AIo0tzGr14pgfAW2ZfPChKO21Z9MGf/gk=
-github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
-github.com/aserto-dev/azm v0.0.15 h1:GO222vhAfcEeJJBTuWJBHNm1lIMA6qxkFjiVHN5pPeo=
-github.com/aserto-dev/azm v0.0.15/go.mod h1:oNNvPYysttO/otHryQDm6u71jAt/Qhs9izhrde7HlD0=
+github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
+github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
+github.com/aserto-dev/azm v0.0.16 h1:TqYF/tO29fakkXR0ZPUOXC6mJI9zO9DRsGKo4q1qlUc=
+github.com/aserto-dev/azm v0.0.16/go.mod h1:oNNvPYysttO/otHryQDm6u71jAt/Qhs9izhrde7HlD0=
 github.com/aserto-dev/errors v0.0.6 h1:iH5fkJwBGFPbcdS4B8mwvNdwODlhDEXXPduZtjLh6vo=
 github.com/aserto-dev/errors v0.0.6/go.mod h1:kenI1gamsemaR2wS+M2un0kXIJ9exTrmeRT/fCFwlWc=
 github.com/aserto-dev/go-directory v0.30.4 h1:MifQgaA2ino54mWcmJAbJZSmUPNqat0V/u60EpFd9Vs=
 github.com/aserto-dev/go-directory v0.30.4/go.mod h1:jHn6ckERuci6nWz9vCzAbYoF2UnbOwEzl++sA1L3nyc=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
-github.com/bufbuild/protovalidate-go v0.4.0 h1:ModSkCLEW07fiyGtdtMXKY+Gz3oPFKSfiaSCgL+FtpU=
-github.com/bufbuild/protovalidate-go v0.4.0/go.mod h1:QqeUPLVYEKQc+/rkoUXFqXW03zPBfrEfIbX+zmA0VxA=
+github.com/bufbuild/protovalidate-go v0.4.1 h1:ye/8S72WbEklCeltPkSEeT8Eu1A7P/gmMsmapkwqTFk=
+github.com/bufbuild/protovalidate-go v0.4.1/go.mod h1:+p5FXfOjSEgLz5WBDTOMPMdQPXqALEERbJZU7huDCtA=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
@@ -60,8 +60,8 @@ github.com/gonvenience/wrap v1.1.2 h1:xPKxNwL1HCguwyM+HlP/1CIuc9LRd7k8RodLwe9YTZ
 github.com/gonvenience/wrap v1.1.2/go.mod h1:GiryBSXoI3BAAhbWD1cZVj7RZmtiu0ERi/6R6eJfslI=
 github.com/gonvenience/ytbx v1.4.4 h1:jQopwyaLsVGuwdxSiN4WkXjsEaFNPJ3V4lUj7eyEpzo=
 github.com/gonvenience/ytbx v1.4.4/go.mod h1:w37+MKCPcCMY/jpPNmEklD4xKqrOAVBO6kIWW2+uI6M=
-github.com/google/cel-go v0.18.1 h1:V/lAXKq4C3BYLDy/ARzMtpkEEYfHQpZzVyzy69nEUjs=
-github.com/google/cel-go v0.18.1/go.mod h1:PVAybmSnWkNMUZR/tEWFUiJ1Np4Hz0MHsZJcgC4zln4=
+github.com/google/cel-go v0.18.2 h1:L0B6sNBSVmt0OyECi8v6VOS74KOc9W/tLiWKfZABvf4=
+github.com/google/cel-go v0.18.2/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=

pkg/bdb/boltdb.go

@@ -2,13 +2,13 @@ package bdb
 
 import (
 	"net/http"
-	"os"
 	"path/filepath"
 	"time"
 
 	"github.com/aserto-dev/azm/cache"
 	"github.com/aserto-dev/azm/model"
 	cerr "github.com/aserto-dev/errors"
+	"github.com/aserto-dev/go-edge-ds/pkg/fs"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -59,13 +59,9 @@ func (s *BoltDB) Open() error {
 	}
 
 	dbDir := filepath.Dir(s.config.DBPath)
-	exists, err := filePathExists(dbDir)
-	if err != nil {
-		return errors.Wrap(err, "failed to determine if store path/file exists")
-	}
-	if !exists {
-		if err = os.MkdirAll(dbDir, 0700); err != nil {
-			return errors.Wrapf(err, "failed to create directory '%s'", dbDir)
+	if !fs.DirExists(dbDir) {
+		if err := fs.EnsureDirPath(dbDir); err != nil {
+			return err
 		}
 	}
 
@@ -91,17 +87,6 @@ func (s *BoltDB) Close() {
 	}
 }
 
-// filePathExists, internal helper function to detect if the file path exists.
-func filePathExists(path string) (bool, error) {
-	if _, err := os.Stat(path); err == nil {
-		return true, nil
-	} else if os.IsNotExist(err) {
-		return false, nil
-	} else {
-		return false, errors.Wrapf(err, "failed to stat file [%s]", path)
-	}
-}
-
 func (s *BoltDB) DB() *bolt.DB {
 	return s.db
 }

pkg/bdb/metadata.go

@@ -31,6 +31,11 @@ func UpdateMetadata[T any, M MessageM[T]](ctx context.Context, tx *bolt.Tx, path
 		if err := SetFieldProperty(msg, "CreatedAt", ts); err != nil {
 			return nil, err
 		}
+		// if new instance set Etag to empty string.
+		if err := SetFieldProperty(msg, "Etag", ""); err != nil {
+			return nil, err
+		}
+
 	case err != nil:
 		return nil, err
 	default:

pkg/bdb/migrate/mig/mig.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/aserto-dev/go-edge-ds/pkg/bdb"
+	"github.com/aserto-dev/go-edge-ds/pkg/fs"
 	"github.com/rs/zerolog"
 
 	"github.com/Masterminds/semver"
@@ -197,6 +198,13 @@ func Backup(db *bolt.DB, version *semver.Version) error {
 }
 
 func OpenDB(cfg *bdb.Config) (*bolt.DB, error) {
+	dbDir := filepath.Dir(cfg.DBPath)
+	if !fs.DirExists(dbDir) {
+		if err := fs.EnsureDirPath(dbDir); err != nil {
+			return nil, err
+		}
+	}
+
 	db, err := bolt.Open(cfg.DBPath, 0644, &bolt.Options{
 		Timeout: cfg.RequestTimeout,
 	})

pkg/bdb/migrate/mig003/migrate.go

@@ -3,8 +3,10 @@ package mig003
 import (
 	"bytes"
 	"context"
+	"hash/fnv"
 	"os"
 	"path/filepath"
+	"strconv"
 
 	"github.com/aserto-dev/azm/migrate"
 	v3 "github.com/aserto-dev/azm/v3"
@@ -39,7 +41,10 @@ const (
 )
 
 var fnMap = []func(*zerolog.Logger, *bolt.DB, *bolt.DB) error{
+	mig.CreateBucket(bdb.SystemPath),
+
 	mig.CreateBucket(bdb.ManifestPath),
+	createModel(),
 	migrateModel(),
 
 	mig.DeleteBucket(bdb.ObjectTypesPath),
@@ -70,13 +75,62 @@ func Migrate(log *zerolog.Logger, roDB, rwDB *bolt.DB) error {
 	return nil
 }
 
+func createModel() func(*zerolog.Logger, *bolt.DB, *bolt.DB) error {
+	return func(log *zerolog.Logger, roDB *bolt.DB, rwDB *bolt.DB) error {
+		// skip when roDB is set.
+		if roDB != nil {
+			log.Debug().Msg("SKIP CreateModel")
+			return nil
+		}
+
+		log.Info().Str("version", Version).Msg("CreateModel")
+		ctx := context.Background()
+
+		manifestBuf := new(bytes.Buffer)
+
+		model, err := v3.Load(bytes.NewReader(manifestBuf.Bytes()))
+		if err != nil {
+			return err
+		}
+
+		h := fnv.New64a()
+		h.Reset()
+		_, _ = h.Write(manifestBuf.Bytes())
+
+		md := &dsm3.Metadata{
+			UpdatedAt: timestamppb.Now(),
+			Etag:      strconv.FormatUint(h.Sum64(), 10),
+		}
+		if err := rwDB.Update(func(tx *bolt.Tx) error {
+			if err := ds.Manifest(md).Set(ctx, tx, manifestBuf); err != nil {
+				return errors.Errorf("failed to set manifest")
+			}
+
+			if err := ds.Manifest(md).SetModel(ctx, tx, model); err != nil {
+				return errors.Errorf("failed to set model")
+			}
+
+			return nil
+		}); err != nil {
+			return err
+		}
+
+		return nil
+	}
+}
+
 // migrateModel,
 // 1) creates a manifest file from the metadata objects in the db
 // 2) computes the in-memory model
 // 2) persists the manifest file byte-stream in the store
 // 3) persists the serialized model in the store.
 func migrateModel() func(*zerolog.Logger, *bolt.DB, *bolt.DB) error {
 	return func(log *zerolog.Logger, roDB *bolt.DB, rwDB *bolt.DB) error {
+		// skip when roDB is nil.
+		if roDB == nil {
+			log.Debug().Msg("SKIP MigrateModel")
+			return nil
+		}
 
 		log.Info().Str("version", Version).Msg("MigrateModel")
 		ctx := context.Background()
@@ -178,6 +232,12 @@ func getMetadata(ctx context.Context, roDB *bolt.DB) (*migrate.Metadata, error)
 // updateObjects, read values from read-only backup, write to new bucket.
 func updateObjects(path bdb.Path) func(*zerolog.Logger, *bolt.DB, *bolt.DB) error {
 	return func(log *zerolog.Logger, roDB *bolt.DB, rwDB *bolt.DB) error {
+		// skip when roDB is nil.
+		if roDB == nil {
+			log.Debug().Msg("SKIP UpdateObjects")
+			return nil
+		}
+
 		log.Info().Str("version", Version).Msg("UpdateObjects")
 
 		if err := roDB.View(func(rtx *bolt.Tx) error {
@@ -228,6 +288,12 @@ func updateObjects(path bdb.Path) func(*zerolog.Logger, *bolt.DB, *bolt.DB) erro
 // updateRelations, read values from read-only backup, write to new bucket.
 func updateRelations(path bdb.Path, d ds.Direction) func(*zerolog.Logger, *bolt.DB, *bolt.DB) error {
 	return func(log *zerolog.Logger, roDB *bolt.DB, rwDB *bolt.DB) error {
+		// skip when roDB is nil.
+		if roDB == nil {
+			log.Debug().Msg("SKIP UpdateRelations")
+			return nil
+		}
+
 		log.Info().Str("version", Version).Msg("UpdateRelations")
 
 		if err := roDB.View(func(rtx *bolt.Tx) error {

pkg/bdb/migrate/migrate.go

@@ -10,6 +10,7 @@ import (
 	"github.com/aserto-dev/go-edge-ds/pkg/bdb/migrate/mig001"
 	"github.com/aserto-dev/go-edge-ds/pkg/bdb/migrate/mig002"
 	"github.com/aserto-dev/go-edge-ds/pkg/bdb/migrate/mig003"
+	"github.com/aserto-dev/go-edge-ds/pkg/fs"
 
 	"github.com/Masterminds/semver"
 	"github.com/rs/zerolog"
@@ -38,10 +39,18 @@ var (
 // higher  returns false, error
 // errors: returns false, error.
 func CheckSchemaVersion(config *bdb.Config, logger *zerolog.Logger, reqVersion *semver.Version) (bool, error) {
+	if !fs.FileExists(config.DBPath) {
+		if err := create(config, logger, reqVersion); err != nil {
+			return false, err
+		}
+		return true, nil
+	}
+
 	boltdb, err := bdb.New(config, logger)
 	if err != nil {
 		return false, err
 	}
+
 	if err := boltdb.Open(); err != nil {
 		return false, err
 	}
@@ -51,6 +60,7 @@ func CheckSchemaVersion(config *bdb.Config, logger *zerolog.Logger, reqVersion *
 	if err != nil {
 		return false, err
 	}
+
 	logger.Info().Str("current", curVersion.String()).Msg("schema_version")
 
 	switch {
@@ -127,6 +137,35 @@ func getCurrent(config *bdb.Config, logger *zerolog.Logger) (*semver.Version, er
 	return mig.GetVersion(boltdb.DB())
 }
 
+func create(config *bdb.Config, log *zerolog.Logger, version *semver.Version) error {
+	rwDB, err := mig.OpenDB(config)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		log.Debug().Str("db_path", rwDB.Path()).Msg("close-rw")
+		if err := rwDB.Close(); err != nil {
+			log.Error().Err(err).Msg("close rwDB")
+		}
+		rwDB = nil
+	}()
+
+	// create flow is signaled by roDB == nil.
+	if err := execute(log, nil, rwDB, version); err != nil {
+		return err
+	}
+
+	if err := mig.SetVersion(rwDB, version); err != nil {
+		return err
+	}
+
+	if err := rwDB.Sync(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func migrate(config *bdb.Config, log *zerolog.Logger, curVersion, nextVersion *semver.Version) error {
 	rwDB, err := mig.OpenDB(config)
 	if err != nil {

pkg/ds/manifest.go

@@ -84,6 +84,12 @@ func (m *manifest) Set(ctx context.Context, tx *bolt.Tx, buf *bytes.Buffer) erro
 // SetModel, persists the model cache in the _manifest bucket
 // _metadata/{name}/{version}/model.
 func (m *manifest) SetModel(ctx context.Context, tx *bolt.Tx, mod *model.Model) error {
+	if mod.Metadata == nil {
+		mod.Metadata = &model.Metadata{}
+	}
+	mod.Metadata.ETag = m.Metadata.Etag
+	mod.Metadata.UpdatedAt = m.Metadata.UpdatedAt.AsTime()
+
 	if _, err := bdb.SetAny[model.Model](ctx, tx, bdb.ManifestPath, bdb.ModelKey, mod); err != nil {
 		return err
 	}

pkg/fs/fs.go

@@ -0,0 +1,28 @@
+package fs
+
+import (
+	"os"
+)
+
+func FileExists(path string) bool {
+	fsInfo, err := os.Stat(path)
+	if err == nil && !fsInfo.IsDir() {
+		return true
+	}
+	return false
+}
+
+func DirExists(path string) bool {
+	fsInfo, err := os.Stat(path)
+	if err == nil && fsInfo.IsDir() {
+		return true
+	}
+	return false
+}
+
+func EnsureDirPath(path string) error {
+	if !DirExists(path) {
+		return os.MkdirAll(path, 0700)
+	}
+	return nil
+}

tests/runner_test.go

@@ -36,7 +36,12 @@ func TestMain(m *testing.M) {
 	ctx := context.Background()
 	logger := zerolog.New(io.Discard)
 
-	dbPath := path.Join(os.TempDir(), "edge-ds", "test-eds.db")
+	dirPath := os.TempDir()
+	if err := os.MkdirAll(dirPath, 0700); err != nil {
+		panic(err)
+	}
+
+	dbPath := path.Join(dirPath, "edge-ds", "test-eds.db")
 	os.Remove(dbPath)
 	fmt.Println(dbPath)