diff --git a/assetdb.go b/assetdb.go index 8db4f82..1e284db 100644 --- a/assetdb.go +++ b/assetdb.go @@ -21,24 +21,19 @@ type AssetDB struct { // If the edge is provided, the entity is created and linked to the source entity using the specified edge. // It returns the newly created entity and an error, if any. func (as *AssetDB) Create(edge *types.Edge, asset oam.Asset) (*types.Entity, error) { - e, err := as.Repo.CreateEntity(asset) + e, err := as.Repo.CreateAsset(asset) if err != nil || edge == nil || edge.FromEntity == nil || edge.Relation == nil { return e, err } edge.ToEntity = e - _, err = as.Repo.Link(edge) + _, err = as.Repo.CreateEdge(edge) if err != nil { return nil, err } return e, nil } -// UpdateEntityLastSeen updates the entity last seen field to the current time by its ID. -func (as *AssetDB) UpdateEntityLastSeen(id string) error { - return as.Repo.UpdateEntityLastSeen(id) -} - // DeleteEntity removes an entity in the database by its ID. func (as *AssetDB) DeleteEntity(id string) error { return as.Repo.DeleteEntity(id) @@ -69,11 +64,11 @@ func (as *AssetDB) FindEntitiesByType(atype oam.AssetType, since time.Time) ([]* return as.Repo.FindEntitiesByType(atype, since) } -// Link creates an edge between two entities in the database. +// CreateEdge creates an edge between two entities in the database. // The link is established by creating a new Edge in the database, linking the two entities. // Returns the created edge as a types.Edge or an error if the link creation fails. -func (as *AssetDB) Link(edge *types.Edge) (*types.Edge, error) { - return as.Repo.Link(edge) +func (as *AssetDB) CreateEdge(edge *types.Edge) (*types.Edge, error) { + return as.Repo.CreateEdge(edge) } // IncomingEdges finds all edges pointing to the entity for the specified labels, if any. @@ -90,12 +85,12 @@ func (as *AssetDB) OutgoingEdges(entity *types.Entity, since time.Time, labels . return as.Repo.OutgoingEdges(entity, since, labels...) } -// CreateEntityTag creates a new entity tag in the database. +// CreateEntityProperty creates a new entity tag in the database. // It takes an oam.Property as input and persists it in the database. // The entity tag is serialized to JSON and stored in the Content field of the EntityTag struct. // Returns the created entity tag as a types.EntityTag or an error if the creation fails. -func (as *AssetDB) CreateEntityTag(entity *types.Entity, property oam.Property) (*types.EntityTag, error) { - return as.Repo.CreateEntityTag(entity, property) +func (as *AssetDB) CreateEntityProperty(entity *types.Entity, property oam.Property) (*types.EntityTag, error) { + return as.Repo.CreateEntityProperty(entity, property) } // GetEntityTags finds all tags for the entity with the specified names and last seen after the since parameter. @@ -112,12 +107,12 @@ func (as *AssetDB) DeleteEntityTag(id string) error { return as.Repo.DeleteEntityTag(id) } -// CreateEdgeTag creates a new edge tag in the database. +// CreateEdgeProperty creates a new edge tag in the database. // It takes an oam.Property as input and persists it in the database. // The edge tag is serialized to JSON and stored in the Content field of the EdgeTag struct. // Returns the created edge tag as a types.EdgeTag or an error if the creation fails. -func (as *AssetDB) CreateEdgeTag(edge *types.Edge, property oam.Property) (*types.EdgeTag, error) { - return as.Repo.CreateEdgeTag(edge, property) +func (as *AssetDB) CreateEdgeProperty(edge *types.Edge, property oam.Property) (*types.EdgeTag, error) { + return as.Repo.CreateEdgeProperty(edge, property) } // GetEdgeTags finds all tags for the edge with the specified names and last seen after the since parameter. diff --git a/assetdb_test.go b/assetdb_test.go index f8929c7..9b7cb0c 100644 --- a/assetdb_test.go +++ b/assetdb_test.go @@ -64,7 +64,7 @@ func TestAssetDB(t *testing.T) { } if tc.expectedError == nil { - mockAssetDB.On("CreateEntity", tc.discovered).Return(tc.expected, tc.expectedError) + mockAssetDB.On("CreateAsset", tc.discovered).Return(tc.expected, tc.expectedError) } e := &types.Edge{ @@ -74,7 +74,7 @@ func TestAssetDB(t *testing.T) { } if tc.source != nil && tc.relation != "" { - mockAssetDB.On("Link", e).Return(&types.Edge{}, nil) + mockAssetDB.On("CreateEdge", e).Return(&types.Edge{}, nil) } result, err := adb.Create(e, tc.discovered) @@ -384,14 +384,14 @@ func (m *mockAssetDB) GetDBType() string { return args.String(0) } -func (m *mockAssetDB) CreateEntity(asset oam.Asset) (*types.Entity, error) { - args := m.Called(asset) +func (m *mockAssetDB) CreateEntity(entity *types.Entity) (*types.Entity, error) { + args := m.Called(entity) return args.Get(0).(*types.Entity), args.Error(1) } -func (m *mockAssetDB) UpdateEntityLastSeen(id string) error { - args := m.Called(id) - return args.Error(0) +func (m *mockAssetDB) CreateAsset(asset oam.Asset) (*types.Entity, error) { + args := m.Called(asset) + return args.Get(0).(*types.Entity), args.Error(1) } func (m *mockAssetDB) FindEntityById(id string) (*types.Entity, error) { @@ -414,7 +414,7 @@ func (m *mockAssetDB) DeleteEntity(id string) error { return args.Error(0) } -func (m *mockAssetDB) Link(edge *types.Edge) (*types.Edge, error) { +func (m *mockAssetDB) CreateEdge(edge *types.Edge) (*types.Edge, error) { args := m.Called(edge) return args.Get(0).(*types.Edge), args.Error(1) } @@ -439,7 +439,12 @@ func (m *mockAssetDB) DeleteEdge(id string) error { return args.Error(0) } -func (m *mockAssetDB) CreateEntityTag(entity *types.Entity, property oam.Property) (*types.EntityTag, error) { +func (m *mockAssetDB) CreateEntityTag(entity *types.Entity, tag *types.EntityTag) (*types.EntityTag, error) { + args := m.Called(entity, tag) + return args.Get(0).(*types.EntityTag), args.Error(1) +} + +func (m *mockAssetDB) CreateEntityProperty(entity *types.Entity, property oam.Property) (*types.EntityTag, error) { args := m.Called(entity, property) return args.Get(0).(*types.EntityTag), args.Error(1) } @@ -459,7 +464,12 @@ func (m *mockAssetDB) DeleteEntityTag(id string) error { return args.Error(0) } -func (m *mockAssetDB) CreateEdgeTag(edge *types.Edge, property oam.Property) (*types.EdgeTag, error) { +func (m *mockAssetDB) CreateEdgeTag(edge *types.Edge, tag *types.EdgeTag) (*types.EdgeTag, error) { + args := m.Called(edge, tag) + return args.Get(0).(*types.EdgeTag), args.Error(1) +} + +func (m *mockAssetDB) CreateEdgeProperty(edge *types.Edge, property oam.Property) (*types.EdgeTag, error) { args := m.Called(edge, property) return args.Get(0).(*types.EdgeTag), args.Error(1) } diff --git a/cache/cache.go b/cache/cache.go index bdfa69d..2c6f451 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -5,14 +5,11 @@ package cache import ( - "errors" "sync" "time" "github.com/caffix/queue" - assetdb "github.com/owasp-amass/asset-db" "github.com/owasp-amass/asset-db/repository" - "github.com/owasp-amass/asset-db/repository/sqlrepo" ) type Cache struct { @@ -20,28 +17,23 @@ type Cache struct { start time.Time freq time.Duration done chan struct{} - cdone chan struct{} cache repository.Repository db repository.Repository queue queue.Queue } -func New(database repository.Repository, done chan struct{}) (*Cache, error) { - if db := assetdb.New(sqlrepo.SQLiteMemory, ""); db != nil { - c := &Cache{ - start: time.Now(), - freq: 10 * time.Minute, - done: done, - cdone: make(chan struct{}, 1), - cache: db.Repo, - db: database, - queue: queue.NewQueue(), - } - - go c.processDBCallbacks() - return c, nil +func New(cache, database repository.Repository) (*Cache, error) { + c := &Cache{ + start: time.Now(), + freq: 10 * time.Minute, + done: make(chan struct{}, 1), + cache: cache, + db: database, + queue: queue.NewQueue(), } - return nil, errors.New("failed to create the cache repository") + + go c.processDBCallbacks() + return c, nil } // StartTime returns the time that the cache was created. @@ -60,7 +52,7 @@ func (c *Cache) Close() error { } } - close(c.cdone) + close(c.done) for { if c.queue.Empty() { break @@ -79,8 +71,6 @@ func (c *Cache) appendToDBQueue(callback func()) { select { case <-c.done: return - case <-c.cdone: - return default: } c.queue.Append(callback) @@ -92,8 +82,6 @@ loop: select { case <-c.done: break loop - case <-c.cdone: - break loop case <-c.queue.Signal(): element, ok := c.queue.Next() diff --git a/cache/edge.go b/cache/edge.go index 62d1c55..68f1722 100644 --- a/cache/edge.go +++ b/cache/edge.go @@ -11,18 +11,18 @@ import ( "github.com/owasp-amass/asset-db/types" ) -// Link implements the Repository interface. -func (c *Cache) Link(edge *types.Edge) (*types.Edge, error) { +// CreateEdge implements the Repository interface. +func (c *Cache) CreateEdge(edge *types.Edge) (*types.Edge, error) { c.Lock() defer c.Unlock() - e, err := c.cache.Link(edge) + e, err := c.cache.CreateEdge(edge) if err != nil { return nil, err } c.appendToDBQueue(func() { - _, _ = c.db.Link(edge) + _, _ = c.db.CreateEdge(edge) }) return e, nil @@ -80,14 +80,14 @@ func (c *Cache) DeleteEdge(id string) error { return } - edges, err := c.db.OutgoingEdges(s, time.Time{}, edge.Relation.Label()) + edges, err := c.db.OutgoingEdges(s[0], time.Time{}, edge.Relation.Label()) if err != nil || len(edges) == 0 { return } var target *types.Edge for _, e := range edges { - if e.ID == o.ID && reflect.DeepEqual(e.Relation, o.Relation) { + if e.ID == o[0].ID && reflect.DeepEqual(e.Relation, edge.Relation) { target = e break } diff --git a/cache/entity.go b/cache/entity.go index a997663..d0102a3 100644 --- a/cache/entity.go +++ b/cache/entity.go @@ -12,22 +12,26 @@ import ( ) // CreateEntity implements the Repository interface. -func (c *Cache) CreateEntity(asset oam.Asset) (*types.Entity, error) { +func (c *Cache) CreateEntity(input *types.Entity) (*types.Entity, error) { c.Lock() defer c.Unlock() - entity, err := c.cache.CreateEntity(asset) + entity, err := c.cache.CreateEntity(input) if err != nil { return nil, err } if tag, found := c.checkCacheEntityTag(entity, "cache_create_entity"); !found { - if last, err := time.Parse("2006-01-02 15:04:05", tag.Value()); err == nil && time.Now().Add(-1*c.freq).After(last) { + if last, err := time.Parse("2006-01-02 15:04:05", tag.Property.Value()); err == nil && time.Now().Add(-1*c.freq).After(last) { _ = c.cache.DeleteEntityTag(tag.ID) _ = c.createCacheEntityTag(entity, "cache_create_entity") c.appendToDBQueue(func() { - _, _ = c.db.CreateEntity(asset) + _, _ = c.db.CreateEntity(&types.Entity{ + CreatedAt: input.CreatedAt, + LastSeen: input.LastSeen, + Asset: input.Asset, + }) }) } } @@ -35,28 +39,28 @@ func (c *Cache) CreateEntity(asset oam.Asset) (*types.Entity, error) { return entity, nil } -// UpdateEntityLastSeen implements the Repository interface. -func (c *Cache) UpdateEntityLastSeen(id string) error { +// CreateAsset implements the Repository interface. +func (c *Cache) CreateAsset(asset oam.Asset) (*types.Entity, error) { c.Lock() defer c.Unlock() - err := c.cache.UpdateEntityLastSeen(id) + entity, err := c.cache.CreateAsset(asset) if err != nil { - return err + return nil, err } - entity, err := c.cache.FindEntityById(id) - if err != nil { - return nil - } + if tag, found := c.checkCacheEntityTag(entity, "cache_create_asset"); !found { + if last, err := time.Parse("2006-01-02 15:04:05", tag.Property.Value()); err == nil && time.Now().Add(-1*c.freq).After(last) { + _ = c.cache.DeleteEntityTag(tag.ID) + _ = c.createCacheEntityTag(entity, "cache_create_asset") - c.appendToDBQueue(func() { - if e, err := c.db.FindEntityByContent(entity.Asset, time.Time{}); err == nil && len(e) == 1 { - _ = c.db.UpdateEntityLastSeen(e[0].ID) + c.appendToDBQueue(func() { + _, _ = c.db.CreateAsset(asset) + }) } - }) + } - return nil + return entity, nil } // FindEntityById implements the Repository interface. @@ -103,7 +107,11 @@ func (c *Cache) FindEntityByContent(asset oam.Asset, since time.Time) ([]*types. var results []*types.Entity for _, entity := range dbentities { - if e, err := c.cache.CreateEntity(entity.Asset); err == nil { + if e, err := c.cache.CreateEntity(&types.Entity{ + CreatedAt: entity.CreatedAt, + LastSeen: entity.LastSeen, + Asset: entity.Asset, + }); err == nil { results = append(results, e) if tags, err := c.cache.GetEntityTags(entity, c.start, "cache_find_entity_by_content"); err == nil && len(tags) > 0 { for _, tag := range tags { @@ -152,7 +160,11 @@ func (c *Cache) FindEntitiesByType(atype oam.AssetType, since time.Time) ([]*typ var results []*types.Entity for _, entity := range dbentities { - if e, err := c.cache.CreateEntity(entity.Asset); err == nil { + if e, err := c.cache.CreateEntity(&types.Entity{ + CreatedAt: entity.CreatedAt, + LastSeen: entity.LastSeen, + Asset: entity.Asset, + }); err == nil { results = append(results, e) if tags, err := c.cache.GetEntityTags(entity, c.start, "cache_find_entities_by_type"); err == nil && len(tags) > 0 { for _, tag := range tags { diff --git a/cache/tag.go b/cache/tag.go index 8d7f57e..a393d27 100644 --- a/cache/tag.go +++ b/cache/tag.go @@ -14,17 +14,42 @@ import ( ) // CreateEntityTag implements the Repository interface. -func (c *Cache) CreateEntityTag(entity *types.Entity, property oam.Property) (*types.EntityTag, error) { +func (c *Cache) CreateEntityTag(entity *types.Entity, input *types.EntityTag) (*types.EntityTag, error) { c.Lock() defer c.Unlock() - tag, err := c.cache.CreateEntityTag(entity, property) + tag, err := c.cache.CreateEntityTag(entity, input) if err != nil { return nil, err } c.appendToDBQueue(func() { - _, _ = c.db.CreateEntityTag(entity, property) + if e, err := c.db.FindEntityByContent(entity.Asset, time.Time{}); err == nil && len(e) == 1 { + _, _ = c.db.CreateEntityTag(e[0], &types.EntityTag{ + CreatedAt: input.CreatedAt, + LastSeen: input.LastSeen, + Property: input.Property, + }) + } + }) + + return tag, nil +} + +// CreateEntityProperty implements the Repository interface. +func (c *Cache) CreateEntityProperty(entity *types.Entity, property oam.Property) (*types.EntityTag, error) { + c.Lock() + defer c.Unlock() + + tag, err := c.cache.CreateEntityProperty(entity, property) + if err != nil { + return nil, err + } + + c.appendToDBQueue(func() { + if e, err := c.db.FindEntityByContent(entity.Asset, time.Time{}); err == nil && len(e) == 1 { + _, _ = c.db.CreateEntityProperty(e[0], property) + } }) return tag, nil @@ -68,9 +93,9 @@ func (c *Cache) DeleteEntityTag(id string) error { c.appendToDBQueue(func() { if e, err := c.db.FindEntityByContent(entity.Asset, time.Time{}); err == nil && len(e) == 1 { - if tags, err := c.db.GetEntityTags(e, time.Time{}, tag.Name()); err == nil && len(tags) > 0 { + if tags, err := c.db.GetEntityTags(e[0], time.Time{}, tag.Property.Name()); err == nil && len(tags) > 0 { for _, t := range tags { - if t.Value() == tag.Value() { + if t.Property.Value() == tag.Property.Value() { _ = c.db.DeleteEntity(t.ID) break } @@ -83,17 +108,112 @@ func (c *Cache) DeleteEntityTag(id string) error { } // CreateEdgeTag implements the Repository interface. -func (c *Cache) CreateEdgeTag(edge *types.Edge, property oam.Property) (*types.EdgeTag, error) { +func (c *Cache) CreateEdgeTag(edge *types.Edge, input *types.EdgeTag) (*types.EdgeTag, error) { + c.Lock() + defer c.Unlock() + + tag, err := c.cache.CreateEdgeTag(edge, input) + if err != nil { + return nil, err + } + + edge2, err := c.cache.FindEdgeById(tag.Edge.ID) + if err != nil { + return nil, err + } + + sub, err := c.cache.FindEntityById(edge2.FromEntity.ID) + if err != nil { + return nil, err + } + + obj, err := c.cache.FindEntityById(edge2.ToEntity.ID) + if err != nil { + return nil, err + } + + c.appendToDBQueue(func() { + s, err := c.db.FindEntityByContent(sub.Asset, time.Time{}) + if err != nil || len(s) != 1 { + return + } + + o, err := c.db.FindEntityByContent(obj.Asset, time.Time{}) + if err != nil || len(o) != 1 { + return + } + + edges, err := c.db.OutgoingEdges(s[0], time.Time{}, edge.Relation.Label()) + if err != nil || len(edges) == 0 { + return + } + + var target *types.Edge + for _, e := range edges { + if e.ID == o[0].ID && reflect.DeepEqual(e.Relation, edge2.Relation) { + target = e + break + } + } + if target != nil { + _, _ = c.db.CreateEdgeProperty(target, input.Property) + } + }) + + return tag, nil +} + +// CreateEdgeProperty implements the Repository interface. +func (c *Cache) CreateEdgeProperty(edge *types.Edge, property oam.Property) (*types.EdgeTag, error) { c.Lock() defer c.Unlock() - tag, err := c.cache.CreateEdgeTag(edge, property) + tag, err := c.cache.CreateEdgeProperty(edge, property) + if err != nil { + return nil, err + } + + edge2, err := c.cache.FindEdgeById(tag.Edge.ID) + if err != nil { + return nil, err + } + + sub, err := c.cache.FindEntityById(edge2.FromEntity.ID) + if err != nil { + return nil, err + } + + obj, err := c.cache.FindEntityById(edge2.ToEntity.ID) if err != nil { return nil, err } c.appendToDBQueue(func() { - _, _ = c.db.CreateEdgeTag(edge, property) + s, err := c.db.FindEntityByContent(sub.Asset, time.Time{}) + if err != nil || len(s) != 1 { + return + } + + o, err := c.db.FindEntityByContent(obj.Asset, time.Time{}) + if err != nil || len(o) != 1 { + return + } + + edges, err := c.db.OutgoingEdges(s[0], time.Time{}, edge.Relation.Label()) + if err != nil || len(edges) == 0 { + return + } + + var target *types.Edge + for _, e := range edges { + if e.ID == o[0].ID && reflect.DeepEqual(e.Relation, edge2.Relation) { + target = e + break + } + } + if target != nil { + _, _ = c.db.CreateEdgeProperty(target, property) + } }) return tag, nil @@ -130,12 +250,17 @@ func (c *Cache) DeleteEdgeTag(id string) error { return nil } - sub, err := c.cache.FindEntityById(tag.Edge.FromEntity.ID) + edge2, err := c.cache.FindEdgeById(tag.Edge.ID) + if err != nil { + return nil + } + + sub, err := c.cache.FindEntityById(edge2.FromEntity.ID) if err != nil { return nil } - obj, err := c.cache.FindEntityById(tag.Edge.ToEntity.ID) + obj, err := c.cache.FindEntityById(edge2.ToEntity.ID) if err != nil { return nil } @@ -151,14 +276,14 @@ func (c *Cache) DeleteEdgeTag(id string) error { return } - edges, err := c.db.OutgoingEdges(s, time.Time{}, tag.Edge.Relation.Label()) + edges, err := c.db.OutgoingEdges(s[0], time.Time{}, tag.Edge.Relation.Label()) if err != nil || len(edges) == 0 { return } var target *types.Edge for _, e := range edges { - if e.ID == o.ID && reflect.DeepEqual(e.Relation, o.Relation) { + if e.ID == o[0].ID && reflect.DeepEqual(e.Relation, edge2.Relation) { target = e break } @@ -167,7 +292,7 @@ func (c *Cache) DeleteEdgeTag(id string) error { return } - if tags, err := c.db.GetEdgeTags(target, time.Time{}, tag.Name()); err == nil && len(tags) > 0 { + if tags, err := c.db.GetEdgeTags(target, time.Time{}, tag.Property.Name()); err == nil && len(tags) > 0 { for _, t := range tags { if tag.Property.Value() == t.Property.Value() { _ = c.db.DeleteEdgeTag(t.ID) @@ -181,7 +306,7 @@ func (c *Cache) DeleteEdgeTag(id string) error { } func (c *Cache) createCacheEntityTag(entity *types.Entity, name string) error { - _, err := c.cache.CreateEntityTag(entity, &property.SimpleProperty{ + _, err := c.cache.CreateEntityProperty(entity, &property.SimpleProperty{ PropertyName: name, PropertyValue: time.Now().Format("2006-01-02 15:04:05"), }) @@ -196,7 +321,7 @@ func (c *Cache) checkCacheEntityTag(entity *types.Entity, name string) (*types.E } func (c *Cache) createCacheEdgeTag(edge *types.Edge, name string) error { - _, err := c.cache.CreateEdgeTag(edge, &property.SimpleProperty{ + _, err := c.cache.CreateEdgeProperty(edge, &property.SimpleProperty{ PropertyName: name, PropertyValue: time.Now().Format("2006-01-02 15:04:05"), }) diff --git a/go.mod b/go.mod index 0b942dc..78c5616 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,14 @@ module github.com/owasp-amass/asset-db go 1.23.1 require ( - github.com/caffix/stringset v0.2.0 + github.com/caffix/queue v0.3.1 github.com/glebarez/sqlite v1.11.0 github.com/owasp-amass/open-asset-model v0.12.0 github.com/rubenv/sql-migrate v1.7.0 github.com/stretchr/testify v1.9.0 gorm.io/datatypes v1.2.4 gorm.io/driver/postgres v1.5.9 - gorm.io/driver/sqlite v1.5.4 + gorm.io/driver/sqlite v1.5.6 gorm.io/gorm v1.25.12 ) @@ -29,7 +29,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-sqlite3 v1.14.19 // indirect + github.com/mattn/go-sqlite3 v1.14.24 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect @@ -41,8 +41,8 @@ require ( golang.org/x/text v0.20.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gorm.io/driver/mysql v1.5.7 // indirect - modernc.org/libc v1.61.0 // indirect + modernc.org/libc v1.61.1 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect - modernc.org/sqlite v1.33.1 // indirect + modernc.org/sqlite v1.34.1 // indirect ) diff --git a/go.sum b/go.sum index 53d266a..044651c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/caffix/queue v0.3.1 h1:eg4V7cGomH76/Xq60MdEhSVZvO6E3iOPlacMwZOjiSA= +github.com/caffix/queue v0.3.1/go.mod h1:AnCrUsy3cwDjLJvNv7FkQx/0Q9Hb7n7agRrUP0OiYl4= github.com/caffix/stringset v0.2.0 h1:kN6xnvL8jzx2YhQNOYr6A6hFzUK+iikt1JtJ2MS2LC8= github.com/caffix/stringset v0.2.0/go.mod h1:8PZ6GIPpMP5+r5hr790/05w3v9xI+gXRxRzJCZL57lQ= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -44,8 +46,8 @@ github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= -github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/microsoft/go-mssqldb v0.17.0 h1:Fto83dMZPnYv1Zwx5vHHxpNraeEaUlQ/hhHLgZiaenE= github.com/microsoft/go-mssqldb v0.17.0/go.mod h1:OkoNGhGEs8EZqchVTtochlXruEhEOaO4S0d2sB5aeGQ= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= @@ -96,23 +98,23 @@ gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= -gorm.io/driver/sqlite v1.5.4 h1:IqXwXi8M/ZlPzH/947tn5uik3aYQslP9BVveoax0nV0= -gorm.io/driver/sqlite v1.5.4/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= +gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= +gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= gorm.io/driver/sqlserver v1.4.1 h1:t4r4r6Jam5E6ejqP7N82qAJIJAht27EGT41HyPfXRw0= gorm.io/driver/sqlserver v1.4.1/go.mod h1:DJ4P+MeZbc5rvY58PnmN1Lnyvb5gw5NPzGshHDnJLig= gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= -modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= -modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.21.0 h1:kKPI3dF7RIag8YcToh5ZwDcVMIv6VGa0ED5cvh0LMW4= -modernc.org/ccgo/v4 v4.21.0/go.mod h1:h6kt6H/A2+ew/3MW/p6KEoQmrq/i3pr0J/SiwiaF/g0= +modernc.org/cc/v4 v4.23.1 h1:WqJoPL3x4cUufQVHkXpXX7ThFJ1C4ik80i2eXEXbhD8= +modernc.org/cc/v4 v4.23.1/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.22.3 h1:C7AW89Zw3kygesTQWBzApwIn9ldM+cb/plrTIKq41Os= +modernc.org/ccgo/v4 v4.22.3/go.mod h1:Dz7n0/UkBbH3pnYaxgi1mFSfF4REqUOZNziphZASx6k= modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= modernc.org/gc/v2 v2.5.0 h1:bJ9ChznK1L1mUtAQtxi0wi5AtAs5jQuw4PrPHO5pb6M= modernc.org/gc/v2 v2.5.0/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= -modernc.org/libc v1.61.0 h1:eGFcvWpqlnoGwzZeZe3PWJkkKbM/3SUGyk1DVZQ0TpE= -modernc.org/libc v1.61.0/go.mod h1:DvxVX89wtGTu+r72MLGhygpfi3aUGgZRdAYGCAVVud0= +modernc.org/libc v1.61.1 h1:F8JngdWfVzqfNpff2apn7JpBkjq1ss8Ue4KuUdLDM7Q= +modernc.org/libc v1.61.1/go.mod h1:4QGjNyX3h+rn7V5oHpJY2yH0QN6frt1X+5BkXzwLPCo= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= @@ -121,8 +123,8 @@ modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM= -modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k= +modernc.org/sqlite v1.34.1 h1:u3Yi6M0N8t9yKRDwhXcyp1eS5/ErhPTBggxWFuR6Hfk= +modernc.org/sqlite v1.34.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/migrations/postgres/001_schema_init.sql b/migrations/postgres/001_schema_init.sql index a87d7f2..f489e6b 100644 --- a/migrations/postgres/001_schema_init.sql +++ b/migrations/postgres/001_schema_init.sql @@ -3,19 +3,19 @@ CREATE TABLE IF NOT EXISTS entities( entity_id INT GENERATED ALWAYS AS IDENTITY, created_at TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, - last_seen TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, etype VARCHAR(255), content JSONB, PRIMARY KEY(entity_id) ); -CREATE INDEX idx_entities_last_seen ON entities (last_seen); +CREATE INDEX idx_entities_updated_at ON entities (updated_at); CREATE INDEX idx_entities_etype ON entities (etype); CREATE TABLE IF NOT EXISTS entity_tags( tag_id INT GENERATED ALWAYS AS IDENTITY, created_at TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, - last_seen TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, ttype VARCHAR(255), content JSONB, entity_id INT, @@ -26,13 +26,13 @@ CREATE TABLE IF NOT EXISTS entity_tags( ON DELETE CASCADE ); -CREATE INDEX idx_enttag_last_seen ON entity_tags (last_seen); +CREATE INDEX idx_enttag_updated_at ON entity_tags (updated_at); CREATE INDEX idx_enttag_entity_id ON entity_tags (entity_id); CREATE TABLE IF NOT EXISTS edges( edge_id INT GENERATED ALWAYS AS IDENTITY, created_at TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, - last_seen TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, etype VARCHAR(255), content JSONB, from_entity_id INT, @@ -48,14 +48,14 @@ CREATE TABLE IF NOT EXISTS edges( ON DELETE CASCADE ); -CREATE INDEX idx_edge_last_seen ON edges (last_seen); +CREATE INDEX idx_edge_updated_at ON edges (updated_at); CREATE INDEX idx_edge_from_entity_id ON edges (from_entity_id); CREATE INDEX idx_edge_to_entity_id ON edges (to_entity_id); CREATE TABLE IF NOT EXISTS edge_tags( tag_id INT GENERATED ALWAYS AS IDENTITY, created_at TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, - last_seen TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP without time zone DEFAULT CURRENT_TIMESTAMP, ttype VARCHAR(255), content JSONB, edge_id INT, @@ -66,24 +66,24 @@ CREATE TABLE IF NOT EXISTS edge_tags( ON DELETE CASCADE ); -CREATE INDEX idx_edgetag_last_seen ON edge_tags (last_seen); +CREATE INDEX idx_edgetag_updated_at ON edge_tags (updated_at); CREATE INDEX idx_edgetag_edge_id ON edge_tags (edge_id); -- +migrate Down DROP INDEX IF EXISTS idx_edgetag_edge_id; -DROP INDEX IF EXISTS idx_edgetag_last_seen; +DROP INDEX IF EXISTS idx_edgetag_updated_at; DROP TABLE edge_tags; DROP INDEX IF EXISTS idx_edge_to_entity_id; DROP INDEX IF EXISTS idx_edge_from_entity_id; -DROP INDEX IF EXISTS idx_edge_last_seen; +DROP INDEX IF EXISTS idx_edge_updated_at; DROP TABLE edges; DROP INDEX IF EXISTS idx_enttag_entity_id; -DROP INDEX IF EXISTS idx_enttag_last_seen; +DROP INDEX IF EXISTS idx_enttag_updated_at; DROP TABLE entity_tags; DROP INDEX IF EXISTS idx_entities_etype; -DROP INDEX IF EXISTS idx_entities_last_seen; +DROP INDEX IF EXISTS idx_entities_updated_at; DROP TABLE entities; diff --git a/migrations/sqlite3/001_schema_init.sql b/migrations/sqlite3/001_schema_init.sql index 4e72627..a5450d9 100644 --- a/migrations/sqlite3/001_schema_init.sql +++ b/migrations/sqlite3/001_schema_init.sql @@ -6,18 +6,18 @@ PRAGMA foreign_keys = ON; CREATE TABLE IF NOT EXISTS entities( entity_id INTEGER PRIMARY KEY, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - last_seen DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, etype TEXT, content TEXT ); -CREATE INDEX idx_entities_last_seen ON entities (last_seen); +CREATE INDEX idx_entities_updated_at ON entities (updated_at); CREATE INDEX idx_entities_etype ON entities (etype); CREATE TABLE IF NOT EXISTS entity_tags( tag_id INTEGER PRIMARY KEY, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - last_seen DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, ttype TEXT, content TEXT, entity_id INTEGER, @@ -26,13 +26,13 @@ CREATE TABLE IF NOT EXISTS entity_tags( ON DELETE CASCADE ); -CREATE INDEX idx_enttag_last_seen ON entity_tags (last_seen); +CREATE INDEX idx_enttag_updated_at ON entity_tags (updated_at); CREATE INDEX idx_enttag_entity_id ON entity_tags (entity_id); CREATE TABLE IF NOT EXISTS edges( edge_id INTEGER PRIMARY KEY, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - last_seen DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, etype TEXT, content TEXT, from_entity_id INTEGER, @@ -45,14 +45,14 @@ CREATE TABLE IF NOT EXISTS edges( ON DELETE CASCADE ); -CREATE INDEX idx_edge_last_seen ON edges (last_seen); +CREATE INDEX idx_edge_updated_at ON edges (updated_at); CREATE INDEX idx_edge_from_entity_id ON edges (from_entity_id); CREATE INDEX idx_edge_to_entity_id ON edges (to_entity_id); CREATE TABLE IF NOT EXISTS edge_tags( tag_id INTEGER PRIMARY KEY, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - last_seen DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, ttype TEXT, content TEXT, edge_id INTEGER, @@ -61,24 +61,24 @@ CREATE TABLE IF NOT EXISTS edge_tags( ON DELETE CASCADE ); -CREATE INDEX idx_edgetag_last_seen ON edge_tags (last_seen); +CREATE INDEX idx_edgetag_updated_at ON edge_tags (updated_at); CREATE INDEX idx_edgetag_edge_id ON edge_tags (edge_id); -- +migrate Down DROP INDEX IF EXISTS idx_edgetag_edge_id; -DROP INDEX IF EXISTS idx_edgetag_last_seen; +DROP INDEX IF EXISTS idx_edgetag_updated_at; DROP TABLE edge_tags; DROP INDEX IF EXISTS idx_edge_to_entity_id; DROP INDEX IF EXISTS idx_edge_from_entity_id; -DROP INDEX IF EXISTS idx_edge_last_seen; +DROP INDEX IF EXISTS idx_edge_updated_at; DROP TABLE edges; -DROP INDEX IF EXISTS idx_enttag_last_seen; -DROP INDEX IF EXISTS idx_entprop_last_seen; +DROP INDEX IF EXISTS idx_enttag_entity_id; +DROP INDEX IF EXISTS idx_enttag_updated_at; DROP TABLE entity_tags; DROP INDEX IF EXISTS idx_entities_etype; -DROP INDEX IF EXISTS idx_entities_last_seen; +DROP INDEX IF EXISTS idx_entities_updated_at; DROP TABLE entities; diff --git a/repository/sqlrepo/edge.go b/repository/sqlrepo/edge.go index 39c1ce9..78fc252 100644 --- a/repository/sqlrepo/edge.go +++ b/repository/sqlrepo/edge.go @@ -52,13 +52,17 @@ func (sql *sqlRepository) CreateEdge(edge *types.Edge) (*types.Edge, error) { } r := Edge{ - CreatedAt: edge.CreatedAt, - LastSeen: edge.LastSeen, Type: string(edge.Relation.RelationType()), Content: jsonContent, FromEntityID: fromEntityId, ToEntityID: toEntityId, } + if !edge.CreatedAt.IsZero() { + r.CreatedAt = edge.CreatedAt.UTC() + } + if !edge.LastSeen.IsZero() { + r.UpdatedAt = edge.LastSeen.UTC() + } result := sql.db.Create(&r) if err := result.Error; err != nil { @@ -94,10 +98,10 @@ func (sql *sqlRepository) isDuplicateEdge(edge *types.Edge) (*types.Edge, bool) func (sql *sqlRepository) edgeSeen(rel *types.Edge) error { id, err := strconv.ParseInt(rel.ID, 10, 64) if err != nil { - return fmt.Errorf("failed to update last seen for ID %s could not parse id; err: %w", rel.ID, err) + return fmt.Errorf("failed to update updated_at for ID %s could not parse id; err: %w", rel.ID, err) } - result := sql.db.Exec("UPDATE edges SET last_seen = current_timestamp WHERE edge_id = ?", id) + result := sql.db.Exec("UPDATE edges SET updated_at = current_timestamp WHERE edge_id = ?", id) if err := result.Error; err != nil { return err } @@ -130,7 +134,7 @@ func (sql *sqlRepository) IncomingEdges(entity *types.Entity, since time.Time, l if since.IsZero() { result = sql.db.Where("to_entity_id = ?", entityId).Find(&edges) } else { - result = sql.db.Where("to_entity_id = ? AND last_seen >= ?", entityId, since.UTC()).Find(&edges) + result = sql.db.Where("to_entity_id = ? AND updated_at >= ?", entityId, since.UTC()).Find(&edges) } if err := result.Error; err != nil { return nil, err @@ -171,7 +175,7 @@ func (sql *sqlRepository) OutgoingEdges(entity *types.Entity, since time.Time, l if since.IsZero() { result = sql.db.Where("from_entity_id = ?", entityId).Find(&edges) } else { - result = sql.db.Where("from_entity_id = ? AND last_seen >= ?", entityId, since.UTC()).Find(&edges) + result = sql.db.Where("from_entity_id = ? AND updated_at >= ?", entityId, since.UTC()).Find(&edges) } if err := result.Error; err != nil { return nil, err @@ -224,8 +228,8 @@ func toEdge(r Edge) *types.Edge { return &types.Edge{ ID: strconv.FormatUint(r.ID, 10), - CreatedAt: r.CreatedAt, - LastSeen: r.LastSeen, + CreatedAt: r.CreatedAt.In(time.UTC).Local(), + LastSeen: r.UpdatedAt.In(time.UTC).Local(), Relation: rel, FromEntity: &types.Entity{ ID: strconv.FormatUint(r.FromEntityID, 10), diff --git a/repository/sqlrepo/edge_test.go b/repository/sqlrepo/edge_test.go index a0e0f10..7267e34 100644 --- a/repository/sqlrepo/edge_test.go +++ b/repository/sqlrepo/edge_test.go @@ -20,12 +20,12 @@ func TestUnfilteredRelations(t *testing.T) { source := domain.FQDN{Name: "owasp.com"} dest1 := domain.FQDN{Name: "www.example.owasp.org"} - sourceEntity, err := store.CreateEntity(source) + sourceEntity, err := store.CreateAsset(source) if err != nil { t.Fatalf("failed to create asset: %s", err) } - dest1Entity, err := store.CreateEntity(dest1) + dest1Entity, err := store.CreateAsset(dest1) if err != nil { t.Fatalf("failed to create asset: %s", err) } @@ -46,7 +46,7 @@ func TestUnfilteredRelations(t *testing.T) { ip, _ := netip.ParseAddr("192.168.1.100") dest2 := network.IPAddress{Address: ip, Type: "IPv4"} - dest2Entity, err := store.CreateEntity(dest2) + dest2Entity, err := store.CreateAsset(dest2) if err != nil { t.Fatalf("failed to create asset: %s", err) } @@ -64,9 +64,9 @@ func TestUnfilteredRelations(t *testing.T) { ToEntity: dest2Entity, } - _, err = store.Link(edge1) + _, err = store.CreateEdge(edge1) assert.NoError(t, err) - r2Rel, err := store.Link(edge2) + r2Rel, err := store.CreateEdge(edge2) assert.NoError(t, err) // Outgoing relations with no filter returns all outgoing relations. @@ -102,7 +102,7 @@ func TestUnfilteredRelations(t *testing.T) { time.Sleep(1000 * time.Millisecond) // Store a duplicate relation and validate last_seen is updated - rr, err := store.Link(edge2) + rr, err := store.CreateEdge(edge2) assert.NoError(t, err) assert.NotNil(t, rr) if rr.LastSeen.UnixNano() <= r2Rel.LastSeen.UnixNano() { diff --git a/repository/sqlrepo/entity.go b/repository/sqlrepo/entity.go index f61fa1f..844552b 100644 --- a/repository/sqlrepo/entity.go +++ b/repository/sqlrepo/entity.go @@ -25,29 +25,26 @@ func (sql *sqlRepository) CreateEntity(input *types.Entity) (*types.Entity, erro } entity := Entity{ - CreatedAt: input.CreatedAt, - LastSeen: input.LastSeen, - Type: string(input.Asset.AssetType()), - Content: jsonContent, + Type: string(input.Asset.AssetType()), + Content: jsonContent, } // ensure that duplicate entities are not entered into the database - if entities, err := sql.FindEntityByContent(input.Asset, time.Time{}); err == nil && len(entities) > 0 { - for _, e := range entities { - if input.Asset.AssetType() == e.Asset.AssetType() { - if id, err := strconv.ParseUint(e.ID, 10, 64); err == nil { - entity.ID = id - entity.CreatedAt = e.CreatedAt - - if sql.UpdateEntityLastSeen(e.ID) == nil { - if f, err := sql.FindEntityById(e.ID); err == nil && f != nil { - entity.LastSeen = f.LastSeen - break - } - } - } + if entities, err := sql.FindEntityByContent(input.Asset, time.Time{}); err == nil && len(entities) == 1 { + e := entities[0] + + if input.Asset.AssetType() == e.Asset.AssetType() { + if id, err := strconv.ParseUint(e.ID, 10, 64); err == nil { + entity.ID = id } } + } else { + if !input.CreatedAt.IsZero() { + entity.CreatedAt = input.CreatedAt.UTC() + } + if !input.LastSeen.IsZero() { + entity.UpdatedAt = input.LastSeen.UTC() + } } result := sql.db.Save(&entity) @@ -57,8 +54,8 @@ func (sql *sqlRepository) CreateEntity(input *types.Entity) (*types.Entity, erro return &types.Entity{ ID: strconv.FormatUint(entity.ID, 10), - CreatedAt: entity.CreatedAt, - LastSeen: entity.LastSeen, + CreatedAt: entity.CreatedAt.In(time.UTC).Local(), + LastSeen: entity.UpdatedAt.In(time.UTC).Local(), Asset: input.Asset, }, nil } @@ -73,7 +70,7 @@ func (sql *sqlRepository) CreateAsset(asset oam.Asset) (*types.Entity, error) { // UpdateEntityLastSeen performs an update on the entity. func (sql *sqlRepository) UpdateEntityLastSeen(id string) error { - result := sql.db.Exec("UPDATE entities SET last_seen = current_timestamp WHERE entity_id = ?", id) + result := sql.db.Exec("UPDATE entities SET updated_at = current_timestamp WHERE entity_id = ?", id) if err := result.Error; err != nil { return err } @@ -113,10 +110,6 @@ func (sql *sqlRepository) FindEntityByContent(assetData oam.Asset, since time.Ti Content: jsonContent, } - if !since.IsZero() { - entity.LastSeen = since - } - jsonQuery, err := entity.JSONQuery() if err != nil { return nil, err @@ -127,7 +120,7 @@ func (sql *sqlRepository) FindEntityByContent(assetData oam.Asset, since time.Ti if since.IsZero() { result = sql.db.Where("etype = ?", entity.Type).Find(&entities, jsonQuery) } else { - result = sql.db.Where("etype = ? AND last_seen >= ?", entity.Type, since.UTC()).Find(&entities, jsonQuery) + result = sql.db.Where("etype = ? AND updated_at >= ?", entity.Type, since.UTC()).Find(&entities, jsonQuery) } if err := result.Error; err != nil { return nil, err @@ -142,8 +135,8 @@ func (sql *sqlRepository) FindEntityByContent(assetData oam.Asset, since time.Ti storedEntities = append(storedEntities, &types.Entity{ ID: strconv.FormatUint(e.ID, 10), - CreatedAt: e.CreatedAt, - LastSeen: e.LastSeen, + CreatedAt: e.CreatedAt.In(time.UTC).Local(), + LastSeen: e.UpdatedAt.In(time.UTC).Local(), Asset: assetData, }) } @@ -173,8 +166,8 @@ func (sql *sqlRepository) FindEntityById(id string) (*types.Entity, error) { return &types.Entity{ ID: strconv.FormatUint(entity.ID, 10), - CreatedAt: entity.CreatedAt, - LastSeen: entity.LastSeen, + CreatedAt: entity.CreatedAt.In(time.UTC).Local(), + LastSeen: entity.UpdatedAt.In(time.UTC).Local(), Asset: assetData, }, nil } @@ -190,7 +183,7 @@ func (sql *sqlRepository) FindEntitiesByType(atype oam.AssetType, since time.Tim if since.IsZero() { result = sql.db.Where("etype = ?", atype).Find(&entities) } else { - result = sql.db.Where("etype = ? AND last_seen >= ?", atype, since.UTC()).Find(&entities) + result = sql.db.Where("etype = ? AND updated_at >= ?", atype, since.UTC()).Find(&entities) } if err := result.Error; err != nil { return nil, err @@ -201,8 +194,8 @@ func (sql *sqlRepository) FindEntitiesByType(atype oam.AssetType, since time.Tim if f, err := e.Parse(); err == nil { results = append(results, &types.Entity{ ID: strconv.FormatUint(e.ID, 10), - CreatedAt: e.CreatedAt, - LastSeen: e.LastSeen, + CreatedAt: e.CreatedAt.In(time.UTC).Local(), + LastSeen: e.UpdatedAt.In(time.UTC).Local(), Asset: f, }) } diff --git a/repository/sqlrepo/entity_test.go b/repository/sqlrepo/entity_test.go index 9ef8737..141e02e 100644 --- a/repository/sqlrepo/entity_test.go +++ b/repository/sqlrepo/entity_test.go @@ -176,13 +176,13 @@ func TestMain(m *testing.M) { func TestLastSeenUpdates(t *testing.T) { ip, _ := netip.ParseAddr("45.73.25.1") asset := &network.IPAddress{Address: ip, Type: "IPv4"} - a1, err := store.CreateEntity(asset) + a1, err := store.CreateAsset(asset) assert.NoError(t, err) // Nanoseconds are truncated by the database, so we need to sleep for a bit. time.Sleep(1000 * time.Millisecond) - a2, err := store.CreateEntity(asset) + a2, err := store.CreateAsset(asset) assert.NoError(t, err) assert.Equal(t, a1.ID, a2.ID) assert.Equal(t, a1.CreatedAt, a2.CreatedAt) @@ -239,7 +239,7 @@ func TestRepository(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - sourceEntity, err := store.CreateEntity(tc.sourceAsset) + sourceEntity, err := store.CreateAsset(tc.sourceAsset) assert.NoError(t, err) assert.NotEqual(t, sourceEntity, nil) @@ -298,7 +298,7 @@ func TestRepository(t *testing.T) { t.Fatalf("failed to find entity by type: did not receive entity %s", sourceEntity.Asset) } - destinationEntity, err := store.CreateEntity(tc.destinationAsset) + destinationEntity, err := store.CreateAsset(tc.destinationAsset) assert.NoError(t, err) assert.NotEqual(t, destinationEntity, nil) @@ -308,7 +308,7 @@ func TestRepository(t *testing.T) { ToEntity: destinationEntity, } - e, err := store.Link(edge) + e, err := store.CreateEdge(edge) assert.NoError(t, err) assert.NotEqual(t, e, nil) diff --git a/repository/sqlrepo/models.go b/repository/sqlrepo/models.go index b858e7b..422738b 100644 --- a/repository/sqlrepo/models.go +++ b/repository/sqlrepo/models.go @@ -29,7 +29,7 @@ import ( type Entity struct { ID uint64 `gorm:"primaryKey;column:entity_id"` CreatedAt time.Time `gorm:"type:datetime;default:CURRENT_TIMESTAMP();column:created_at"` - LastSeen time.Time `gorm:"type:datetime;default:CURRENT_TIMESTAMP();column:last_seen"` + UpdatedAt time.Time `gorm:"type:datetime;default:CURRENT_TIMESTAMP();column:updated_at"` Type string `gorm:"column:etype"` Content datatypes.JSON } @@ -38,7 +38,7 @@ type Entity struct { type EntityTag struct { ID uint64 `gorm:"primaryKey;column:tag_id"` CreatedAt time.Time `gorm:"type:datetime;default:CURRENT_TIMESTAMP();column:created_at"` - LastSeen time.Time `gorm:"type:datetime;default:CURRENT_TIMESTAMP();column:last_seen"` + UpdatedAt time.Time `gorm:"type:datetime;default:CURRENT_TIMESTAMP();column:updated_at"` Type string `gorm:"column:ttype"` Content datatypes.JSON EntityID uint64 `gorm:"column:entity_id"` @@ -48,7 +48,7 @@ type EntityTag struct { type Edge struct { ID uint64 `gorm:"primaryKey;column:edge_id"` CreatedAt time.Time `gorm:"type:datetime;default:CURRENT_TIMESTAMP();column:created_at"` - LastSeen time.Time `gorm:"type:datetime;default:CURRENT_TIMESTAMP();column:last_seen"` + UpdatedAt time.Time `gorm:"type:datetime;default:CURRENT_TIMESTAMP();column:updated_at"` Type string `gorm:"column:etype"` Content datatypes.JSON FromEntityID uint64 `gorm:"column:from_entity_id"` @@ -61,7 +61,7 @@ type Edge struct { type EdgeTag struct { ID uint64 `gorm:"primaryKey;column:tag_id"` CreatedAt time.Time `gorm:"type:datetime;default:CURRENT_TIMESTAMP();column:created_at"` - LastSeen time.Time `gorm:"type:datetime;default:CURRENT_TIMESTAMP();column:last_seen"` + UpdatedAt time.Time `gorm:"type:datetime;default:CURRENT_TIMESTAMP();column:updated_at"` Type string `gorm:"column:ttype"` Content datatypes.JSON EdgeID uint64 `gorm:"column:edge_id"` diff --git a/repository/sqlrepo/tag.go b/repository/sqlrepo/tag.go index e8cd953..161dc9b 100644 --- a/repository/sqlrepo/tag.go +++ b/repository/sqlrepo/tag.go @@ -29,11 +29,9 @@ func (sql *sqlRepository) CreateEntityTag(entity *types.Entity, input *types.Ent } tag := EntityTag{ - CreatedAt: input.CreatedAt, - LastSeen: input.LastSeen, - Type: string(input.Property.PropertyType()), - Content: jsonContent, - EntityID: entityid, + Type: string(input.Property.PropertyType()), + Content: jsonContent, + EntityID: entityid, } // ensure that duplicate entity tags are not entered into the database @@ -42,17 +40,17 @@ func (sql *sqlRepository) CreateEntityTag(entity *types.Entity, input *types.Ent if input.Property.PropertyType() == t.Property.PropertyType() && input.Property.Value() == t.Property.Value() { if id, err := strconv.ParseUint(t.ID, 10, 64); err == nil { tag.ID = id - tag.CreatedAt = t.CreatedAt - - if sql.UpdateEntityTagLastSeen(t.ID) == nil { - if f, err := sql.FindEntityTagById(t.ID); err == nil && f != nil { - tag.LastSeen = f.LastSeen - break - } - } + break } } } + } else { + if !input.CreatedAt.IsZero() { + tag.CreatedAt = input.CreatedAt.UTC() + } + if !input.LastSeen.IsZero() { + tag.UpdatedAt = input.LastSeen.UTC() + } } result := sql.db.Save(&tag) @@ -62,8 +60,8 @@ func (sql *sqlRepository) CreateEntityTag(entity *types.Entity, input *types.Ent return &types.EntityTag{ ID: strconv.FormatUint(tag.ID, 10), - CreatedAt: tag.CreatedAt, - LastSeen: tag.LastSeen, + CreatedAt: tag.CreatedAt.In(time.UTC).Local(), + LastSeen: tag.UpdatedAt.In(time.UTC).Local(), Property: input.Property, Entity: entity, }, nil @@ -79,7 +77,7 @@ func (sql *sqlRepository) CreateEntityProperty(entity *types.Entity, prop oam.Pr // UpdateEntityTagLastSeen performs an update on the entity tag. func (sql *sqlRepository) UpdateEntityTagLastSeen(id string) error { - result := sql.db.Exec("UPDATE entity_tags SET last_seen = current_timestamp WHERE tag_id = ?", id) + result := sql.db.Exec("UPDATE entity_tags SET updated_at = current_timestamp WHERE tag_id = ?", id) if err := result.Error; err != nil { return err } @@ -108,8 +106,8 @@ func (sql *sqlRepository) FindEntityTagById(id string) (*types.EntityTag, error) return &types.EntityTag{ ID: strconv.FormatUint(tag.ID, 10), - CreatedAt: tag.CreatedAt, - LastSeen: tag.LastSeen, + CreatedAt: tag.CreatedAt.In(time.UTC).Local(), + LastSeen: tag.UpdatedAt.In(time.UTC).Local(), Property: data, Entity: &types.Entity{ID: strconv.FormatUint(tag.EntityID, 10)}, }, nil @@ -129,7 +127,7 @@ func (sql *sqlRepository) GetEntityTags(entity *types.Entity, since time.Time, n if since.IsZero() { result = sql.db.Where("entity_id = ?", entityId).Find(&tags) } else { - result = sql.db.Where("entity_id = ? AND last_seen >= ?", entityId, since.UTC()).Find(&tags) + result = sql.db.Where("entity_id = ? AND updated_at >= ?", entityId, since.UTC()).Find(&tags) } if err := result.Error; err != nil { return nil, err @@ -157,8 +155,8 @@ func (sql *sqlRepository) GetEntityTags(entity *types.Entity, since time.Time, n if found { results = append(results, &types.EntityTag{ ID: strconv.Itoa(int(t.ID)), - CreatedAt: t.CreatedAt, - LastSeen: t.LastSeen, + CreatedAt: t.CreatedAt.In(time.UTC).Local(), + LastSeen: t.UpdatedAt.In(time.UTC).Local(), Property: prop, Entity: entity, }) @@ -202,11 +200,9 @@ func (sql *sqlRepository) CreateEdgeTag(edge *types.Edge, input *types.EdgeTag) } tag := EdgeTag{ - CreatedAt: input.CreatedAt, - LastSeen: input.LastSeen, - Type: string(input.Property.PropertyType()), - Content: jsonContent, - EdgeID: edgeid, + Type: string(input.Property.PropertyType()), + Content: jsonContent, + EdgeID: edgeid, } // ensure that duplicate edge tags are not entered into the database @@ -215,17 +211,17 @@ func (sql *sqlRepository) CreateEdgeTag(edge *types.Edge, input *types.EdgeTag) if input.Property.PropertyType() == t.Property.PropertyType() && input.Property.Value() == t.Property.Value() { if id, err := strconv.ParseUint(t.ID, 10, 64); err == nil { tag.ID = id - tag.CreatedAt = t.CreatedAt - - if sql.UpdateEdgeTagLastSeen(t.ID) == nil { - if f, err := sql.FindEdgeTagById(t.ID); err == nil && f != nil { - tag.LastSeen = f.LastSeen - break - } - } + break } } } + } else { + if !input.CreatedAt.IsZero() { + tag.CreatedAt = input.CreatedAt.UTC() + } + if !input.LastSeen.IsZero() { + tag.UpdatedAt = input.LastSeen.UTC() + } } result := sql.db.Save(&tag) @@ -235,8 +231,8 @@ func (sql *sqlRepository) CreateEdgeTag(edge *types.Edge, input *types.EdgeTag) return &types.EdgeTag{ ID: strconv.FormatUint(tag.ID, 10), - CreatedAt: tag.CreatedAt, - LastSeen: tag.LastSeen, + CreatedAt: tag.CreatedAt.In(time.UTC).Local(), + LastSeen: tag.UpdatedAt.In(time.UTC).Local(), Property: input.Property, Edge: edge, }, nil @@ -252,7 +248,7 @@ func (sql *sqlRepository) CreateEdgeProperty(edge *types.Edge, prop oam.Property // UpdateEdgeTagLastSeen performs an update on the edge tag. func (sql *sqlRepository) UpdateEdgeTagLastSeen(id string) error { - result := sql.db.Exec("UPDATE edge_tags SET last_seen = current_timestamp WHERE tag_id = ?", id) + result := sql.db.Exec("UPDATE edge_tags SET updated_at = current_timestamp WHERE tag_id = ?", id) if err := result.Error; err != nil { return err } @@ -286,8 +282,8 @@ func (sql *sqlRepository) FindEdgeTagById(id string) (*types.EdgeTag, error) { return &types.EdgeTag{ ID: strconv.FormatUint(tag.ID, 10), - CreatedAt: tag.CreatedAt, - LastSeen: tag.LastSeen, + CreatedAt: tag.CreatedAt.In(time.UTC).Local(), + LastSeen: tag.UpdatedAt.In(time.UTC).Local(), Property: data, Edge: edge, }, nil @@ -307,7 +303,7 @@ func (sql *sqlRepository) GetEdgeTags(edge *types.Edge, since time.Time, names . if since.IsZero() { result = sql.db.Where("edge_id = ?", edgeId).Find(&tags) } else { - result = sql.db.Where("edge_id = ? AND last_seen >= ?", edgeId, since.UTC()).Find(&tags) + result = sql.db.Where("edge_id = ? AND updated_at >= ?", edgeId, since.UTC()).Find(&tags) } if err := result.Error; err != nil { return nil, err @@ -335,8 +331,8 @@ func (sql *sqlRepository) GetEdgeTags(edge *types.Edge, since time.Time, names . if found { results = append(results, &types.EdgeTag{ ID: strconv.Itoa(int(t.ID)), - CreatedAt: t.CreatedAt, - LastSeen: t.LastSeen, + CreatedAt: t.CreatedAt.In(time.UTC).Local(), + LastSeen: t.UpdatedAt.In(time.UTC).Local(), Property: prop, Edge: edge, }) diff --git a/repository/sqlrepo/tag_test.go b/repository/sqlrepo/tag_test.go index d9d72b1..1089ef0 100644 --- a/repository/sqlrepo/tag_test.go +++ b/repository/sqlrepo/tag_test.go @@ -17,24 +17,24 @@ import ( ) func TestEntityTag(t *testing.T) { - entity, err := store.CreateEntity(&domain.FQDN{Name: "utica.edu"}) + entity, err := store.CreateAsset(&domain.FQDN{Name: "utica.edu"}) assert.NoError(t, err) - now := time.Now().Truncate(time.Second).UTC() + now := time.Now().Truncate(time.Second) prop := &property.SimpleProperty{ PropertyName: "test", PropertyValue: "foo", } - ct, err := store.CreateEntityTag(entity, prop) + ct, err := store.CreateEntityProperty(entity, prop) assert.NoError(t, err) assert.Equal(t, ct.Property.Name(), prop.PropertyName) assert.Equal(t, ct.Property.Value(), prop.PropertyValue) assert.Equal(t, oam.SimpleProperty, ct.Property.PropertyType()) - if now.After(ct.CreatedAt.UTC()) { + if now.After(ct.CreatedAt) { t.Errorf("tag.CreatedAt: %s, expected to be after: %s", ct.CreatedAt.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)) } - if now.After(ct.LastSeen.UTC()) { + if now.After(ct.LastSeen) { t.Errorf("tag.LastSeen: %s, expected to be after: %s", ct.LastSeen.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)) } @@ -46,7 +46,7 @@ func TestEntityTag(t *testing.T) { assert.Equal(t, ct.Property.Value(), tag.Property.Value()) time.Sleep(time.Second) - ct2, err := store.CreateEntityTag(entity, prop) + ct2, err := store.CreateEntityProperty(entity, prop) assert.NoError(t, err) if ct2.LastSeen.UnixNano() < ct.LastSeen.UnixNano() { t.Errorf("ct2.LastSeen: %s, ct.LastSeen: %s", ct2.LastSeen.Format(time.RFC3339Nano), ct.LastSeen.Format(time.RFC3339Nano)) @@ -54,7 +54,7 @@ func TestEntityTag(t *testing.T) { time.Sleep(time.Second) prop.PropertyValue = "bar" - ct3, err := store.CreateEntityTag(entity, prop) + ct3, err := store.CreateEntityProperty(entity, prop) assert.NoError(t, err) assert.Equal(t, ct3.Property.Value(), prop.PropertyValue) if ct3.CreatedAt.UnixNano() < ct2.CreatedAt.UnixNano() { @@ -84,13 +84,13 @@ func TestEntityTag(t *testing.T) { } func TestEdgeTag(t *testing.T) { - e1, err := store.CreateEntity(&domain.FQDN{Name: "owasp.org"}) + e1, err := store.CreateAsset(&domain.FQDN{Name: "owasp.org"}) assert.NoError(t, err) - e2, err := store.CreateEntity(&domain.FQDN{Name: "www.owasp.org"}) + e2, err := store.CreateAsset(&domain.FQDN{Name: "www.owasp.org"}) assert.NoError(t, err) - edge, err := store.Link(&types.Edge{ + edge, err := store.CreateEdge(&types.Edge{ Relation: &relation.BasicDNSRelation{ Name: "dns_record", Header: relation.RRHeader{RRType: 5}, @@ -100,21 +100,21 @@ func TestEdgeTag(t *testing.T) { }) assert.NoError(t, err) - now := time.Now().Truncate(time.Second).UTC() + now := time.Now().Truncate(time.Second) prop := &property.SimpleProperty{ PropertyName: "test", PropertyValue: "foo", } - ct, err := store.CreateEdgeTag(edge, prop) + ct, err := store.CreateEdgeProperty(edge, prop) assert.NoError(t, err) assert.Equal(t, ct.Property.Name(), prop.PropertyName) assert.Equal(t, ct.Property.Value(), prop.PropertyValue) assert.Equal(t, oam.SimpleProperty, ct.Property.PropertyType()) - if now.After(ct.CreatedAt.UTC()) { + if now.After(ct.CreatedAt) { t.Errorf("tag.CreatedAt: %s, expected to be after: %s", ct.CreatedAt.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)) } - if now.After(ct.LastSeen.UTC()) { + if now.After(ct.LastSeen) { t.Errorf("tag.LastSeen: %s, expected to be after: %s", ct.LastSeen.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)) } @@ -126,7 +126,7 @@ func TestEdgeTag(t *testing.T) { assert.Equal(t, ct.Property.Value(), tag.Property.Value()) time.Sleep(time.Second) - ct2, err := store.CreateEdgeTag(edge, prop) + ct2, err := store.CreateEdgeProperty(edge, prop) assert.NoError(t, err) if ct2.LastSeen.UnixNano() < ct.LastSeen.UnixNano() { t.Errorf("ct2.LastSeen: %s, ct.LastSeen: %s", ct2.LastSeen.Format(time.RFC3339Nano), ct.LastSeen.Format(time.RFC3339Nano)) @@ -134,7 +134,7 @@ func TestEdgeTag(t *testing.T) { time.Sleep(time.Second) prop.PropertyValue = "bar" - ct3, err := store.CreateEdgeTag(edge, prop) + ct3, err := store.CreateEdgeProperty(edge, prop) assert.NoError(t, err) assert.Equal(t, ct3.Property.Value(), prop.PropertyValue) if ct3.CreatedAt.UnixNano() < ct2.CreatedAt.UnixNano() {