diff --git a/project.clj b/project.clj index 393e67d284..8be7114416 100644 --- a/project.clj +++ b/project.clj @@ -210,7 +210,8 @@ "git" "symbolic-ref" "--short" "HEAD")))})}] - :profiles {:dev {:dependencies [[puppetlabs/trapperkeeper ~trapperkeeper-version + :profiles {:dev {:jvm-opts ["-XX:-OmitStackTraceInFastThrow"] + :dependencies [[puppetlabs/trapperkeeper ~trapperkeeper-version :classifier "test"] [puppetlabs/kitchensink ~trapperkeeper-version :classifier "test"] @@ -228,7 +229,8 @@ :global-vars {*warn-on-reflection* true} :jvm-opts [ ;; actually print stack traces instead of useless ;; "Full report at: /tmp/clojure-8187773283812483853.edn" - "-Dclojure.main.report=stderr"]} + "-Dclojure.main.report=stderr" + "-XX:-OmitStackTraceInFastThrow"]} :next-clojure {:dependencies [[org.clojure/clojure "1.12.0-master-SNAPSHOT"]] :repositories [["snapshots" "https://oss.sonatype.org/content/repositories/snapshots/"]]} :jmx {:jvm-opts ["-Dcom.sun.management.jmxremote" diff --git a/resources/ctia/public/doc/bulk-bundle.org b/resources/ctia/public/doc/bulk-bundle.org index a6c6b338ba..702588f997 100644 --- a/resources/ctia/public/doc/bulk-bundle.org +++ b/resources/ctia/public/doc/bulk-bundle.org @@ -667,10 +667,18 @@ When a bundle is submitted: 1. All entities that have already been imported with the external ID whose prefix has been configured with the `ctia.store.external-key-prefixes` property are searched. 2. If they are identified by transient IDs, a mapping table between transient and stored IDs is built. -3. Only new entities are created in the same way as the `/bulk` API endpoint with transient IDs resolutions. Existing entities are not modified. +3. New entities are created in the same way as the `/bulk` API endpoint with transient IDs and external ids resolutions. If query parameter `patch-existing=true`, then existing entities are similarly patched with `result=updated`; otherwise, existing entities are not modified with `result=exists`. If more than one entity is referenced by the same external ID, an error is reported. +If provided query parameter `asset_properties-merge-strategy=merge-overriding-previous`, then when patching asset properties, +existing asset properties will be retrieved and combined with the asset properties in the request bundle +as if by concatenating existing and new properties together in a single list, +removing properties to the left of a property with the same name, +then sorting the list lexicographically by name before using this list to patch the existing entity. + +If `asset_properties-merge-strategy=ignore-existing`, then asset properties will be patched to their new values as they appear in the request bundle. + Example for an incident along with its context, note the transient ids #+begin_src HTTP POST /ctia/bundle/import HTTP/1.1 @@ -778,7 +786,7 @@ accept: application/json |----------------+----------------------------------------------------------| | `:id` | The real ID | | `:original_id` | Provided ID if different from real ID (ex: transient ID) | -| `:result` | `error`, `created` or `exists` | +| `:result` | `error`, `created`, `exists` or `updated` | | `:external_id` | External ID used to identify the entity | | `:error` | Error message | diff --git a/src/ctia/auth.clj b/src/ctia/auth.clj index fbf2c4a5de..ae511cf904 100644 --- a/src/ctia/auth.clj +++ b/src/ctia/auth.clj @@ -39,13 +39,16 @@ (def denied-identity-singleton (->DeniedIdentity)) +(s/defschema AuthIdentity + (s/protocol IIdentity)) + (s/defschema IdentityMap {:client-id (s/maybe s/Str) :login (s/maybe s/Str) :groups [s/Str]}) (s/defn ident->map :- (s/maybe IdentityMap) - [ident] + [ident :- (s/maybe AuthIdentity)] (when ident {:login (login ident) :groups (groups ident) diff --git a/src/ctia/bulk/core.clj b/src/ctia/bulk/core.clj index f82819717c..4e9f6e00f3 100644 --- a/src/ctia/bulk/core.clj +++ b/src/ctia/bulk/core.clj @@ -42,7 +42,9 @@ (s/defn create-fn "return the create function provided an entity type key" - [k auth-identity params + [k + auth-identity :- auth/AuthIdentity + params {{:keys [get-store]} :StoreService} :- APIHandlerServices] #(-> (get-store k) (store/create-record @@ -50,9 +52,37 @@ (auth/ident->map auth-identity) params))) -(s/defn create-entities +(s/defschema EntitiesResult + [(s/conditional + string? schemas/ID + :else {(s/optional-key :error) (s/conditional + string? s/Str + :else {:type (s/conditional + string? s/Str + :else s/Keyword) + :reason s/Str + (s/optional-key :index) s/Str + (s/optional-key :index_uuid) s/Str}) + (s/optional-key :msg) s/Str + (s/optional-key :entity) (s/pred map?) + (s/optional-key :type) (s/conditional + string? s/Str + :else s/Keyword) + (s/optional-key :id) (s/maybe s/Str) + s/Keyword s/Any})]) + +(s/defschema EnvelopedEntities+TempIDs + (s/maybe + {:data EntitiesResult + (s/optional-key :tempids) TempIDs})) + +(s/defn create-entities :- EnvelopedEntities+TempIDs "Create many entities provided their type and returns a list of ids" - [new-entities entity-type tempids auth-identity params + [new-entities :- flows/Entities + entity-type :- s/Keyword + tempids :- TempIDs + auth-identity :- auth/AuthIdentity + params services :- APIHandlerServices] (when (seq new-entities) (let [{:keys [realize-fn new-spec]} (get (all-entities) entity-type)] @@ -82,9 +112,11 @@ (st/assoc s/Keyword s/Any)) s/Keyword s/Any}) -(s/defn read-entities +(s/defn read-entities :- [(s/maybe (s/pred map?))] "Retrieve many entities of the same type provided their ids and common type" - [ids entity-type auth-identity + [ids :- [s/Str] + entity-type :- s/Keyword + auth-identity :- auth/AuthIdentity {{:keys [get-store]} :StoreService :as services} :- ReadEntitiesServices] (let [store (get-store entity-type)] @@ -125,9 +157,19 @@ concat (map #(to-long-id % services) not-found))))) +(defn make-patch-bulk-enveloped-result + [fm] + (-> fm + flows/make-enveloped-result + (update :data #(mapv (fn [{:keys [error id] :as result}] + (if error result id)) + %)))) + (s/defn delete-fn "return the delete function provided an entity type key" - [k auth-identity params + [k + auth-identity :- auth/AuthIdentity + params {{:keys [get-store]} :StoreService} :- APIHandlerServices] #(-> (get-store k) (store/bulk-delete @@ -137,7 +179,9 @@ (s/defn update-fn "return the update function provided an entity type key" - [k auth-identity params + [k + auth-identity :- auth/AuthIdentity + params {{:keys [get-store]} :StoreService} :- APIHandlerServices] #(-> (get-store k) (store/bulk-update @@ -159,7 +203,10 @@ (s/defn delete-entities "delete many entities provided their type and returns a list of ids" - [entity-ids entity-type auth-identity params + [entity-ids + entity-type + auth-identity :- auth/AuthIdentity + params services :- APIHandlerServices] (when (seq entity-ids) (let [get-fn #(read-entities % entity-type auth-identity services)] @@ -176,7 +223,10 @@ (s/defn update-entities "update many entities provided their type and returns errored and successed entities' ids" - [entities entity-type auth-identity params + [entities + entity-type + auth-identity :- auth/AuthIdentity + params services :- APIHandlerServices] (when (seq entities) (let [get-fn #(read-entities % entity-type auth-identity services) @@ -196,24 +246,32 @@ (s/defn patch-entities "patch many entities provided their type and returns errored and successed entities' ids" - [patches entity-type auth-identity params - services :- APIHandlerServices] + [patches + entity-type + tempids :- TempIDs + auth-identity :- auth/AuthIdentity + params + services :- APIHandlerServices + {:keys [make-result]} :- {(s/optional-key :make-result) s/Any}] (when (seq patches) (let [get-fn #(read-entities % entity-type auth-identity services) {:keys [realize-fn new-spec]} (get (all-entities) entity-type)] (flows/patch-flow - :services services - :get-fn get-fn - :realize-fn realize-fn - :update-fn (update-fn entity-type auth-identity params services) - :long-id-fn #(with-long-id % services) - :entity-type entity-type - :identity auth-identity - :patch-operation :replace - :partial-entities patches - :spec new-spec - :make-result make-bulk-result - :get-success-entities (get-success-entities-fn :updated))))) + :services services + :get-fn get-fn + :realize-fn realize-fn + :update-fn (update-fn entity-type auth-identity params services) + :long-id-fn #(with-long-id % services) + :entity-type entity-type + :identity auth-identity + :patch-operation :replace + :partial-entities patches + :tempids tempids + :spec new-spec + :make-result (or make-result make-bulk-result) + :get-success-entities (get-success-entities-fn :updated))))) + +(s/defschema BulkEntities {s/Keyword flows/Entities}) (defn gen-bulk-from-fn "Kind of fmap but adapted for bulk @@ -237,7 +295,7 @@ (catch java.util.concurrent.ExecutionException e (throw (.getCause e))))) -(defn merge-tempids +(s/defn merge-tempids :- TempIDs "Merges tempids from all entities {:entity-type1 {:data [] :tempids {transientid1 id1 @@ -255,7 +313,7 @@ The create-entities set the enveloped-result? to True in the flow configuration to get :data and :tempids for each entity in the result." - [entities-by-type] + [entities-by-type :- {s/Keyword EnvelopedEntities+TempIDs}] (into {} (map (fn [[_ v]] (:tempids v))) entities-by-type)) @@ -278,15 +336,19 @@ (bad-request! (str "Bulk max number of entities: " (get-bulk-max-size get-in-config))))) +(s/defschema BulkRefs {s/Keyword EntitiesResult}) + (s/defschema BulkRefs+TempIDs - {:bulk-refs {s/Keyword [s/Any]} + {:bulk-refs BulkRefs :tempids TempIDs}) +(s/defschema BulkRefsAssocTempIDs + (st/assoc BulkRefs (s/optional-key :tempids) TempIDs)) + (s/defn import-bulks-with :- BulkRefs+TempIDs "Import each new-bulk in order while accumulating tempids." - [f :- (s/=> {s/Keyword {:data [s/Any] - :tempids TempIDs}} - (s/named (s/pred map?) 'new-bulk) + [f :- (s/=> {s/Keyword EnvelopedEntities+TempIDs} + BulkEntities TempIDs) new-bulks tempids :- TempIDs] @@ -300,7 +362,7 @@ :tempids tempids} new-bulks)) -(s/defn create-bulk +(s/defn create-bulk :- BulkRefsAssocTempIDs "Creates entities in bulk. To define relationships between entities, transient IDs can be used. They are automatically converted into real IDs. @@ -308,7 +370,7 @@ 1. Creates all entities except Relationships 2. Creates Relationships with mapping between transient and real IDs" ([new-bulk login services :- APIHandlerServices] (create-bulk new-bulk {} login {} services)) - ([new-bulk + ([new-bulk :- BulkEntities tempids :- TempIDs login params @@ -334,22 +396,40 @@ (seq tempids) (assoc :tempids tempids))))) (s/defn fetch-bulk - [bulk auth-identity + [bulk + auth-identity :- auth/AuthIdentity services :- APIHandlerServices] (ent/un-store-map (gen-bulk-from-fn read-entities bulk auth-identity services))) (s/defn delete-bulk - [bulk auth-identity params + [bulk + auth-identity :- auth/AuthIdentity + params services :- APIHandlerServices] (gen-bulk-from-fn delete-entities bulk auth-identity params services)) (s/defn update-bulk - [bulk auth-identity params + [bulk + auth-identity :- auth/AuthIdentity + params services :- APIHandlerServices] (gen-bulk-from-fn update-entities bulk auth-identity params services)) (s/defn patch-bulk - [bulk auth-identity params - services :- APIHandlerServices] - (gen-bulk-from-fn patch-entities bulk auth-identity params services)) + ([bulk + tempids :- TempIDs + auth-identity :- auth/AuthIdentity + params + services :- APIHandlerServices] + (patch-bulk bulk tempids auth-identity params services {})) + ([bulk + tempids :- TempIDs + auth-identity :- auth/AuthIdentity + params + services :- APIHandlerServices + {:keys [enveloped-result?] :as opts}] + (let [entities (gen-bulk-from-fn patch-entities bulk tempids auth-identity params services (select-keys opts [:make-result]))] + (cond-> entities + enveloped-result? (-> (update-vals :data) + (assoc :tempids (into tempids (merge-tempids entities)))))))) diff --git a/src/ctia/bulk/routes.clj b/src/ctia/bulk/routes.clj index e05e1ca9c5..2cfba3f811 100644 --- a/src/ctia/bulk/routes.clj +++ b/src/ctia/bulk/routes.clj @@ -150,6 +150,7 @@ :auth-identity auth-identity (core/validate-bulk-size! bulk services) (ok (core/patch-bulk bulk + {} ;; transient ids only supported via PATCH bundle/import auth-identity (common/wait_for->refresh wait_for) services)))) diff --git a/src/ctia/bundle/core.clj b/src/ctia/bundle/core.clj index c604336470..389b1d9e5a 100644 --- a/src/ctia/bundle/core.clj +++ b/src/ctia/bundle/core.clj @@ -8,7 +8,9 @@ [ctia.bulk.core :as bulk] [ctia.bundle.schemas :refer [BundleImportData BundleImportResult EntityImportData - FindByExternalIdsServices]] + FindByExternalIdsServices AssetPropertiesMergeStrategy]] + [ctia.entity.asset-properties :refer [AssetProperties]] + [ctia.entity.entities :as entities] [ctia.domain.entities :as ent :refer [with-long-id]] [ctia.lib.collection :as coll] [ctia.properties :as p] @@ -19,7 +21,9 @@ [ctia.store :as store] [ctia.store-service.schemas :refer [GetStoreFn]] [ctim.domain.id :as id] - [schema.core :as s]) + [ring.util.http-response :refer [bad-request!]] + [schema.core :as s] + [schema-tools.core :as st]) (:import [java.time Instant] [java.util UUID] @@ -75,10 +79,10 @@ (seq filtered-ext-ids) (assoc :external_ids filtered-ext-ids)))) (s/defn all-pages - "Retrieves all external ids using pagination." + "Retrieves all entities by external ids using pagination." [entity-type external-ids - auth-identity + auth-identity :- auth/AuthIdentity get-store :- GetStoreFn] (loop [ext-ids external-ids entities []] @@ -91,14 +95,16 @@ (auth/ident->map auth-identity) paging)) acc-entities (into entities results) - matched-ext-ids (into #{} (mapcat :external_ids results)) - remaining-ext-ids (remove matched-ext-ids ext-ids)] + matched-ext-ids (into #{} (mapcat :external_ids) results) + remaining-ext-ids (into [] (remove matched-ext-ids) ext-ids)] (if next-page (recur remaining-ext-ids acc-entities) acc-entities)))) (s/defn find-by-external-ids - [import-data entity-type auth-identity + [import-data + entity-type :- s/Keyword + auth-identity :- auth/AuthIdentity {{:keys [get-store]} :StoreService} :- FindByExternalIdsServices] (let [external-ids (mapcat :external_ids import-data)] (log/debugf "Searching %s matching these external_ids %s" @@ -109,14 +115,47 @@ (all-pages entity-type external-ids auth-identity get-store)) []))) +(s/defn find-by-asset_refs :- {s/Str (s/pred map?)} + "Returns a map from asset_ref to entity of entity-type that has + entry `:asset_ref asset_ref`." + [asset_refs :- #{s/Str} + entity-type :- (s/enum :asset-properties :asset-mapping) + auth-identity :- auth/AuthIdentity + {{:keys [get-store]} :StoreService} :- FindByExternalIdsServices] + (if (empty? asset_refs) + {} + (let [_ (log/debugf "Searching %s matching these asset_refs %s" + entity-type + (pr-str asset_refs)) + entities (loop [asset_refs asset_refs + entities {}] + (if (empty? asset_refs) + entities + (let [query {:all-of {:asset_ref asset_refs}} + paging {:limit find-by-external-ids-limit} + {results :data + {next-page :next} :paging} (-> (get-store entity-type) + (store/list-records + query + (auth/ident->map auth-identity) + paging)) + entities (into entities (map (juxt :asset_ref identity)) + results)] + (if next-page + (recur (apply disj asset_refs (map :asset_ref results)) + entities) + entities))))] + (debug (format "Results searching %s for asset_refs %s:" entity-type (pr-str asset_refs)) + entities)))) + (defn by-external-id "Index entities by external_id Ex: - {{:external_id \"ctia-1\"} {:external_id \"ctia-1\" - :entity {...}} - {:external_id \"ctia-2\"} {:external_id \"ctia-2\" - :entity {...}}}" + {{:external_id \"ctia-1\"} #{{:external_id \"ctia-1\" + :entity {...}}} + {:external_id \"ctia-2\"} #{{:external_id \"ctia-2\" + :entity {...}}}}" [entities] (let [entity-with-external-id (->> entities @@ -162,14 +201,23 @@ (s/defn with-existing-entity :- EntityImportData "If the entity has already been imported, update the import data with its ID. If more than one old entity is linked to an external id, - an error is reported." + an error is reported. If realized :id is provided, asserts that entity + must exist, otherwise errors." [{:keys [external_ids] :as entity-data} :- EntityImportData find-by-external-id :- (s/=> s/Any (s/named s/Any 'external_id)) + id->old-entity :- {s/Str (s/pred map?)} services :- HTTPShowServices] - (if-let [old-entities (mapcat find-by-external-id external_ids)] - (let [old-entity (some-> old-entities - first + (if-some [realized-id (when-some [new-id (get-in entity-data [:new-entity :id])] + (when-not (schemas/transient-id? new-id) + new-id))] + (if-some [old-entity (id->old-entity realized-id)] + (assoc entity-data :result "exists" :id (:id old-entity)) + (assoc entity-data :error {:type :unresolvable-id + :reason (str "Long id must already correspond to an entity: " realized-id)} + :result "error")) + (let [old-entities (mapcat find-by-external-id external_ids) + old-entity (some-> (first old-entities) :entity (with-long-id services) ent/un-store)] @@ -184,9 +232,9 @@ pr-str)))) (cond-> entity-data ;; only one entity linked to the external ID - old-entity (assoc :result "exists" - :id (:id old-entity)))) - entity-data)) + old-entity (-> (assoc :result "exists" + :id (:id old-entity)) + (assoc-in [:new-entity :id] (:id old-entity))))))) (s/defschema WithExistingEntitiesServices (csu/open-service-schema @@ -200,28 +248,41 @@ (s/defn with-existing-entities :- [EntityImportData] "Add existing entities to the import data map." - [import-data entity-type identity-map - services :- WithExistingEntitiesServices] - (let [entities-by-external-id + [import-data + entity-type + auth-identity :- auth/AuthIdentity + {{:keys [get-store]} :StoreService :as services} :- WithExistingEntitiesServices] + (let [{:keys [realized-entities unrealized-entities]} (group-by (fn [{{:keys [id]} :new-entity}] + (if (schemas/non-transient-id? id) + :realized-entities + :unrealized-entities)) + import-data) + entities-by-external-id (by-external-id - (find-by-external-ids import-data + (find-by-external-ids unrealized-entities entity-type - identity-map + auth-identity services)) find-by-external-id-fn (fn [external_id] (when external_id (get entities-by-external-id - {:external_id external_id})))] - (map #(with-existing-entity % find-by-external-id-fn services) + {:external_id external_id}))) + id->old-entity (into {} (comp (remove nil?) + (map (juxt :id identity))) + (bulk/read-entities (map (comp :id :new-entity) realized-entities) + entity-type + auth-identity + services))] + (map #(with-existing-entity % find-by-external-id-fn id->old-entity services) import-data))) (s/defn prepare-import :- BundleImportData "Prepares the import data by searching all existing - entities based on their external IDs. Only new entities - will be imported" + entities based on their external IDs. New entities + will be created, and existing entities will be patched." [bundle-entities external-key-prefixes - auth-identity + auth-identity :- auth/AuthIdentity services :- APIHandlerServices] (map-kv (fn [k v] (let [entity-type (bulk/entity-type-from-bulk-key k)] @@ -230,42 +291,77 @@ (with-existing-entities entity-type auth-identity services)))) bundle-entities)) -(defn create? +(s/defn create? :- s/Bool "Whether the provided entity should be created or not" [{:keys [result]}] ;; Add only new entities without error (not (contains? #{"error" "exists"} result))) +(s/defn patch? :- s/Bool + "Whether the provided entity should be patched or not" + [{:keys [result]}] + (= "exists" result)) + (s/defn prepare-bulk - "Creates the bulk data structure with all entities to create." - [bundle-import-data :- BundleImportData] - (map-kv - (fn [_ v] - (->> v - (filter create?) - (remove nil?) - (map :new-entity))) - bundle-import-data)) - -(s/defn with-bulk-result - "Set the bulk result to the bundle import data" + :- {:creates-bulk bulk/BulkEntities + :create-bundle-import-data BundleImportData + :patches-bulk bulk/BulkEntities + :patch-bundle-import-data BundleImportData + :unsubmitted-result BundleImportData} + "Creates separate bulk structures with entities to create or patch. + Returns several bundles in order to preserve correspondence between submission order + and results order for the delicate processing in `with-bulk-result`. + + If patch-existing is false, then :patches-bulk will be empty and moved to :unsubmitted-result + along with errors." [bundle-import-data :- BundleImportData - bulk-result] - (map-kv (fn [k v] - (let [{submitted true - not-submitted false} (group-by create? v)] - (concat - ;; Only submitted entities are processed - (map (fn [entity-import-data - {:keys [error msg] :as entity-bulk-result}] + tempids :- TempIDs + patch-existing :- s/Bool] + (reduce-kv (fn [acc k vs] + (reduce (fn [acc v] + (let [op (cond + (create? v) :creates-bulk + (and (patch? v) patch-existing) :patches-bulk + :else :unsubmitted-result)] + (-> acc + (update-in [op k] (fnil conj []) + (cond-> v + (not= :unsubmitted-result op) :new-entity)) + (cond-> + (not= :unsubmitted-result op) + (update-in [(case op + :creates-bulk :create-bundle-import-data + :patches-bulk :patch-bundle-import-data) + k] + (fnil conj []) v))))) + acc vs)) + {:creates-bulk {} + :create-bundle-import-data {} + :patches-bulk {} + :patch-bundle-import-data {} + :unsubmitted-result {}} + bundle-import-data)) + +(s/defn with-bulk-result :- BundleImportData + "Set the bulk result to the bundle import data, all of which + were submitted and have results in the same order as bulk-result." + [bundle-import-data :- BundleImportData + bulk-result :- bulk/BulkRefs] + (map-kv (fn [k submissions] + (let [results (get bulk-result k)] + ;; guaranteed via `prepare-{upsert}-bulk` + (assert (= (count submissions) (count results)) + [submissions results]) + (mapv (s/fn :- EntityImportData + [entity-import-data + {:keys [error msg] :as entity-bulk-result}] (cond-> entity-import-data error (assoc :error error :result "error") msg (assoc :msg msg) (not error) (assoc :id entity-bulk-result - :result "created"))) - submitted (get bulk-result k)) - not-submitted))) + :result (if (create? entity-import-data) "created" "updated")))) + submissions results))) bundle-import-data)) (s/defn build-response :- BundleImportResult @@ -288,27 +384,161 @@ (log/warn error))) response) +(defn entity->bundle-keys + "For given entity key returns corresponding keys that may be present in Bundle schema. + e.g. :asset => [:assets :asset_refs]" + [entity-key] + (let [{:keys [entity plural]} (get (entities/all-entities) entity-key) + kw->snake-case-str (fn [kw] (-> kw name (string/replace #"-" "_")))] + [(-> plural kw->snake-case-str keyword) + (-> entity kw->snake-case-str (str "_refs") keyword)])) + +(s/defn prep-bundle-schema :- s/Any + "Remove keys of disabled entities from Bundle schema" + [{{:keys [entity-enabled?]} :FeaturesService} :- APIHandlerServices] + (->> (entities/all-entities) + keys + (remove entity-enabled?) + (mapcat entity->bundle-keys) + (apply st/dissoc NewBundle))) + +(s/defschema AssetPropertiesProperties (st/get-in AssetProperties [:properties])) + +(s/defn merge-asset_properties-properties :- AssetPropertiesProperties + "Right-most properties win after concatenating old...new. Return in order of :name." + [new-properties :- AssetPropertiesProperties + old-properties :- AssetPropertiesProperties] + (-> (sorted-map) + (into (map (juxt :name identity)) + (concat old-properties new-properties)) + vals)) + +(s/defn with-existing-asset-entity :- EntityImportData + "If entity has result error, do nothing. + If entity has result exists (via :old-entity), then merge properties with existing. + If entity is scheduled for creation, but already exists via :asset_ref, merge properties + with existing and schedule for patching. + + Ensure tempids includes transient mapping for created Assets, if any. Will + resolve asset_ref on :new-entity if needed." + [{:keys [id new-entity result] :as import-data} :- EntityImportData + tempids :- TempIDs + bulk-asset-kw :- (s/enum :asset_properties :asset_mappings) + asset_ref->old-entity :- {s/Str (s/pred map?)} + merge-strategy :- AssetPropertiesMergeStrategy] + (or (when (not= "error" result) + (let [asset_ref (get tempids (:asset_ref new-entity) (:asset_ref new-entity))] + (if (and (schemas/transient-id? asset_ref) + (= "exists" result)) + (-> import-data + (assoc :error {:type :unresolvable-transient-id + :reason (str "Unresolvable asset_ref: " asset_ref)} + :result "error")) + (when-some [{:keys [id] :as old-entity} (or ;; already resolved by :external_ids or realized :id + (:old-entity import-data) + (asset_ref->old-entity asset_ref))] + (-> import-data + (assoc :old-entity old-entity + :id id + :result "exists" + :new-entity (-> new-entity + (assoc :id id :asset_ref asset_ref) + (cond-> + (and (= :merge-overriding-previous merge-strategy) + (= :asset_properties bulk-asset-kw) + (contains? new-entity :properties)) + (update :properties + merge-asset_properties-properties + (:properties old-entity)))))))))) + import-data)) + +(s/defn resolve-asset-properties+mappings :- BundleImportData + [bundle-import-data :- BundleImportData + tempids :- TempIDs + auth-identity :- auth/AuthIdentity + merge-strategy :- AssetPropertiesMergeStrategy + services :- FindByExternalIdsServices] + (let [resolve* (s/fn :- BundleImportData + [bundle-import-data :- BundleImportData + bulk-asset-kw :- (s/enum :asset_mappings :asset_properties)] + (let [asset_refs (into #{} (keep (fn [{:keys [id] {:keys [asset_ref]} :new-entity :as import-data}] + (when (and (nil? id) asset_ref) + (let [asset_ref (get tempids asset_ref asset_ref)] + (when-not (schemas/transient-id? asset_ref) + asset_ref))))) + (bulk-asset-kw bundle-import-data)) + asset_ref->old-entity (find-by-asset_refs asset_refs + (bulk/entity-type-from-bulk-key bulk-asset-kw) + auth-identity + services)] + (cond-> bundle-import-data + (seq (bulk-asset-kw bundle-import-data)) + (update bulk-asset-kw + (fn [bulk-assets] + (mapv #(with-existing-asset-entity % tempids bulk-asset-kw asset_ref->old-entity merge-strategy) + bulk-assets))))))] + (-> bundle-import-data + (resolve* :asset_mappings) + (resolve* :asset_properties)))) + +(defn bundle-import-data->tempids + [bundle-import-data + tempids] + (into tempids (map entities-import-data->tempids) + (vals bundle-import-data))) + (s/defn import-bundle :- BundleImportResult - [bundle :- NewBundle - external-key-prefixes :- (s/maybe s/Str) - auth-identity :- (s/protocol auth/IIdentity) - {{:keys [get-in-config]} :ConfigService - :as services} :- APIHandlerServices] - (let [bundle-entities (select-keys bundle bundle-entity-keys) - bundle-import-data (prepare-import bundle-entities - external-key-prefixes - auth-identity - services) - bulk (debug "Bulk" (prepare-bulk bundle-import-data)) - tempids (->> bundle-import-data - (map (fn [[_ entities-import-data]] - (entities-import-data->tempids entities-import-data))) - (apply merge {}))] - (debug "Import bundle response" - (->> (bulk/create-bulk bulk tempids auth-identity (bulk-params get-in-config) services) - (with-bulk-result bundle-import-data) + ([bundle :- (st/optional-keys-schema NewBundle) + external-key-prefixes :- (s/maybe s/Str) + auth-identity :- auth/AuthIdentity + services :- APIHandlerServices] + (import-bundle bundle external-key-prefixes auth-identity services {})) + ([bundle :- (st/optional-keys-schema NewBundle) + external-key-prefixes :- (s/maybe s/Str) + auth-identity :- auth/AuthIdentity + {{:keys [get-in-config]} :ConfigService + :as services} :- APIHandlerServices + {:keys [patch-existing asset_properties-merge-strategy] + :or {patch-existing false + asset_properties-merge-strategy :ignore-existing}} :- {(s/optional-key :patch-existing) s/Bool + (s/optional-key :asset_properties-merge-strategy) AssetPropertiesMergeStrategy}] + (let [bundle-entities (select-keys bundle bundle-entity-keys) + ;; the hard case is when patching asset_ref on an Asset that we also create in this bundle. + ;; even harder, the same Asset could be used to patch a relationship's source_ref. + ;; handled by processing the bundle as separate groups of entities in dependency order + {:keys [bulk-refs]} (bulk/import-bulks-with + (fn [bundle-entities tempids] + (let [bundle-import-data (prepare-import bundle-entities external-key-prefixes auth-identity services) + tempids (bundle-import-data->tempids bundle-import-data tempids) + bundle-import-data (resolve-asset-properties+mappings + bundle-import-data tempids auth-identity asset_properties-merge-strategy services) + tempids (bundle-import-data->tempids bundle-import-data tempids) + {:keys [creates-bulk create-bundle-import-data + patches-bulk patch-bundle-import-data + unsubmitted-result]} (debug "Bulk" (prepare-bulk bundle-import-data tempids patch-existing)) + {:keys [tempids] :as create-bulk-refs + :or {tempids {}}} (bulk/create-bulk creates-bulk tempids auth-identity (bulk-params get-in-config) services) + create-result (with-bulk-result create-bundle-import-data (dissoc create-bulk-refs :tempids)) + patch-result (when patch-existing + (let [patch-bulk-refs (bulk/patch-bulk patches-bulk tempids auth-identity (bulk-params get-in-config) services + {:enveloped-result? true + :make-result bulk/make-patch-bulk-enveloped-result})] + (with-bulk-result patch-bundle-import-data (dissoc patch-bulk-refs :tempids))))] + (-> (merge-with into create-result patch-result unsubmitted-result) + ;; cram back into the format that bulk/import-bulks-with expects. + (update-vals #(hash-map :data % + :tempids tempids))))) + (keep not-empty + [(dissoc bundle-entities :relationships :asset_mappings :asset_properties) + ;; create assets before processing entities with asset_ref + (select-keys bundle-entities [:asset_mappings :asset_properties]) + ;; create all non-relationships before processing {source,target}_ref + (select-keys bundle-entities [:relationships])]) + {})] + (debug "Import bundle response" + (-> bulk-refs build-response - log-errors)))) + log-errors))))) (defn bundle-max-size [get-in-config] (bulk/get-bulk-max-size get-in-config)) @@ -481,7 +711,9 @@ :source "ctia"}) (s/defn export-bundle - [ids identity params + [ids + identity :- auth/AuthIdentity + params {{:keys [send-event]} :RiemannService :as services} :- APIHandlerServices] (if (seq ids) diff --git a/src/ctia/bundle/routes.clj b/src/ctia/bundle/routes.clj index 52e71fb5a5..c9eaa03e81 100644 --- a/src/ctia/bundle/routes.clj +++ b/src/ctia/bundle/routes.clj @@ -1,18 +1,18 @@ (ns ctia.bundle.routes (:refer-clojure :exclude [identity]) (:require - [clojure.string :as str] [ctia.lib.compojure.api.core :refer [GET POST context routes]] [ctia.bundle.core :refer [bundle-max-size bundle-size import-bundle - export-bundle]] - [ctia.bundle.schemas :refer [BundleImportResult + export-bundle + prep-bundle-schema]] + [ctia.bundle.schemas :refer [AssetPropertiesMergeStrategy + BundleImportResult NewBundleExport BundleExportIds BundleExportOptions BundleExportQuery]] - [ctia.entity.entities :as entities] [ctia.http.routes.common :as common] [ctia.schemas.core :refer [APIHandlerServices NewBundle]] [ring.swagger.json-schema :refer [describe]] @@ -69,24 +69,6 @@ :read-casebook :list-casebooks}) -(defn- entity->bundle-keys - "For given entity key returns corresponding keys that may be present in Bundle schema. - e.g. :asset => [:assets :asset_refs]" - [entity-key] - (let [{:keys [entity plural]} (get (entities/all-entities) entity-key) - kw->snake-case-str (fn [kw] (-> kw name (str/replace #"-" "_")))] - [(-> plural kw->snake-case-str keyword) - (-> entity kw->snake-case-str (str "_refs") keyword)])) - -(s/defn prep-bundle-schema :- s/Any - "Remove keys of disabled entities from Bundle schema" - [{{:keys [entity-enabled?]} :FeaturesService} :- APIHandlerServices] - (->> (entities/all-entities) - keys - (remove entity-enabled?) - (mapcat entity->bundle-keys) - (apply st/dissoc NewBundle))) - (s/defn bundle-routes [{{:keys [get-in-config]} :ConfigService :as services} :- APIHandlerServices] (routes @@ -113,7 +95,8 @@ :auth-identity identity (ok (export-bundle (:ids body) identity query services)))) - (let [capabilities #{:create-actor + (let [bundle-schema (prep-bundle-schema services) + capabilities #{:create-actor :create-asset :create-asset-mapping :create-asset-properties @@ -137,17 +120,36 @@ (POST "/import" [] :return BundleImportResult :body [bundle - (prep-bundle-schema services) - {:description "a Bundle to import"}] + (st/optional-keys-schema bundle-schema) + {:description "a Bundle to import, partial entities allowed for existing entities"}] :query-params [{external-key-prefixes :- (describe s/Str "Comma separated list of external key prefixes") - nil}] - :summary "POST many new entities using a single HTTP call" + nil} + {patch-existing :- (describe s/Bool + (str "If true, existing entities will be patched with result=updated. Otherwise, existing entities will be " + "ignored with result-existing.")) + false} + {asset_properties-merge-strategy :- + (describe AssetPropertiesMergeStrategy + (str "Only relevant if patch-existing=true.\n\n" + "If ignore-existing, then asset properties will be patched to their new " + "values as they appear in the request bundle.\n\n" + "If merge-overriding-previous, then existing asset properties " + "will be retrieved and combined with the asset properties in the request bundle " + "as if by concatenating existing and new properties together in a single list, " + "removing properties to the left of a property with the same name, " + "then sorting the list lexicographically by name before using this list to patch the existing entity." + "\n\n" + " Defaults to ignore-existing")) + :ignore-existing}] + :summary "POST many new and partial entities using a single HTTP call" :auth-identity auth-identity :description (common/capabilities->description capabilities) :capabilities capabilities (let [max-size (bundle-max-size get-in-config)] (if (< max-size (bundle-size bundle)) (bad-request (str "Bundle max nb of entities: " max-size)) - (ok (import-bundle bundle external-key-prefixes auth-identity services))))))))) + (ok (import-bundle bundle external-key-prefixes auth-identity services + {:patch-existing patch-existing + :asset_properties-merge-strategy asset_properties-merge-strategy}))))))))) diff --git a/src/ctia/bundle/schemas.clj b/src/ctia/bundle/schemas.clj index eaf8637f3c..112b7bbf4f 100644 --- a/src/ctia/bundle/schemas.clj +++ b/src/ctia/bundle/schemas.clj @@ -9,7 +9,7 @@ (st/optional-keys {:id s/Str :original_id s/Str - :result (s/enum "error" "created" "exists") + :result (s/enum "error" "created" "exists" "updated") :type s/Keyword :external_ids [s/Str] :error s/Any @@ -58,3 +58,6 @@ {:StoreService {:get-store GetStoreFn s/Keyword s/Any} s/Keyword s/Any}) + +(s/defschema AssetPropertiesMergeStrategy + (s/enum :ignore-existing :merge-overriding-previous)) diff --git a/src/ctia/flows/crud.clj b/src/ctia/flows/crud.clj index 5b491f1ba8..ee5315368a 100644 --- a/src/ctia/flows/crud.clj +++ b/src/ctia/flows/crud.clj @@ -24,9 +24,11 @@ (:import java.util.UUID)) +(s/defschema Entities [{s/Keyword s/Any}]) + (s/defschema FlowMap {:create-event-fn (s/pred fn?) - :entities [{s/Keyword s/Any}] + :entities Entities :entity-type s/Keyword :flow-type (s/enum :create :update :delete) :services APIHandlerServices @@ -365,11 +367,15 @@ ((:apply-event-hooks HooksService) event)) fm) +(s/defn make-enveloped-result + [{:keys [entities tempids]} :- FlowMap] + (cond-> {:data entities} + (seq tempids) (assoc :tempids tempids))) + (s/defn make-create-result :- s/Any - [{:keys [entities enveloped-result? tempids]} :- FlowMap] + [{:keys [entities enveloped-result?] :as fm} :- FlowMap] (if enveloped-result? - (cond-> {:data entities} - (seq tempids) (assoc :tempids tempids)) + (make-enveloped-result fm) entities)) (defn patch-entity @@ -384,20 +390,19 @@ (s/defn patch-entities :- FlowMap [{:keys [get-prev-entity entities - patch-operation] + patch-operation + tempids] :as fm} :- FlowMap] (let [patch-fn (case patch-operation :add coll/add-colls :remove coll/remove-colls :replace coll/replace-colls coll/replace-colls) - patched (for [partial-entity entities - :let [prev-entity (some->> partial-entity - :id - get-prev-entity)]] - (cond->> partial-entity - (some? prev-entity) - (patch-entity patch-fn prev-entity)))] + patched (for [{:keys [id] :as partial-entity} entities + :let [prev-entity (get-prev-entity id)]] + (cond->> partial-entity + (some? prev-entity) + (patch-entity patch-fn prev-entity)))] (assoc fm :entities patched))) (defn create-flow @@ -539,16 +544,19 @@ identity patch-operation partial-entities + tempids long-id-fn spec get-success-entities make-result] - :or {get-success-entities default-success-entities}}] + :or {get-success-entities default-success-entities + tempids {}}}] (let [ids (map :id partial-entities) prev-entity-fn (prev-entity get-fn ids)] (-> {:flow-type :update :entity-type entity-type :entities partial-entities + :tempids tempids :services services :get-prev-entity prev-entity-fn :patch-operation patch-operation diff --git a/src/ctia/schemas/core.clj b/src/ctia/schemas/core.clj index 470eec2494..3ddcd44049 100644 --- a/src/ctia/schemas/core.clj +++ b/src/ctia/schemas/core.clj @@ -286,6 +286,11 @@ (st/dissoc (csu/recursive-open-schema-version CTIMBundle) :casebooks)) +(s/defschema PartialBundle + (st/optional-keys-schema + (st/dissoc + (csu/recursive-open-schema-version CTIMBundle) :casebooks))) + ;; common (s/defschema TLP @@ -334,9 +339,18 @@ s/Keyword s/Any} s/Keyword s/Any}) -(defn transient-id? - [id] - (and id (some? (re-matches id/transient-id-re id)))) +(s/defn transient-id? :- s/Bool + "True only if id is a transient id." + [id :- (s/maybe s/Str)] + (boolean + (and id (re-matches id/transient-id-re id)))) + +(s/defn non-transient-id? :- s/Bool + "True only if id is a long or short id." + [id :- (s/maybe s/Str)] + (boolean + (and id (or (re-matches id/short-id-re id) + (re-matches id/long-id-re id))))) (s/defschema ESSortMode (s/enum "max" "min" "sum" "avg" "median")) diff --git a/src/ctia/stores/es/crud.clj b/src/ctia/stores/es/crud.clj index f8f3ffb972..ba4864ced2 100644 --- a/src/ctia/stores/es/crud.clj +++ b/src/ctia/stores/es/crud.clj @@ -425,7 +425,8 @@ It returns the documents with full hits meta data including the real index in wh first (into meta))) prepared) - bulk-res (when prepared + bulk-res (if-not prepared + {} (try (format-bulk-res (ductile.doc/bulk-index-docs conn diff --git a/test/ctia/bulk/core_test.clj b/test/ctia/bulk/core_test.clj index d1ea39cc2b..029233fade 100644 --- a/test/ctia/bulk/core_test.clj +++ b/test/ctia/bulk/core_test.clj @@ -259,11 +259,13 @@ "visible entities that the user is not allowed to write on are returned as forbidden errors"))] (testing "bulk-patch shall properly patch submitties entitites" (let [other-group-res (sut/patch-bulk bulk-patch + {} other-group-ident {:refresh "true"} services) {:keys [sightings indicators]} (sut/patch-bulk bulk-patch + {} ident {:refresh "true"} services)] @@ -279,7 +281,7 @@ (is (= "patched indicator" (:source (read-record indicator-store indicator-id ident-map {}))))))) - (testing "bulk-update shall properly update submitties entitites" + (testing "bulk-update shall properly update submitted entitites" (let [other-group-res (sut/update-bulk bulk-update other-group-ident {:refresh "true"} @@ -334,14 +336,14 @@ (let [intermediate-tempids (atom []) incident1 (assoc incident-minimal :id "transientid1") incident2 (assoc incident-minimal :id "transientid2")] - (is (= {:bulk-refs {:incidents [incident1 incident2]} + (is (= {:bulk-refs {:incidents (mapv :id [incident1 incident2])} :tempids {"foo" "bar" "transientid1" "id1" "transientid2" "id2"}} (sut/import-bulks-with (fn [{:keys [incidents]} tempids] (swap! intermediate-tempids conj tempids) - {:incidents {:data incidents + {:incidents {:data (mapv :id incidents) :tempids (into tempids (map (fn [{:keys [id]}] {id (subs id (count "transient"))})) diff --git a/test/ctia/bundle/core_asset_ref_entities_test.clj b/test/ctia/bundle/core_asset_ref_entities_test.clj index 8910775fb8..25be49c7ce 100644 --- a/test/ctia/bundle/core_asset_ref_entities_test.clj +++ b/test/ctia/bundle/core_asset_ref_entities_test.clj @@ -8,20 +8,29 @@ These tests are to ensure that such a relationship is observed when these types of objects when they created via Bundle Import" (:require - [clojure.test :refer [deftest is testing use-fixtures join-fixtures]] + [clojure.test :refer [deftest is testing use-fixtures]] [clojure.walk :as walk] [ctia.auth.threatgrid :refer [map->Identity]] [ctia.bulk.core :as bulk] [ctia.bundle.core :as bundle] [ctia.store :as store] + [ctia.test-helpers.store :refer [test-for-each-store-with-app]] [ctia.test-helpers.auth :as auth] [ctia.test-helpers.core :as th] [ctim.examples.bundles :refer [bundle-maximal]] - [puppetlabs.trapperkeeper.app :as app])) + [ctia.test-helpers.fake-whoami-service :as whoami-helpers] + [ctia.auth.capabilities :refer [all-capabilities]] + [puppetlabs.trapperkeeper.app :as app] + [schema.test :refer [validate-schemas]])) + +(use-fixtures :once + validate-schemas + whoami-helpers/fixture-server) (def ^:private login (map->Identity {:login "foouser" - :groups ["foogroup"]})) + :groups ["foogroup"] + :capabilities (all-capabilities)})) (defn- set-transient-asset-refs [x] (walk/prewalk @@ -46,34 +55,6 @@ ;; in order to test association of Asset to AssetMappings/AssetProperties set-transient-asset-refs)) -(deftest bulk-for-asset-related-entities - (testing "delay creation of :asset-mapping and :asset-properties, until all - transient IDs for :asset are resolved" - (th/fixture-ctia-with-app - (fn [app] - (let [services (app/service-graph app)] - (testing "Passing a Bundle with Assets with transient IDs, should skip - the creation of AssetMappings and AssetProperties (initially)" - (with-redefs [bulk/gen-bulk-from-fn - (fn [_ bulk _ _ _ _] - (is (empty? - (select-keys - bulk [:asset_mappings - :asset_properties]))) - ;; Doesn't make sence to continue from here; once we - ;; asserted that asset-mapping and asset-properties - ;; won't be explicitly created (until we create Assets - ;; with non-transient IDs), we achieved the goal of - ;; this test. - (throw (Exception. "stopped intentionally")))] - (is (thrown-with-msg? - Exception #"stopped intentionally" - (bundle/import-bundle - bundle-ents - nil ;; external-key-prefixes - login - services)))))))))) - (deftest asset-refs-test (th/fixture-ctia-with-app (fn [app] @@ -106,44 +87,61 @@ (is (every? (partial contains? asset-refs) refs)))))))) (deftest validate-asset-refs-test - (testing "Bundle with asset_refs that have no correspoding Asset" + (testing "Bundle with asset_refs that have no corresponding Asset" (let [;; assign non-transient ID to the asset in the Bundle, - ;; leaving :asset_refs pointing to a transient ID that would never resolve - bundle (walk/prewalk - #(if (and (map? %) - (-> % :type (= "asset"))) - (assoc % :id "http://ex.tld/ctia/asset/asset-61884b14-e273-4930-a5ff-dcce69207724") - %) - bundle-ents)] - (th/fixture-ctia-with-app + ;; leaving :asset_ref's pointing to a transient ID that would never resolve + bundle (update bundle-ents :assets + #(into #{} (map (fn [asset] + (assoc asset :id "http://ex.tld/ctia/asset/asset-61884b14-e273-4930-a5ff-dcce69207724"))) + %))] + (test-for-each-store-with-app (fn [app] - (let [services (app/service-graph app)] - (is - (thrown? Exception - (bundle/import-bundle - bundle - nil ;; external-key-prefixes - login - services)))))))) + (th/set-capabilities! app "foouser" ["foogroup"] "user" (all-capabilities)) + (whoami-helpers/set-whoami-response app + "45c1f5e3f05d0" + "foouser" + "foogroup" + "user") + (let [create-response (th/POST app + "ctia/bundle/import" + :body bundle + :headers {"Authorization" "45c1f5e3f05d0"}) + res (:parsed-body create-response)] + (when (is (= 200 (:status create-response))) + (let [actual-results (map #(dissoc % :error) (sort-by :type (:results res)))] + (is (= [{:result "error", :type :asset + :external_ids ["http://ex.tld/ctia/asset/asset-61884b14-e273-4930-a5ff-dcce69207724"]} + {:result "error", :type :asset-mapping + :external_ids ["http://ex.tld/ctia/asset-mapping/asset-mapping-636ef2cc-1cb0-47ee-afd4-ecc1fe4be451"]} + {:result "error", :type :asset-properties + :external_ids ["http://ex.tld/ctia/asset-properties/asset-properties-97c3dbb5-6deb-4eed-b6d7-b77fa632cc7b"]}] + actual-results) + (pr-str actual-results))))))))) (testing "Bundle with asset_refs that aren't transient" - (let [;; :asset_refs that are non-transient should still be allowed + (let [;; :asset_ref's that are non-transient should still be allowed bundle (walk/prewalk #(if (and (map? %) (contains? % :asset_ref)) (assoc % :asset_ref "http://ex.tld/ctia/asset/asset-61884b14-e273-4930-a5ff-dcce69207724") %) bundle-ents)] - (th/fixture-ctia-with-app + (test-for-each-store-with-app (fn [app] - (let [services (app/service-graph app) - {:keys [results]} (bundle/import-bundle - bundle - nil ;; external-key-prefixes - login - services) + (th/set-capabilities! app "foouser" ["foogroup"] "user" (all-capabilities)) + (whoami-helpers/set-whoami-response app + "45c1f5e3f05d0" + "foouser" + "foogroup" + "user") + (let [create-response (th/POST app + "ctia/bundle/import" + :body bundle + :headers {"Authorization" "45c1f5e3f05d0"}) + {:keys [results]} (:parsed-body create-response) num-created (->> results (map :result) - (keep (partial = "created")) + (keep #{"created"}) count)] - (is (= (count bundle-ents) - num-created)))))))) + (when (is (= 200 (:status create-response))) + (is (= (count bundle-ents) + num-created))))))))) diff --git a/test/ctia/bundle/core_test.clj b/test/ctia/bundle/core_test.clj index 1a7efe66fe..6f0d3bd1e3 100644 --- a/test/ctia/bundle/core_test.clj +++ b/test/ctia/bundle/core_test.clj @@ -8,17 +8,17 @@ [ctia.test-helpers.http :refer [app->HTTPShowServices]] [ctia.test-helpers.es :as es-helpers])) -(use-fixtures :each - es-helpers/fixture-properties:es-store - h/fixture-ctia-fast) - (deftest local-entity?-test - (are [x y] (= x (sut/local-entity? y (app->HTTPShowServices (h/get-current-app)))) - false nil - false "" - false "http://unknown.site/ctia/indicator/indicator-56067199-47c0-4294-8957-13d6b265bdc4" - true "indicator-56067199-47c0-4294-8957-13d6b265bdc4" - true "http://localhost:57254/ctia/indicator/indicator-56067199-47c0-4294-8957-13d6b265bdc4")) + (es-helpers/fixture-properties:es-store + (fn [] + (h/fixture-ctia-with-app + (fn [app] + (are [x y] (= x (sut/local-entity? y (app->HTTPShowServices app))) + false nil + false "" + false "http://unknown.site/ctia/indicator/indicator-56067199-47c0-4294-8957-13d6b265bdc4" + true "indicator-56067199-47c0-4294-8957-13d6b265bdc4" + true "http://localhost:57254/ctia/indicator/indicator-56067199-47c0-4294-8957-13d6b265bdc4")))))) (deftest clean-bundle-test (is (= {:b '(1 2 3) :d '(1 3)} @@ -59,52 +59,89 @@ :related_to [:source_ref]}))))) (deftest with-existing-entity-test - (testing "with-existing-entity" - (let [app (h/get-current-app) - http-show-services (app->HTTPShowServices app) - - indicator-id-1 (make-id "indicator") - indicator-id-2 (make-id "indicator") - indicator-id-3 (make-id "indicator") - new-indicator {:id indicator-id-3 - :external_ids ["swe-alarm-indicator-1"]} - find-by-ext-ids (fn [existing-ids] - (constantly - (map (fn [old-id] - {:entity {:id old-id}}) - existing-ids))) - test-fn (fn [{:keys [msg expected existing-ids log?]}] - (with-log - (testing msg - (is (= expected - (sut/with-existing-entity - new-indicator - (find-by-ext-ids existing-ids) - http-show-services))) - (is (= log? - (logged? 'ctia.bundle.core - :warn - #"More than one entity is linked to the external ids"))))))] - (test-fn {:msg "no existing external id" - :expected {:id indicator-id-3 - :external_ids ["swe-alarm-indicator-1"]} - :existing-ids [] - :log? false}) - (test-fn {:msg "1 existing external id" - :expected (with-long-id {:result "exists" - :external_ids ["swe-alarm-indicator-1"] - :id indicator-id-1} - http-show-services) - :existing-ids [indicator-id-1] - :log? false}) - (test-fn {:msg "more than 1 existing external id" - :expected (with-long-id {:result "exists" - :external_ids ["swe-alarm-indicator-1"] - :id indicator-id-2} - http-show-services) - :existing-ids [indicator-id-2 - indicator-id-1] - :log? true})))) + (es-helpers/fixture-properties:es-store + (fn [] + (h/fixture-ctia-with-app + (fn [app] + (testing "with-existing-entity" + (let [http-show-services (app->HTTPShowServices app) + + indicator-id-1 (make-id "indicator") + indicator-id-2 (make-id "indicator") + indicator-id-3 (make-id "indicator") + indicator-original-id-1 (str "transient:" indicator-id-1) + new-indicator {:id indicator-original-id-1 + :external_ids ["swe-alarm-indicator-1"]} + find-by-ext-ids (fn [existing-ids] + (fn [external_id] + (map (fn [old-id] + {:external_id external_id + :entity {:id old-id}}) + existing-ids))) + test-fn (fn [{new-indicator* :new-indicator :keys [msg expected existing-ids id->old-entity log?] + :or {log? false + existing-ids [] + id->old-entity {} + new-indicator* new-indicator}}] + (with-log + (testing msg + (is (= expected + (sut/with-existing-entity + new-indicator* + (find-by-ext-ids existing-ids) + id->old-entity + http-show-services))) + (is (= log? + (logged? 'ctia.bundle.core + :warn + #"More than one entity is linked to the external ids"))))))] + (test-fn {:msg "no existing external id" + :expected {:id indicator-original-id-1 + :external_ids ["swe-alarm-indicator-1"]} + :existing-ids [] + :log? false}) + (let [{:keys [id] :as new-indicator} (with-long-id (assoc new-indicator :id indicator-id-1) + http-show-services)] + (doseq [existing-ids [[] + [indicator-id-2] + [indicator-id-2 + indicator-id-1]]] + (testing (str "existing-ids:" (pr-str existing-ids)) + (test-fn {:msg "existing long id" + :new-indicator {:new-entity new-indicator} + :id->old-entity {id {:id id}} + :expected {:result "exists" + :id id + :new-entity new-indicator} + :existing-ids existing-ids}) + (test-fn {:msg "non-existing long id" + :new-indicator {:new-entity new-indicator} + :id->old-entity {} + :expected {:result "error" + :error {:type :unresolvable-id + :reason (str "Long id must already correspond to an entity: " + id)} + :new-entity new-indicator} + :existing-ids existing-ids})))) + (test-fn {:msg "1 existing external id" + :expected (with-long-id {:result "exists" + :external_ids ["swe-alarm-indicator-1"] + :id indicator-id-1 + :new-entity (with-long-id {:id indicator-id-1} + http-show-services)} + http-show-services) + :existing-ids [indicator-id-1] + :log? false}) + (test-fn {:msg "more than 1 existing external id" + :expected (with-long-id {:result "exists" + :external_ids ["swe-alarm-indicator-1"] + :id indicator-id-2 + :new-entity (with-long-id {:id indicator-id-2} + http-show-services)} + http-show-services) + :existing-ids [indicator-id-2 + indicator-id-1] + :log? true})))))))) (deftest filter-external-ids-test (let [external-ids ["ctia-indicator-1" "cisco-indicator-1" "indicator-1"]] @@ -213,3 +250,32 @@ :external_ids external_ids} :type :sighting :external_ids external_ids}}))) + +(deftest merge-asset_properties-properties-test + (let [[old1 old2 old3 new1 new2] (map (comp str gensym) + '[old1 old2 old3 new1 new2])] + (is (= [{:name "bar" :value new2} + {:name "baz" :value old3} + {:name "foo" :value new1}] + (sut/merge-asset_properties-properties + (shuffle [{:name "foo" :value new1} + {:name "bar" :value new2}]) + (shuffle [{:name "foo" :value old1} + {:name "bar" :value old2} + {:name "baz" :value old3}])))) + (testing "right-most wins in both new and old" + (is (= [{:name "bar" :value new2} + {:name "baz" :value old1} + {:name "foo" :value new2}] + (sut/merge-asset_properties-properties + [{:name "foo" :value new1} + {:name "foo" :value new1} + {:name "foo" :value new1} + {:name "foo" :value new2} + {:name "bar" :value new2}] + [{:name "foo" :value old1} + {:name "bar" :value old2} + {:name "baz" :value old3} + {:name "baz" :value old3} + {:name "baz" :value old3} + {:name "baz" :value old1}])))))) diff --git a/test/ctia/bundle/routes_test.clj b/test/ctia/bundle/routes_test.clj index 9110d3eb76..866797e9c6 100644 --- a/test/ctia/bundle/routes_test.clj +++ b/test/ctia/bundle/routes_test.clj @@ -12,6 +12,7 @@ [ctia.bundle.routes :as bundle.routes] [ctia.test-helpers.core :as helpers :refer [deep-dissoc-entity-ids GET POST DELETE]] + [ctia.test-helpers.http :refer [app->APIHandlerServices]] [ctia.test-helpers.fake-whoami-service :as whoami-helpers] [ctia.test-helpers.store :refer [test-for-each-store-with-app]] [ctim.domain.id :as id] @@ -122,7 +123,7 @@ original-entity] (testing (str "Entity " external_id) (is (= (:id original-entity) original_id) - "The orignal ID is in the result") + "The original ID is in the result") (is (contains? (set (:external_ids original-entity)) external_id) "The external ID is in the result") @@ -133,6 +134,7 @@ external_id) :headers {"Authorization" "45c1f5e3f05d0"}) [entity :as entities] (:parsed-body response)] + (assert (is (= 200 (:status response)))) (is (= 1 (count entities)) "Only one entity is linked to the external ID") (is (= id (:id entity)) @@ -145,6 +147,7 @@ (encode id)) :headers {"Authorization" "45c1f5e3f05d0"}) entity (:parsed-body response)] + (assert (is (= 200 (:status response)))) (is (= (assoc original-entity :id id :schema_version ctim-schema-version) @@ -158,6 +161,18 @@ (filter #(= (:original_id %) original-id)) first)) +(s/defn find-id-by-original-id :- s/Str + [msg bundle-result original-id] + (let [{:keys [id error] :as result} (find-result-by-original-id bundle-result original-id)] + (or id + (throw (ex-info (str "Missing long id for transient " + original-id + (some->> error (str ": "))) + (cond-> {:msg msg + :original-id original-id + :bundle-result bundle-result} + result (assoc :result result))))))) + (defn resolve-ids "Resolves transient IDs in the target_ref and the source_ref of a relationship" @@ -237,13 +252,14 @@ response (POST app "ctia/bundle/import" :body new-bundle + :query-params {"patch-existing" false} :headers {"Authorization" "45c1f5e3f05d0"}) bundle-result (:parsed-body response)] - (is (= 200 (:status response))) - (is (= (count-bundle-entities new-bundle) - (count-bundle-result-entities (:results bundle-result) - "created")) - "All entities are created"))) + (when (is (= 200 (:status response))) + (is (= (count-bundle-entities new-bundle) + (count-bundle-result-entities (:results bundle-result) + "created")) + "All entities are created")))) (testing "Create" (let [bundle {:type "bundle" :source "source" @@ -254,57 +270,70 @@ response (POST app "ctia/bundle/import" :body bundle + :query-params {"patch-existing" false} :headers {"Authorization" "45c1f5e3f05d0"}) bundle-result (:parsed-body response)] - (is (= 200 (:status response))) - - (is (every? #(= "created" %) - (map :result (:results bundle-result))) - "All entities are created") - - (doseq [entity (concat indicators - sightings - (map #(resolve-ids bundle-result %) - relationships))] - (validate-entity-record - app - (find-result-by-original-id bundle-result (:id entity)) - entity)))) - (testing "Update" - (let [bundle + (when (is (= 200 (:status response))) + + (is (every? #(= "created" %) + (map :result (:results bundle-result))) + "All entities are created") + + (doseq [entity (concat indicators + sightings + (map #(resolve-ids bundle-result %) + relationships))] + (validate-entity-record + app + (find-result-by-original-id bundle-result (:id entity)) + entity))))) + (testing "Update with partial" + (let [updated-indicators (set (map with-modified-description indicators)) + bundle {:type "bundle" :source "source" - :indicators (set (map with-modified-description indicators)) + ;; partial entity updates are allowed (:producer is a required Indicator entry) + :indicators (into #{} (map #(select-keys % [:id :external_ids :description])) updated-indicators) :sightings (set (map with-modified-description sightings)) :relationships (set (map with-modified-description relationships))} - response (POST app - "ctia/bundle/import" - :body bundle - :headers {"Authorization" "45c1f5e3f05d0"}) - bundle-result (:parsed-body response)] - (is (= 200 (:status response))) - - (is (pos? (count (:results bundle-result)))) - - (is (every? #(= "exists" %) - (map :result (:results bundle-result))) - "All existing entities are not updated") - - (doseq [entity (concat indicators - sightings - (map #(resolve-ids bundle-result %) - relationships))] - (validate-entity-record - app - (find-result-by-original-id bundle-result (:id entity)) - entity)))) + test-update-with-partial (s/fn [patch-existing :- s/Bool] + (let [response (POST app + "ctia/bundle/import" + :body bundle + :query-params (cond-> {} + patch-existing (assoc "patch-existing" patch-existing)) + :headers {"Authorization" "45c1f5e3f05d0"}) + bundle-result (:parsed-body response)] + (when (is (= 200 (:status response))) + (is (pos? (count (:results bundle-result)))) + (is (every? #(= (if patch-existing "updated" "exists") %) + (map :result (:results bundle-result))) + "All existing entities are updated") + (doseq [entity (if patch-existing + (concat updated-indicators + (:sightings bundle) + (map #(resolve-ids bundle-result %) + (:relationships bundle))) + (concat indicators + sightings + (map #(resolve-ids bundle-result %) + relationships)))] + (validate-entity-record + app + (find-result-by-original-id bundle-result (:id entity)) + entity)))))] + ;; order of these tests is important since they operate on the same entities + (testing "no patch existing" + (test-update-with-partial false)) + (testing "patch existing" + (test-update-with-partial true)))) (testing "Update and create" (let [indicator (mk-indicator 2000) sighting (first sightings) - relationship (mk-relationship 2000 - sighting - indicator - "sighting-of") + relationship (mk-relationship 200 + sighting + indicator + "sighting-of") bundle {:type "bundle" :source "source" @@ -314,18 +343,17 @@ response (POST app "ctia/bundle/import" :body bundle + :query-params {"patch-existing" true} :headers {"Authorization" "45c1f5e3f05d0"}) bundle-result (:parsed-body response)] - (is (= 200 (:status response))) - - (is (pos? (count (:results bundle-result)))) - - (doseq [entity [indicator sighting - (resolve-ids bundle-result relationship)]] - (validate-entity-record - app - (find-result-by-original-id bundle-result (:id entity)) - entity)))) + (when (is (= 200 (:status response))) + (is (pos? (count (:results bundle-result)))) + (doseq [entity [indicator sighting + (resolve-ids bundle-result relationship)]] + (validate-entity-record + app + (find-result-by-original-id bundle-result (:id entity)) + entity))))) (testing "Bundle with missing entities" (let [relationship (mk-relationship 2001 (first sightings) @@ -336,91 +364,141 @@ :relationships [relationship]} response-create (POST app "ctia/bundle/import" - :query-params {"external-key-prefixes" "custom-"} + :query-params {"external-key-prefixes" "custom-" + "patch-existing" false} :body bundle :headers {"Authorization" "45c1f5e3f05d0"}) bundle-result-create (:parsed-body response-create)] - (is (= 200 (:status response-create))) - (is (= [{:original_id (:id relationship), - :result "error", - :type :relationship, - :error (str "A relationship cannot be created if a " - "source or a target ref is still a transient " - "ID (The source or target entity is probably " - "not provided in the bundle)")}] - (filter (fn [r] (= (:result r) "error")) - (:results bundle-result-create))) - (str "A relationship cannot be created if the source and the " - "target entities referenced by a transient ID are not " - "included in the bundle.")))) + (when (is (= 200 (:status response-create))) + (is (= [{:original_id (:id relationship), + :result "error", + :type :relationship, + :error (str "A relationship cannot be created if a " + "source or a target ref is still a transient " + "ID (The source or target entity is probably " + "not provided in the bundle)")}] + (:results bundle-result-create)))))) (testing "Custom external prefix keys" (let [bundle {:type "bundle" :source "source" - :indicators (hash-set - (assoc (first indicators) - :external_ids - ["custom-2"]))} + :indicators #{(assoc (first indicators) + :external_ids + ["custom-2"])}} response-create (POST app "ctia/bundle/import" - :query-params {"external-key-prefixes" "custom-"} + :query-params {"external-key-prefixes" "custom-" + "patch-existing" false} :body bundle :headers {"Authorization" "45c1f5e3f05d0"}) bundle-result-create (:parsed-body response-create) response-update (POST app "ctia/bundle/import" - :query-params {"external-key-prefixes" "custom-"} + :query-params {"external-key-prefixes" "custom-" + "patch-existing" true} :body bundle :headers {"Authorization" "45c1f5e3f05d0"}) bundle-result-update (:parsed-body response-update)] - (is (= 200 (:status response-create))) - (is (= 200 (:status response-update))) - - (is (pos? (count (:results bundle-result-create)))) - (is (pos? (count (:results bundle-result-update)))) - - (is (every? #(= "created" %) - (map :result (:results bundle-result-create))) - "All new entities are created") - (is (every? #(= "exists" %) - (map :result (:results bundle-result-update))) - "All existing entities are not updated"))) + (when (is (= 200 (:status response-create))) + (is (pos? (count (:results bundle-result-update)))) + (is (every? #(= "created" %) + (map :result (:results bundle-result-create))) + "All new entities are created")) + + (when (is (= 200 (:status response-update))) + (is (pos? (count (:results bundle-result-create)))) + (is (every? #(= "updated" %) + (map :result (:results bundle-result-update))) + "All existing entities are updated")))) + (testing "schema failures" + (testing "Fail on creating partial entities" + (let [partial-indicator (-> (first indicators) + (assoc :external_ids ["custom-3"]) + (dissoc :producer)) + bundle {:type "bundle" + :source "source" + :indicators #{partial-indicator}} + response-create (POST app + "ctia/bundle/import" + :query-params {"external-key-prefixes" "custom-" + "patch-existing" false} + :body bundle + :headers {"Authorization" "45c1f5e3f05d0"}) + bundle-result-create (:parsed-body response-create)] + (when (is (= 200 (:status response-create))) + (let [[{:keys [result error msg]}] (:results bundle-result-create)] + (is (= 1 (count (:results bundle-result-create)))) + (is (= "error" result)) + (is (= "Entity validation Error" error)) + (is (str/ends-with? msg "- failed: (contains? % :producer) spec: :new-indicator/map\n")))))) + (testing "Fail on patching bad partial entities" + (let [bundle {:type "bundle" + :source "source" + :indicators #{(-> (first indicators) + (assoc :external_ids ["custom-3"]))}} + response-create (POST app + "ctia/bundle/import" + :query-params {"external-key-prefixes" "custom-" + "patch-existing" false} + :body bundle + :headers {"Authorization" "45c1f5e3f05d0"}) + bundle-result-create (:parsed-body response-create) + response-update (POST app + "ctia/bundle/import" + :query-params {"external-key-prefixes" "custom-" + "patch-existing" true} + :body (update bundle :indicators #(into #{} (map (fn [e] + (assoc e :producer {:something :bad}))) + %)) + :headers {"Authorization" "45c1f5e3f05d0"}) + bundle-result-update (:parsed-body response-update)] + (when (is (= 200 (:status response-create))) + (is (pos? (count (:results bundle-result-create)))) + (is (every? #(= "created" %) + (map :result (:results bundle-result-create))) + "All new entities are created")) + + (when (is (= 400 (:status response-update))) + (is (= {:errors {:indicators #{{:producer "(not (instance? java.lang.String {:something :bad}))"}}}} + bundle-result-update)))))) (testing "Partial results with errors" (let [indicator-store-state (-> (get-store :indicator) :state) indexname (:index indicator-store-state) ;; close indicator index to produce ES errors on that store - _ (es-index/close! (:conn indicator-store-state) indexname) - bundle {:type "bundle" - :source "source" - :sightings [(mk-sighting 10) - (mk-sighting 11)] - ;; Remove external_ids to avoid errors - ;; coming from the search operation - :indicators [(dissoc (mk-indicator 10) - :external_ids)]} - response-create (POST app - "ctia/bundle/import" - :body bundle - :headers {"Authorization" "45c1f5e3f05d0"}) - bundle-result-create (:parsed-body response-create)] - (is (= 200 (:status response-create))) - (is (every? #(= "created" %) - (->> (:results bundle-result-create) - (filter #(= "sighting" %)) - (map :result))) - "All valid entities are created") - (doseq [entity (:sightings bundle)] - (validate-entity-record - app - (find-result-by-original-id bundle-result-create (:id entity)) - entity)) - (let [indicators (filter - #(= :indicator (:type %)) - (:results bundle-result-create))] - (is (seq indicators) - "The result collection for indicators is not empty") - (is (every? #(contains? % :error) indicators))) - ;; reopen index to enable cleaning - (es-index/open! (:conn indicator-store-state) indexname))))))) + _ (es-index/close! (:conn indicator-store-state) indexname)] + (try (let [bundle {:type "bundle" + :source "source" + :sightings [(mk-sighting 10) + (mk-sighting 11)] + ;; Remove external_ids to avoid errors + ;; coming from the search operation + :indicators [(dissoc (mk-indicator 10) + :external_ids)]} + response-create (POST app + "ctia/bundle/import" + :body bundle + :query-params {"patch-existing" false} + :headers {"Authorization" "45c1f5e3f05d0"}) + bundle-result-create (:parsed-body response-create)] + (when (is (= 200 (:status response-create))) + (is (every? #(= "created" %) + (->> (:results bundle-result-create) + (filter #(= "sighting" %)) + (map :result))) + "All valid entities are created") + (doseq [entity (:sightings bundle)] + (validate-entity-record + app + (find-result-by-original-id bundle-result-create (:id entity)) + entity)) + (let [indicators (filter + #(= :indicator (:type %)) + (:results bundle-result-create))] + (is (seq indicators) + "The result collection for indicators is not empty") + (is (every? #(contains? % :error) indicators))))) + (finally + ;; reopen index to enable cleaning + (es-index/open! (:conn indicator-store-state) indexname))))))))) (deftest ^:encoding bundle-import-non-utf-8-encoding (test-for-each-store-with-app @@ -452,26 +530,6 @@ (get :description)))))))))) (deftest bundle-import-should-not-allow-disabled-entities - (testing "disabled entities should be removed from Bundle schema" - (let [selected-keys #{:asset :target-record :asset-properties} - set-of (fn [model] (set (repeat 3 model))) - fake-bundle {:id "http://ex.tld/ctia/bundle/bundle-5023697b-3857-4652-9b53-ccda297f9c3e" - :source "source" - :type "bundle" - :assets (set-of asset-maximal) - :asset_refs #{"http://ex.tld/ctia/asset/asset-5023697b-3857-4652-9b53-ccda297f9c3e"} - :asset_properties (set-of asset-properties-maximal) - :asset_properties_refs #{"http://ex.tld/ctia/asset-properties/asset-properties-5023697b-3857-4652-9b53-ccda297f9c3e"} - :target_record_refs #{"http://ex.tld/ctia/target-record/target-record-5023697b-3857-4652-9b53-ccda297f9c3e"} - :target_records (set-of target-record-maximal)} - api-handler-svcs {:FeaturesService {:entity-enabled? #(contains? selected-keys %)}}] - (s/set-fn-validation! false) ;; otherwise it fails for incomplete APIHandlerServices passed into `prep-bundle-schema` - (is (map? (s/validate (bundle.routes/prep-bundle-schema api-handler-svcs) fake-bundle))) - (is (thrown? Exception - (s/validate - (bundle.routes/prep-bundle-schema api-handler-svcs) - (assoc fake-bundle :incidents (set-of incident-maximal)))) - "Bundle schema with a key that's not explicitly allowed shouldn't validate"))) (testing "Attempts to import bundle with disabled entities should fail" (let [disable [:asset :asset-properties :actor :sighting]] (helpers/with-config-transformer @@ -486,12 +544,32 @@ "foouser" "foogroup" "user") + ;; moved inside a context with `app` so APIHandlerServices can be stubbed + (testing "disabled entities should be removed from Bundle schema" + (let [selected-keys #{:asset :target-record :asset-properties} + set-of (fn [model] (set (repeat 3 model))) + fake-bundle {:id "http://ex.tld/ctia/bundle/bundle-5023697b-3857-4652-9b53-ccda297f9c3e" + :source "source" + :type "bundle" + :assets (set-of asset-maximal) + :asset_refs #{"http://ex.tld/ctia/asset/asset-5023697b-3857-4652-9b53-ccda297f9c3e"} + :asset_properties (set-of asset-properties-maximal) + :asset_properties_refs #{"http://ex.tld/ctia/asset-properties/asset-properties-5023697b-3857-4652-9b53-ccda297f9c3e"} + :target_record_refs #{"http://ex.tld/ctia/target-record/target-record-5023697b-3857-4652-9b53-ccda297f9c3e"} + :target_records (set-of target-record-maximal)} + api-handler-svcs (assoc-in (app->APIHandlerServices app) [:FeaturesService :entity-enabled?] #(contains? selected-keys %))] + (is (map? (s/validate (core/prep-bundle-schema api-handler-svcs) fake-bundle))) + (is (thrown? clojure.lang.ExceptionInfo + (s/validate + (core/prep-bundle-schema api-handler-svcs) + (assoc fake-bundle :incidents (set-of incident-maximal)))) + "Bundle schema with a key that's not explicitly allowed shouldn't validate"))) (let [new-bundle (deep-dissoc-entity-ids bundle-maximal) resp (POST app "ctia/bundle/import" :body new-bundle :headers {"Authorization" "45c1f5e3f05d0"}) disallowed-keys-expected (->> disable - (mapcat #'bundle.routes/entity->bundle-keys) + (mapcat core/entity->bundle-keys) set) disallowed-keys-res (->> resp :body @@ -556,7 +634,7 @@ (repeat (* 10 core/find-by-external-ids-limit)) (map #(assoc % :id (id/make-transient-id nil)))) more-indicators (map mk-indicator (range 1 10)) - all-indicators (set (concat duplicated-indicators more-indicators)) + all-indicators (into (set duplicated-indicators) more-indicators) duplicated-external-id (-> duplicated-indicators first :external_ids first) all-external-ids (mapcat :external_ids all-indicators) bundle {:type "bundle" @@ -581,9 +659,7 @@ count) max-matched))) (is (= (set all-external-ids) - (->> matched-entities - (mapcat :external_ids) - set)) + (into #{} (mapcat :external_ids) matched-entities)) "all-pages must match at least one entity for each existing external-id"))))) (deftest find-by-external-ids-test @@ -600,30 +676,29 @@ bundle {:type "bundle" :source "source" :indicators (set (map mk-indicator (range nb-entities)))} - response-create (POST app - "ctia/bundle/import" - :body bundle - :headers {"Authorization" "45c1f5e3f05d0"}) + do-import #(POST app + "ctia/bundle/import" + :body bundle + :query-params {"patch-existing" true} + :headers {"Authorization" "45c1f5e3f05d0"}) + response-create (do-import) bundle-result-create (:parsed-body response-create) - response-update (POST app - "ctia/bundle/import" - :body bundle - :query-params {"external-key-prefixes" "ctia-"} - :headers {"Authorization" "45c1f5e3f05d0"}) + response-update (do-import) bundle-result-update (:parsed-body response-update)] (is (= 200 (:status response-create))) (is (= 200 (:status response-update))) (is (= nb-entities - (count (:results bundle-result-create)))) + (count (:results bundle-result-create))) + bundle-result-create) (is (= nb-entities (count (:results bundle-result-update)))) (is (every? #(= "created" %) (map :result (:results bundle-result-create))) "All new entities are created") - (is (every? #(= "exists" %) + (is (every? #(= "updated" %) (map :result (:results bundle-result-update))) - "All existing entities are not updated"))))) + "All existing entities are updated"))))) (def bundle-fixture-1 (let [indicators [(mk-indicator 0) @@ -1240,49 +1315,173 @@ "foogroup" "user") - (testing "relationships are created for asset mappings/properties" - (let [new-bundle (-> bundle-minimal - (assoc :assets #{{:asset_type "device" - :valid_time {:start_time #inst "2023-03-02T19:14:46.658-00:00"} - :schema_version "1.0.19" - :type "asset" - :source "something" - :external_ids ["transient:89497b1a-1e42-4258-81f0-1d394fe1a90f"] - :title "something" - :source_uri "https://something" - :id "transient:89497b1a-1e42-4258-81f0-1d394fe1a90f" - :timestamp #inst "2023-03-02T19:14:46.658-00:00"}} - :asset_mappings #{{:asset_type "device" - :valid_time {:start_time #inst "2023-03-02T19:14:46.660-00:00"} - :stability "Managed" - :schema_version ctim-schema-version - :observable {:value "something" :type "hostname"} - :asset_ref "transient:89497b1a-1e42-4258-81f0-1d394fe1a90f" - :type "asset-mapping" - :source "Something" - :source_uri "https://something" - :id "transient:07b82ae5-0757-4e72-bda4-9a4cd62986e1" - :timestamp #inst "2023-03-02T19:14:46.660-00:00" - :specificity "Unique" - :confidence "Unknown"}} - :asset_properties #{{:properties [{:name "something" :value "66"}] - :valid_time {:start_time #inst "2023-03-02T19:14:46.660-00:00"} - :schema_version ctim-schema-version - :asset_ref "transient:89497b1a-1e42-4258-81f0-1d394fe1a90f" - :type "asset-properties" - :source "somethintg" - :source_uri "https://something" - :id "transient:8ae8d2b0-950b-402d-b053-935da85582a3" - :timestamp #inst "2023-03-02T19:14:46.783-00:00"}} - :relationships #{{:source_ref "https://private.intel.int.iroh.site:443/ctia/incident/incident-4fb91401-36a5-46d1-b0aa-01af02f00a7a", :target_ref "transient:07b82ae5-0757-4e72-bda4-9a4cd62986e1", :relationship_type "related-to", :source "IROH Risk Score Service"} - {:source_ref "https://private.intel.int.iroh.site:443/ctia/incident/incident-4fb91401-36a5-46d1-b0aa-01af02f00a7a", :target_ref "transient:8ae8d2b0-950b-402d-b053-935da85582a3", :relationship_type "related-to", :source "IROH Risk Score Service"} - {:source_ref "https://private.intel.int.iroh.site:443/ctia/incident/incident-4fb91401-36a5-46d1-b0aa-01af02f00a7a", :target_ref "transient:89497b1a-1e42-4258-81f0-1d394fe1a90f", :relationship_type "related-to", :source "IROH Risk Score Service"}})) - response (POST app - "ctia/bundle/import" - :body new-bundle - :headers {"Authorization" "45c1f5e3f05d0"}) - {:keys [results]} (:parsed-body response)] - (is (= 200 (:status response))) - (is (= 6 (count results))) - (is (every? (comp #{"created"} :result) results) - (pr-str (mapv :result results)))))))) + (let [[oldv1 oldv2 newv1 newv2 newv3] (map (comp str gensym) '[oldv1 oldv2 newv1 newv2 newv3]) + [relationship1-original-id + asset_mapping1-original-id + asset_property1-original-id + asset1-original-id + asset2-original-id] (map #(str "transient:" % "_" (random-uuid)) + '[relationship1-original-id + asset_mapping1-original-id + asset_property1-original-id + asset1-original-id + asset2-original-id]) + asset1 {:asset_type "device" + :valid_time {:start_time #inst "2023-03-02T19:14:46.658-00:00"} + :schema_version "1.0.19" + :type "asset" + :source "something" + :external_ids [(str "asset-1" (random-uuid))] + :title "something" + :source_uri "https://something" + :id asset1-original-id + :timestamp #inst "2023-03-02T19:14:46.658-00:00"} + asset_mapping1 {:asset_type "device" + :valid_time {:start_time #inst "2023-03-02T19:14:46.660-00:00"} + :stability "Managed" + :schema_version ctim-schema-version + :observable {:value "something" :type "hostname"} + :asset_ref asset1-original-id + :type "asset-mapping" + :source "Something" + :source_uri "https://something" + :id asset_mapping1-original-id + :timestamp #inst "2023-03-02T19:14:46.660-00:00" + :specificity "Unique" + :confidence "Unknown"} + old-properties [{:name "something1" :value oldv1} + {:name "something2" :value oldv2}] + asset_property1 {:properties old-properties + :valid_time {:start_time #inst "2023-03-02T19:14:46.660-00:00"} + :schema_version ctim-schema-version + :asset_ref asset1-original-id + :type "asset-properties" + :source "something" + :source_uri "https://something" + :id asset_property1-original-id + :timestamp #inst "2023-03-02T19:14:46.783-00:00"} + new-bundle (-> bundle-minimal + (assoc :assets #{asset1} + :asset_mappings #{asset_mapping1} + :asset_properties #{asset_property1} + :relationships #{{:id relationship1-original-id + :source_ref "https://private.intel.int.iroh.site:443/ctia/incident/incident-4fb91401-36a5-46d1-b0aa-01af02f00a7a" + :target_ref asset_mapping1-original-id, :relationship_type "related-to", :source "IROH Risk Score Service"} + {:source_ref "https://private.intel.int.iroh.site:443/ctia/incident/incident-4fb91401-36a5-46d1-b0aa-01af02f00a7a" + :target_ref asset_property1-original-id, :relationship_type "related-to", :source "IROH Risk Score Service"} + {:source_ref "https://private.intel.int.iroh.site:443/ctia/incident/incident-4fb91401-36a5-46d1-b0aa-01af02f00a7a" + :target_ref asset1-original-id, :relationship_type "related-to", :source "IROH Risk Score Service"}})) + create-response (POST app + "ctia/bundle/import" + :body new-bundle + :headers {"Authorization" "45c1f5e3f05d0"}) + {create-results :results :as create-bundle-results} (:parsed-body create-response) + ;; resolve in order of creation/patch for easier debugging + asset1-id (find-id-by-original-id :asset1-id create-bundle-results asset1-original-id) + asset_property1-id (find-id-by-original-id :asset_property1-id create-bundle-results asset_property1-original-id) + asset_mapping1-id (find-id-by-original-id :asset_mapping1-id create-bundle-results asset_mapping1-original-id) + relationship1-id (find-id-by-original-id :relationship1-id create-bundle-results relationship1-original-id)] + (testing "relationships are created for asset mappings/properties" + (when (is (= 200 (:status create-response))) + (is (= 6 (count create-results))) + (is (every? (comp #{"created"} :result) create-results) + (pr-str (mapv :result create-results))))) + (testing "asset-properties and asset-mappings are patched" + (let [new-properties [{:name "something1" :value newv1} + {:name "something-else" :value newv2}] + updated-asset_property1 (-> asset_property1 + (select-keys [:id :asset_ref :type]) + (assoc :properties new-properties)) + updated-asset_mapping1 asset_mapping1 + update-bundle (-> bundle-minimal + (assoc :assets #{asset1} + :asset_mappings #{updated-asset_mapping1} + :asset_properties #{updated-asset_property1})) + test-merge-strategy (fn [msg query-params expected-properties] + (testing msg + (let [update-response (POST app + "ctia/bundle/import" + :body update-bundle + :query-params (assoc query-params "patch-existing" true) + :headers {"Authorization" "45c1f5e3f05d0"}) + {update-results :results :as update-bundle-result} (:parsed-body update-response)] + (when (is (= 200 (:status update-response))) + (is (= 3 (count update-results)) update-results) + (is (every? (comp #{"updated"} :result) update-results) + (pr-str (mapv :result update-results))) + (let [get-stored (fn [entity] + (let [realized-id (some (fn [{:keys [original_id id]}] + (when (= original_id (:id entity)) + id)) + update-results) + _ (assert realized-id [(:id entity) update-results]) + response (GET app + (format "ctia/%s/%s" + (name (:type entity)) + (encode realized-id)) + :headers {"Authorization" "45c1f5e3f05d0"})] + (:parsed-body response)))] + (testing ":asset_mappings" + (let [stored (get-stored updated-asset_mapping1)] + ;; no interesting merging logic on asset mappings + (is (= (dissoc stored :id :schema_version :asset_ref :owner :groups :timestamp) + (-> updated-asset_mapping1 + (dissoc :id :schema_version :asset_ref :timestamp) + (assoc :tlp "green") + (assoc-in [:valid_time :end_time] #inst "2525-01-01T00:00:00.000-00:00")))))) + (testing ":asset_properties" + (let [stored (get-stored updated-asset_property1)] + (is (= expected-properties + (:properties stored))) + (is (= #inst "2525-01-01T00:00:00.000-00:00" + (get-in stored [:valid_time :end_time]))))))))))] + ;; the order in which we test these merge strategies is important since we're patching the same entities. + (test-merge-strategy "with asset_properties-merge-strategy=merge-overriding-previous" + {"asset_properties-merge-strategy" "merge-overriding-previous"} + [{:name "something-else" :value newv2} + {:name "something1" :value newv1} + {:name "something2" :value oldv2}]) + (test-merge-strategy "no asset_properties merge strategy" + {} + new-properties))) + (testing "patched relationships, asset mappings/properties resolve refs after creating other entities" + (let [;; on existing entities, patch asset_ref to a newly created asset, + ;; and :{source/target}_ref to newly created entities + asset2-external-id (str "asset-2-" (random-uuid)) + asset2 (assoc asset1 :id asset2-original-id :external_ids [asset2-external-id]) + updated-asset_property1 {:id asset_property1-id + :asset_ref asset2-original-id} + updated-asset_mapping1 {:id asset_mapping1-id + :asset_ref asset2-original-id} + updated-relationship1 {:id relationship1-id + :source_ref asset2-original-id + :target_ref asset2-original-id} + create+update-bundle (-> bundle-minimal + (assoc :assets #{asset2} + :asset_properties #{updated-asset_property1} + :asset_mappings #{updated-asset_mapping1} + :relationships #{updated-relationship1})) + create+update-response (POST app + "ctia/bundle/import" + :body create+update-bundle + :query-params {"patch-existing" true} + :headers {"Authorization" "45c1f5e3f05d0"}) + {create+update-results :results :as create+update-bundle-result} (:parsed-body create+update-response)] + (when (is (= 200 (:status create+update-response))) + (let [asset2-id (find-id-by-original-id :asset2-id create+update-bundle-result asset2-original-id)] + (is (= 4 (count create+update-results))) + (is (= #{{:id asset_mapping1-id + :result "updated" + :type :asset-mapping} + {:id asset_property1-id + :result "updated" + :type :asset-properties} + {:id asset2-id + :original_id asset2-original-id + :result "created" + :type :asset + :external_ids [asset2-external-id]} + {:id relationship1-id + :result "updated" + :type :relationship}} + (set create+update-results))))))))))) diff --git a/test/ctia/schemas/core_test.clj b/test/ctia/schemas/core_test.clj new file mode 100644 index 0000000000..2524eca5bc --- /dev/null +++ b/test/ctia/schemas/core_test.clj @@ -0,0 +1,19 @@ +(ns ctia.schemas.core-test + (:require [clojure.test :refer [deftest is]] + [ctia.schemas.core :as sut])) + +(deftest transient-id?-test + (is (not (sut/transient-id? nil))) + (is (not (sut/transient-id? ""))) + (is (not (sut/transient-id? "incident-1234"))) + (is (not (sut/transient-id? "incident-0e14cef7-fbd9-4c06-a6d6-332ad82d5b32"))) + (is (not (sut/transient-id? "http://localhost:3000/ctia/incident/incident-0e14cef7-fbd9-4c06-a6d6-332ad82d5b32"))) + (is (sut/transient-id? "transient:incident-1234"))) + +(deftest non-transient-id?-test + (is (not (sut/non-transient-id? nil))) + (is (not (sut/non-transient-id? ""))) + (is (not (sut/non-transient-id? "incident-1234"))) + (is (sut/non-transient-id? "incident-0e14cef7-fbd9-4c06-a6d6-332ad82d5b32")) + (is (sut/non-transient-id? "http://localhost:3000/ctia/incident/incident-0e14cef7-fbd9-4c06-a6d6-332ad82d5b32")) + (is (not (sut/non-transient-id? "transient:incident-1234"))))