CMR-9187 - Sending provider events to ordering (#1934)

* Calling CRM Ordering when a provider is created or updated so that it can resync the copy of providers that it has * Update to transmit access control to use same client id as ordering Co-authored-by: K. Lucia Z. <[email protected]>
nasa · Jul 21, 2023 · 3238156 · 3238156
1 parent 7081007
commit 3238156
Show file tree

Hide file tree

Showing 12 changed files with 132 additions and 19 deletions.
diff --git a/.gitignore b/.gitignore
@@ -35,6 +35,7 @@ schema-validation-lib/build.xml
 /target
 pom.xml
 browse-scaler/src/*.zip
+cmr-exchange/*/target
 
 ###############################
 ### macOS

diff --git a/dev-system/resources/ordering_examples/api b/dev-system/resources/ordering_examples/api
@@ -0,0 +1,9 @@
+{
+  "data": {
+    "syncProviders": {
+      "added": [],
+      "deleted": [],
+      "message": "0 providers changed"
+    }
+  }
+}
diff --git a/dev-system/src/cmr/dev_system/control.clj b/dev-system/src/cmr/dev_system/control.clj
@@ -95,6 +95,16 @@
            :headers {"Content-Type" "application/csv; charset=utf-8"}}
           (route/not-found "KMS resource not found\n"))))
 
+    ;; Retrive Ordering resources
+    (POST "/ordering/:filename" [filename]
+      (let [resource (io/resource (str "ordering_examples/" filename))]
+        (info (format "Responding to ordering request for [%s]" filename))
+        (if resource
+          {:status 200
+           :body (slurp resource)
+           :headers {"Content-Type" "application/csv; charset=utf-8"}}
+          (route/not-found "ordering resource not found\n"))))
+
     ;; For debugging. Gets the state of the world in relations to ACLs and what's indexed
     (GET "/acl-state" []
       {:status 200

diff --git a/ingest-app/src/cmr/ingest/services/provider_service.clj b/ingest-app/src/cmr/ingest/services/provider_service.clj
@@ -5,7 +5,8 @@
   (:require
    [cmr.common.services.errors :as errors]
    [cmr.ingest.data.ingest-events :as ingest-events]
-   [cmr.transmit.metadata-db :as mdb]))
+   [cmr.transmit.metadata-db :as mdb]
+   [cmr.transmit.ordering :as ordering]))
 
 (defn verify-empty-provider
   "Throws error if provider still has collections."
@@ -35,8 +36,9 @@
   [context provider]
   (let [response (mdb/create-provider-raw context provider)]
     (when (successful? response)
+      (ordering/notify-ordering context)
       (ingest-events/publish-provider-event
-        context (ingest-events/provider-create-event (:provider-id provider))))
+       context (ingest-events/provider-create-event (:provider-id provider))))
     response))
 
 (defn read-provider
@@ -49,6 +51,7 @@
   [context provider]
   (let [response (mdb/update-provider-raw context provider)]
     (when (successful? response)
+      (ordering/notify-ordering context)
       (ingest-events/publish-provider-event
         context (ingest-events/provider-update-event (:provider-id provider))))
     response))
@@ -58,6 +61,7 @@
   [context provider-id]
   (let [response (mdb/delete-provider-raw context provider-id)]
     (when (successful? response)
+      (ordering/notify-ordering context)
       (ingest-events/publish-provider-event
         context (ingest-events/provider-delete-event provider-id)))
     response))

diff --git a/ingest-app/src/cmr/ingest/system.clj b/ingest-app/src/cmr/ingest/system.clj
@@ -118,7 +118,7 @@
               :public-conf (public-conf)
               :queue-broker (queue-broker/create-queue-broker (config/queue-config))}]
      (transmit-config/system-with-connections
-       sys [:metadata-db :indexer :access-control :echo-rest :search :kms :urs]))))
+       sys [:metadata-db :indexer :access-control :echo-rest :search :kms :ordering :urs]))))
 
 (defn start
   "Performs side effects to initialize the system, acquire resources,

diff --git a/message-queue-lib/src/cmr/message_queue/queue/memory_queue.clj b/message-queue-lib/src/cmr/message_queue/queue/memory_queue.clj
@@ -175,13 +175,13 @@
       (println "Handling" (pr-str msg) "from queue" queue-name)
       {:status :success})
 
-    (queue/subscribe running-qb "a" (partial message-handler "a"))
-    (queue/subscribe running-qb "b" (partial message-handler "b"))
-    (queue/subscribe running-qb "c" (partial message-handler "c")))
+    (queue-protocol/subscribe running-qb "a" (partial message-handler "a"))
+    (queue-protocol/subscribe running-qb "b" (partial message-handler "b"))
+    (queue-protocol/subscribe running-qb "c" (partial message-handler "c")))
 
-  (queue/publish-to-queue running-qb "a" {:id 1})
-  (queue/publish-to-queue running-qb "b" {:id 2})
-  (queue/publish-to-queue running-qb "c" {:id 3})
+  (queue-protocol/publish-to-queue running-qb "a" {:id 1})
+  (queue-protocol/publish-to-queue running-qb "b" {:id 2})
+  (queue-protocol/publish-to-queue running-qb "c" {:id 3})
 
-  (queue/publish-to-exchange running-qb "e1" {:id 4})
-  (queue/publish-to-exchange running-qb "e2" {:id 5}))
+  (queue-protocol/publish-to-exchange running-qb "e1" {:id 4})
+  (queue-protocol/publish-to-exchange running-qb "e2" {:id 5}))
diff --git a/schemas/project.clj b/schemas/project.clj
@@ -5,9 +5,11 @@
             :name "Apache License, Version 2.0"
             :url "http://www.apache.org/licenses/LICENSE-2.0"}
   :dependencies [[org.clojure/clojure "1.10.0"]]
+  :plugins [[lein-shell "0.5.0"]]
   :profiles {:security {:plugins [[com.livingsocial/lein-dependency-check "1.4.1"]]
                         :dependency-check {:output-format [:all]
                                            :suppression-file "resources/security/suppression.xml"}}
+
              :test {:plugins [[lein-shell "0.5.0"]]}}
 
   :aliases {;; Kaocha test aliases
@@ -18,7 +20,7 @@
             "ci-test" ["kaocha"]
             "ci-itest" ["itest"]
             "ci-utest" ["utest"]
-
+            "lint" ["shell" "echo" "lint is NOT supported for this project"]
             ;; The following aliases are needed for the CMR build process.
             "generate-static" ["with-profile" "+test" "shell" "echo" "NO OP"]
             "check-sec" ["with-profile" "security" "dependency-check"]})
diff --git a/site-templates/project.clj b/site-templates/project.clj
@@ -1,15 +1,14 @@
 (defproject gov.nasa.earthdata/cmr-site-templates "0.1.1-SNAPSHOT"
   :description "Selmer templates for CMR documentation, directory pages, and various static web content"
   :url "https://github.com/nasa/Common-Metadata-Repository/site-templates"
-  :license {
-            :name "Apache License, Version 2.0"
+  :license {:name "Apache License, Version 2.0"
             :url "http://www.apache.org/licenses/LICENSE-2.0"}
   :dependencies [[org.clojure/clojure "1.10.0"]]
   :profiles {:security {:plugins [[com.livingsocial/lein-dependency-check "1.4.1"]]
                         :dependency-check {:output-format [:all]
                                            :suppression-file "resources/security/suppression.xml"}}
              :test {:plugins [[lein-shell "0.5.0"]]}}
-
+  :plugins [[lein-shell "0.5.0"]]
   :aliases {;; Kaocha test aliases
             ;; refer to tests.edn for test configuration
             "kaocha" ["check"]
@@ -18,6 +17,7 @@
             "ci-test" ["kaocha"]
             "ci-itest" ["itest"]
             "ci-utest" ["utest"]
+            "lint" ["shell" "echo" "lint is NOT supported in this project"]
 
             ;; The following aliases are needed for the CMR build process.
             "generate-static" ["with-profile" "+test" "shell" "echo" "NO OP"]

diff --git a/transmit-lib/src/cmr/transmit/access_control.clj b/transmit-lib/src/cmr/transmit/access_control.clj
@@ -2,10 +2,10 @@
   "This contains functions for interacting with the access control API."
   (:require
    [cheshire.core :as json]
-   [clojure.string :as string]
    [cmr.transmit.config :as config]
    [cmr.transmit.connection :as conn]
-   [cmr.transmit.http-helper :as h]))
+   [cmr.transmit.http-helper :as h]
+   [cmr.transmit.config :as transmit-config]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; URL functions
@@ -134,7 +134,7 @@
   (let [token (:token context)
         headers (-> token
                     (when {config/token-header token})
-                    (as-> item (merge {:client-id "cmr-internal"
+                    (as-> item (merge {:client-id transmit-config/cmr-client-id
                                        :content-type "application/json"} item)))
         body (json/generate-string {:user-token user-token})]
     (h/request context :access-control

diff --git a/transmit-lib/src/cmr/transmit/config.clj b/transmit-lib/src/cmr/transmit/config.clj
@@ -13,7 +13,6 @@
   between public and private communications."
   (:require
    [camel-snake-kebab.core :as csk]
-   [cheshire.core :as json]
    [cmr.common.config :as cfg :refer [defconfig]]
    [cmr.common.util :as util]
    [cmr.transmit.connection :as conn]))
@@ -37,6 +36,10 @@
   "The test user's password"
   "Password101")
 
+(def cmr-client-id
+  "A client id to use as a header when communicating to other CMR services"
+  "cmr-internal")
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (def token-header
@@ -87,6 +90,8 @@
 (def-app-conn-config indexer {:port 3004})
 (def-app-conn-config ingest {:port 3002})
 (def-app-conn-config kms {:port 2999, :relative-root-url "/kms"})
+;; nothing is here, no respones is needed at this time
+(def-app-conn-config ordering {:port 2999, :relative-root-url "/ordering/api"})
 (def-app-conn-config metadata-db {:port 3001})
 ;; CMR open search is 3010
 (def-app-conn-config search {:port 3003})
@@ -243,6 +248,10 @@
          :host (kms-host)
          :port (kms-port)
          :context (kms-relative-root-url)}
+   :ordering {:protocol (ordering-protocol)
+              :host (ordering-host)
+              :port (ordering-port)
+              :context (ordering-relative-root-url)}
    :urs {:protocol (urs-protocol)
          :host (urs-host)
          :port (urs-port)

diff --git a/transmit-lib/src/cmr/transmit/ordering.clj b/transmit-lib/src/cmr/transmit/ordering.clj
@@ -0,0 +1,59 @@
+(ns cmr.transmit.ordering
+  "Handle all communications to the CMR-Ordering application, a graphql app which
+   processes orders for Earthdata Search and CMR. This application needs to know
+   when CMR has changed a provider so that it can download the latest provider
+   list"
+
+  (:require
+   [clj-http.client :as client]
+   [cmr.common.api.context :as ctxt]
+   [cmr.common.log :as log :refer (debug info warn error)]
+   [cmr.common.mime-types :as mime]
+   [cmr.transmit.config :as config]
+   [cmr.transmit.connection :as conn]
+   [clojure.core.async :as async]
+   [clojure.string :as string]))
+
+;; GraphQl query to call mutation of data
+;; query='mutation SyncProvider {syncProviders {added,deleted,message}}'
+;; server="https://cmr.sit.earthdata.nasa.gov/ordering/api"
+;; curl -s \
+;;   --header 'Content-Type: application/json' \
+;;   --data "{\"query\": \"$query\"}" \
+;;   "$server"
+
+(def order-provider-sync-message
+  "mutation SyncProvider {syncProviders {added,deleted,message}}")
+
+(defn- context->just-token
+  "graph-ql does not require bearer in the token"
+  [context]
+  (-> context
+      :token
+      (string/replace-first "Bearer:" "")
+      (string/replace-first "bearer:" "")
+      string/trim))
+
+(defn- send-to-ordering
+  "A generic message send action"
+  [context message]
+  (let [conn (config/context->app-connection context :ordering)
+        url (conn/root-url conn)
+        token (context->just-token context)
+        params (merge
+                (config/conn-params conn)
+                {:body (format "{\"query\": \"%s\"}" message)
+                 :headers (merge
+                           (ctxt/context->http-headers context)
+                           {:content-type mime/json
+                            :client-id config/cmr-client-id
+                            config/token-header token})
+                 :throw-exceptions false})
+        response (client/post url params)]
+      (info (format "Provider change event, notifying [%s] - response is [%s]\n" url response))))
+
+(defn notify-ordering
+  "Sent a message to the sync mutation at cmr-ordering"
+  [context]
+  (debug "sending sync message to cmr-ordering")
+  (async/go (send-to-ordering context order-provider-sync-message)))
diff --git a/transmit-lib/test/cmr/transmit/test/ordering.clj b/transmit-lib/test/cmr/transmit/test/ordering.clj
@@ -0,0 +1,19 @@
+(ns cmr.transmit.test.ordering
+  "Contains unit tests for verifying ordering notifications."
+  (:require [clojure.test :refer :all]
+            [clj-http.client :as client]
+            [cmr.common.util :refer [are3]]
+            [cmr.transmit.ordering :as ordering]))
+
+(deftest just-token-test
+  (testing "Test token prep function to ensure graphql gets the correct form"
+    (are3
+     [expected provided]
+     (do (let [context {:token provided}]
+           (is (= expected (#'cmr.transmit.ordering/context->just-token context)))))
+     "No bearer" "1234" "1234"
+     "Upper case bearer" "1234" "Bearer:1234"
+     "Lower case bearer" "1234" "bearer:1234"
+     "Spaces around bearer" "1234" " bearer: 1234"
+     "nothing" "" ""
+     "bearer but no token" "" " Bearer: ")))