CMR-10023: Part 1

search-app/dev/user.clj

@@ -19,7 +19,6 @@
 
 (def system nil)
 
-
 (defn create-system
   []
   ;; Set the default job start delay to avoid jobs kicking off with tests etc.

search-app/src/cmr/search/api/association.clj

@@ -3,13 +3,12 @@
   (:require
    [cheshire.core :as json]
    [cmr.common-app.api.enabled :as common-enabled]
-   [cmr.common.log :refer (info)]
+   [cmr.common.log :refer (debug info)]
    [cmr.common.mime-types :as mt]
    [cmr.common.util :as util]
    [cmr.search.services.association-service :as assoc-service]
    [cmr.search.services.association-validation :as assoc-validation]
-   [compojure.core :refer :all]
-   [compojure.route :as route]))
+   [compojure.core :refer :all]))
 
 (defn- validate-association-content-type
   "Validates that content type sent with a association is JSON."
@@ -65,7 +64,7 @@
   [context headers body concept-type concept-id]
   (common-enabled/validate-write-enabled context "search")
   (validate-association-content-type headers)
-  (info (format "Dissociating %s [%s] from collections: %s by client: %s."
+  (debug (format "Dissociating %s [%s] from collections: %s by client: %s."
                 (name concept-type) concept-id body (:client-id context)))
   (if (and (> (count (map :concept-id (json/parse-string body true))) 1)
            (= :variable concept-type)) 

search-app/src/cmr/search/api/community_usage_metrics.clj

@@ -7,10 +7,8 @@
    [cmr.acl.core :as acl]
    [cmr.common-app.api.routes :as cr]
    [cmr.common.mime-types :as mt]
-   [cmr.common.services.errors :as errors]
    [cmr.search.services.community-usage-metrics.metrics-service :as metrics-service]
-   [compojure.core :refer :all]
-   [compojure.route :as route]))
+   [compojure.core :refer :all]))
 
 (defn- community-usage-metrics-response
   "Creates a successful community usage metrics response with the given data response"

search-app/src/cmr/search/api/concepts_lookup.clj

@@ -3,7 +3,7 @@
   (:require
    [cmr.common-app.api.routes :as common-routes]
    [cmr.common.concepts :as concepts]
-   [cmr.common.log :refer (debug info warn error)]
+   [cmr.common.log :refer (debug)]
    [cmr.common.mime-types :as mt]
    [cmr.common.services.errors :as svc-errors]
    [cmr.search.api.core :as core-api]
@@ -73,13 +73,13 @@
 (defn- find-concept-by-concept-id*
   "Perfrom the retrieval of concept by concept id and revision id"
   ([ctx result-format concept-id]
-   (info (format "Search for concept with cmr-concept-id [%s]" concept-id))
+   (debug (format "Search for concept with cmr-concept-id [%s]" concept-id))
    (core-api/search-response
     ctx
     (query-svc/find-concept-by-id ctx result-format concept-id)))
 
   ([ctx result-format concept-id revision-id]
-   (info (format "Search for concept with cmr-concept-id [%s] and revision-id [%s]"
+   (debug (format "Search for concept with cmr-concept-id [%s] and revision-id [%s]"
                  concept-id
                  revision-id))
    (core-api/search-response

search-app/src/cmr/search/api/concepts_search.clj

@@ -3,13 +3,12 @@
   (:require
    [cheshire.core :as json]
    [clojure.string :as string]
-   [clojure.walk :as walk]
    [cmr.common-app.api.launchpad-token-validation :refer [get-token-type]]
    [cmr.common-app.api.routes :as common-routes]
    [cmr.common-app.config :as common-app-config]
    [cmr.common-app.services.search :as search]
    [cmr.common.cache :as cache]
-   [cmr.common.config :as cfg :refer [defconfig]]
+   [cmr.common.config :refer [defconfig]]
    [cmr.common.generics :as common-generic]
    [cmr.common.log :refer (debug info warn error)]
    [cmr.common.mime-types :as mt]
@@ -20,8 +19,7 @@
    [cmr.search.services.query-service :as query-svc]
    [cmr.search.services.result-format-helper :as rfh]
    [cmr.search.validators.all-granule-validation :as all-gran-validation]
-   [compojure.core :refer :all]
-   [inflections.core :as inf]))
+   [compojure.core :refer :all]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;; Constants and Utility Functions
@@ -287,7 +285,7 @@
   "Retrieves a timeline of granules within each collection found."
   [ctx path-w-extension params headers query-string]
   (let [params (core-api/process-params :granule params path-w-extension headers mt/json)
-        _ (info (format "Getting granule timeline from client %s, token_type %s with params %s."
+        _ (debug (format "Getting granule timeline from client %s, token_type %s with params %s."
                         (:client-id ctx) (get-token-type (:token ctx)) (pr-str params)))
         search-params (lp/process-legacy-psa params)]
     (core-api/search-response ctx (query-svc/get-granule-timeline ctx search-params))))
@@ -299,7 +297,7 @@
         _ (when (= :stac (:result-format params))
             (svc-errors/throw-service-error
              :bad-request "search by JSON query is not allowed with STAC result format"))
-        _ (info (format "Searching for concepts from client %s in format %s with AQL: %s and query parameters %s."
+        _ (debug (format "Searching for concepts from client %s in format %s with AQL: %s and query parameters %s."
                         (:client-id ctx) (rfh/printable-result-format (:result-format params)) aql params))
         results (query-svc/find-concepts-by-aql ctx params aql)]
     (core-api/search-response ctx results)))
@@ -320,7 +318,7 @@
   "Invokes query service to search for collections that are deleted and returns the response"
   [ctx path-w-extension params headers]
   (let [params (core-api/process-params :collection params path-w-extension headers mt/xml)]
-    (info (format "Searching for deleted collections from client %s in format %s with params %s."
+    (debug (format "Searching for deleted collections from client %s in format %s with params %s."
                   (:client-id ctx) (rfh/printable-result-format (:result-format params))
                   (pr-str params)))
     (core-api/search-response ctx (query-svc/get-deleted-collections ctx params))))
@@ -329,7 +327,7 @@
   "Invokes query service to search for granules that are deleted and returns the response"
   [ctx path-w-extension params headers]
   (let [params (core-api/process-params nil params path-w-extension headers mt/xml)]
-    (info (format "Searching for deleted granules from client %s in format %s with params %s."
+    (debug (format "Searching for deleted granules from client %s in format %s with params %s."
                   (:client-id ctx) (rfh/printable-result-format (:result-format params))
                   (pr-str params)))
     (core-api/search-response ctx (query-svc/get-deleted-granules ctx params))))
@@ -361,7 +359,7 @@
                                (json/parse-string true)
                                :scroll_id)]
     (do
-      (info (format "Clear scroll: %s" short-scroll-id))
+      (debug (format "Clear scroll: %s" short-scroll-id))
       ;; if the short scroll id is valid, retrieve the real scroll id
       (if-let [scroll-id (->> short-scroll-id
                               (core-api/get-scroll-id-and-search-params-from-cache context)

search-app/src/cmr/search/api/generic_association.clj

@@ -4,7 +4,7 @@
    [cheshire.core :as json]
    [cmr.common-app.api.enabled :as common-enabled]
    [cmr.common.concepts :as common-concepts]
-   [cmr.common.log :refer (info)]
+   [cmr.common.log :refer (debug)]
    [cmr.common.mime-types :as mt]
    [cmr.common.util :as util]
    [cmr.search.services.generic-association-service :as generic-assoc-service]
@@ -43,7 +43,7 @@
   [context headers body concept-id revision-id]
   (common-enabled/validate-write-enabled context "search")
   (validate-association-content-type headers)
-  (info (format "Associate concept [%s] revision [%s] on concepts: %s by client: %s."
+  (debug (format "Associate concept [%s] revision [%s] on concepts: %s by client: %s."
                 concept-id revision-id body (:client-id context)))
   (let [concept-type (common-concepts/concept-id->type concept-id)
         results (generic-assoc-service/associate-to-concepts context concept-type concept-id revision-id body)
@@ -56,7 +56,7 @@
   [context headers body concept-id revision-id]
   (common-enabled/validate-write-enabled context "search")
   (validate-association-content-type headers)
-  (info (format "Dissociating concept [%s] revision [%s] from concepts: %s by client: %s."
+  (debug (format "Dissociating concept [%s] revision [%s] from concepts: %s by client: %s."
                 concept-id revision-id body (:client-id context)))
   (let [concept-type (common-concepts/concept-id->type concept-id)
         results (generic-assoc-service/dissociate-from-concepts context concept-type concept-id revision-id body)

search-app/src/cmr/search/api/providers.clj

@@ -3,7 +3,7 @@
   (:require
    [cheshire.core :as json]
    [cmr.common-app.api.routes :as common-routes]
-   [cmr.common.log :refer (debug info warn error)]
+   [cmr.common.log :refer (debug)]
    [cmr.common.mime-types :as mt]
    [cmr.search.api.core :as core-api]
    [cmr.search.services.provider-service :as provider-service]
@@ -31,7 +31,7 @@
   "Invokes query service to retrieve provider holdings and returns the response"
   [ctx path-w-extension params headers]
   (let [params (core-api/process-params nil params path-w-extension headers mt/json)
-        _ (info (format "Searching for provider holdings from client %s in format %s with params %s."
+        _ (debug (format "Searching for provider holdings from client %s in format %s with params %s."
                         (:client-id ctx) (rfh/printable-result-format (:result-format params))
                         (pr-str params)))
         [provider-holdings provider-holdings-formatted]

search-app/src/cmr/search/api/services.clj

@@ -2,8 +2,7 @@
   "Defines the API for associating/dissociating services with collections in the CMR."
   (:require
    [cmr.search.api.association :as association]
-   [compojure.core :refer :all]
-   [compojure.route :as route]))
+   [compojure.core :refer :all]))
 
 (def service-api-routes
   (context "/services" []

search-app/src/cmr/search/api/tags.clj

@@ -5,15 +5,13 @@
    [clojure.string :as string]
    [cmr.acl.core :as acl]
    [cmr.common-app.api.enabled :as common-enabled]
-   [cmr.common-app.api.launchpad-token-validation :as lt-validation]
-   [cmr.common.log :refer (debug info warn error)]
+   [cmr.common.log :refer (debug)]
    [cmr.common.mime-types :as mt]
    [cmr.common.services.errors :as errors]
    [cmr.common.util :as util]
    [cmr.search.services.tagging-service :as tagging-service]
    [cmr.transmit.config :as transmit-config]
-   [compojure.core :refer :all]
-   [compojure.route :as route]))
+   [compojure.core :refer :all]))
 
 (defn- validate-tag-content-type
   "Validates that content type sent with a tag is JSON"
@@ -81,7 +79,7 @@
   (verify-tag-modification-permission context :update)
   (common-enabled/validate-write-enabled context "search")
   (validate-tag-content-type headers)
-  (info (format "Tagging [%s] on collections: %s by client: %s."
+  (debug (format "Tagging [%s] on collections: %s by client: %s."
                 tag-key body (:client-id context)))
   (tag-api-response (tagging-service/associate-tag-to-collections context tag-key body)))
 
@@ -91,7 +89,7 @@
   (verify-tag-modification-permission context :update)
   (common-enabled/validate-write-enabled context "search")
   (validate-tag-content-type headers)
-  (info (format "Dissociating tag [%s] from collections: %s by client: %s."
+  (debug (format "Dissociating tag [%s] from collections: %s by client: %s."
                 tag-key body (:client-id context)))
   (tag-api-response (tagging-service/dissociate-tag-to-collections context tag-key body)))
 
@@ -101,7 +99,7 @@
   (verify-tag-modification-permission context :update)
   (common-enabled/validate-write-enabled context "search")
   (validate-tag-content-type headers)
-  (info (format "Tagging [%s] on collections by query: %s by client: %s."
+  (debug (format "Tagging [%s] on collections by query: %s by client: %s."
                 tag-key body (:client-id context)))
   (tag-api-response (tagging-service/associate-tag-by-query context tag-key body)))
 
@@ -111,7 +109,7 @@
   (verify-tag-modification-permission context :update)
   (common-enabled/validate-write-enabled context "search")
   (validate-tag-content-type headers)
-  (info (format "Dissociating tag [%s] from collections by query: %s by client: %s."
+  (debug (format "Dissociating tag [%s] from collections by query: %s by client: %s."
                 tag-key body (:client-id context)))
   (tag-api-response (tagging-service/dissociate-tag-by-query context tag-key body)))
 

search-app/src/cmr/search/api/tools.clj

@@ -2,8 +2,7 @@
   "Defines the API for associating/dissociating tools with collections in the CMR."
   (:require
    [cmr.search.api.association :as association]
-   [compojure.core :refer :all]
-   [compojure.route :as route]))
+   [compojure.core :refer :all]))
 
 (def tool-api-routes
   (context "/tools" []

search-app/src/cmr/search/data/complex_to_simple_converters/has_granules.clj

@@ -2,7 +2,6 @@
   (:require
    [cmr.elastic-utils.search.query-transform :as c2s]
    [cmr.common.services.search.query-model :as cqm]
-   [cmr.search.models.query :as qm]
    [cmr.search.services.query-execution.has-granules-results-feature :as has-granules-base]))
 
 ;; The following protocol implementation ensures that a c.s.m.q.HasGranulesCondition record in our

search-app/src/cmr/search/data/complex_to_simple_converters/has_granules_or_cwic.clj

@@ -2,7 +2,6 @@
   (:require
    [cmr.elastic-utils.search.query-transform :as c2s]
    [cmr.common.services.search.query-model :as cqm]
-   [cmr.search.models.query :as qm]
    [cmr.search.services.query-execution.has-granules-or-cwic-results-feature :as has-gran-or-cwic-base]
    [cmr.search.services.query-execution.has-granules-results-feature :as has-granules-base]))
 

search-app/src/cmr/search/data/complex_to_simple_converters/two_d_coordinate_system.clj

@@ -3,8 +3,7 @@
   protocol for two d coordinate system related search fields."
   (:require [cmr.common.services.search.query-model :as qm]
             [cmr.elastic-utils.search.es-group-query-conditions :as gc]
-            [cmr.elastic-utils.search.query-transform :as c2s]
-            [cmr.common.services.errors :as errors]))
+            [cmr.elastic-utils.search.query-transform :as c2s]))
 
 (defprotocol ConvertCoordinateCondition
   (coordinate-cond->condition

search-app/src/cmr/search/data/elastic_search_index.clj

@@ -3,7 +3,7 @@
   (:require
    [clojure.string :as string]
    [cmr.common.concepts :as concepts]
-   [cmr.common.config :as cfg :refer [defconfig]]
+   [cmr.common.config :refer [defconfig]]
    [cmr.common.hash-cache :as hcache]
    [cmr.common.services.errors :as e]
    [cmr.common.services.search.query-model :as qm]

search-app/src/cmr/search/data/metadata_retrieval/metadata_cache.clj

@@ -17,7 +17,7 @@
    [cmr.common.hash-cache :as hash-cache]
    [cmr.common.memory-db.connection]
    [cmr.common.jobs :refer [defjob]]
-   [cmr.common.log :as log :refer [debug info]]
+   [cmr.common.log :refer [debug info]]
    [cmr.common.redis-log-util :as rl-util]
    [cmr.common.services.errors :as errors]
    [cmr.common.util :as u]
@@ -75,8 +75,8 @@
 (defn refresh-cache
   "Refreshes the collection metadata cache"
   [context]
-  (info "Refreshing metadata cache")
-  (let [incremental-since-refresh-date (str (clj-time.core/now))
+  (let [start-time (System/currentTimeMillis)
+        incremental-since-refresh-date (str (clj-time.core/now))
         concepts-tuples (cmn-coll-metadata-cache/fetch-collections-from-elastic context)
         new-cache-value (reduce #(merge %1 (concept-tuples->cache-map context %2))
                                 {}
@@ -91,15 +91,15 @@
                           cmn-coll-metadata-cache/collection-metadata-cache-fields-key
                           (vec (remove nil? (keys new-cache-value))))
     (hash-cache/set-values rcache cmn-coll-metadata-cache/cache-key new-cache-value)
-    (info "Metadata cache refresh complete. Cache Size:"
-          (hash-cache/cache-size rcache cmn-coll-metadata-cache/cache-key))))
+    (info "Metadata cache refresh complete. Took %d ms. Cache Size:"
+          ((System/currentTimeMillis) - start-time) (hash-cache/cache-size rcache cmn-coll-metadata-cache/cache-key))))
 
 (defn update-cache-job
   "Updates the collection metadata cache by querying elastic search for updates since the
   last time the cache was updated."
   [context]
-    (info "Updating collection metadata cache")
-    (let [cache (hash-cache/context->cache context cmn-coll-metadata-cache/cache-key)
+    (let [start-time (System/currentTimeMillis)
+          cache (hash-cache/context->cache context cmn-coll-metadata-cache/cache-key)
           incremental-since-refresh-date (str (t/now))
           concepts-tuples (cmn-coll-metadata-cache/fetch-updated-collections-from-elastic context)
           new-cache-value (reduce #(merge %1 (concept-tuples->cache-map context %2))
@@ -121,7 +121,7 @@
       (hash-cache/set-values cache
                              cmn-coll-metadata-cache/cache-key
                              new-cache-value)
-      (info "Metadata cache update complete. Cache Size:" (hash-cache/cache-size cache cmn-coll-metadata-cache/cache-key))))
+      (info "Metadata cache update complete. Took %d ms. Cache Size:" ((System/currentTimeMillis) - start-time) (hash-cache/cache-size cache cmn-coll-metadata-cache/cache-key))))
 
 (defn in-memory-db?
   "Checks to see if the database in the context is an in-memory db."

search-app/src/cmr/search/data/query_order_by_expense.clj

@@ -2,9 +2,7 @@
   "Sorts query conditions within a query by their execution expense. Elasticsearch recommend putting
   cached filters before uncached filters. Expensive queries like exact spatial intersection should
   be placed last."
-  (:require [cmr.common.services.errors :as errors]
-            [cmr.search.models.query :as qm]
-            [cmr.elastic-utils.search.es-query-order-by-expense :as qobe])
+  (:require [cmr.elastic-utils.search.es-query-order-by-expense :as qobe])
   (:import [cmr.search.models.query
             CollectionQueryCondition
             SpatialCondition

search-app/test/cmr/search/test/unit/data/query_to_elastic_converters/keyword_wildcards.clj

@@ -2,7 +2,7 @@
   "This test tests the pure functions get-validate-keyword-wildcards-msg"
   (:require
     [clojure.test :refer :all]
-    [cmr.common.util :as u :refer [are3]]
+    [cmr.common.util :refer [are3]]
     [cmr.search.data.query-to-elastic :as query-to-elastic]))
 
 (deftest get-validate-keyword-wildcards-msg-test

search-app/test/cmr/search/test/unit/results_handlers/opendata_results_handler.clj

@@ -2,7 +2,7 @@
   "This tests the opendata-results-handler namespace."
   (:require
    [clojure.test :refer :all]
-   [cmr.common.util :as util :refer [are3]]
+   [cmr.common.util :refer [are3]]
    [cmr.search.results-handlers.opendata-results-handler :as opendata-results-handler]))
 
 (deftest distribution-is-downloadable

search-app/test/cmr/search/test/unit/results_handlers/results_handler_util.clj

@@ -1,6 +1,6 @@
 (ns cmr.search.test.unit.results-handlers.results-handler-util
   (:require [clojure.test :refer :all]
-            [cmr.common.util :as util :refer [are3]]
+            [cmr.common.util :refer [are3]]
             [cmr.search.results-handlers.results-handler-util :as rs-util]))
 
 (deftest convert-associations-test

search-app/test/cmr/search/test/unit/results_handlers/stac_results_handler.clj

@@ -3,8 +3,7 @@
   (:require
    [clojure.test :refer :all]
    [cmr.common.util :as util :refer [are3]]
-   [cmr.search.results-handlers.stac-results-handler :as stac-results-handler]
-   [ring.util.codec :as codec]))
+   [cmr.search.results-handlers.stac-results-handler :as stac-results-handler]))
 
 (def ^:private metadata-link
   "example metadata-link for test"