Merge pull request #1642 from openforis/development

Merge development into main Production 2023-06-16

src/clj/collect_earth_online/db/projects.clj

@@ -4,20 +4,21 @@
    java.util.Date
    java.util.UUID)
   (:require
-   [clojure.set                                   :as set]
-   [clojure.string                                :as str]
+   [clojure.java.io :as io]
+   [clojure.set                                   :as    set]
+   [clojure.string                                :as    str]
    [collect-earth-online.generators.clj-point     :refer [generate-point-plots generate-point-samples]]
-   [collect-earth-online.generators.external-file :refer [generate-file-plots generate-file-samples]]
+   [collect-earth-online.generators.external-file :refer [generate-file-plots generate-file-samples zip-shape-files]]
    [collect-earth-online.utils.geom               :refer [make-geo-json-polygon]]
-   [collect-earth-online.utils.part-utils         :as pu]
+   [collect-earth-online.utils.part-utils         :as    pu]
    [collect-earth-online.views                    :refer [data-response]]
    [triangulum.database                           :refer [call-sql
                                                           insert-rows!
                                                           p-insert-rows!
                                                           sql-primitive]]
    [triangulum.logging                            :refer [log]]
-   [triangulum.type-conversion                    :as tc]
-   [triangulum.utils                              :as u]))
+   [triangulum.type-conversion                    :as    tc]
+   [triangulum.utils                              :as    u]))
 
 ;;;
 ;;; Auth functions
@@ -346,6 +347,10 @@
   (when-let [assigned-plots (assign-user-plots current-plots design-settings)]
     (p-insert-rows! "plot_assignments" (assign-qaqc assigned-plots design-settings))))
 
+(defn- copy-template-plots [project-id template-id design-settings]
+  (call-sql "copy_template_plots" template-id project-id)
+  (assign-plots design-settings (call-sql "get_plot_centers_by_project" project-id)))
+
 (defn- create-project-plots! [project-id
                               plot-distribution
                               num-plots
@@ -464,7 +469,7 @@
     (try
       ;; Create or copy plots
       (if (and (pos? project-template) use-template-plots)
-        (call-sql "copy_template_plots" project-template project-id)
+        (copy-template-plots project-id project-template design-settings)
         (create-project-plots! project-id
                                plot-distribution
                                num-plots
@@ -932,3 +937,16 @@
                                               ".csv")}
          :body (str/join "\n" (cons headers-out data-rows))})
       (data-response "Project not found."))))
+
+(defn create-shape-files!
+  [{:keys [params]}]
+  (let [project-id (:projectId params)
+        zip-file (zip-shape-files project-id)
+        file-name (last (str/split zip-file #"/"))]
+    (if zip-file
+      {:headers {"Content-Type" "application/zip"
+                 "Content-Disposition" (str "attachment; filename=" file-name)}
+       :body (io/file zip-file)
+       :status 200}
+      {:status 500
+       :body "Error generating shape files."})))

src/clj/collect_earth_online/generators/external_file.clj

@@ -261,20 +261,22 @@
   [folder-name project-id table-name db-config]
   (sh-wrapper-quoted folder-name {}
                          (pgsql2shp-string db-config
-                                           (str project-id "-plots")
-                                           (str "\"SELECT * FROM " table-name "_shapes WHERE p_id=" project-id "\""))
-                         (str "7z a " project-id "-" table-name ".zip " project-id "-plots*")))
-
+                                           (str project-id "-" table-name)
+                                           (str "\"SELECT * FROM get_" table-name "_shapes(" project-id ")\""))))
 
 (defn create-shape-files
-  [table-name project-id]
-  (let [folder-name (str tmp-dir "/ceo-tmp-" project-id "-" table-name "/")
-        db-config   (get-config :database)
-        zip-name    (str project-id "-" table-name ".zip")]
+  [folder-name table-name project-id]
+  (let [shape-folder-name (str folder-name table-name "-shape-files/")
+        db-config   (get-config :database)]
+    (sh-wrapper folder-name {} (str "rm -rf " shape-folder-name) (str "mkdir " shape-folder-name))
+    (export-table-to-file shape-folder-name project-id table-name db-config)))
+
+(defn zip-shape-files
+  [project-id]
+  (let [folder-name (str tmp-dir "/ceo-tmp-" project-id "-files/")]
     (sh-wrapper tmp-dir {} (str "rm -rf " folder-name) (str "mkdir " folder-name))
-    (sh-wrapper-quoted folder-name {}
-                         (pgsql2shp-string db-config
-                                           (str project-id "-plots")
-                                           (str "\"SELECT * FROM " table-name "_shapes WHERE project_id=" project-id "\""))
-                         (str "7z a " zip-name " " project-id "-plots*"))
-    (str folder-name zip-name)))
+    (create-shape-files folder-name "plot" project-id)
+    (create-shape-files folder-name "sample" project-id)
+    (sh-wrapper tmp-dir {}
+                (str "7z a " folder-name "/files" ".zip " folder-name "/*"))
+    (str folder-name "files.zip")))

src/clj/collect_earth_online/routing.clj

@@ -104,6 +104,9 @@
    [:post "/update-project"]                 {:handler     projects/update-project!
                                               :auth-type   :admin
                                               :auth-action :block}
+   [:get "/create-shape-files"]              {:handler projects/create-shape-files!
+                                              :auth-type :user
+                                              :auth-action :block}
    ;; Plots API
    [:get  "/get-collection-plot"]            {:handler     plots/get-collection-plot
                                               :auth-type   :collect

src/js/geodash/ImageCollectionAssetDesigner.js

@@ -35,7 +35,7 @@ export default function ImageCollectionAssetDesigner({ isDual = false, prefixPat
       />
       <GDTextArea
         dataKey="visParams"
-        placeholder={'{"bands": "B4, B3, B2", \n"min":0, \n"max": 0.3}'}
+        placeholder={'{"bands": "B4,B3,B2", \n"min":0, \n"max": 0.3}'}
         prefixPath={prefixPath}
         title="Image Parameters (JSON format)"
       />

src/js/project/ManageProject.js

@@ -310,6 +310,12 @@ class ProjectManagement extends React.Component {
               type="button"
               value="Download Sample Data"
             />
+            <input
+              className="btn btn-outline-lightgreen btn-sm w-100"
+              onClick={() => window.open(`/create-shape-files?projectId=${id}`, "_blank")}
+              type="button"
+              value="Download Shape Files"
+            />
           </div>
         </div>
       </div>

src/js/reviewInstitution.js

@@ -1261,6 +1261,18 @@ function Project({ project, isAdmin, deleteProject }) {
               S
             </button>
           </div>
+          <div className="col-1 pl-0">
+            <button
+              className="btn btn-sm btn-outline-lightgreen btn-block"
+              onClick={() =>
+                window.open(`/create-shape-files?projectId=${project.id}`, "_blank")
+              }
+              title="Download Shape Files"
+              type="button"
+            >
+              Shapes
+            </button>
+          </div>
         </>
       )}
     </div>

src/py/gee/planet.py

@@ -154,7 +154,7 @@ def add_similar_features(feature, geometry, buffer):
     return features_layer(features, name)
 
 
-def getPlanetMapID(api_key, geometry, start, end=None, layerCount=1, item_types=['PSScene3Band', 'PSScene4Band'], buffer=0.5, addsimilar=True):
+def getPlanetMapID(api_key, geometry, start, end=None, layerCount=1, item_types=['PSScene'], buffer=0.5, addsimilar=True):
     fullList = []
     global PLANET_API_KEY
     PLANET_API_KEY = api_key

src/py/gee/routes.py

@@ -47,12 +47,16 @@ def image(requestDict):
 
 
 def imageCollection(requestDict):
+    visParams = safeParseJSON(getDefault(requestDict, 'visParams', {}))
+    if visParams.get("bands"):
+        bands = visParams.get("bands").replace(' ', '')
+        visParams.update({"bands": bands})
     values = imageCollectionToMapId(
         getDefault(requestDict, 'assetId', None),
-        safeParseJSON(getDefault(requestDict, 'visParams', {})),
+        visParams,
         getDefault(requestDict, 'reducer', 'Mean'),
-        getDefault(requestDict, 'startDate', None),
-        getDefault(requestDict, 'endDate', None)
+        getDefault(requestDict, 'startDate', '2022-01-01'),
+        getDefault(requestDict, 'endDate', '2022-12-31')
     )
     return values
 
@@ -148,12 +152,11 @@ def featureCollection(requestDict):
 def getPlanetTile(requestDict):
     values = getPlanetMapID(
         getDefault(requestDict, 'apiKey'),
-        getDefault(requestDict, 'geometry'), getDefault(
-            requestDict, 'startDate'),
+        getDefault(requestDict, 'geometry'),
+        getDefault(requestDict, 'startDate'),
         getDefault(requestDict, 'endDate', None),
         getDefault(requestDict, 'layerCount', 1),
-        getDefault(requestDict, 'itemTypes', [
-                    'PSScene3Band', 'PSScene4Band']),
+        getDefault(requestDict, 'itemTypes', ['PSScene']),
         float(getDefault(requestDict, 'buffer', 0.5)),
         bool(strtobool(getDefault(requestDict, 'addsimilar', 'True')))
     )

src/sql/functions/plots.sql

@@ -539,3 +539,59 @@ CREATE OR REPLACE FUNCTION get_plot_centers_by_project(_project_id integer)
     WHERE project_rid = _project_id
 
 $$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION get_plot_shapes(_project_id integer)
+ RETURNS TABLE (project_id integer,
+                plot_id    integer,
+                plot_geom  geometry(Geometry, 4326)
+ ) AS $$
+
+    WITH plot_geoms AS (SELECT project_rid, plot_distribution, plot_shape, plot_size, plot_uid,
+                               ST_Transform(plot_geom, 3857) AS plot_geom
+                        FROM projects AS pr
+                        INNER JOIN plots AS pl
+                        ON pr.project_uid = pl.project_rid
+                        WHERE project_uid = _project_id),
+
+         plot_boundaries AS (SELECT plot_uid,
+                                    CASE
+                                      WHEN plot_distribution = 'shp'
+                                      THEN plot_geom
+                                      WHEN plot_shape = 'circle'
+                                      THEN ST_Buffer(plot_geom, plot_size/2)
+                                      WHEN plot_shape = 'square'
+                                      THEN ST_MakeEnvelope(ST_X(plot_geom) - plot_size/2,
+                                                           ST_Y(plot_geom) - plot_size/2,
+                                                           ST_X(plot_geom) + plot_size/2,
+                                                           ST_Y(plot_geom) + plot_size/2,
+                                                           3857)
+                                      ELSE plot_geom
+                                    END AS plot_boundary
+                             FROM plot_geoms)
+
+      SELECT project_rid, plot_uid, ST_Transform(plot_geom, 4326) AS plot_geom
+      FROM plot_geoms
+      INNER JOIN plot_boundaries
+      USING (plot_uid)
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION get_sample_shapes(_project_id integer)
+ RETURNS TABLE (project_id integer,
+                plot_id integer,
+                sample_id integer,
+                sample_geom geometry(Geometry, 4326)
+ ) AS $$
+
+    (SELECT project_rid, plot_rid, sample_uid, sample_geom
+       FROM samples s
+       INNER JOIN plots pl
+       ON pl.plot_uid = s.plot_rid
+       WHERE pl.project_rid = _project_id)
+    UNION
+    (SELECT project_rid, plot_rid, sample_uid, sample_geom
+       FROM ext_samples s
+       INNER JOIN plots pl
+       ON pl.plot_uid = s.plot_rid
+       WHERE pl.project_rid = _project_id)
+
+$$ LANGUAGE SQL;