From 25e5cc8af44d7db7f1e62611ae4e32b38a48843b Mon Sep 17 00:00:00 2001 From: Arthur G Luz Date: Fri, 26 May 2023 10:15:37 -0300 Subject: [PATCH 1/6] Fixing default values and Bands for image collection asset widget (#1632) --- src/js/geodash/ImageCollectionAssetDesigner.js | 2 +- src/py/gee/routes.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/js/geodash/ImageCollectionAssetDesigner.js b/src/js/geodash/ImageCollectionAssetDesigner.js index 8ac2e4f98..f877f29f5 100644 --- a/src/js/geodash/ImageCollectionAssetDesigner.js +++ b/src/js/geodash/ImageCollectionAssetDesigner.js @@ -35,7 +35,7 @@ export default function ImageCollectionAssetDesigner({ isDual = false, prefixPat /> diff --git a/src/py/gee/routes.py b/src/py/gee/routes.py index cac9dfe95..6c04e3d02 100644 --- a/src/py/gee/routes.py +++ b/src/py/gee/routes.py @@ -47,12 +47,16 @@ def image(requestDict): def imageCollection(requestDict): + visParams = safeParseJSON(getDefault(requestDict, 'visParams', {})) + if visParams.get("bands"): + bands = visParams.get("bands").replace(' ', '') + visParams.update({"bands": bands}) values = imageCollectionToMapId( getDefault(requestDict, 'assetId', None), - safeParseJSON(getDefault(requestDict, 'visParams', {})), + visParams, getDefault(requestDict, 'reducer', 'Mean'), - getDefault(requestDict, 'startDate', None), - getDefault(requestDict, 'endDate', None) + getDefault(requestDict, 'startDate', '2022-01-01'), + getDefault(requestDict, 'endDate', '2022-12-31') ) return values From 5e256ab50cd9649af6636626e3fd43ca4dcf139a Mon Sep 17 00:00:00 2001 From: Arthur G Luz Date: Mon, 5 Jun 2023 11:42:41 -0300 Subject: [PATCH 2/6] Changing deprecated default item types for planet API (#1638) --- src/py/gee/planet.py | 2 +- src/py/gee/routes.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/py/gee/planet.py b/src/py/gee/planet.py index 0d82109e6..e788c93b7 100644 --- a/src/py/gee/planet.py +++ b/src/py/gee/planet.py @@ -154,7 +154,7 @@ def add_similar_features(feature, geometry, buffer): return features_layer(features, name) -def getPlanetMapID(api_key, geometry, start, end=None, layerCount=1, item_types=['PSScene3Band', 'PSScene4Band'], buffer=0.5, addsimilar=True): +def getPlanetMapID(api_key, geometry, start, end=None, layerCount=1, item_types=['PSScene'], buffer=0.5, addsimilar=True): fullList = [] global PLANET_API_KEY PLANET_API_KEY = api_key diff --git a/src/py/gee/routes.py b/src/py/gee/routes.py index 6c04e3d02..93ec8f1f2 100644 --- a/src/py/gee/routes.py +++ b/src/py/gee/routes.py @@ -152,12 +152,11 @@ def featureCollection(requestDict): def getPlanetTile(requestDict): values = getPlanetMapID( getDefault(requestDict, 'apiKey'), - getDefault(requestDict, 'geometry'), getDefault( - requestDict, 'startDate'), + getDefault(requestDict, 'geometry'), + getDefault(requestDict, 'startDate'), getDefault(requestDict, 'endDate', None), getDefault(requestDict, 'layerCount', 1), - getDefault(requestDict, 'itemTypes', [ - 'PSScene3Band', 'PSScene4Band']), + getDefault(requestDict, 'itemTypes', ['PSScene']), float(getDefault(requestDict, 'buffer', 0.5)), bool(strtobool(getDefault(requestDict, 'addsimilar', 'True'))) ) From 46d1b0c42794aa3b0de225870869b9bdef4f0911 Mon Sep 17 00:00:00 2001 From: Arthur G Luz Date: Mon, 5 Jun 2023 11:42:56 -0300 Subject: [PATCH 3/6] Download shape files (#1633) --- src/clj/collect_earth_online/db/projects.clj | 26 +++++++-- .../generators/external_file.clj | 30 +++++----- src/clj/collect_earth_online/routing.clj | 3 + src/js/project/ManageProject.js | 6 ++ src/js/reviewInstitution.js | 12 ++++ src/sql/functions/plots.sql | 56 +++++++++++++++++++ 6 files changed, 113 insertions(+), 20 deletions(-) diff --git a/src/clj/collect_earth_online/db/projects.clj b/src/clj/collect_earth_online/db/projects.clj index 97998efb0..c8b99796b 100644 --- a/src/clj/collect_earth_online/db/projects.clj +++ b/src/clj/collect_earth_online/db/projects.clj @@ -4,20 +4,21 @@ java.util.Date java.util.UUID) (:require - [clojure.set :as set] - [clojure.string :as str] + [clojure.java.io :as io] + [clojure.set :as set] + [clojure.string :as str] [collect-earth-online.generators.clj-point :refer [generate-point-plots generate-point-samples]] - [collect-earth-online.generators.external-file :refer [generate-file-plots generate-file-samples]] + [collect-earth-online.generators.external-file :refer [generate-file-plots generate-file-samples zip-shape-files]] [collect-earth-online.utils.geom :refer [make-geo-json-polygon]] - [collect-earth-online.utils.part-utils :as pu] + [collect-earth-online.utils.part-utils :as pu] [collect-earth-online.views :refer [data-response]] [triangulum.database :refer [call-sql insert-rows! p-insert-rows! sql-primitive]] [triangulum.logging :refer [log]] - [triangulum.type-conversion :as tc] - [triangulum.utils :as u])) + [triangulum.type-conversion :as tc] + [triangulum.utils :as u])) ;;; ;;; Auth functions @@ -932,3 +933,16 @@ ".csv")} :body (str/join "\n" (cons headers-out data-rows))}) (data-response "Project not found.")))) + +(defn create-shape-files! + [{:keys [params]}] + (let [project-id (:projectId params) + zip-file (zip-shape-files project-id) + file-name (last (str/split zip-file #"/"))] + (if zip-file + {:headers {"Content-Type" "application/zip" + "Content-Disposition" (str "attachment; filename=" file-name)} + :body (io/file zip-file) + :status 200} + {:status 500 + :body "Error generating shape files."}))) diff --git a/src/clj/collect_earth_online/generators/external_file.clj b/src/clj/collect_earth_online/generators/external_file.clj index d5db058da..6a1614b9f 100644 --- a/src/clj/collect_earth_online/generators/external_file.clj +++ b/src/clj/collect_earth_online/generators/external_file.clj @@ -261,20 +261,22 @@ [folder-name project-id table-name db-config] (sh-wrapper-quoted folder-name {} (pgsql2shp-string db-config - (str project-id "-plots") - (str "\"SELECT * FROM " table-name "_shapes WHERE p_id=" project-id "\"")) - (str "7z a " project-id "-" table-name ".zip " project-id "-plots*"))) - + (str project-id "-" table-name) + (str "\"SELECT * FROM get_" table-name "_shapes(" project-id ")\"")))) (defn create-shape-files - [table-name project-id] - (let [folder-name (str tmp-dir "/ceo-tmp-" project-id "-" table-name "/") - db-config (get-config :database) - zip-name (str project-id "-" table-name ".zip")] + [folder-name table-name project-id] + (let [shape-folder-name (str folder-name table-name "-shape-files/") + db-config (get-config :database)] + (sh-wrapper folder-name {} (str "rm -rf " shape-folder-name) (str "mkdir " shape-folder-name)) + (export-table-to-file shape-folder-name project-id table-name db-config))) + +(defn zip-shape-files + [project-id] + (let [folder-name (str tmp-dir "/ceo-tmp-" project-id "-files/")] (sh-wrapper tmp-dir {} (str "rm -rf " folder-name) (str "mkdir " folder-name)) - (sh-wrapper-quoted folder-name {} - (pgsql2shp-string db-config - (str project-id "-plots") - (str "\"SELECT * FROM " table-name "_shapes WHERE project_id=" project-id "\"")) - (str "7z a " zip-name " " project-id "-plots*")) - (str folder-name zip-name))) + (create-shape-files folder-name "plot" project-id) + (create-shape-files folder-name "sample" project-id) + (sh-wrapper tmp-dir {} + (str "7z a " folder-name "/files" ".zip " folder-name "/*")) + (str folder-name "files.zip"))) diff --git a/src/clj/collect_earth_online/routing.clj b/src/clj/collect_earth_online/routing.clj index 89d4aa578..b29384061 100644 --- a/src/clj/collect_earth_online/routing.clj +++ b/src/clj/collect_earth_online/routing.clj @@ -104,6 +104,9 @@ [:post "/update-project"] {:handler projects/update-project! :auth-type :admin :auth-action :block} + [:get "/create-shape-files"] {:handler projects/create-shape-files! + :auth-type :user + :auth-action :block} ;; Plots API [:get "/get-collection-plot"] {:handler plots/get-collection-plot :auth-type :collect diff --git a/src/js/project/ManageProject.js b/src/js/project/ManageProject.js index 6b9227e32..e1f090904 100644 --- a/src/js/project/ManageProject.js +++ b/src/js/project/ManageProject.js @@ -310,6 +310,12 @@ class ProjectManagement extends React.Component { type="button" value="Download Sample Data" /> + window.open(`/create-shape-files?projectId=${id}`, "_blank")} + type="button" + value="Download Shape Files" + /> diff --git a/src/js/reviewInstitution.js b/src/js/reviewInstitution.js index 923c66ce2..72d6b291c 100644 --- a/src/js/reviewInstitution.js +++ b/src/js/reviewInstitution.js @@ -1261,6 +1261,18 @@ function Project({ project, isAdmin, deleteProject }) { S +
+ +
)} diff --git a/src/sql/functions/plots.sql b/src/sql/functions/plots.sql index afb894c6f..a0471445e 100644 --- a/src/sql/functions/plots.sql +++ b/src/sql/functions/plots.sql @@ -539,3 +539,59 @@ CREATE OR REPLACE FUNCTION get_plot_centers_by_project(_project_id integer) WHERE project_rid = _project_id $$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION get_plot_shapes(_project_id integer) + RETURNS TABLE (project_id integer, + plot_id integer, + plot_geom geometry(Geometry, 4326) + ) AS $$ + + WITH plot_geoms AS (SELECT project_rid, plot_distribution, plot_shape, plot_size, plot_uid, + ST_Transform(plot_geom, 3857) AS plot_geom + FROM projects AS pr + INNER JOIN plots AS pl + ON pr.project_uid = pl.project_rid + WHERE project_uid = _project_id), + + plot_boundaries AS (SELECT plot_uid, + CASE + WHEN plot_distribution = 'shp' + THEN plot_geom + WHEN plot_shape = 'circle' + THEN ST_Buffer(plot_geom, plot_size/2) + WHEN plot_shape = 'square' + THEN ST_MakeEnvelope(ST_X(plot_geom) - plot_size/2, + ST_Y(plot_geom) - plot_size/2, + ST_X(plot_geom) + plot_size/2, + ST_Y(plot_geom) + plot_size/2, + 3857) + ELSE plot_geom + END AS plot_boundary + FROM plot_geoms) + + SELECT project_rid, plot_uid, ST_Transform(plot_geom, 4326) AS plot_geom + FROM plot_geoms + INNER JOIN plot_boundaries + USING (plot_uid) +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION get_sample_shapes(_project_id integer) + RETURNS TABLE (project_id integer, + plot_id integer, + sample_id integer, + sample_geom geometry(Geometry, 4326) + ) AS $$ + + (SELECT project_rid, plot_rid, sample_uid, sample_geom + FROM samples s + INNER JOIN plots pl + ON pl.plot_uid = s.plot_rid + WHERE pl.project_rid = _project_id) + UNION + (SELECT project_rid, plot_rid, sample_uid, sample_geom + FROM ext_samples s + INNER JOIN plots pl + ON pl.plot_uid = s.plot_rid + WHERE pl.project_rid = _project_id) + +$$ LANGUAGE SQL; From b284c2821902f4205aba8590b9d322c50e902939 Mon Sep 17 00:00:00 2001 From: Arthur G Luz Date: Tue, 6 Jun 2023 11:42:58 -0300 Subject: [PATCH 4/6] COL-464: Fix NICFI images not loading when set to newest (#1639) --- src/js/geodash/MapWidget.js | 21 +++++++++++++++++++-- src/js/utils/mercator.js | 7 +++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/js/geodash/MapWidget.js b/src/js/geodash/MapWidget.js index e75aa2ae2..e0d7a642c 100644 --- a/src/js/geodash/MapWidget.js +++ b/src/js/geodash/MapWidget.js @@ -17,13 +17,20 @@ export default class MapWidget extends React.Component { timeOutRefs: [], overlayValue: 100, opacityValue: 100, + newestNicfiLayer: "", }; } /// Lifecycle componentDidMount() { - this.initMap(); + fetch("/get-nicfi-dates") + .then((response) => (response.ok ? response.json() : Promise.reject(response))) + .then((layers) => { + this.setState({ newestNicfiLayer: layers[0] }) + this.initMap(); + }) + .catch((error) => console.error(error)); } componentDidUpdate(prevProps, prevState) { @@ -159,7 +166,17 @@ export default class MapWidget extends React.Component { this.props.imageryList.find((imagery) => imagery.title === "Open Street Map") || this.props.imageryList[0]; const basemapLayer = new TileLayer({ - source: mercator.createSource(sourceConfig, id, attribution, isProxied), + source: mercator.createSource(sourceConfig, + id, + attribution, + isProxied, + [[-180, -90], + [180, -90], + [180, 90], + [-180, 90], + [-180, -90]], + false, + this.state.newestNicfiLayer), }); const plotSampleLayer = new VectorLayer({ source: this.props.vectorSource, diff --git a/src/js/utils/mercator.js b/src/js/utils/mercator.js index c3223471d..820f642d2 100644 --- a/src/js/utils/mercator.js +++ b/src/js/utils/mercator.js @@ -220,7 +220,8 @@ mercator.createSource = ( [-180, 90], [-180, -90], ], - show = false + show = false, + newestNicfiLayer = "" ) => { const { type } = sourceConfig; if (isProxied) { @@ -252,14 +253,16 @@ mercator.createSource = ( attributions: attribution, }); } else if (type === "PlanetNICFI") { + const dataLayer = (sourceConfig.time === "newest") ? newestNicfiLayer : sourceConfig.time; return new XYZ({ url: "get-nicfi-tiles?z={z}&x={x}&y={y}" + - `&dataLayer=${sourceConfig.time}` + + `&dataLayer=${dataLayer}` + `&band=${sourceConfig.band}` + `&imageryId=${imageryId}`, attributions: attribution, }); + } else if (type === "PlanetDaily") { // make ajax call to get layerid then add xyz layer const theJson = { From 9264fc06932ae780f1f8d26ffcb556040af3cda9 Mon Sep 17 00:00:00 2001 From: Arthur G Luz Date: Fri, 9 Jun 2023 13:58:47 -0300 Subject: [PATCH 5/6] Fixing issues with QA/QC when using a project template (#1640) --- src/clj/collect_earth_online/db/projects.clj | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/clj/collect_earth_online/db/projects.clj b/src/clj/collect_earth_online/db/projects.clj index c8b99796b..6345019cd 100644 --- a/src/clj/collect_earth_online/db/projects.clj +++ b/src/clj/collect_earth_online/db/projects.clj @@ -347,6 +347,10 @@ (when-let [assigned-plots (assign-user-plots current-plots design-settings)] (p-insert-rows! "plot_assignments" (assign-qaqc assigned-plots design-settings)))) +(defn- copy-template-plots [project-id template-id design-settings] + (call-sql "copy_template_plots" template-id project-id) + (assign-plots design-settings (call-sql "get_plot_centers_by_project" project-id))) + (defn- create-project-plots! [project-id plot-distribution num-plots @@ -465,7 +469,7 @@ (try ;; Create or copy plots (if (and (pos? project-template) use-template-plots) - (call-sql "copy_template_plots" project-template project-id) + (copy-template-plots project-id project-template design-settings) (create-project-plots! project-id plot-distribution num-plots From b2b33165f18e3354cc656c1eb653e95ad7e0437b Mon Sep 17 00:00:00 2001 From: Arthur G Luz Date: Tue, 13 Jun 2023 10:51:48 -0300 Subject: [PATCH 6/6] Revert "COL-464: Fix NICFI images not loading when set to newest (#1639)" (#1641) This reverts commit b284c2821902f4205aba8590b9d322c50e902939. --- src/js/geodash/MapWidget.js | 21 ++------------------- src/js/utils/mercator.js | 7 ++----- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/src/js/geodash/MapWidget.js b/src/js/geodash/MapWidget.js index e0d7a642c..e75aa2ae2 100644 --- a/src/js/geodash/MapWidget.js +++ b/src/js/geodash/MapWidget.js @@ -17,20 +17,13 @@ export default class MapWidget extends React.Component { timeOutRefs: [], overlayValue: 100, opacityValue: 100, - newestNicfiLayer: "", }; } /// Lifecycle componentDidMount() { - fetch("/get-nicfi-dates") - .then((response) => (response.ok ? response.json() : Promise.reject(response))) - .then((layers) => { - this.setState({ newestNicfiLayer: layers[0] }) - this.initMap(); - }) - .catch((error) => console.error(error)); + this.initMap(); } componentDidUpdate(prevProps, prevState) { @@ -166,17 +159,7 @@ export default class MapWidget extends React.Component { this.props.imageryList.find((imagery) => imagery.title === "Open Street Map") || this.props.imageryList[0]; const basemapLayer = new TileLayer({ - source: mercator.createSource(sourceConfig, - id, - attribution, - isProxied, - [[-180, -90], - [180, -90], - [180, 90], - [-180, 90], - [-180, -90]], - false, - this.state.newestNicfiLayer), + source: mercator.createSource(sourceConfig, id, attribution, isProxied), }); const plotSampleLayer = new VectorLayer({ source: this.props.vectorSource, diff --git a/src/js/utils/mercator.js b/src/js/utils/mercator.js index 820f642d2..c3223471d 100644 --- a/src/js/utils/mercator.js +++ b/src/js/utils/mercator.js @@ -220,8 +220,7 @@ mercator.createSource = ( [-180, 90], [-180, -90], ], - show = false, - newestNicfiLayer = "" + show = false ) => { const { type } = sourceConfig; if (isProxied) { @@ -253,16 +252,14 @@ mercator.createSource = ( attributions: attribution, }); } else if (type === "PlanetNICFI") { - const dataLayer = (sourceConfig.time === "newest") ? newestNicfiLayer : sourceConfig.time; return new XYZ({ url: "get-nicfi-tiles?z={z}&x={x}&y={y}" + - `&dataLayer=${dataLayer}` + + `&dataLayer=${sourceConfig.time}` + `&band=${sourceConfig.band}` + `&imageryId=${imageryId}`, attributions: attribution, }); - } else if (type === "PlanetDaily") { // make ajax call to get layerid then add xyz layer const theJson = {