Merge pull request #1634 from openforis/development

Merge development to main Production 2023-05-19

src/clj/collect_earth_online/db/doi.clj

@@ -0,0 +1,92 @@
+(ns collect-earth-online.db.doi
+  (:require [clj-http.client :as http]
+            [clojure.java.io :as io]
+            [triangulum.database :refer [call-sql
+                                         sql-primitive]]
+            [triangulum.config  :refer [get-config]]
+            [triangulum.type-conversion :as tc]
+            [collect-earth-online.generators.external-file :refer [create-shape-files]])
+  (:import java.time.format.DateTimeFormatter
+           java.time.LocalDateTime))
+
+(def base-url (:url (get-config :zenodo)))
+
+(defn req-headers
+  []
+  (let [auth-token (:api-key (get-config :zenodo))]
+    {"Authorization" (str "Bearer " auth-token)}))
+
+(defn get-zenodo-deposition
+  [deposition-id]
+  (let [headers (req-headers)]
+    (http/get (str base-url "/deposit/depositions/" deposition-id) {:headers headers :as :json})))
+
+(defn create-contributors-list
+  [users institution-name]
+  (map (fn [user]
+         {:name        (:email user)
+          :type        "DataCurator"
+          :affiliation institution-name})
+       users))
+
+(defn create-zenodo-deposition!
+  [institution-name
+   project-name
+   user
+   users-assigned
+   description]
+  (let [headers  (req-headers)
+        metadata {:upload_type      "dataset"
+                  :title            (str institution-name "_" project-name)
+                  :publication_date (-> (DateTimeFormatter/ofPattern "yyyy-MM-dd")
+                                        (.format (LocalDateTime/now)))
+                  :creators         [{:name        (:email user)
+                                      :affiliation institution-name}]
+                  :description      description
+                  :contributors     (create-contributors-list users-assigned institution-name)}]
+    (http/post (str base-url "/deposit/depositions")
+               {:form-params  {:metadata metadata}
+                :content-type :json
+                :as           :json
+                :headers      headers})))
+
+(defn upload-deposition-file!
+  [bucket-url project-id table-name]
+  (let [headers    (req-headers)
+        shape-file (create-shape-files table-name project-id)]
+    (http/put (str bucket-url "/" table-name "-" project-id ".zip")
+              {:content-type :multipart/form-data
+               :headers      headers
+               :multipart    [{:name "Content/type" :content "application/zip"}
+                              {:name "file" :content (io/file shape-file)}]})))
+
+(defn insert-doi!
+  [{:keys [id metadata] :as doi-info}
+   project-id user-id]
+  (let [doi-path (-> metadata :prereserve_doi :doi)]
+    (-> "insert_doi"
+        (call-sql id project-id user-id doi-path doi-info)
+        first
+        sql-primitive)))
+
+(defn create-doi
+  [{:keys [params]}]
+  (let [user-id          (:userId params -1)
+        project-id       (tc/val->int (:projectId params))
+        project-name     (:projectName params)
+        institution-name (:institution params)
+        description      (:description params)
+        creator          (first (call-sql "get_user_by_id" user-id))
+        contributors     (call-sql "select_assigned_users_by_project" project-id)]
+    (->
+     (create-zenodo-deposition! institution-name project-name creator contributors description)
+     :body
+     (insert-doi! project-id user-id))))
+
+(defn upload-doi-files!
+  [{:keys [params]}]
+  (let [project-id (:projectId params)
+        doi        (first (call-sql "select_doi_by_project" project-id))
+        bucket     (-> doi :links :bucket)
+        options    (:options params)]
+    (upload-deposition-file! bucket project-id options)))

src/clj/collect_earth_online/db/geodash.clj

@@ -16,15 +16,15 @@
 
 (require-python '[sys :bind-ns])
 (py. (get-attr sys "path") "append" "src/py")
-(require-python '[gee.routes :as gee]
-                '[gee.utils :refer [initialize]])
+(require-python '[gee.routes :as gee :reload]
+                '[gee.utils :as utils :reload])
 
 (defonce last-initialized (atom 0))
 
 (defn- check-initialized []
   (when (> (- (System/currentTimeMillis) @last-initialized) max-age)
     (let [{:keys [ee-account ee-key-path]} (get-config :gee)]
-      (initialize ee-account ee-key-path)
+      (utils/initialize ee-account ee-key-path)
       (reset! last-initialized (System/currentTimeMillis)))))
 
 (defn- parse-py-errors [e]
@@ -52,7 +52,6 @@
    "imageCollection"        gee/imageCollection
    "imageCollectionByIndex" gee/imageCollectionByIndex
    "statistics"             gee/statistics
-   "timeSeriesByAsset"      gee/timeSeriesByAsset
    "timeSeriesByIndex"      gee/timeSeriesByIndex})
 
 (defn gateway-request [{:keys [params json-params]}]

src/clj/collect_earth_online/db/projects.clj

@@ -143,26 +143,27 @@
 (defn get-template-by-id [{:keys [params]}]
   (let [project-id (tc/val->int (:projectId params))
         project    (first (call-sql "select_project_by_id" project-id))]
-    (data-response {:imageryId          (:imagery_id project)
-                    :name               (:name project)
-                    :description        (:description project)
-                    :aoiFeatures        (tc/jsonb->clj (:aoi_features project))
-                    :aoiFileName        (:aoi_file_name project)
-                    :plotDistribution   (:plot_distribution project)
-                    :numPlots           (:num_plots project)
-                    :plotSpacing        (:plot_spacing project)
-                    :plotShape          (:plot_shape project)
-                    :plotSize           (:plot_size project)
-                    :plotFileName       (:plot_file_name project)
-                    :sampleDistribution (:sample_distribution project)
-                    :samplesPerPlot     (:samples_per_plot project)
-                    :sampleResolution   (:sample_resolution project)
-                    :sampleFileName     (:sample_file_name project)
-                    :allowDrawnSamples  (:allow_drawn_samples project)
-                    :surveyQuestions    (tc/jsonb->clj (:survey_questions project) [])
-                    :surveyRules        (tc/jsonb->clj (:survey_rules project) [])
-                    :projectOptions     (merge default-options (tc/jsonb->clj (:options project)))
-                    :designSettings     (merge default-settings (tc/jsonb->clj (:design_settings project)))})))
+    (data-response {:imageryId                  (:imagery_id project)
+                    :templateInstitutionId      (:institution_id project)
+                    :name                       (:name project)
+                    :description                (:description project)
+                    :aoiFeatures                (tc/jsonb->clj (:aoi_features project))
+                    :aoiFileName                (:aoi_file_name project)
+                    :plotDistribution           (:plot_distribution project)
+                    :numPlots                   (:num_plots project)
+                    :plotSpacing                (:plot_spacing project)
+                    :plotShape                  (:plot_shape project)
+                    :plotSize                   (:plot_size project)
+                    :plotFileName               (:plot_file_name project)
+                    :sampleDistribution         (:sample_distribution project)
+                    :samplesPerPlot             (:samples_per_plot project)
+                    :sampleResolution           (:sample_resolution project)
+                    :sampleFileName             (:sample_file_name project)
+                    :allowDrawnSamples          (:allow_drawn_samples project)
+                    :surveyQuestions            (tc/jsonb->clj (:survey_questions project) [])
+                    :surveyRules                (tc/jsonb->clj (:survey_rules project) [])
+                    :projectOptions             (merge default-options (tc/jsonb->clj (:options project)))
+                    :designSettings             (merge default-settings (tc/jsonb->clj (:design_settings project)))})))
 
 (defn get-project-stats [{:keys [params]}]
   (let [project-id (tc/val->int (:projectId params))

src/clj/collect_earth_online/db/users.clj

@@ -44,7 +44,7 @@
 
 (defn register [{:keys [params]}]
   (let [reset-key             (str (UUID/randomUUID))
-        email                 (:email params)
+        email                 (str/lower-case (:email params))
         password              (:password params)
         password-confirmation (:passwordConfirmation params)]
     (if-let [error-msg (get-register-errors email password password-confirmation)]
@@ -196,7 +196,7 @@
       (data-response {}))))
 
 (defn update-institution-role [{:keys [params]}]
-  (let [new-user-email   (:newUserEmail params)
+  (let [new-user-email   (str/lower-case (:newUserEmail params))
         account-id       (if-let [id (:accountId params)]
                            (tc/val->int id)
                            (-> (call-sql "get_user_by_email" new-user-email)

src/clj/collect_earth_online/generators/external_file.clj

@@ -3,6 +3,7 @@
             [clojure.set        :as set]
             [clojure.java.io    :as io]
             [clojure.java.shell :as sh]
+            [triangulum.config  :refer [get-config]]
             [triangulum.type-conversion :as tc]
             [triangulum.utils :refer [parse-as-sh-cmd]]
             [collect-earth-online.utils.part-utils :as pu]
@@ -44,6 +45,14 @@
           out
           (pu/init-throw  err))))))
 
+(defn- sh-wrapper-quoted [dir env & commands]
+  (sh/with-sh-dir dir
+    (sh/with-sh-env (merge {:PATH path-env} env)
+      (doseq [cmd commands]
+        (let [{:keys [exit err]} (sh/sh "bash" "-c" cmd)]
+          (when-not (= 0 exit)
+            (pu/init-throw err)))))))
+
 ;;;
 ;;; Load Information
 ;;;
@@ -241,3 +250,31 @@
                (assoc :project_rid project-id)
                (split-ext :extra_plot_info [:project_rid :visible_id :plot_geom])))
          ext-plots)))
+
+(defn pgsql2shp-string
+  [db-config file-name query]
+  (let [db-name (:dbname db-config)
+        db-user (:user db-config)]
+    (str "pgsql2shp -f " file-name " -u " db-user " " db-name " " query)))
+
+(defn export-table-to-file
+  [folder-name project-id table-name db-config]
+  (sh-wrapper-quoted folder-name {}
+                         (pgsql2shp-string db-config
+                                           (str project-id "-plots")
+                                           (str "\"SELECT * FROM " table-name "_shapes WHERE p_id=" project-id "\""))
+                         (str "7z a " project-id "-" table-name ".zip " project-id "-plots*")))
+
+
+(defn create-shape-files
+  [table-name project-id]
+  (let [folder-name (str tmp-dir "/ceo-tmp-" project-id "-" table-name "/")
+        db-config   (get-config :database)
+        zip-name    (str project-id "-" table-name ".zip")]
+    (sh-wrapper tmp-dir {} (str "rm -rf " folder-name) (str "mkdir " folder-name))
+    (sh-wrapper-quoted folder-name {}
+                         (pgsql2shp-string db-config
+                                           (str project-id "-plots")
+                                           (str "\"SELECT * FROM " table-name "_shapes WHERE project_id=" project-id "\""))
+                         (str "7z a " zip-name " " project-id "-plots*"))
+    (str folder-name zip-name)))

src/js/collection.js

@@ -1110,14 +1110,12 @@ function ImageAnalysisPane({ imageryAttribution }) {
 class SideBar extends React.Component {
   checkCanSave = () => {
     const { answerMode, currentPlot, inReviewMode, surveyQuestions } = this.props;
-    const noneAnswered = everyObject(surveyQuestions, ([_id, sq]) => safeLength(sq.answered) === 0);
+    const visibleSurveyQuestions = filterObject(surveyQuestions, ([_id, val]) => val.hideQuestion != true);
+    const noneAnswered = everyObject(visibleSurveyQuestions, ([_id, sq]) => safeLength(sq.answered) === 0);
     const hasSamples = safeLength(currentPlot.samples) > 0;
     const allAnswered = everyObject(
-      surveyQuestions,
-      ([_id, sq]) =>
-        (!sq.cardOrder && sq.parentQuestionId === -1) || // Skip over the duplicate questions
-        safeLength(sq.visible) === safeLength(sq.answered)
-    );
+      visibleSurveyQuestions,
+      ([_id, sq]) => safeLength(sq.visible) === safeLength(sq.answered));
     if (answerMode !== "question") {
       alert("You must be in question mode to save the collection.");
       return false;

src/js/geodash/GraphWidget.js

@@ -111,7 +111,7 @@ export default class GraphWidget extends React.Component {
 
   loadTimeSeries = () => {
     const { widget, plotExtentPolygon } = this.props;
-    const path = this.widgetIsCustom() ? "timeSeriesByAsset" : "timeSeriesByIndex";
+    const path = "timeSeriesByIndex";
     fetch("/geo-dash/gateway-request", {
       method: "POST",
       headers: {

src/js/geodash/StatsWidget.js

@@ -1,7 +1,6 @@
 import React from "react";
-
 import { Polygon } from "ol/geom";
-import { getArea as sphereGetArea } from "ol/sphere";
+import { mercator } from "../utils/mercator";
 
 export default class StatsWidget extends React.Component {
   constructor(props) {
@@ -58,13 +57,8 @@ export default class StatsWidget extends React.Component {
     }
   };
 
-  calculateArea = (poly) => {
-    try {
-      return sphereGetArea(new Polygon([poly]), { projection: "EPSG:4326" }) / 10000;
-    } catch (e) {
-      return "N/A";
-    }
-  };
+  calculateArea = (plotExtentPolygon) => 
+    mercator.calculateArea(new Polygon([plotExtentPolygon]));
 
   renderRow = (label, value, image) => (
     <div className="d-flex align-items-center mb-3">

src/js/project/CreateProjectWizard.js

@@ -171,10 +171,11 @@ export default class CreateProjectWizard extends React.Component {
       .then((response) => (response.ok ? response.json() : Promise.reject(response)))
       .then((data) => {
         this.setState({ templateProject: data });
+        const clearedTemplateAssignments = this.clearTemplateUserAssignments(data);
         const institutionImageryIds = this.context.institutionImagery.map((i) => i.id);
         this.context.setProjectDetails(
           {
-            ...data,
+            ...clearedTemplateAssignments,
             templateProjectId: projectId,
             imageryId: institutionImageryIds.includes(data.imageryId)
               ? data.imageryId
@@ -199,11 +200,8 @@ export default class CreateProjectWizard extends React.Component {
     fetch("/get-project-imagery?projectId=" + projectId)
       .then((response) => (response.ok ? response.json() : Promise.reject(response)))
       .then((data) => {
-        const institutionImageryIds = this.context.institutionImagery.map((i) => i.id);
         this.context.setProjectDetails({
-          projectImageryList: data
-            .map((i) => i.id)
-            .filter((id) => institutionImageryIds.includes(id)),
+          projectImageryList: data.map((i) => i.id)
         });
       });
 
@@ -403,12 +401,17 @@ export default class CreateProjectWizard extends React.Component {
     return errorList.filter((e) => e);
   };
 
+  allAnswersHidden = (answers) => {
+    const hiddenAnswers = filterObject(answers, ([_id, ans]) => ans.hide);
+    return lengthObject(answers) === lengthObject(hiddenAnswers);
+  }
+
   validateSurveyQuestions = () => {
     const { surveyQuestions } = this.context;
     const errorList = [
       lengthObject(surveyQuestions) === 0 && "A survey must include at least one question.",
-      someObject(surveyQuestions, ([_id, sq]) => lengthObject(sq.answers) === 0) &&
-        "All survey questions must contain at least one answer.",
+      someObject(surveyQuestions, ([_id, sq]) => (lengthObject(sq.answers) === 0 || this.allAnswersHidden(sq.answers))) &&
+        "All survey questions must contain at least one (unhidden) answer.",
     ];
     return errorList.filter((e) => e);
   };
@@ -513,6 +516,18 @@ export default class CreateProjectWizard extends React.Component {
     }
   };
 
+
+  clearTemplateUserAssignments = (templateProject) => 
+    this.context.institutionId != templateProject.templateInstitutionId ?
+      {...templateProject,
+       designSettings: {...templateProject.designSettings,
+                        userAssignment: {
+                            userMethod: null,
+                            users: [],
+                            percents: []}}}
+      : templateProject;
+
+
   /// Render Functions
 
   renderStep = (stepName) => {

src/js/reviewInstitution.js

@@ -1296,6 +1296,7 @@ class UserList extends React.Component {
   updateUserInstitutionRole = (accountId, newUserEmail, newInstitutionRole) => {
     const existingRole = (this.state.institutionUserList.find((u) => u.id === accountId) || {})
       .institutionRole;
+    const lowerCaseNewUserEmail = newUserEmail.toLowerCase();
     const adminCount = this.state.institutionUserList.filter(
       (user) => user.institutionRole === "admin"
     ).length;
@@ -1312,7 +1313,7 @@ class UserList extends React.Component {
           },
           body: JSON.stringify({
             accountId,
-            newUserEmail,
+            newUserEmail: lowerCaseNewUserEmail,
             institutionId: this.props.institutionId,
             institutionRole: newInstitutionRole,
           }),