Merge pull request #5045 from bcgov/test-marshal-RQ-2879

PR to test-marshal for bulk download records - 2879
bcgov · Feb 8, 2024 · b127088 · b127088
2 parents a83ca1a + 4ad9dbf
commit b127088
Show file tree

Hide file tree

Showing 7 changed files with 128 additions and 20 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -224,6 +224,8 @@ services:
         - REACT_APP_RECORD_PROCESSING_HRS=${RECORD_PROCESSING_HRS}
         - REACT_APP_DISABLE_REDACT_WEBLINK=${DISABLE_REDACT_WEBLINK}
         - REACT_APP_DISABLE_GATHERINGRECORDS_TAB=${DISABLE_GATHERINGRECORDS_TAB}
+        - REACT_APP_RECORD_DOWNLOAD_LIMIT=${RECORD_DOWNLOAD_LIMIT}
+        - REACT_APP_RECORD_DOWNLOAD_SIZE_LIMIT=${RECORD_DOWNLOAD_SIZE_LIMIT}
     volumes:
       - ".:/app"
       - "/app/node_modules"

diff --git a/forms-flow-web/.sampleenv b/forms-flow-web/.sampleenv
@@ -30,4 +30,7 @@ REACT_APP_FOI_BASE_API_URL=http://IPADDRESS:15000
 ##AXIS base URL
 REACT_APP_AXIS_API_URL=http://flowaxisapidev.gov.bc.ca
 
-RECORD_PROCESSING_HRS=2
+RECORD_PROCESSING_HRS=2
+
+RECORD_DOWNLOAD_LIMIT=200
+RECORD_DOWNLOAD_SIZE_LIMIT=2147483648
diff --git a/forms-flow-web/Dockerfile b/forms-flow-web/Dockerfile
@@ -35,6 +35,8 @@ ARG REACT_APP_FOI_RECORD_FORMATS
 ARG REACT_APP_RECORD_PROCESSING_HRS
 ARG REACT_APP_DISABLE_REDACT_WEBLINK
 ARG REACT_APP_DISABLE_GATHERINGRECORDS_TAB
+ARG REACT_APP_RECORD_DOWNLOAD_LIMIT
+ARG REACT_APP_RECORD_DOWNLOAD_SIZE_LIMIT
 
 ENV NODE_ENV ${NODE_ENV}
 ENV GENERATE_SOURCEMAP false
@@ -66,6 +68,8 @@ ENV REACT_APP_DISABLE_REDACT_WEBLINK ${REACT_APP_DISABLE_REDACT_WEBLINK}
 ENV REACT_APP_DISABLE_GATHERINGRECORDS_TAB ${REACT_APP_DISABLE_GATHERINGRECORDS_TAB}
 
 ENV REACT_APP_SOCKETIO_CONNECT_NONCE ${REACT_APP_SOCKETIO_CONNECT_NONCE}
+ENV REACT_APP_RECORD_DOWNLOAD_LIMIT ${REACT_APP_RECORD_DOWNLOAD_LIMIT}
+ENV REACT_APP_RECORD_DOWNLOAD_SIZE_LIMIT ${REACT_APP_RECORD_DOWNLOAD_SIZE_LIMIT}
 # add `/app/node_modules/.bin` to $PATH
 ENV PATH /forms-flow-web/app/node_modules/.bin:$PATH
 

diff --git a/forms-flow-web/Dockerfile.local b/forms-flow-web/Dockerfile.local
@@ -32,6 +32,8 @@ ARG REACT_APP_FOI_RECORD_FORMATS
 ARG REACT_APP_RECORD_PROCESSING_HRS
 ARG REACT_APP_DISABLE_REDACT_WEBLINK
 ARG REACT_APP_DISABLE_GATHERINGRECORDS_TAB
+ARG REACT_APP_RECORD_DOWNLOAD_LIMIT
+ARG REACT_APP_RECORD_DOWNLOAD_SIZE_LIMIT
 
 ENV NODE_ENV ${NODE_ENV}
 ENV GENERATE_SOURCEMAP ${GENERATE_SOURCEMAP}
@@ -59,6 +61,8 @@ ENV REACT_APP_FOI_RECORD_FORMATS ${REACT_APP_FOI_RECORD_FORMATS}
 ENV REACT_APP_RECORD_PROCESSING_HRS ${REACT_APP_RECORD_PROCESSING_HRS}
 ENV REACT_APP_DISABLE_REDACT_WEBLINK ${REACT_APP_DISABLE_REDACT_WEBLINK}
 ENV REACT_APP_DISABLE_GATHERINGRECORDS_TAB ${REACT_APP_DISABLE_GATHERINGRECORDS_TAB}
+ENV REACT_APP_RECORD_DOWNLOAD_LIMIT ${REACT_APP_RECORD_DOWNLOAD_LIMIT}
+ENV REACT_APP_RECORD_DOWNLOAD_SIZE_LIMIT ${REACT_APP_RECORD_DOWNLOAD_SIZE_LIMIT}
 # add `/app/node_modules/.bin` to $PATH
 ENV PATH /forms-flow-web/app/node_modules/.bin:$PATH
 

diff --git a/forms-flow-web/src/components/FOI/customComponents/Records/index.js b/forms-flow-web/src/components/FOI/customComponents/Records/index.js
@@ -83,6 +83,7 @@ import {
   faTrash,
   faPenToSquare,
   faLinkSlash,
+  faDownload,
 } from "@fortawesome/free-solid-svg-icons";
 import Dialog from "@material-ui/core/Dialog";
 import DialogActions from "@material-ui/core/DialogActions";
@@ -96,6 +97,8 @@ import {
   RECORD_PROCESSING_HRS,
   OSS_S3_CHUNK_SIZE,
   DISABLE_REDACT_WEBLINK,
+  RECORD_DOWNLOAD_LIMIT,
+  RECORD_DOWNLOAD_SIZE_LIMIT
 } from "../../../../constants/constants";
 import {
   removeDuplicateFiles,
@@ -1103,27 +1106,33 @@ export const RecordsLog = ({
     );
   }
 
-  const downloadAllDocuments = async () => {
+  const downloadSelectedDocuments = async () => {
     let blobs = [];
     var completed = 0;
     let failed = 0;
-    var exporting = records.filter((record) => !record.isduplicate);
-    for (let record of exporting) {
-      if (record.attachments)
-        for (let attachment of record.attachments) {
-          if (!attachment.isduplicate) exporting.push(attachment);
+    var selected = records.filter((record) => record.isselected);
+
+    for(let record of records) {
+      if(record.attachments && !record.isselected) {
+        for(let attachment of record.attachments) {
+          if(attachment.isselected) {
+            selected.push(record);
+            break;
+          }
         }
+      }
     }
     const toastID = toast.loading(
-      "Exporting files (" + completed + "/" + exporting.length + ")"
+      "Downloading files (" + completed + "/" + selected.length + ")"
     );
     try {
-      for (let record of exporting) {
+      for (let record of selected) {
         var filepath = record.s3uripath;
         var filename = record.filename;
-        if (record.isredactionready && record.isconverted) {
-          filepath = filepath.substr(0, filepath.lastIndexOf(".")) + ".pdf";
-          filename += ".pdf";
+        if (record.isduplicate) {
+          let duplicatefiles = selected.filter((_record) => _record.filename == record.filename);
+          if(duplicatefiles.length > 1)
+            filename = filename.substring(0, filename.lastIndexOf(".")) + "_" + record.attributes.divisions[0].divisionname + "_" + Date.now() + filename.substring(filename.lastIndexOf("."));
         }
         const response = await getFOIS3DocumentPreSignedUrl(
           filepath.split("/").slice(4).join("/"),
@@ -1143,7 +1152,7 @@ export const RecordsLog = ({
           completed++;
           toast.update(toastID, {
             render:
-              "Exporting files (" + completed + "/" + exporting.length + ")",
+              "Downloading files (" + completed + "/" + selected.length + ")",
             isLoading: true,
           });
         });
@@ -1155,7 +1164,7 @@ export const RecordsLog = ({
       render:
         failed > 0
           ? failed.length + " file(s) failed to download"
-          : exporting.length + " Files exported",
+          : selected.length + " Files exported",
       type: failed > 0 ? "error" : "success",
     };
     toast.update(toastID, {
@@ -1168,11 +1177,12 @@ export const RecordsLog = ({
       draggable: true,
       closeButton: true,
     });
-    const zipfile = await downloadZip(blobs).blob();
-    var currentFilter = divisionFilters.find(
-      (division) => division.divisionid === filterValue
-    ).divisionname;
-    saveAs(zipfile, requestNumber + " Records - " + currentFilter + ".zip");
+    if(blobs.length == 1) {
+      saveAs(blobs[0].input, blobs[0].name);
+    } else {
+      const zipfile = await downloadZip(blobs).blob();
+      saveAs(zipfile, requestNumber + " Records - " + ".zip");
+    }
   };
 
   const retryDocument = (record) => {
@@ -1269,6 +1279,11 @@ export const RecordsLog = ({
         setModalFor("delete");
         setModal(true);
         break;
+      case "downloadselected":
+        downloadSelectedDocuments();
+        setModalFor("download");
+        setModal(false);
+        break;
       case "retry":
         retryDocument(_record);
         setModal(false);
@@ -1512,6 +1527,53 @@ export const RecordsLog = ({
     return false;
   };
 
+  const displaySizeLimit = () => {
+    if(RECORD_DOWNLOAD_SIZE_LIMIT >= (1024*1024*1024))
+      return (RECORD_DOWNLOAD_SIZE_LIMIT/(1024*1024*1024)).toFixed(1) + "GB";
+    if(RECORD_DOWNLOAD_SIZE_LIMIT >= (1024*1024))
+      return (RECORD_DOWNLOAD_SIZE_LIMIT/(1024*1024)).toFixed(1) + "MB";
+    if(RECORD_DOWNLOAD_SIZE_LIMIT >= 1024 )
+      return (RECORD_DOWNLOAD_SIZE_LIMIT/(1024)).toFixed(1) + "KB";
+    return RECORD_DOWNLOAD_SIZE_LIMIT + "Bytes";
+  }
+
+  const isSelectLimitReached = () => {
+    let fileCount = 0;
+    let totalFileSize = 0;
+
+    const recordlimit = RECORD_DOWNLOAD_LIMIT == 0 ? 1000000000 : RECORD_DOWNLOAD_LIMIT;
+    const sizelimt = RECORD_DOWNLOAD_SIZE_LIMIT == 0 ? 999999999999 : RECORD_DOWNLOAD_SIZE_LIMIT;
+
+    for (let record of records) {
+      if (record.isselected) {
+        fileCount++;
+        totalFileSize += record.attributes.filesize;
+
+        if(fileCount > recordlimit || totalFileSize > sizelimt)
+          return true;
+      } else {
+        if(record.attachments) {
+          for(let attachment of record.attachments) {
+            if(attachment.isselected) {
+              fileCount++;
+              totalFileSize += record.attributes.filesize;
+
+              if(fileCount > recordlimit || totalFileSize > sizelimt)
+              return true;
+
+              break;
+            }
+          }
+        }
+      }
+    }
+
+    if(fileCount == 0)
+      return true;
+    else
+      return false;
+  };
+
   function intersection(setA, setB) {
     const _intersection = new Set();
     for (const elem of setB) {
@@ -2077,6 +2139,37 @@ export const RecordsLog = ({
                   </button>
                 </span>
               </Tooltip>
+              <Tooltip
+                title={
+                  isSelectLimitReached() ? (
+                    <div style={{ fontSize: "11px" }}>
+                      To download records:{" "}
+                      <ul>
+                        <li>at least one record must be selected</li>
+                        <li>download size limit is {displaySizeLimit()}</li>
+                        <li>download records limit is {RECORD_DOWNLOAD_LIMIT}</li>
+                      </ul>
+                    </div>
+                  ) : (
+                    <div style={{ fontSize: "11px" }}>Download</div>
+                  )
+                }
+                sx={{ fontSize: "11px" }}
+              >
+                <span>
+                  <button
+                    className={` btn`}
+                    onClick={() => handlePopupButtonClick("downloadselected")}
+                    // title="Delete"
+                    disabled={isSelectLimitReached()}
+                    style={
+                      isSelectLimitReached() ? { pointerEvents: "none" } : {}
+                    }
+                  >
+                    <FontAwesomeIcon icon={faDownload} size="lg" color="#38598A" />
+                  </button>
+                </span>
+              </Tooltip>
             </Grid>
             <Grid
               container

diff --git a/forms-flow-web/src/constants/constants.js b/forms-flow-web/src/constants/constants.js
@@ -65,3 +65,5 @@ export const RECORD_PROCESSING_HRS = (window._env_ && window._env_.REACT_APP_REC
 export const DISABLE_REDACT_WEBLINK = (window._env_ && window._env_.REACT_APP_DISABLE_REDACT_WEBLINK) || process.env.REACT_APP_DISABLE_REDACT_WEBLINK || 'false';
 export const DISABLE_GATHERINGRECORDS_TAB = (window._env_ && window._env_.REACT_APP_DISABLE_GATHERINGRECORDS_TAB) || process.env.REACT_APP_DISABLE_GATHERINGRECORDS_TAB || 'false';
 
+export const RECORD_DOWNLOAD_LIMIT = (window._env_ && window._env_.REACT_APP_RECORD_DOWNLOAD_LIMIT) || process.env.REACT_APP_RECORD_DOWNLOAD_LIMIT || 200;
+export const RECORD_DOWNLOAD_SIZE_LIMIT = (window._env_ && window._env_.REACT_APP_RECORD_DOWNLOAD_SIZE_LIMIT) || process.env.REACT_APP_RECORD_DOWNLOAD_SIZE_LIMIT || 2147483648;
diff --git a/request-management-api/request_api/models/FOIRawRequests.py b/request-management-api/request_api/models/FOIRawRequests.py
@@ -656,7 +656,7 @@ def addadditionalfilter(cls, basequery, additionalfilter=None, userid=None, isia
                     return basequery.filter(
                         and_(
                             FOIRawRequest.status.notin_(['Archived']),
-                            or_(and_(FOIRawRequest.isiaorestricted == False, FOIRawRequest.assignedgroup.in_(ProcessingTeamWithKeycloackGroup.list())), and_(FOIRawRequest.isiaorestricted == True, FOIRawRequest.assignedto == userid))))
+                            or_(and_(FOIRawRequest.isiaorestricted == False, FOIRawRequest.assignedgroup.in_(ProcessingTeamWithKeycloackGroup.list()), FOIRawRequest.assignedgroup.in_(tuple(groups))), and_(FOIRawRequest.isiaorestricted == True, FOIRawRequest.assignedto == userid))))
                 return basequery.filter(
                     and_(
                         FOIRawRequest.status.notin_(['Archived']),