Merge branch 'beta-refactored' of github.com:kobotoolbox/kpi into 436…

…4-remove-invalid-default-values

dependencies/pip/dev_requirements.in

@@ -13,6 +13,7 @@ pytest-cov
 pytest-django
 pytest-env
 pytest-xdist
+freezegun
 
 # KoboCAT
 httmock

dependencies/pip/dev_requirements.txt

@@ -8,7 +8,7 @@
     # via -r dependencies/pip/requirements.in
 -e git+https://github.com/trevoriancox/django-dont-vary-on.git@01a804122b7ddcdc22f50b40993f91c27b03bef6#egg=django-dont-vary-on
     # via -r dependencies/pip/requirements.in
--e git+https://github.com/kobotoolbox/formpack.git@451df4cd2a0d614be69a3b3309259c67369f7efb#egg=formpack
+-e git+https://github.com/kobotoolbox/formpack.git@5a8cda8cc37a56a2313c98b88fcacc18049ef477#egg=formpack
     # via -r dependencies/pip/requirements.in
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest
     # via -r dependencies/pip/requirements.in
@@ -267,6 +267,8 @@ flake8-quotes==3.4.0
     # via -r dependencies/pip/dev_requirements.in
 flower==2.0.1
     # via -r dependencies/pip/requirements.in
+freezegun==1.5.1
+    # via -r dependencies/pip/dev_requirements.in
 frozenlist==1.4.1
     # via
     #   aiohttp
@@ -521,6 +523,7 @@ python-dateutil==2.9.0.post0
     #   -r dependencies/pip/requirements.in
     #   botocore
     #   celery
+    #   freezegun
     #   pandas
     #   python-crontab
 python3-openid==3.2.0
@@ -685,4 +688,3 @@ yubico-client==1.13.0
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
-backports-zoneinfo==0.2.1; python_version < '3.9'

dependencies/pip/requirements.in

@@ -2,7 +2,7 @@
 # https://github.com/bndr/pipreqs is a handy utility, too.
 
 # formpack
--e git+https://github.com/kobotoolbox/formpack.git@451df4cd2a0d614be69a3b3309259c67369f7efb#egg=formpack
+-e git+https://github.com/kobotoolbox/formpack.git@5a8cda8cc37a56a2313c98b88fcacc18049ef477#egg=formpack
 
 # More up-to-date version of django-digest than PyPI seems to have.
 # Also, python-digest is an unlisted dependency thereof.
@@ -109,6 +109,3 @@ modilabs-python-utils
 djangorestframework-csv
 djangorestframework-jsonp
 pandas
-
-# Python 3.8 support
-backports.zoneinfo; python_version < '3.9'

dependencies/pip/requirements.txt

@@ -8,7 +8,7 @@
     # via -r dependencies/pip/requirements.in
 -e git+https://github.com/trevoriancox/django-dont-vary-on.git@01a804122b7ddcdc22f50b40993f91c27b03bef6#egg=django-dont-vary-on
     # via -r dependencies/pip/requirements.in
--e git+https://github.com/kobotoolbox/formpack.git@451df4cd2a0d614be69a3b3309259c67369f7efb#egg=formpack
+-e git+https://github.com/kobotoolbox/formpack.git@5a8cda8cc37a56a2313c98b88fcacc18049ef477#egg=formpack
     # via -r dependencies/pip/requirements.in
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest
     # via -r dependencies/pip/requirements.in
@@ -530,4 +530,3 @@ yarl==1.9.4
     # via aiohttp
 yubico-client==1.13.0
     # via django-trench
-backports-zoneinfo==0.2.1; python_version < '3.9'

jsapp/js/assetUtils.ts

@@ -451,17 +451,17 @@ export function findRow(assetContent: AssetContent, rowName: string) {
   return assetContent?.survey?.find((row) => getRowName(row) === rowName);
 }
 
-export function findRowByQpath(assetContent: AssetContent, qpath: string) {
-  return assetContent?.survey?.find((row) => row.$qpath === qpath);
+export function findRowByXpath(assetContent: AssetContent, xpath: string) {
+  return assetContent?.survey?.find((row) => row.$xpath === xpath);
 }
 
 export function getRowType(assetContent: AssetContent, rowName: string) {
   const foundRow = findRow(assetContent, rowName);
   return foundRow?.type;
 }
 
-export function getRowNameByQpath(assetContent: AssetContent, qpath: string) {
-  const foundRow = findRowByQpath(assetContent, qpath);
+export function getRowNameByXpath(assetContent: AssetContent, xpath: string) {
+  const foundRow = findRowByXpath(assetContent, xpath);
   if (foundRow) {
     return getRowName(foundRow);
   }
@@ -552,9 +552,8 @@ export function injectSupplementalRowsIntoListOfRows(
   // Step 4: Inject all the extra columns immediately after source question
   const outputWithCols: string[] = [];
   output.forEach((col: string) => {
-    const qpath = col.replace(/\//g, '-');
     outputWithCols.push(col);
-    (extraColsBySource[qpath] || []).forEach((extraCol) => {
+    (extraColsBySource[col] || []).forEach((extraCol) => {
       outputWithCols.push(`_supplementalDetails/${extraCol.dtpath}`);
     });
   });
@@ -702,7 +701,7 @@ export function getAssetSubmissionProcessingUrl(
   return undefined;
 }
 
-/** Returns a list of all rows (their `qpath`s) activated for advanced features. */
+/** Returns a list of all rows (their `xpath`s) activated for advanced features. */
 export function getAssetProcessingRows(assetUid: string) {
   const foundAsset = assetStore.getAsset(assetUid);
   if (foundAsset?.advanced_submission_schema?.properties) {
@@ -723,9 +722,9 @@ export function getAssetProcessingRows(assetUid: string) {
   return undefined;
 }
 
-export function isRowProcessingEnabled(assetUid: string, qpath: string) {
+export function isRowProcessingEnabled(assetUid: string, xpath: string) {
   const processingRows = getAssetProcessingRows(assetUid);
-  return Array.isArray(processingRows) && processingRows.includes(qpath);
+  return Array.isArray(processingRows) && processingRows.includes(xpath);
 }
 
 export function isAssetProcessingActivated(assetUid: string) {

jsapp/js/components/permissions/userAssetPermsEditor.component.tsx

@@ -478,10 +478,10 @@ export default class UserAssetPermsEditor extends React.Component<
         false,
         true
       );
-      for (const [, qPath] of Object.entries(flatPaths)) {
+      for (const [, path] of Object.entries(flatPaths)) {
         output.push({
-          value: qPath,
-          label: qPath,
+          value: path,
+          label: path,
         });
       }
     }

jsapp/js/components/processing/analysis/analysisContent.component.tsx

@@ -14,7 +14,7 @@ export default function AnalysisContent() {
 
   // We only want to display analysis questions for this survey question
   const filteredQuestions = analysisQuestions.state.questions.filter(
-    (question) => question.qpath === singleProcessingStore.currentQuestionQpath
+    (question) => question.xpath === singleProcessingStore.currentQuestionXpath
   );
 
   return (

jsapp/js/components/processing/analysis/analysisHeader.component.tsx

@@ -46,7 +46,7 @@ export default function AnalysisHeader() {
           analysisQuestions?.dispatch({
             type: 'addQuestion',
             payload: {
-              qpath: singleProcessingStore.currentQuestionQpath,
+              xpath: singleProcessingStore.currentQuestionXpath,
               type: definition.type,
             },
           });

jsapp/js/components/processing/analysis/analysisQuestions.actions.ts

@@ -8,7 +8,7 @@ export type AnalysisQuestionsAction =
   // Sets all the quetsion with new ones (useful for initialising)
   | {type: 'setQuestions'; payload: {questions: AnalysisQuestionInternal[]}}
   // Creates a draft question of given type with new uid assigned
-  | {type: 'addQuestion'; payload: {qpath: string; type: AnalysisQuestionType}}
+  | {type: 'addQuestion'; payload: {xpath: string; type: AnalysisQuestionType}}
   // Opens question for editing, i.e. causes the editor to be opened for given
   // question
   | {type: 'startEditingQuestion'; payload: {uuid: string}}
@@ -43,7 +43,7 @@ export type AnalysisQuestionsAction =
   | {
       type: 'updateResponseCompleted';
       payload: {
-        qpath: string;
+        xpath: string;
         apiResponse: SubmissionProcessingDataResponse;
       };
     }

jsapp/js/components/processing/analysis/analysisQuestions.reducer.ts

@@ -73,7 +73,7 @@ export const analysisQuestionsReducer: AnalysisQuestionReducerType = (
       }
 
       const newQuestion: AnalysisQuestionInternal = {
-        qpath: action.payload.qpath,
+        xpath: action.payload.xpath,
         type: action.payload.type,
         labels: {_default: ''},
         uuid: newUuid,
@@ -186,7 +186,7 @@ export const analysisQuestionsReducer: AnalysisQuestionReducerType = (
     }
     case 'updateResponseCompleted': {
       const newQuestions = applyUpdateResponseToInternalQuestions(
-        action.payload.qpath,
+        action.payload.xpath,
         action.payload.apiResponse,
         state.questions
       );

jsapp/js/components/processing/analysis/analysisTab.component.tsx

@@ -70,7 +70,7 @@ export default function AnalysisTab() {
             await fetchGetUrl<SubmissionProcessingDataResponse>(processingUrl);
 
           questions = applyUpdateResponseToInternalQuestions(
-            singleProcessingStore.currentQuestionQpath,
+            singleProcessingStore.currentQuestionXpath,
             apiResponse,
             questions
           );

jsapp/js/components/processing/analysis/constants.ts

@@ -72,7 +72,7 @@ export interface AnalysisQuestionBase {
   uuid: string;
   options?: AnalysisQuestionOptions;
   /** The survey question that this analysis questions is for. */
-  qpath: string;
+  xpath: string;
 }
 
 /** Analysis question definition from the asset's schema (i.e. from Back end) */
@@ -160,7 +160,7 @@ export interface SubmissionAnalysisResponse extends AnalysisQuestionBase {
  * This is the payload of a request made to update a question response.
  */
 export interface AnalysisResponseUpdateRequest {
-  [qpath: string]:
+  [xpath: string]:
     | {
         qual: AnalysisRequest[];
       }
@@ -173,7 +173,7 @@ export interface AnalysisResponseUpdateRequest {
  * response.
  */
 export interface SubmissionProcessingDataResponse {
-  [qpath: string]: {
+  [xpath: string]: {
     qual: AnalysisResponse[];
   };
 }

jsapp/js/components/processing/analysis/list/analysisQuestionsList.component.tsx

@@ -35,7 +35,7 @@ export default function AnalysisQuestionsList() {
           // hide them at this point (not filtering the whole list beforehand),
           // because we need the indexes to match the whole list. And FYI all
           // analysis questions live on a single list :)
-          if (question.qpath !== singleProcessingStore.currentQuestionQpath) {
+          if (question.xpath !== singleProcessingStore.currentQuestionXpath) {
             return null;
           }
 

jsapp/js/components/processing/analysis/responseForms/integerResponseForm.component.tsx

@@ -52,7 +52,7 @@ export default function IntegerResponseForm(props: IntegerResponseFormProps) {
 
     updateResponseAndReducer(
       analysisQuestions.dispatch,
-      question.qpath,
+      question.xpath,
       props.uuid,
       question.type,
       response

jsapp/js/components/processing/analysis/responseForms/selectMultipleResponseForm.component.tsx

@@ -63,7 +63,7 @@ export default function SelectMultipleResponseForm(
     // Update endpoint and reducer
     updateResponseAndReducer(
       analysisQuestions.dispatch,
-      question.qpath,
+      question.xpath,
       props.uuid,
       question.type,
       newResponse

jsapp/js/components/processing/analysis/responseForms/selectOneResponseForm.component.tsx

@@ -57,7 +57,7 @@ export default function SelectOneResponseForm(
     // Update endpoint and reducer
     updateResponseAndReducer(
       analysisQuestions.dispatch,
-      question.qpath,
+      question.xpath,
       props.uuid,
       question.type,
       newResponse

jsapp/js/components/processing/analysis/responseForms/tagsResponseForm.component.tsx

@@ -55,7 +55,7 @@ export default function TagsResponseForm(props: TagsResponseFormProps) {
     // Update endpoint and reducer
     updateResponseAndReducer(
       analysisQuestions.dispatch,
-      question.qpath,
+      question.xpath,
       props.uuid,
       question.type,
       newTags

jsapp/js/components/processing/analysis/responseForms/textResponseForm.component.tsx

@@ -52,7 +52,7 @@ export default function TextResponseForm(props: TextResponseFormProps) {
 
     updateResponseAndReducer(
       analysisQuestions.dispatch,
-      question.qpath,
+      question.xpath,
       props.uuid,
       question.type,
       response

jsapp/js/components/processing/analysis/utils.ts

@@ -49,7 +49,7 @@ export function convertQuestionsFromInternalToSchema(
       options: question.options,
       choices: question.additionalFields?.choices,
       scope: 'by_question#survey',
-      qpath: question.qpath,
+      xpath: question.xpath,
     };
   });
 }
@@ -64,7 +64,7 @@ export function convertQuestionsFromSchemaToInternal(
 ): AnalysisQuestionInternal[] {
   return questions.map((question) => {
     const output: AnalysisQuestionInternal = {
-      qpath: question.qpath,
+      xpath: question.xpath,
       uuid: question.uuid,
       type: question.type,
       labels: question.labels,
@@ -85,12 +85,12 @@ export function convertQuestionsFromSchemaToInternal(
  * internal questions list using the API endpoint response.
  */
 export function applyUpdateResponseToInternalQuestions(
-  qpath: string,
+  xpath: string,
   updateResp: SubmissionProcessingDataResponse,
   questions: AnalysisQuestionInternal[]
 ): AnalysisQuestionInternal[] {
   const newQuestions = clonedeep(questions);
-  const analysisResponses = updateResp[qpath]?.qual || [];
+  const analysisResponses = updateResp[xpath]?.qual || [];
   newQuestions.forEach((question) => {
     const foundResponse = analysisResponses.find(
       (analResp) => question.uuid === analResp.uuid
@@ -191,15 +191,15 @@ export async function updateSurveyQuestions(
 async function updateResponse(
   processingUrl: string,
   submissionUid: string,
-  qpath: string,
+  xpath: string,
   analysisQuestionUuid: string,
   analysisQuestionType: AnalysisQuestionType,
   newResponse: string | string[] | number | null
 ) {
   try {
     const payload: AnalysisResponseUpdateRequest = {
       submission: submissionUid,
-      [qpath]: {
+      [xpath]: {
         qual: [
           {
             uuid: analysisQuestionUuid,
@@ -219,7 +219,7 @@ async function updateResponse(
 
     return {
       apiResponse: apiResponse,
-      qpath: qpath,
+      xpath: xpath,
     };
   } catch (err) {
     return Promise.reject(err);
@@ -243,7 +243,7 @@ async function updateResponse(
  */
 export async function updateResponseAndReducer(
   dispatch: React.Dispatch<AnalysisQuestionsAction>,
-  surveyQuestionQpath: string,
+  surveyQuestionXpath: string,
   analysisQuestionUuid: string,
   analysisQuestionType: AnalysisQuestionType,
   response: string | string[]
@@ -284,7 +284,7 @@ export async function updateResponseAndReducer(
     const result = await updateResponse(
       processingUrl,
       singleProcessingStore.currentSubmissionEditId,
-      surveyQuestionQpath,
+      surveyQuestionXpath,
       analysisQuestionUuid,
       analysisQuestionType,
       actualResponse