Skip to content

Commit

Permalink
Merge pull request #893 from EyeSeeTea/development
Browse files Browse the repository at this point in the history
  • Loading branch information
@SferaDev
SferaDev authored Aug 30, 2021
2 parents 42668c1 + 79ad7b6 commit d26dc90
Show file tree
Hide file tree
Showing 34 changed files with 3,008 additions and 2,595 deletions.
2 changes: 1 addition & 1 deletion .env
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
BROWSER=false
PORT=8081
SKIP_PREFLIGHT_CHECK=true
REACT_APP_DHIS2_BASE_URL=http://localhost:8080
REACT_APP_DHIS2_BASE_URL=https://dev.eyeseetea.com/play/
1 change: 1 addition & 0 deletions .prettierignore
View file
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
src/react-app.d.ts
src/locales
10 changes: 8 additions & 2 deletions i18n/en.pot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ msgstr ""
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=(n != 1)\n"
"POT-Creation-Date: 2021-04-20T10:05:20.063Z\n"
"PO-Revision-Date: 2021-04-20T10:05:20.063Z\n"
"POT-Creation-Date: 2021-08-09T06:46:23.536Z\n"
"PO-Revision-Date: 2021-08-09T06:46:23.536Z\n"

msgid ""
"THIS NEW RELEASE INCLUDES SHARING SETTINGS PER INSTANCES. FOR THIS VERSION "
Expand Down Expand Up @@ -1949,6 +1949,9 @@ msgstr[1] ""
msgid "Synchronization History"
msgstr ""

msgid "Server error on aggregation"
msgstr ""

msgid "Aggregate Data For HMIS"
msgstr ""

Expand Down Expand Up @@ -1976,6 +1979,9 @@ msgstr ""
msgid "Do you want to proceed?"
msgstr ""

msgid "Syncing process will stop if you leave the current page"
msgstr ""

msgid "Retrieving information from the system..."
msgstr ""

Expand Down
8 changes: 7 additions & 1 deletion i18n/es.po
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
msgid ""
msgstr ""
"Project-Id-Version: i18next-conv\n"
"POT-Creation-Date: 2021-04-20T09:30:54.460Z\n"
"POT-Creation-Date: 2021-07-29T11:01:03.713Z\n"
"PO-Revision-Date: 2020-07-10T06:53:30.625Z\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
Expand Down Expand Up @@ -1953,6 +1953,9 @@ msgstr[1] ""
msgid "Synchronization History"
msgstr ""

msgid "Server error on aggregation"
msgstr ""

msgid "Aggregate Data For HMIS"
msgstr ""

Expand Down Expand Up @@ -1980,6 +1983,9 @@ msgstr ""
msgid "Do you want to proceed?"
msgstr ""

msgid "Syncing process will stop if you leave the current page"
msgstr ""

msgid "Retrieving information from the system..."
msgstr ""

Expand Down
8 changes: 7 additions & 1 deletion i18n/fr.po
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
msgid ""
msgstr ""
"Project-Id-Version: i18next-conv\n"
"POT-Creation-Date: 2021-04-20T09:30:54.460Z\n"
"POT-Creation-Date: 2021-07-29T11:01:03.713Z\n"
"PO-Revision-Date: 2020-07-10T06:53:30.625Z\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
Expand Down Expand Up @@ -1953,6 +1953,9 @@ msgstr[1] ""
msgid "Synchronization History"
msgstr ""

msgid "Server error on aggregation"
msgstr ""

msgid "Aggregate Data For HMIS"
msgstr ""

Expand Down Expand Up @@ -1980,6 +1983,9 @@ msgstr ""
msgid "Do you want to proceed?"
msgstr ""

msgid "Syncing process will stop if you leave the current page"
msgstr ""

msgid "Retrieving information from the system..."
msgstr ""

Expand Down
8 changes: 7 additions & 1 deletion i18n/pt.po
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
msgid ""
msgstr ""
"Project-Id-Version: i18next-conv\n"
"POT-Creation-Date: 2021-04-20T09:30:54.460Z\n"
"POT-Creation-Date: 2021-07-29T11:01:03.713Z\n"
"PO-Revision-Date: 2020-07-10T06:53:30.625Z\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
Expand Down Expand Up @@ -1953,6 +1953,9 @@ msgstr[1] ""
msgid "Synchronization History"
msgstr ""

msgid "Server error on aggregation"
msgstr ""

msgid "Aggregate Data For HMIS"
msgstr ""

Expand Down Expand Up @@ -1980,6 +1983,9 @@ msgstr ""
msgid "Do you want to proceed?"
msgstr ""

msgid "Syncing process will stop if you leave the current page"
msgstr ""

msgid "Retrieving information from the system..."
msgstr ""

Expand Down
6 changes: 3 additions & 3 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "metadata-synchronization",
"description": "Advanced metadata & data synchronization utility",
"version": "2.10.1",
"version": "2.11.0",
"license": "GPL-3.0",
"author": "EyeSeeTea team",
"homepage": ".",
Expand All @@ -18,8 +18,8 @@
"@dhis2/d2-ui-forms": "7.0.4",
"@dhis2/ui-core": "4.14.0",
"@dhis2/ui-widgets": "2.1.0",
"@eyeseetea/d2-api": "1.8.1",
"@eyeseetea/d2-ui-components": "2.6.2",
"@eyeseetea/d2-api": "1.9.2",
"@eyeseetea/d2-ui-components": "2.6.7",
"@material-ui/core": "4.11.0",
"@material-ui/icons": "4.9.1",
"@material-ui/lab": "4.0.0-alpha.56",
Expand Down
110 changes: 72 additions & 38 deletions src/data/aggregated/AggregatedD2ApiRepository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,50 +42,83 @@ export class AggregatedD2ApiRepository implements AggregatedRepository {

if (dataSet.length === 0 && dataElementGroup.length === 0) return { dataValues: [] };

const orgUnit = cleanOrgUnitPaths(orgUnitPaths);
const orgUnits = cleanOrgUnitPaths(orgUnitPaths);
const attributeOptionCombo = !allAttributeCategoryOptions
? attributeCategoryOptions
: undefined;

const [defaultCategoryOptionCombo] = await this.getDefaultIds("categoryOptionCombos");

const dimensions = _.uniqBy(
[
...dataSet.map(id => ({ type: "dataSet", id })),
...dataElementGroup.map(id => ({ type: "dataElementGroup", id })),
],
({ id }) => id
);

try {
const { dataValues = [] } = await this.api.dataValues
.getSet({
dataElementIdScheme: "UID",
orgUnitIdScheme: "UID",
categoryOptionComboIdScheme: "UID",
includeDeleted: false,
startDate: startDate.format("YYYY-MM-DD"),
endDate: endDate.format("YYYY-MM-DD"),
attributeOptionCombo,
dataSet,
dataElementGroup,
orgUnit,
lastUpdated: lastUpdated ? moment(lastUpdated).format("YYYY-MM-DD") : undefined,
// Chunked request by orgUnits and dimensions (dataSets and dataElementGroups) to avoid 414
const dataValues = await promiseMap(_.chunk(orgUnits, 100), orgUnit =>
promiseMap(_.chunk(dimensions, 50), dimensions => {
const dataSet = dimensions
.filter(({ type }) => type === "dataSet")
.map(({ id }) => id);
const dataElementGroup = dimensions
.filter(({ type }) => type === "dataElementGroup")
.map(({ id }) => id);

return this.api.dataValues
.getSet({
dataElementIdScheme: "UID",
orgUnitIdScheme: "UID",
categoryOptionComboIdScheme: "UID",
includeDeleted: false,
startDate: startDate.format("YYYY-MM-DD"),
endDate: endDate.format("YYYY-MM-DD"),
attributeOptionCombo,
dataSet: [dataSet.join(",")],
dataElementGroup: [dataElementGroup.join(",")],
orgUnit: [orgUnit.join(",")],
lastUpdated: lastUpdated
? moment(lastUpdated).format("YYYY-MM-DD")
: undefined,
})
.map(({ data }) =>
data.dataValues.map(
({
dataElement,
period,
orgUnit,
categoryOptionCombo,
attributeOptionCombo,
value,
comment,
}) => ({
dataElement,
period,
orgUnit,
value,
comment,
categoryOptionCombo:
categoryOptionCombo ?? defaultCategoryOptionCombo,
attributeOptionCombo:
attributeOptionCombo ?? defaultCategoryOptionCombo,
})
)
)
.getData();
})
.getData();

const [defaultCategoryOptionCombo] = await this.getDefaultIds("categoryOptionCombos");
);

return {
dataValues: dataValues.map(
({
dataElement,
period,
orgUnit,
categoryOptionCombo,
attributeOptionCombo,
value,
comment,
}) => ({
dataElement,
period,
orgUnit,
value,
comment,
categoryOptionCombo: categoryOptionCombo ?? defaultCategoryOptionCombo,
attributeOptionCombo: attributeOptionCombo ?? defaultCategoryOptionCombo,
})
),
dataValues: _(dataValues)
.flatten()
.flatten()
.uniqBy(({ orgUnit, period, dataElement, categoryOptionCombo }) =>
[orgUnit, period, dataElement, categoryOptionCombo].join("-")
)
.value(),
};
} catch (error) {
console.error(error);
Expand Down Expand Up @@ -122,8 +155,8 @@ export class AggregatedD2ApiRepository implements AggregatedRepository {
if (dimensionIds.length === 0 || orgUnits.length === 0) {
return { dataValues: [] };
} else if (aggregationType) {
const result = await promiseMap(_.chunk(periods, 300), period =>
promiseMap(_.chunk(dimensionIds, Math.max(10, 300 - period.length)), ids => {
const result = await promiseMap(_.chunk(periods, 5), period =>
promiseMap(_.chunk(dimensionIds, Math.max(10, 100 - period.length)), ids => {
return this.api
.get<AggregatedPackage>("/analytics/dataValueSet.json", {
dimension: _.compact([
Expand Down Expand Up @@ -272,6 +305,7 @@ export class AggregatedD2ApiRepository implements AggregatedRepository {
}

async delete(data: AggregatedPackage): Promise<SynchronizationResult> {
// TODO: To be refactored. Requires authority F_SKIP_DATA_IMPORT_AUDIT.
return await this.save(data, { strategy: "DELETES", skipAudit: true });
}

Expand Down
Loading

0 comments on commit d26dc90

Please sign in to comment.