Full data dump export

src/backend/app.ts

@@ -48,6 +48,7 @@ export function createAppConstants() {
     cityLevelPastWeekGeneralResultsKey: 'city_level_past_week_general_results.json',
     postalCodeLevelGeneralResultsKey: 'postalcode_level_general_results.json',
     dailyTotalsKey: 'daily_totals.json',
+    responsesFullKey: 'responses_full.json',
   };
 }
 

src/backend/appMocks.ts

@@ -80,6 +80,7 @@ export function createMockApp(overrides: Partial<App> = {}): App {
     cityLevelPastWeekGeneralResultsKey: 'city_level_past_week_general_results.json',
     postalCodeLevelGeneralResultsKey: 'postalcode_level_general_results.json',
     dailyTotalsKey: 'daily_totals.json',
+    responsesFullKey: 'responses_full.json',
   };
 
   const mockApp: App = {

src/backend/cli.ts

@@ -14,6 +14,7 @@ import {
   fetchPostalCodeLevelGeneralResults,
   pushPostalCodeLevelGeneralResults,
 } from './dataExports/postalCodeLevelGeneralResults';
+import { fetchResponses, pushResponses } from './dataExports/responses';
 
 const writeFile = promisify(fs.writeFile);
 
@@ -46,12 +47,17 @@ const dataExportHandlers: { [K in keyof AppConstants]?: DataExportHandler } = {
     push: pushPostalCodeLevelGeneralResults,
   },
   [app.constants.dailyTotalsKey]: { fetch: fetchDailyTotals, push: pushDailyTotals },
+  [app.constants.responsesFullKey]: {
+    fetch: fetchResponses,
+    push: pushResponses,
+  },
 };
 
 interface CommonArgs {
   env: 'dev' | 'prod';
 }
 
+// prettier-ignore
 // eslint-disable-next-line @typescript-eslint/no-unused-expressions
 yargs
   .command(
@@ -95,7 +101,11 @@ yargs
         process.exit(1);
       }
     },
-  ).argv;
+  )
+  .strict()
+  .demandCommand()
+  .help()
+  .argv;
 
 interface DumpArgs extends CommonArgs {
   filename: string;

src/backend/dataExports/openDataIndex.ts

@@ -9,6 +9,7 @@ const openDataConstantKeys: Array<keyof AppConstants> = [
   'cityLevelPastWeekGeneralResultsKey',
   'postalCodeLevelGeneralResultsKey',
   'dailyTotalsKey',
+  'responsesFullKey',
   'lowPopulationPostalCodesKey',
   'populationPerCityKey',
   'postalCodeAreasKey',

src/backend/dataExports/responses.ts

@@ -0,0 +1,117 @@
+import { App, s3PutJsonHelper } from '../app';
+import { LowPopulationPostalCodes, PostalCodeCityMappings } from '../../common/model';
+
+export async function exportResponses(app: App) {
+  const responses = await fetchResponses(app);
+
+  await pushResponses(app, responses);
+}
+
+export async function fetchResponses(app: App) {
+  const responsesResult = await queryResponses(app);
+
+  const postalCodeCityMappings = await app.s3Sources.fetchPostalCodeCityMappings();
+  const lowPopulationPostalCodes = await app.s3Sources.fetchLowPopulationPostalCodes();
+
+  const responses = mapResponses(responsesResult.Items, postalCodeCityMappings, lowPopulationPostalCodes);
+
+  return responses;
+}
+
+interface ResponsesQuery {
+  response_id: string;
+  timestamp: string;
+  participant_id: string;
+  app_version: string;
+  country_code: string;
+  fever: string;
+  cough: string;
+  breathing_difficulties: string;
+  muscle_pain: string;
+  headache: string;
+  sore_throat: string;
+  rhinitis: string;
+  stomach_issues: string;
+  sensory_issues: string;
+  healthcare_contact: string;
+  general_wellbeing: string;
+  longterm_medication: string;
+  smoking: string;
+  corona_suspicion: string;
+  age_group: string;
+  gender: string;
+  postal_code: string;
+  duration: string;
+  abuse_score: string;
+}
+
+export const responsesQuery = `
+  SELECT *
+  FROM
+    responses
+  WHERE
+    country_code = 'FI'
+    OR
+    country_code = ''
+  ORDER BY timestamp ASC
+`;
+
+export async function queryResponses(app: App) {
+  return app.athenaClient.query<ResponsesQuery>({
+    sql: responsesQuery,
+    db: app.constants.athenaDb,
+  });
+}
+
+export function mapResponses(
+  responses: ResponsesQuery[],
+  postalCodeCityMappings: PostalCodeCityMappings,
+  lowPopulationPostalCodes: LowPopulationPostalCodes,
+) {
+  // TODO: Map and filter the responses
+  const outputs = [];
+
+  for (const response of responses) {
+    // Normalize postal code
+    const postal_code = lowPopulationPostalCodes.data[response.postal_code] || response.postal_code;
+
+    // Filter by valid postal areas
+    // TODO: Use the new `postalcode_areas.json
+    if (postal_code in postalCodeCityMappings.data) {
+      // Filter keys
+      const { response_id, app_version, abuse_score, ...output } = response;
+
+      outputs.push({
+        ...output,
+        postal_code,
+        // Timestamps to be shown to the hour
+        timestamp: `${response.timestamp.slice(0, 13)}:00:00.000Z`,
+        // Two age groups, _under 50_ and _over 50_
+        age_group: Number(response.age_group) < 50 ? 'under50' : 'over50',
+        // Two genders, male and female. Other genders coalesce to male
+        gender: response.gender === 'female' ? 'female' : 'male',
+        duration: Number(response.duration) || 0,
+      });
+    }
+  }
+
+  return outputs;
+}
+
+type Responses = ReturnType<typeof mapResponses>;
+
+export async function pushResponses(app: App, responses: Responses) {
+  await s3PutJsonHelper(app.s3Client, {
+    Bucket: app.constants.openDataBucket,
+    // TODO:
+    Key: app.constants.responsesFullKey,
+    Body: {
+      meta: {
+        description: 'TODO',
+        generated: new Date().toISOString(),
+        link: `https://${app.constants.domainName}/${app.constants.responsesFullKey}`,
+      },
+      data: responses,
+    },
+  });
+}