Updated feedback

amonsour · Jun 23, 2023 · 1313f0a · 1313f0a
1 parent 60007de
commit 1313f0a
Show file tree

Hide file tree

Showing 6 changed files with 68 additions and 19 deletions.
diff --git a/libs/db-update/src/lib/airtable/feedback.service.ts b/libs/db-update/src/lib/airtable/feedback.service.ts
@@ -26,32 +26,54 @@ export class FeedbackService {
     const latestDataDate: Date | null = (await this.feedbackModel
       .findOne({}, { date: 1 })
       .sort({ date: -1 }))?.date;
-
+
+    const createdDateThreshold = dayjs(latestDataDate || '2020-01-01')
+      .utc(false)
+      .add(1, 'day') as DateType;
+
+    const startDate = dayjs(endDate).subtract(14, 'days').startOf('day') as DateType;
+
     const dateRange = {
-      start: dayjs(latestDataDate || '2020-01-01')
-        .utc(false)
-        .add(1, 'day') as DateType,
-      end: (endDate || dayjs().utc(true).subtract(1, 'day')) as DateType,
+      start: createdDateThreshold < startDate ? startDate : createdDateThreshold,
+      end: endDate,
     } as DateRange;
 
-    const feedbackData = (await this.airtableClient.getFeedback(dateRange))
-      .map(
-        (feedbackData) =>
-          ({
-            _id: new Types.ObjectId(),
-            ...feedbackData,
-          } as Feedback)
-      )
-      .sort((current, next) => current.date.getTime() - next.date.getTime());
+    const [craFeedbackData, liveFeedbackData] = await Promise.all([
+      this.airtableClient.getFeedback(dateRange),
+      this.airtableClient.getLiveFeedback(dateRange)
+    ]);
 
+    const feedbackData = [...craFeedbackData, ...liveFeedbackData].map((data) => ({
+      _id: new Types.ObjectId(),
+      ...data,
+    }) as Feedback);
+
     if (feedbackData.length === 0) {
       this.logger.log('Feedback data already up-to-date.');
       return;
     }
+
+    const existingUniqueIds = new Set<string>();
+    const existingFeedbackData = await this.feedbackModel
+      .find({ unique_id: { $in: feedbackData.map((data) => data.unique_id) } })
+      .lean();
+
+    existingFeedbackData.forEach((feedback) => {
+      existingUniqueIds.add(feedback.unique_id);
+    });
+
+    const filteredFeedbackData = feedbackData
+    .filter((data) => !existingUniqueIds.has(data.unique_id))
+    .sort((current, next) => current.date.getTime() - next.date.getTime());
+
+    if (filteredFeedbackData.length === 0) {
+      this.logger.log('Feedback data already up-to-date.');
+      return;
+    }
 
     return await this.feedbackModel
-      .insertMany(feedbackData)
-      .then(() => this.logger.log('Successfully updated Feedback data'));
+      .insertMany(filteredFeedbackData)
+      .then(() => this.logger.log(`Successfully updated ${filteredFeedbackData.length} Feedback data`));
   }
 
   async repopulateFeedback() {

diff --git a/libs/db/src/lib/schemas/feedback.schema.d.ts b/libs/db/src/lib/schemas/feedback.schema.d.ts
@@ -10,8 +10,11 @@ export declare type FeedbackDocument = Feedback & Document;
 export declare class Feedback {
     _id: Types.ObjectId;
     airtable_id: string;
+    unique_id: string;
     url: string;
     date: Date;
+    created_time: Date;
+    time_received: string;
     tags?: string[];
     status?: string;
     whats_wrong?: string;

diff --git a/libs/db/src/lib/schemas/feedback.schema.ts b/libs/db/src/lib/schemas/feedback.schema.ts
@@ -12,12 +12,21 @@ export class Feedback implements IFeedback {
   @Prop({ type: String })
   airtable_id? = '';
 
+  @Prop({ type: String })
+  unique_id? = '';
+
   @Prop({ type: String, required: true, index: true })
   url = '';
 
   @Prop({ type: Date, required: true, index: true })
   date: Date = new Date(0);
 
+  @Prop({ type: Date, required: true, index: true })
+  created_time: Date = new Date(0);
+
+  @Prop({ type: String })
+  time_received? = '';
+
   @Prop({ type: String })
   lang = '';
 

diff --git a/libs/external-data/src/lib/airtable/index.ts b/libs/external-data/src/lib/airtable/index.ts
@@ -416,10 +416,13 @@ export class AirtableClient {
 
     return (await this.selectAll(query))
       .filter(({ fields }) => fields['URL'] && fields['Date'])
-      .map(({ id, fields }) => ({
+      .map(({ id, fields, createdTime }) => ({
         airtable_id: id,
+        unique_id: fields['Unique ID'],
         url: fields['URL'].replace(/^https:\/\//i, ''),
         date: dayjs.utc(fields['Date']).toDate(),
+        time_received: fields['Time received'],
+        created_time: dayjs.utc(String(createdTime)).toDate(),
         lang: fields['Lang'],
         comment: fields['Comment'],
         tags: fields['Lookup_tags'],
@@ -439,11 +442,14 @@ export class AirtableClient {
     });
 
     return (await this.selectAll(query))
-      .filter(({ fields }) => fields['URL'] && fields['Date'])
-      .map(({ id, fields }) => ({
+      .filter(({ fields }) => fields['Institution'].toUpperCase() === 'CRA' && fields['URL'] && fields['Date'])
+      .map(({ id, fields, createdTime  }) => ({
         airtable_id: id,
+        unique_id: fields['Unique ID'],
         url: fields['URL'].replace(/^https:\/\//i, ''),
         date: dayjs.utc(fields['Date']).toDate(),
+        time_received: fields['Time received'],
+        created_time: dayjs.utc(String(createdTime)).toDate(),
         lang: fields['Lang'],
         comment: fields['Comment'],
         tags: fields['Lookup_tags'],

diff --git a/libs/external-data/src/lib/airtable/types.ts b/libs/external-data/src/lib/airtable/types.ts
@@ -84,8 +84,11 @@ export interface PageData {
 
 export interface FeedbackData {
   airtable_id: string;
+  unique_id: string;
   url: string;
   date: Date;
+  created_time: Date;
+  time_received: string;
   tags?: string[];
   status?: string;
   whats_wrong?: string;

diff --git a/libs/types-common/src/lib/schema.types.ts b/libs/types-common/src/lib/schema.types.ts
@@ -75,10 +75,16 @@ export interface IFeedback {
 
   airtable_id?: string;
 
+  unique_id?: string;
+
   url: string;
 
   date: Date;
 
+  created_time: Date;
+
+  time_received?: string;
+
   lang: string;
 
   comment: string;