feat: add enableTransactionsFilterByDate option (#728)

eshaham · Nov 23, 2022 · c44b617 · c44b617
1 parent 4fe2184
commit c44b617
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 7 deletions.
diff --git a/src/scrapers/base-isracard-amex.ts b/src/scrapers/base-isracard-amex.ts
@@ -224,8 +224,9 @@ async function fetchTransactions(page: Page, options: ExtendedScraperOptions, st
         if (!options.combineInstallments) {
           allTxns = fixInstallments(allTxns);
         }
-        allTxns = filterOldTransactions(allTxns, startMoment, options.combineInstallments || false);
-
+        if (options.outputData?.enableTransactionsFilterByDate ?? true) {
+          allTxns = filterOldTransactions(allTxns, startMoment, options.combineInstallments || false);
+        }
         accountTxns[account.accountNumber] = {
           accountNumber: account.accountNumber,
           index: account.index,

diff --git a/src/scrapers/base-scraper.ts b/src/scrapers/base-scraper.ts
@@ -120,6 +120,18 @@ export interface ScraperOptions {
    * if set, will set the timeout in milliseconds of puppeteer's `page.setDefaultTimeout`.
    */
   defaultTimeout?: number;
+
+  /**
+   * Options for manipulation of output data
+   */
+  outputData?: OutputDataOptions;
+}
+
+export interface OutputDataOptions {
+  /**
+   * if true, the result wouldn't be filtered out by date, and you will return unfiltered scrapped data.
+   */
+  enableTransactionsFilterByDate?: boolean;
 }
 
 export enum ScaperProgressTypes {

diff --git a/src/scrapers/beyahad-bishvilha.ts b/src/scrapers/beyahad-bishvilha.ts
@@ -116,7 +116,9 @@ async function fetchTransactions(page: Page, options: ScraperOptions) {
   const accountTransactions = convertTransactions(rawTransactions.filter((item) => !!item) as ScrapedTransaction[]);
 
   debug('filer out old transactions');
-  const txns = filterOldTransactions(accountTransactions, startMoment, false);
+  const txns = (options.outputData?.enableTransactionsFilterByDate ?? true) ?
+    filterOldTransactions(accountTransactions, startMoment, false) :
+    accountTransactions;
   debug(`found ${txns.length} valid transactions out of ${accountTransactions.length} transactions for account ending with ${accountNumber.substring(accountNumber.length - 2)}`);
 
   return {

diff --git a/src/scrapers/max.ts b/src/scrapers/max.ts
@@ -212,13 +212,15 @@ function addResult(allResults: Record<string, Transaction[]>, result: Record<str
   return clonedResults;
 }
 
-function prepareTransactions(txns: Transaction[], startMoment: moment.Moment, combineInstallments: boolean) {
+function prepareTransactions(txns: Transaction[], startMoment: moment.Moment, combineInstallments: boolean, enableTransactionsFilterByDate: boolean) {
   let clonedTxns = Array.from(txns);
   if (!combineInstallments) {
     clonedTxns = fixInstallments(clonedTxns);
   }
   clonedTxns = sortTransactionsByDate(clonedTxns);
-  clonedTxns = filterOldTransactions(clonedTxns, startMoment, combineInstallments || false);
+  clonedTxns = enableTransactionsFilterByDate ?
+    filterOldTransactions(clonedTxns, startMoment, combineInstallments || false) :
+    clonedTxns;
   return clonedTxns;
 }
 
@@ -239,7 +241,8 @@ async function fetchTransactions(page: Page, options: ScraperOptions) {
 
   Object.keys(allResults).forEach((accountNumber) => {
     let txns = allResults[accountNumber];
-    txns = prepareTransactions(txns, startMoment, options.combineInstallments || false);
+    txns = prepareTransactions(txns, startMoment, options.combineInstallments || false,
+      (options.outputData?.enableTransactionsFilterByDate ?? true));
     allResults[accountNumber] = txns;
   });
 

diff --git a/src/scrapers/visa-cal.ts b/src/scrapers/visa-cal.ts
@@ -272,7 +272,9 @@ async function fetchTransactionsForAccount(page: Page, startDate: Moment, accoun
   }
 
   debug('filer out old transactions');
-  const txns = filterOldTransactions(accountTransactions, startDate, scraperOptions.combineInstallments || false);
+  const txns = (scraperOptions.outputData?.enableTransactionsFilterByDate ?? true) ?
+    filterOldTransactions(accountTransactions, startDate, scraperOptions.combineInstallments || false) :
+    accountTransactions;
   debug(`found ${txns.length} valid transactions out of ${accountTransactions.length} transactions for account ending with ${accountNumber.substring(accountNumber.length - 2)}`);
   return {
     accountNumber,