feat: fetch credit card line of credit utilization

src/scrapers/base-isracard-amex.ts

@@ -61,6 +61,11 @@ interface ScrapedAccount {
   processedDate: string;
 }
 
+interface AccountCredit {
+  creditUtilization: number;
+  creditLimit: number;
+}
+
 interface ScrapedLoginValidation {
   Header: {
     Status: string;
@@ -84,6 +89,19 @@ interface ScrapedAccountsWithinPageResponse {
   };
 }
 
+interface ScrapedCreditDataWithinPageResponse {
+  Header: {
+    Status: string;
+  };
+  RikuzNetuneyCreditDigiBean: {
+    cardsTotal: {
+      cardNumberTail: string;
+      nitzulLoCredit: string;
+      misgeretKolelet: string;
+    }[];
+  };
+}
+
 interface ScrapedCurrentCardTransactions {
   txnIsrael?: ScrapedTransaction[];
   txnAbroad?: ScrapedTransaction[];
@@ -98,6 +116,39 @@ interface ScrapedTransactionData {
   }>;
 }
 
+function getCreditUtilizationUrl(servicesUrl: string) {
+  return buildUrl(servicesUrl, {
+    queryParams: {
+      reqName: 'RikuzNetuneyCreditDigi',
+    },
+  });
+}
+
+async function fetchCreditUtilization(
+  page: Page,
+  servicesUrl: string,
+): Promise<_.Dictionary<AccountCredit> | null> {
+  const dataUrl = getCreditUtilizationUrl(servicesUrl);
+  const dataResult = await fetchGetWithinPage<ScrapedCreditDataWithinPageResponse>(
+    page,
+    dataUrl,
+  );
+
+  if (!dataResult) {
+    throw new Error('Failed to fetch credit utilization data, empty response');
+  }
+
+  return _.fromPairs(
+    dataResult.RikuzNetuneyCreditDigiBean.cardsTotal.map((item) => [
+      item.cardNumberTail,
+      {
+        creditUtilization: parseFloat(item.nitzulLoCredit),
+        creditLimit: parseFloat(item.misgeretKolelet.replace(/,/g, '')),
+      },
+    ]),
+  );
+}
+
 function getAccountsUrl(servicesUrl: string, monthMoment: Moment) {
   const billingDate = monthMoment.format('YYYY-MM-DD');
   return buildUrl(servicesUrl, {
@@ -284,6 +335,12 @@ function getExtraScrap(accountsWithIndex: ScrapedAccountsWithIndex[], page: Page
 async function fetchAllTransactions(page: Page, options: ExtendedScraperOptions, startMoment: Moment) {
   const futureMonthsToScrape = options.futureMonthsToScrape ?? 1;
   const allMonths = getAllMonthMoments(startMoment, futureMonthsToScrape);
+  let creditUtilization: _.Dictionary<AccountCredit> | null = null;
+  if (options.includeCreditUtilization) {
+    debug('Getting credit utilization data');
+    creditUtilization = await fetchCreditUtilization(page, options.servicesUrl);
+  }
+
   const results: ScrapedAccountsWithIndex[] = await Promise.all(allMonths.map(async (monthMoment) => {
     return fetchTransactions(page, options, startMoment, monthMoment);
   }));
@@ -306,10 +363,17 @@ async function fetchAllTransactions(page: Page, options: ExtendedScraperOptions,
   });
 
   const accounts = Object.keys(combinedTxns).map((accountNumber) => {
-    return {
+    const account: TransactionsAccount = {
       accountNumber,
       txns: combinedTxns[accountNumber],
     };
+    if (creditUtilization && creditUtilization[accountNumber]) {
+      account.credit = {
+        creditUtilization: creditUtilization[accountNumber].creditUtilization,
+        creditLimit: creditUtilization[accountNumber].creditLimit,
+      };
+    }
+    return account;
   });
 
   return {

src/scrapers/interface.ts

@@ -70,6 +70,11 @@ export interface ScraperOptions {
    */
   combineInstallments?: boolean;
 
+  /**
+   * if set to true, will fetch the credit utilization of the account (credit card only)
+   */
+  includeCreditUtilization?: boolean;
+
   /**
    * additional arguments to pass to the browser instance. The list of flags can be found in
    *

src/scrapers/max.ts

@@ -1,3 +1,4 @@
+import _ from 'lodash';
 import buildUrl from 'build-url';
 import moment, { Moment } from 'moment';
 import { Page, LoadEvent } from 'puppeteer';
@@ -7,7 +8,10 @@ import { waitForRedirect } from '../helpers/navigation';
 import { waitUntilElementFound, elementPresentOnPage, clickButton } from '../helpers/elements-interactions';
 import getAllMonthMoments from '../helpers/dates';
 import { fixInstallments, sortTransactionsByDate, filterOldTransactions } from '../helpers/transactions';
-import { Transaction, TransactionStatuses, TransactionTypes } from '../transactions';
+import {
+  Transaction, TransactionStatuses,
+  TransactionTypes, TransactionsAccount,
+} from '../transactions';
 import { getDebug } from '../helpers/debug';
 import { ScraperOptions } from './interface';
 import { SHEKEL_CURRENCY, DOLLAR_CURRENCY, EURO_CURRENCY } from '../constants';
@@ -95,6 +99,51 @@ function getTransactionsUrl(monthMoment: Moment) {
   });
 }
 
+function getCreditUtilizationUrl() {
+  return buildUrl(BASE_API_ACTIONS_URL, {
+    path: '/api/registered/getHomePageData',
+  });
+}
+
+interface ScrapedCreditDataWithinPageResponse {
+  Result: {
+    UserCards: {
+      Cards: {
+        Last4Digits: string;
+        CreditLimit: number;
+        OpenToBuy: number;
+      }[];
+    };
+  };
+}
+
+interface AccountCredit {
+  creditUtilization: number;
+  creditLimit: number;
+}
+
+async function fetchCreditUtilization(
+  page: Page,
+): Promise<_.Dictionary<AccountCredit>> {
+  const dataUrl = getCreditUtilizationUrl();
+  const dataResult = await fetchGetWithinPage<ScrapedCreditDataWithinPageResponse>(
+    page,
+    dataUrl,
+  );
+  if (!dataResult) {
+    throw new Error('Failed to fetch credit utilization data, empty response');
+  }
+  return _.fromPairs(
+    dataResult.Result.UserCards.Cards.map((item) => [
+      item.Last4Digits,
+      {
+        creditUtilization: item.CreditLimit - item.OpenToBuy,
+        creditLimit: item.CreditLimit,
+      },
+    ]),
+  );
+}
+
 interface FetchCategoryResult {
   result? : Array<{
     id: number;
@@ -343,11 +392,23 @@ class MaxScraper extends BaseScraperWithBrowser<ScraperSpecificCredentials> {
 
   async fetchData() {
     const results = await fetchTransactions(this.page, this.options);
+    let creditUtilization: _.Dictionary<AccountCredit> | null = null;
+    if (this.options.includeCreditUtilization) {
+      debug('Getting credit utilization data');
+      creditUtilization = await fetchCreditUtilization(this.page);
+    }
     const accounts = Object.keys(results).map((accountNumber) => {
-      return {
+      const account: TransactionsAccount = {
         accountNumber,
         txns: results[accountNumber],
       };
+      if (creditUtilization && creditUtilization[accountNumber]) {
+        account.credit = {
+          creditUtilization: creditUtilization[accountNumber].creditUtilization,
+          creditLimit: creditUtilization[accountNumber].creditLimit,
+        };
+      }
+      return account;
     });
 
     return {

src/scrapers/visa-cal.ts

@@ -21,6 +21,7 @@ import { ScraperScrapingResult } from './interface';
 
 const LOGIN_URL = 'https://www.cal-online.co.il/';
 const TRANSACTIONS_REQUEST_ENDPOINT = 'https://api.cal-online.co.il/Transactions/api/transactionsDetails/getCardTransactionsDetails';
+const CREDIT_FRAME_ENDPOINT = 'https://api.cal-online.co.il/Frames/api/Frames/GetFrameStatus';
 
 const InvalidPasswordMessage = 'שם המשתמש או הסיסמה שהוזנו שגויים';
 
@@ -85,11 +86,39 @@ interface InitResponse {
     }[];
   };
 }
+
+interface AccountCredit {
+  creditUtilization: number;
+  creditLimit: number;
+}
+
+
 type CurrencySymbol = '₪' | string;
 interface CardTransactionDetailsError {
   title: string;
   statusCode: number;
 }
+
+interface CardFrameStatusError {
+  title: string;
+  statusCode: number;
+  statusDescription: string;
+}
+
+interface CardFrameStatus extends CardFrameStatusError {
+  result: {
+    calIssuedCards: {
+      fictiveMaxAccAmt: number;
+      frameLimitForCardAmount: number;
+      totalUsageAmountForAccountManagementLevel: number;
+      nextTotalDebitForAccount: number;
+    };
+  };
+  statusCode: 1;
+  statusDescription: string;
+  statusTitle: string;
+}
+
 interface CardTransactionDetails extends CardTransactionDetailsError {
   result: {
     bankAccounts: {
@@ -238,6 +267,34 @@ function convertParsedDataToTransactions(parsedData: CardTransactionDetails[]):
     });
 }
 
+async function fetchCreditUtilization(
+  page: Page,
+  Authorization: string,
+  xSiteId: string,
+  card: { cardUniqueId: string, last4Digits: string },
+): Promise<AccountCredit> {
+  const dataResult = await fetchPostWithinPage<CardFrameStatus | CardFrameStatusError>(
+    page,
+    CREDIT_FRAME_ENDPOINT,
+    { cardsForFrameData: [{ cardUniqueId: card.cardUniqueId }] },
+    {
+      Authorization,
+      'X-Site-Id': xSiteId,
+      'Content-Type': 'application/json',
+    },
+  );
+
+  if (dataResult?.statusCode !== 1) {
+    throw new Error(`failed to fetch frame data. Message: ${dataResult?.title || ''}`);
+  }
+
+  const utilizationData = (dataResult as CardFrameStatus).result.calIssuedCards;
+  return {
+    creditUtilization: utilizationData.nextTotalDebitForAccount,
+    creditLimit: utilizationData.frameLimitForCardAmount,
+  };
+}
+
 type ScraperSpecificCredentials = { username: string, password: string };
 
 class VisaCalScraper extends BaseScraperWithBrowser<ScraperSpecificCredentials> {
@@ -376,10 +433,17 @@ class VisaCalScraper extends BaseScraperWithBrowser<ScraperSpecificCredentials>
           filterOldTransactions(transactions, moment(startDate), this.options.combineInstallments || false) :
           transactions;
 
-        return {
+        const account: TransactionsAccount = {
           txns,
           accountNumber: card.last4Digits,
-        } as TransactionsAccount;
+        };
+
+        if (this.options.includeCreditUtilization) {
+          debug('Getting credit utilization data');
+          account.credit = await fetchCreditUtilization(this.page, Authorization, xSiteId, card);
+        }
+
+        return account;
       }),
     );
 

src/transactions.ts

@@ -1,6 +1,15 @@
 
 export interface TransactionsAccount {
   accountNumber: string;
+
+  /**
+   * Relevant only for credit cards, this is the line of credit utilization, and credit limit.
+   * Only fetched if `includeCreditUtilization` is set to true in the scraper options.
+   */
+  credit?: {
+    creditUtilization: number;
+    creditLimit: number;
+  };
   balance?: number;
   txns: Transaction[];
 }