From 70e2a628473e4b387a402a39d2c85c8ed005e711 Mon Sep 17 00:00:00 2001 From: Gal Date: Sat, 13 Jul 2024 22:34:38 +0300 Subject: [PATCH] feat(cal): fetch pending transaction (#794) Co-authored-by: Baruch Odem --- src/scrapers/visa-cal.ts | 170 +++++++++++++++++++++++++++------------ 1 file changed, 120 insertions(+), 50 deletions(-) diff --git a/src/scrapers/visa-cal.ts b/src/scrapers/visa-cal.ts index 4065daf7..5a0d98dc 100644 --- a/src/scrapers/visa-cal.ts +++ b/src/scrapers/visa-cal.ts @@ -20,6 +20,7 @@ import { type ScraperScrapingResult } from './interface'; const LOGIN_URL = 'https://www.cal-online.co.il/'; const TRANSACTIONS_REQUEST_ENDPOINT = 'https://api.cal-online.co.il/Transactions/api/transactionsDetails/getCardTransactionsDetails'; +const PENDING_TRANSACTIONS_REQUEST_ENDPOINT = 'https://api.cal-online.co.il/Transactions/api/approvals/getClearanceRequests'; const InvalidPasswordMessage = 'שם המשתמש או הסיסמה שהוזנו שגויים'; @@ -74,6 +75,26 @@ interface ScrapedTransaction { trnTypeCode: TrnTypeCode; walletProviderCode: 0; walletProviderDesc: ''; + earlyPaymentInd: boolean; +} +interface ScrapedPendingTransaction { + merchantID: string; + merchantName: string; + trnPurchaseDate: string; + walletTranInd: number; + transactionsOrigin: number; + trnAmt: number; + tpaApprovalAmount: unknown; + trnCurrencySymbol: CurrencySymbol; + trnTypeCode: TrnTypeCode; + trnType: string; + branchCodeDesc: string; + transCardPresentInd: boolean; + j5Indicator: string; + numberOfPayments: number; + firstPaymentAmount: number; + transTypeCommentDetails: []; + } interface InitResponse { result: { @@ -120,7 +141,31 @@ interface CardTransactionDetails extends CardTransactionDetailsError { statusDescription: string; statusTitle: string; } +interface CardPendingTransactionDetails extends CardTransactionDetailsError { + result: { + cardsList: { + cardUniqueID: string; + authDetalisList: ScrapedPendingTransaction[]; + }[]; + }; + statusCode: 1; + statusDescription: string; + statusTitle: string; +} +function isPending(transaction: ScrapedTransaction | ScrapedPendingTransaction): transaction is ScrapedPendingTransaction { + return (transaction as ScrapedTransaction).debCrdDate === undefined; // an arbitrary field that only appears in a completed transaction +} + +function isCardTransactionDetails(result: CardTransactionDetails | CardTransactionDetailsError): +result is CardTransactionDetails { + return (result as CardTransactionDetails).result !== undefined; +} + +function isCardPendingTransactionDetails(result: CardPendingTransactionDetails | CardTransactionDetailsError): +result is CardPendingTransactionDetails { + return (result as CardPendingTransactionDetails).result !== undefined; +} async function getLoginFrame(page: Page) { let frame: Frame | null = null; @@ -186,55 +231,66 @@ function createLoginFields(credentials: ScraperSpecificCredentials) { ]; } -function convertParsedDataToTransactions(parsedData: CardTransactionDetails[]): Transaction[] { - const bankAccounts = parsedData - .flatMap((monthData) => monthData.result.bankAccounts); +function convertParsedDataToTransactions(data: CardTransactionDetails[], pendingData?: CardPendingTransactionDetails | null): Transaction[] { + const pendingTransactions = pendingData?.result ? + pendingData.result.cardsList.flatMap((card) => card.authDetalisList) : + []; + const bankAccounts = data + .flatMap((monthData) => monthData.result.bankAccounts); const regularDebitDays = bankAccounts .flatMap((accounts) => accounts.debitDates); const immediateDebitDays = bankAccounts .flatMap((accounts) => accounts.immidiateDebits.debitDays); + const completedTransactions = [...regularDebitDays, ...immediateDebitDays] + .flatMap((debitDate) => debitDate.transactions); + + const all: (ScrapedTransaction | ScrapedPendingTransaction)[] = [...pendingTransactions, ...completedTransactions]; - return [...regularDebitDays, ...immediateDebitDays] - .flatMap((debitDate) => debitDate.transactions) - .map((transaction) => { - const installments = (transaction.curPaymentNum && transaction.numOfPayments && + return all.map((transaction) => { + const numOfPayments = isPending(transaction) ? transaction.numberOfPayments : transaction.numOfPayments; + const installments = numOfPayments ? { - number: transaction.curPaymentNum, - total: transaction.numOfPayments, - }) || - undefined; - - const date = moment(transaction.trnPurchaseDate); - - const chargedAmount = transaction.amtBeforeConvAndIndex * (-1); - const originalAmount = transaction.trnAmt * (-1); - - const result: Transaction = { - identifier: transaction.trnIntId, - type: [TrnTypeCode.regular, TrnTypeCode.standingOrder].includes(transaction.trnTypeCode) ? - TransactionTypes.Normal : - TransactionTypes.Installments, - status: TransactionStatuses.Completed, - date: installments ? - date.add(installments.number - 1, 'month').toISOString() : - date.toISOString(), - processedDate: new Date(transaction.debCrdDate).toISOString(), - originalAmount, - originalCurrency: transaction.trnCurrencySymbol, - chargedAmount, - chargedCurrency: transaction.debCrdCurrencySymbol, - description: transaction.merchantName, - memo: transaction.transTypeCommentDetails.toString(), - category: transaction.branchCodeDesc, - }; - - if (installments) { - result.installments = installments; - } + number: isPending(transaction) ? 1 : transaction.curPaymentNum, + total: numOfPayments, + } : + undefined; + + const date = moment(transaction.trnPurchaseDate); + + let chargedAmount = isPending(transaction) ? transaction.trnAmt * (-1) : transaction.amtBeforeConvAndIndex * (-1); + let originalAmount = transaction.trnAmt * (-1); + + if (transaction.trnTypeCode === TrnTypeCode.credit) { + chargedAmount = isPending(transaction) ? transaction.trnAmt : transaction.amtBeforeConvAndIndex; + originalAmount = transaction.trnAmt; + } - return result; - }); + const result: Transaction = { + identifier: !isPending(transaction) ? transaction.trnIntId : undefined, + type: [TrnTypeCode.regular, TrnTypeCode.standingOrder].includes(transaction.trnTypeCode) ? + TransactionTypes.Normal : + TransactionTypes.Installments, + status: isPending(transaction) ? TransactionStatuses.Pending : TransactionStatuses.Completed, + date: installments ? + date.add(installments.number - 1, 'month').toISOString() : + date.toISOString(), + processedDate: isPending(transaction) ? date.toISOString() : new Date(transaction.debCrdDate).toISOString(), + originalAmount, + originalCurrency: transaction.trnCurrencySymbol, + chargedAmount, + chargedCurrency: !isPending(transaction) ? transaction.debCrdCurrencySymbol : undefined, + description: transaction.merchantName, + memo: transaction.transTypeCommentDetails.toString(), + category: transaction.branchCodeDesc, + }; + + if (installments) { + result.installments = installments; + } + + return result; + }); } type ScraperSpecificCredentials = { username: string, password: string }; @@ -323,11 +379,6 @@ class VisaCalScraper extends BaseScraperWithBrowser }; } - isCardTransactionDetails(result: CardTransactionDetails | CardTransactionDetailsError): - result is CardTransactionDetails { - return (result as CardTransactionDetails).result !== undefined; - } - async fetchData(): Promise { const defaultStartMoment = moment().subtract(1, 'years').subtract(6, 'months').add(1, 'day'); const startDate = this.options.startDate || defaultStartMoment.toDate(); @@ -341,12 +392,23 @@ class VisaCalScraper extends BaseScraperWithBrowser const accounts = await Promise.all( cards.map(async (card) => { - debug(`fetch transactions for card ${card.cardUniqueId}`); - const finalMonthToFetchMoment = moment().add(futureMonthsToScrape, 'month'); const months = finalMonthToFetchMoment.diff(startMoment, 'months'); const allMonthsData: (CardTransactionDetails)[] = []; + + debug(`fetch pending transactions for card ${card.cardUniqueId}`); + let pendingData = await fetchPostWithinPage( + this.page, PENDING_TRANSACTIONS_REQUEST_ENDPOINT, + { cardUniqueIDArray: [card.cardUniqueId] }, + { + Authorization, + 'X-Site-Id': xSiteId, + 'Content-Type': 'application/json', + }, + ); + + debug(`fetch completed transactions for card ${card.cardUniqueId}`); for (let i = 0; i <= months; i += 1) { const month = finalMonthToFetchMoment.clone().subtract(i, 'months'); const monthData = await fetchPostWithinPage( @@ -361,14 +423,22 @@ class VisaCalScraper extends BaseScraperWithBrowser if (monthData?.statusCode !== 1) throw new Error(`failed to fetch transactions for card ${card.last4Digits}. Message: ${monthData?.title || ''}`); - if (!this.isCardTransactionDetails(monthData)) { + if (!isCardTransactionDetails(monthData)) { throw new Error('monthData is not of type CardTransactionDetails'); } allMonthsData.push(monthData); } - const transactions = convertParsedDataToTransactions(allMonthsData); + if (pendingData?.statusCode !== 1 && pendingData?.statusCode !== 96) { + debug(`failed to fetch pending transactions for card ${card.last4Digits}. Message: ${pendingData?.title || ''}`); + pendingData = null; + } else if (!isCardPendingTransactionDetails(pendingData)) { + debug('pendingData is not of type CardTransactionDetails'); + pendingData = null; + } + + const transactions = convertParsedDataToTransactions(allMonthsData, pendingData); debug('filer out old transactions'); const txns = (this.options.outputData?.enableTransactionsFilterByDate ?? true) ?