QT/bigcapital
1
0
Fork 0
mirror of https://github.com/bigcapitalhq/bigcapital.git synced 2026-02-16 04:40:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(server): move updating plaid transactions to background job

Browse Source
This commit is contained in:
Ahmed Bouhuolia
2024-02-03 13:59:46 +02:00
parent b940c6dd17
commit e0ddcb022a
16 changed files with 150 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/server/src/lib/Plaid/Plaid.ts
View File

@@ -23,7 +23,7 @@ const defaultLogger = async (clientMethod, clientMethodArgs, response) => {
// );
// await createPlaidApiEvent(1, 1, clientMethod, clientMethodArgs, response);
console.log(response);
// console.log(response);
};
/**
@@ -39,7 +39,7 @@ const noAccessTokenLogger = async (
clientMethodArgs,
response
) => {
console.log(response);
// console.log(response);
// await createPlaidApiEvent(
// undefined,

29
packages/server/src/lib/Plaid/PlaidFetchTransactionsJob.ts
View File

@@ -1,29 +0,0 @@
import Container, { Service } from 'typedi';
@Service()
export class PlaidFetchTransactionsJob {
/**
* Constructor method.
*/
constructor(agenda) {
agenda.define(
'plaid-update-account-transactions',
{ priority: 'high', concurrency: 2 },
this.handler
);
}
/**
* Triggers the function.
*/
private handler = async (job, done: Function) => {
const {} = job.attrs.data;
try {
done();
} catch (error) {
console.log(error);
done(error);
}
};
}

135
packages/server/src/lib/Plaid/PlaidSyncDB.ts
View File

@@ -1,135 +0,0 @@
import * as R from 'ramda';
import { Inject, Service } from 'typedi';
import async from 'async';
import { forOwn, groupBy } from 'lodash';
import { CreateAccount } from '@/services/Accounts/CreateAccount';
import {
PlaidAccount,
PlaidTransaction,
SyncAccountsTransactionsTask,
} from './_types';
import {
transformPlaidAccountToCreateAccount,
transformPlaidTrxsToCashflowCreate,
} from './utils';
import NewCashflowTransactionService from '@/services/Cashflow/NewCashflowTransactionService';
import HasTenancyService from '@/services/Tenancy/TenancyService';
import { ICashflowNewCommandDTO } from '@/interfaces';
@Service()
export class PlaidSyncDb {
@Inject()
private tenancy: HasTenancyService;
@Inject()
private createAccountService: CreateAccount;
@Inject()
private createCashflowTransactionService: NewCashflowTransactionService;
/**
* Syncs the plaid accounts to the system accounts.
* @param {number} tenantId Tenant ID.
* @param {PlaidAccount[]} plaidAccounts
*/
public syncBankAccounts(tenantId: number, plaidAccounts: PlaidAccount[]) {
const accountCreateDTOs = R.map(transformPlaidAccountToCreateAccount)(
plaidAccounts
);
accountCreateDTOs.map((createDTO) => {
return this.createAccountService.createAccount(tenantId, createDTO);
});
}
/**
* Synsc the Plaid transactions to the system GL entries.
* @param {number} tenantId - Tenant ID.
* @param {number} plaidAccountId - Plaid account ID.
* @param {PlaidTransaction[]} plaidTranasctions - Plaid transactions
*/
public async syncAccountTranactions(
tenantId: number,
plaidAccountId: number,
plaidTranasctions: PlaidTransaction[]
): Promise<void> {
const { Account } = await this.tenancy.models(tenantId);
const cashflowAccount = await Account.query()
.findOne('plaidAccountId', plaidAccountId)
.throwIfNotFound();
const openingEquityBalance = await Account.query().findOne(
'slug',
'opening-balance-equity'
);
// Transformes the Plaid transactions to cashflow create DTOs.
const transformTransaction = transformPlaidTrxsToCashflowCreate(
cashflowAccount.id,
openingEquityBalance.id
);
const accountsCashflowDTO = R.map(transformTransaction)(plaidTranasctions);
// Creating account transaction queue.
const createAccountTransactionsQueue = async.queue(
(cashflowDTO: ICashflowNewCommandDTO) =>
this.createCashflowTransactionService.newCashflowTransaction(
tenantId,
cashflowDTO
),
10
);
accountsCashflowDTO.forEach((cashflowDTO) => {
createAccountTransactionsQueue.push(cashflowDTO);
});
await createAccountTransactionsQueue.drain();
}
/**
* Syncs the accounts transactions in paraller under controlled concurrency.
* @param {number} tenantId
* @param {PlaidTransaction[]} plaidTransactions
*/
public async syncAccountsTransactions(
tenantId: number,
plaidAccountsTransactions: PlaidTransaction[]
): Promise<void> {
const groupedTrnsxByAccountId = groupBy(
plaidAccountsTransactions,
'account_id'
);
const syncAccountsTrnsx = async.queue(
({
tenantId,
plaidAccountId,
plaidTransactions,
}: SyncAccountsTransactionsTask) => {
return this.syncAccountTranactions(
tenantId,
plaidAccountId,
plaidTransactions
);
},
2
);
forOwn(groupedTrnsxByAccountId, (plaidTransactions, plaidAccountId) => {
syncAccountsTrnsx.push({ tenantId, plaidAccountId, plaidTransactions });
});
await syncAccountsTrnsx.drain();
}
/**
* Syncs the Plaid item last transaction cursor.
* @param {number} tenantId -
* @param {string} itemId -
* @param {string} lastCursor -
*/
public async syncTransactionsCursor(
tenantId: number,
plaidItemId: string,
lastCursor: string
) {
const { PlaidItem } = this.tenancy.models(tenantId);
await PlaidItem.query().findById(plaidItemId).patch({ lastCursor });
}
}

104
packages/server/src/lib/Plaid/PlaidUpdateTransactions.ts
View File

@@ -1,104 +0,0 @@
import HasTenancyService from '@/services/Tenancy/TenancyService';
import { Inject, Service } from 'typedi';
import { PlaidClientWrapper } from './Plaid';
import { PlaidSyncDb } from './PlaidSyncDB';
import { PlaidFetchedTransactionsUpdates } from './_types';
@Service()
export class PlaidUpdateTransactions {
@Inject()
private tenancy: HasTenancyService;
@Inject()
private plaidSync: PlaidSyncDb;
/**
* Handles the fetching and storing of new, modified, or removed transactions
* @param {number} tenantId Tenant ID.
* @param {string} plaidItemId the Plaid ID for the item.
*/
public async updateTransactions(tenantId: number, plaidItemId: string) {
// Fetch new transactions from plaid api.
const { added, modified, removed, cursor, accessToken } =
await this.fetchTransactionUpdates(tenantId, plaidItemId);
const request = { access_token: accessToken };
const plaidInstance = new PlaidClientWrapper();
const {
data: { accounts },
} = await plaidInstance.accountsGet(request);
// Update the DB.
await this.plaidSync.syncBankAccounts(tenantId, accounts);
await this.plaidSync.syncAccountsTransactions(
tenantId,
added.concat(modified)
);
await this.plaidSync.syncTransactionsCursor(tenantId, plaidItemId, cursor);
return {
addedCount: added.length,
modifiedCount: modified.length,
removedCount: removed.length,
};
}
/**
* Fetches transactions from the `Plaid API` for a given item.
* @param {number} tenantId - Tenant ID.
* @param {string} plaidItemId - The Plaid ID for the item.
* @returns {Promise<PlaidFetchedTransactionsUpdates>}
*/
public async fetchTransactionUpdates(
tenantId: number,
plaidItemId: string
): Promise<PlaidFetchedTransactionsUpdates> {
// the transactions endpoint is paginated, so we may need to hit it multiple times to
// retrieve all available transactions.
const { PlaidItem } = this.tenancy.models(tenantId);
const plaidItem = await PlaidItem.query().findOne(
'plaidItemId',
plaidItemId
);
if (!plaidItem) {
throw new Error('The given Plaid item id is not found.');
}
const { plaidAccessToken, lastCursor } = plaidItem;
let cursor = lastCursor;
// New transaction updates since "cursor"
let added = [];
let modified = [];
// Removed transaction ids
let removed = [];
let hasMore = true;
const batchSize = 100;
try {
// Iterate through each page of new transaction updates for item
/* eslint-disable no-await-in-loop */
while (hasMore) {
const request = {
access_token: plaidAccessToken,
cursor: cursor,
count: batchSize,
};
const plaidInstance = new PlaidClientWrapper();
const response = await plaidInstance.transactionsSync(request);
const data = response.data;
// Add this page of results
added = added.concat(data.added);
modified = modified.concat(data.modified);
removed = removed.concat(data.removed);
hasMore = data.has_more;
// Update cursor to the next cursor
cursor = data.next_cursor;
}
} catch (err) {
console.error(`Error fetching transactions: ${err.message}`);
cursor = lastCursor;
}
return { added, modified, removed, cursor, accessToken: plaidAccessToken };
}
}

41
packages/server/src/lib/Plaid/_types.ts
View File

@@ -1,41 +0,0 @@
export interface PlaidAccount {
account_id: string;
balances: {
available: number;
current: number;
iso_currency_code: string;
limit: null;
unofficial_currency_code: null;
};
mask: string;
name: string;
official_name: string;
persistent_account_id: string;
subtype: string;
type: string;
}
export interface PlaidTransaction {
account_id: string;
amount: number;
authorized_data: string;
category: string[];
check_number: number | null;
iso_currency_code: string;
transaction_id: string;
transaction_type: string;
}
export interface PlaidFetchedTransactionsUpdates {
added: any[];
modified: any[];
removed: any[];
accessToken: string;
cursor: string;
}
export interface SyncAccountsTransactionsTask {
tenantId: number;
plaidAccountId: number;
plaidTransactions: PlaidTransaction[];
}

2
packages/server/src/lib/Plaid/index.ts
View File

@@ -1 +1 @@
export * from './Plaid';
export * from './Plaid';

41
packages/server/src/lib/Plaid/utils.ts
View File

@@ -1,41 +0,0 @@
import * as R from 'ramda';
import { IAccountCreateDTO, ICashflowNewCommandDTO } from '@/interfaces';
import { PlaidAccount, PlaidTransaction } from './_types';
export const transformPlaidAccountToCreateAccount = (
plaidAccount: PlaidAccount
): IAccountCreateDTO => {
return {
name: plaidAccount.name,
code: '',
description: plaidAccount.official_name,
currencyCode: plaidAccount.balances.iso_currency_code,
accountType: 'cash',
active: true,
plaidAccountId: plaidAccount.account_id,
};
};
export const transformPlaidTrxsToCashflowCreate = R.curry(
(
cashflowAccountId: number,
creditAccountId: number,
plaidTranasction: PlaidTransaction,
): ICashflowNewCommandDTO => {
return {
date: plaidTranasction.authorized_data,
transactionType: '',
description: '',
amount: plaidTranasction.amount,
exchangeRate: 1,
currencyCode: plaidTranasction.iso_currency_code,
creditAccountId,
cashflowAccountId,
// transactionNumber: string;
// referenceNo: string;
};
}
);