Merge pull request #589 from bigcapitalhq/bank-pending-transactions

feat: Pending bank transactions

packages/server/src/api/controllers/Banking/BankAccountsController.ts

@@ -1,9 +1,10 @@
 import { Inject, Service } from 'typedi';
 import { NextFunction, Request, Response, Router } from 'express';
+import { param, query } from 'express-validator';
 import BaseController from '@/api/controllers/BaseController';
 import { GetBankAccountSummary } from '@/services/Banking/BankAccounts/GetBankAccountSummary';
 import { BankAccountsApplication } from '@/services/Banking/BankAccounts/BankAccountsApplication';
-import { param } from 'express-validator';
+import { GetPendingBankAccountTransactions } from '@/services/Cashflow/GetPendingBankAccountTransaction';
 
 @Service()
 export class BankAccountsController extends BaseController {
@@ -13,6 +14,9 @@ export class BankAccountsController extends BaseController {
   @Inject()
   private bankAccountsApp: BankAccountsApplication;
 
+  @Inject()
+  private getPendingTransactionsService: GetPendingBankAccountTransactions;
+
   /**
    * Router constructor.
    */
@@ -20,6 +24,16 @@ export class BankAccountsController extends BaseController {
     const router = Router();
 
     router.get('/:bankAccountId/meta', this.getBankAccountSummary.bind(this));
+    router.get(
+      '/pending_transactions',
+      [
+        query('account_id').optional().isNumeric().toInt(),
+        query('page').optional().isNumeric().toInt(),
+        query('page_size').optional().isNumeric().toInt(),
+      ],
+      this.validationResult,
+      this.getBankAccountsPendingTransactions.bind(this)
+    );
     router.post(
       '/:bankAccountId/disconnect',
       this.disconnectBankAccount.bind(this)
@@ -27,17 +41,13 @@ export class BankAccountsController extends BaseController {
     router.post('/:bankAccountId/update', this.refreshBankAccount.bind(this));
     router.post(
       '/:bankAccountId/pause_feeds',
-      [
-        param('bankAccountId').exists().isNumeric().toInt(),
-      ],
+      [param('bankAccountId').exists().isNumeric().toInt()],
       this.validationResult,
       this.pauseBankAccountFeeds.bind(this)
     );
     router.post(
       '/:bankAccountId/resume_feeds',
-      [
-        param('bankAccountId').exists().isNumeric().toInt(),
-      ],
+      [param('bankAccountId').exists().isNumeric().toInt()],
       this.validationResult,
       this.resumeBankAccountFeeds.bind(this)
     );
@@ -72,6 +82,32 @@ export class BankAccountsController extends BaseController {
     }
   }
 
+  /**
+   * Retrieves the bank account pending transactions.
+   * @param {Request} req
+   * @param {Response} res
+   * @param {NextFunction} next
+   */
+  async getBankAccountsPendingTransactions(
+    req: Request,
+    res: Response,
+    next: NextFunction
+  ) {
+    const { tenantId } = req;
+    const query = this.matchedQueryData(req);
+
+    try {
+      const data =
+        await this.getPendingTransactionsService.getPendingTransactions(
+          tenantId,
+          query
+        );
+      return res.status(200).send(data);
+    } catch (error) {
+      next(error);
+    }
+  }
+
   /**
    * Disonnect the given bank account.
    * @param {Request} req
@@ -128,9 +164,9 @@ export class BankAccountsController extends BaseController {
 
   /**
    * Resumes the bank account feeds sync.
-   * @param {Request} req 
-   * @param {Response} res 
-   * @param {NextFunction} next 
+   * @param {Request} req
+   * @param {Response} res
+   * @param {NextFunction} next
    * @returns {Promise<Response | void>}
    */
   async resumeBankAccountFeeds(
@@ -155,9 +191,9 @@ export class BankAccountsController extends BaseController {
 
   /**
    * Pauses the bank account feeds sync.
-   * @param {Request} req 
-   * @param {Response} res 
-   * @param {NextFunction} next 
+   * @param {Request} req
+   * @param {Response} res
+   * @param {NextFunction} next
    * @returns {Promise<Response | void>}
    */
   async pauseBankAccountFeeds(

packages/server/src/database/migrations/20240811121028_add_pending_column_to_uncategorized_transactions_table.js

@@ -0,0 +1,13 @@
+exports.up = function (knex) {
+  return knex.schema.table('uncategorized_cashflow_transactions', (table) => {
+    table.boolean('pending').defaultTo(false);
+    table.string('pending_plaid_transaction_id').nullable();
+  });
+};
+
+exports.down = function (knex) {
+  return knex.schema.table('uncategorized_cashflow_transactions', (table) => {
+    table.dropColumn('pending');
+    table.dropColumn('pending_plaid_transaction_id');
+  });
+};

packages/server/src/interfaces/CashFlow.ts

@@ -268,6 +268,8 @@ export interface CreateUncategorizedTransactionDTO {
   description?: string;
   referenceNo?: string | null;
   plaidTransactionId?: string | null;
+  pending?: boolean;
+  pendingPlaidTransactionId?: string | null;
   batch?: string;
 }
 
@@ -283,3 +285,17 @@ export interface IUncategorizedTransactionCreatedEventPayload {
   createUncategorizedTransactionDTO: CreateUncategorizedTransactionDTO;
   trx: Knex.Transaction;
 }
+
+export interface IPendingTransactionRemovingEventPayload {
+  tenantId: number;
+  uncategorizedTransactionId: number;
+  pendingTransaction: IUncategorizedCashflowTransaction;
+  trx?: Knex.Transaction;
+}
+
+export interface IPendingTransactionRemovedEventPayload {
+  tenantId: number;
+  uncategorizedTransactionId: number;
+  pendingTransaction: IUncategorizedCashflowTransaction;
+  trx?: Knex.Transaction;
+}

packages/server/src/models/UncategorizedCashflowTransaction.ts

@@ -21,6 +21,7 @@ export default class UncategorizedCashflowTransaction extends mixin(
   plaidTransactionId!: string;
   recognizedTransactionId!: number;
   excludedAt: Date;
+  pending: boolean;
 
   /**
    * Table name.
@@ -46,7 +47,8 @@ export default class UncategorizedCashflowTransaction extends mixin(
       'isDepositTransaction',
       'isWithdrawalTransaction',
       'isRecognized',
-      'isExcluded'
+      'isExcluded',
+      'isPending',
     ];
   }
 
@@ -99,6 +101,14 @@ export default class UncategorizedCashflowTransaction extends mixin(
     return !!this.excludedAt;
   }
 
+  /**
+   * Detarmines whether the transaction is pending.
+   * @returns {boolean}
+   */
+  public get isPending(): boolean {
+    return !!this.pending;
+  }
+
   /**
    * Model modifiers.
    */
@@ -143,6 +153,20 @@ export default class UncategorizedCashflowTransaction extends mixin(
         query.whereNull('categorizeRefType');
         query.whereNull('categorizeRefId');
       },
+
+      /**
+       * Filters the not pending transactions.
+       */
+      notPending(query) {
+        query.where('pending', false);
+      },
+
+      /**
+       * Filters the pending transactions.
+       */
+      pending(query) {
+        query.where('pending', true);
+      },
     };
   }
 

packages/server/src/services/Banking/BankAccounts/GetBankAccountSummary.ts

@@ -52,6 +52,9 @@ export class GetBankAccountSummary {
         q.withGraphJoined('matchedBankTransactions');
         q.whereNull('matchedBankTransactions.id');
 
+        // Exclude the pending transactions.
+        q.modify('notPending');
+
         // Count the results.
         q.count('uncategorized_cashflow_transactions.id as total');
         q.first();
@@ -65,16 +68,32 @@ export class GetBankAccountSummary {
         q.withGraphJoined('recognizedTransaction');
         q.whereNotNull('recognizedTransaction.id');
 
+        // Exclude the pending transactions.
+        q.modify('notPending');
+
         // Count the results.
         q.count('uncategorized_cashflow_transactions.id as total');
         q.first();
       });
-
+    // Retrieves excluded transactions count.
     const excludedTransactionsCount =
       await UncategorizedCashflowTransaction.query().onBuild((q) => {
         q.where('accountId', bankAccountId);
         q.modify('excluded');
 
+        // Exclude the pending transactions.
+        q.modify('notPending');
+
+        // Count the results.
+        q.count('uncategorized_cashflow_transactions.id as total');
+        q.first();
+      });
+    // Retrieves the pending transactions count.
+    const pendingTransactionsCount =
+      await UncategorizedCashflowTransaction.query().onBuild((q) => {
+        q.where('accountId', bankAccountId);
+        q.modify('pending');
+
         // Count the results.
         q.count('uncategorized_cashflow_transactions.id as total');
         q.first();
@@ -83,14 +102,15 @@ export class GetBankAccountSummary {
     const totalUncategorizedTransactions =
       uncategorizedTranasctionsCount?.total || 0;
     const totalRecognizedTransactions = recognizedTransactionsCount?.total || 0;
-
     const totalExcludedTransactions = excludedTransactionsCount?.total || 0;
+    const totalPendingTransactions = pendingTransactionsCount?.total || 0;
 
     return {
       name: bankAccount.name,
       totalUncategorizedTransactions,
       totalRecognizedTransactions,
       totalExcludedTransactions,
+      totalPendingTransactions,
     };
   }
 }

packages/server/src/services/Banking/Plaid/PlaidSyncDB.ts

@@ -25,6 +25,7 @@ import { Knex } from 'knex';
 import uniqid from 'uniqid';
 import { EventPublisher } from '@/lib/EventPublisher/EventPublisher';
 import events from '@/subscribers/events';
+import { RemovePendingUncategorizedTransaction } from '@/services/Cashflow/RemovePendingUncategorizedTransaction';
 
 const CONCURRENCY_ASYNC = 10;
 
@@ -40,7 +41,7 @@ export class PlaidSyncDb {
   private cashflowApp: CashflowApplication;
 
   @Inject()
-  private deleteCashflowTransactionService: DeleteCashflowTransaction;
+  private removePendingTransaction: RemovePendingUncategorizedTransaction;
 
   @Inject()
   private eventPublisher: EventPublisher;
@@ -185,21 +186,22 @@ export class PlaidSyncDb {
     plaidTransactionsIds: string[],
     trx?: Knex.Transaction
   ) {
-    const { CashflowTransaction } = this.tenancy.models(tenantId);
+    const { UncategorizedCashflowTransaction } = this.tenancy.models(tenantId);
 
-    const cashflowTransactions = await CashflowTransaction.query(trx).whereIn(
-      'plaidTransactionId',
-      plaidTransactionsIds
-    );
-    const cashflowTransactionsIds = cashflowTransactions.map(
+    const uncategorizedTransactions =
+      await UncategorizedCashflowTransaction.query(trx).whereIn(
+        'plaidTransactionId',
+        plaidTransactionsIds
+      );
+    const uncategorizedTransactionsIds = uncategorizedTransactions.map(
       (trans) => trans.id
     );
     await bluebird.map(
-      cashflowTransactionsIds,
-      (transactionId: number) =>
-        this.deleteCashflowTransactionService.deleteCashflowTransaction(
+      uncategorizedTransactionsIds,
+      (uncategorizedTransactionId: number) =>
+        this.removePendingTransaction.removePendingTransaction(
           tenantId,
-          transactionId,
+          uncategorizedTransactionId,
           trx
         ),
       { concurrency: CONCURRENCY_ASYNC }

packages/server/src/services/Banking/Plaid/PlaidUpdateTransactions.ts

@@ -73,6 +73,12 @@ export class PlaidUpdateTransactions {
       item,
       trx
     );
+    // Sync removed transactions.
+    await this.plaidSync.syncRemoveTransactions(
+      tenantId,
+      removed?.map((r) => r.transaction_id),
+      trx
+    );
     // Sync bank account transactions.
     await this.plaidSync.syncAccountsTransactions(
       tenantId,

packages/server/src/services/Banking/Plaid/utils.ts

@@ -3,11 +3,11 @@ import {
   Item as PlaidItem,
   Institution as PlaidInstitution,
   AccountBase as PlaidAccount,
+  TransactionBase as PlaidTransactionBase,
 } from 'plaid';
 import {
   CreateUncategorizedTransactionDTO,
   IAccountCreateDTO,
-  PlaidTransaction,
 } from '@/interfaces';
 
 /**
@@ -48,7 +48,7 @@ export const transformPlaidAccountToCreateAccount = R.curry(
 export const transformPlaidTrxsToCashflowCreate = R.curry(
   (
     cashflowAccountId: number,
-    plaidTranasction: PlaidTransaction
+    plaidTranasction: PlaidTransactionBase
   ): CreateUncategorizedTransactionDTO => {
     return {
       date: plaidTranasction.date,
@@ -64,6 +64,8 @@ export const transformPlaidTrxsToCashflowCreate = R.curry(
       accountId: cashflowAccountId,
       referenceNo: plaidTranasction.payment_meta?.reference_number,
       plaidTransactionId: plaidTranasction.transaction_id,
+      pending: plaidTranasction.pending,
+      pendingPlaidTransactionId: plaidTranasction.pending_transaction_id,
     };
   }
 );

packages/server/src/services/Cashflow/GetPendingBankAccountTransaction.ts

@@ -0,0 +1,53 @@
+import { Inject } from 'typedi';
+import { TransformerInjectable } from '@/lib/Transformer/TransformerInjectable';
+import HasTenancyService from '../Tenancy/TenancyService';
+import { GetPendingBankAccountTransactionTransformer } from './GetPendingBankAccountTransactionTransformer';
+
+export class GetPendingBankAccountTransactions {
+  @Inject()
+  private tenancy: HasTenancyService;
+
+  @Inject()
+  private transformer: TransformerInjectable;
+
+  /**
+   * Retrieves the given bank accounts pending transaction.
+   * @param {number} tenantId - Tenant id.
+   * @param {GetPendingTransactionsQuery} filter - Pending transactions query.
+   */
+  async getPendingTransactions(
+    tenantId: number,
+    filter?: GetPendingTransactionsQuery
+  ) {
+    const { UncategorizedCashflowTransaction } = this.tenancy.models(tenantId);
+
+    const _filter = {
+      page: 1,
+      pageSize: 20,
+      ...filter,
+    };
+    const { results, pagination } =
+      await UncategorizedCashflowTransaction.query()
+        .onBuild((q) => {
+          q.modify('pending');
+
+          if (_filter?.accountId) {
+            q.where('accountId', _filter.accountId);
+          }
+        })
+        .pagination(_filter.page - 1, _filter.pageSize);
+
+    const data = await this.transformer.transform(
+      tenantId,
+      results,
+      new GetPendingBankAccountTransactionTransformer()
+    );
+    return { data, pagination };
+  }
+}
+
+interface GetPendingTransactionsQuery {
+  page?: number;
+  pageSize?: number;
+  accountId?: number;
+}

packages/server/src/services/Cashflow/GetPendingBankAccountTransactionTransformer.ts

@@ -0,0 +1,73 @@
+import { Transformer } from '@/lib/Transformer/Transformer';
+import { formatNumber } from '@/utils';
+
+export class GetPendingBankAccountTransactionTransformer extends Transformer {
+  /**
+   * Include these attributes to sale invoice object.
+   * @returns {string[]}
+   */
+  public includeAttributes = (): string[] => {
+    return [
+      'formattedAmount',
+      'formattedDate',
+      'formattedDepositAmount',
+      'formattedWithdrawalAmount',
+    ];
+  };
+
+  /**
+   * Exclude all attributes.
+   * @returns {Array<string>}
+   */
+  public excludeAttributes = (): string[] => {
+    return [];
+  };
+
+  /**
+   * Formattes the transaction date.
+   * @param transaction
+   * @returns {string}
+   */
+  public formattedDate(transaction) {
+    return this.formatDate(transaction.date);
+  }
+
+  /**
+   * Formatted amount.
+   * @param transaction
+   * @returns {string}
+   */
+  public formattedAmount(transaction) {
+    return formatNumber(transaction.amount, {
+      currencyCode: transaction.currencyCode,
+    });
+  }
+
+  /**
+   * Formatted deposit amount.
+   * @param transaction
+   * @returns {string}
+   */
+  protected formattedDepositAmount(transaction) {
+    if (transaction.isDepositTransaction) {
+      return formatNumber(transaction.deposit, {
+        currencyCode: transaction.currencyCode,
+      });
+    }
+    return '';
+  }
+
+  /**
+   * Formatted withdrawal amount.
+   * @param transaction
+   * @returns {string}
+   */
+  protected formattedWithdrawalAmount(transaction) {
+    if (transaction.isWithdrawalTransaction) {
+      return formatNumber(transaction.withdrawal, {
+        currencyCode: transaction.currencyCode,
+      });
+    }
+    return '';
+  }
+}

packages/server/src/services/Cashflow/GetRecongizedTransactions.ts

@@ -34,8 +34,13 @@ export class GetRecognizedTransactionsService {
           q.withGraphFetched('recognizedTransaction.assignAccount');
           q.withGraphFetched('recognizedTransaction.bankRule');
           q.whereNotNull('recognizedTransactionId');
+
+          // Exclude the excluded transactions.
           q.modify('notExcluded');
 
+          // Exclude the pending transactions.
+          q.modify('notPending');
+
           if (_filter.accountId) {
             q.where('accountId', _filter.accountId);
           }

packages/server/src/services/Cashflow/GetUncategorizedTransactions.ts

@@ -51,7 +51,9 @@ export class GetUncategorizedTransactions {
         .onBuild((q) => {
           q.where('accountId', accountId);
           q.where('categorized', false);
+
           q.modify('notExcluded');
+          q.modify('notPending');
 
           q.withGraphFetched('account');
           q.withGraphFetched('recognizedTransaction.assignAccount');

packages/server/src/services/Cashflow/RemovePendingUncategorizedTransaction.ts

@@ -0,0 +1,72 @@
+import { Knex } from 'knex';
+import { Inject, Service } from 'typedi';
+import HasTenancyService from '../Tenancy/TenancyService';
+import UnitOfWork from '../UnitOfWork';
+import { EventPublisher } from '@/lib/EventPublisher/EventPublisher';
+import events from '@/subscribers/events';
+import { ServiceError } from '@/exceptions';
+import { ERRORS } from './constants';
+import {
+  IPendingTransactionRemovedEventPayload,
+  IPendingTransactionRemovingEventPayload,
+} from '@/interfaces';
+
+@Service()
+export class RemovePendingUncategorizedTransaction {
+  @Inject()
+  private tenancy: HasTenancyService;
+
+  @Inject()
+  private uow: UnitOfWork;
+
+  @Inject()
+  private eventPublisher: EventPublisher;
+
+  /**
+   * REmoves the pending uncategorized transaction.
+   * @param {number} tenantId -
+   * @param {number} uncategorizedTransactionId -
+   * @param {Knex.Transaction} trx -
+   * @returns {Promise<void>}
+   */
+  public async removePendingTransaction(
+    tenantId: number,
+    uncategorizedTransactionId: number,
+    trx?: Knex.Transaction
+  ): Promise<void> {
+    const { UncategorizedCashflowTransaction } = this.tenancy.models(tenantId);
+
+    const pendingTransaction = await UncategorizedCashflowTransaction.query(trx)
+      .findById(uncategorizedTransactionId)
+      .throwIfNotFound();
+
+    if (!pendingTransaction.isPending) {
+      throw new ServiceError(ERRORS.TRANSACTION_NOT_PENDING);
+    }
+    return this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => {
+      await this.eventPublisher.emitAsync(
+        events.bankTransactions.onPendingRemoving,
+        {
+          tenantId,
+          uncategorizedTransactionId,
+          pendingTransaction,
+          trx,
+        } as IPendingTransactionRemovingEventPayload
+      );
+      // Removes the pending uncategorized transaction.
+      await UncategorizedCashflowTransaction.query(trx)
+        .findById(uncategorizedTransactionId)
+        .delete();
+
+      await this.eventPublisher.emitAsync(
+        events.bankTransactions.onPendingRemoved,
+        {
+          tenantId,
+          uncategorizedTransactionId,
+          pendingTransaction,
+          trx,
+        } as IPendingTransactionRemovedEventPayload
+      );
+    });
+  }
+}

packages/server/src/services/Cashflow/constants.ts

@@ -15,10 +15,10 @@ export const ERRORS = {
     'UNCATEGORIZED_TRANSACTION_TYPE_INVALID',
   CANNOT_DELETE_TRANSACTION_CONVERTED_FROM_UNCATEGORIZED:
     'CANNOT_DELETE_TRANSACTION_CONVERTED_FROM_UNCATEGORIZED',
-
-  CANNOT_CATEGORIZE_EXCLUDED_TRANSACTION: 'CANNOT_CATEGORIZE_EXCLUDED_TRANSACTION',
-  TRANSACTION_NOT_CATEGORIZED: 'TRANSACTION_NOT_CATEGORIZED'
-
+  CANNOT_CATEGORIZE_EXCLUDED_TRANSACTION:
+    'CANNOT_CATEGORIZE_EXCLUDED_TRANSACTION',
+  TRANSACTION_NOT_CATEGORIZED: 'TRANSACTION_NOT_CATEGORIZED',
+  TRANSACTION_NOT_PENDING: 'TRANSACTION_NOT_PENDING',
 };
 
 export enum CASHFLOW_DIRECTION {

packages/server/src/services/Cashflow/subscribers/DecrementUncategorizedTransactionOnCategorize.ts

@@ -1,11 +1,11 @@
 import { Inject, Service } from 'typedi';
+import PromisePool from '@supercharge/promise-pool';
 import events from '@/subscribers/events';
 import HasTenancyService from '@/services/Tenancy/TenancyService';
 import {
   ICashflowTransactionCategorizedPayload,
   ICashflowTransactionUncategorizedPayload,
 } from '@/interfaces';
-import PromisePool from '@supercharge/promise-pool';
 
 @Service()
 export class DecrementUncategorizedTransactionOnCategorize {
@@ -36,13 +36,17 @@ export class DecrementUncategorizedTransactionOnCategorize {
   public async decrementUnCategorizedTransactionsOnCategorized({
     tenantId,
     uncategorizedTransactions,
-    trx
+    trx,
   }: ICashflowTransactionCategorizedPayload) {
     const { Account } = this.tenancy.models(tenantId);
 
     await PromisePool.withConcurrency(1)
       .for(uncategorizedTransactions)
       .process(async (uncategorizedTransaction) => {
+        // Cannot continue if the transaction is still pending.
+        if (uncategorizedTransaction.isPending) {
+          return;
+        }
         await Account.query(trx)
           .findById(uncategorizedTransaction.accountId)
           .decrement('uncategorizedTransactions', 1);
@@ -56,13 +60,17 @@ export class DecrementUncategorizedTransactionOnCategorize {
   public async incrementUnCategorizedTransactionsOnUncategorized({
     tenantId,
     uncategorizedTransactions,
-    trx
+    trx,
   }: ICashflowTransactionUncategorizedPayload) {
     const { Account } = this.tenancy.models(tenantId);
 
     await PromisePool.withConcurrency(1)
       .for(uncategorizedTransactions)
       .process(async (uncategorizedTransaction) => {
+        // Cannot continue if the transaction is still pending.
+        if (uncategorizedTransaction.isPending) {
+          return;
+        }
         await Account.query(trx)
           .findById(uncategorizedTransaction.accountId)
           .increment('uncategorizedTransactions', 1);
@@ -82,6 +90,9 @@ export class DecrementUncategorizedTransactionOnCategorize {
 
     if (!uncategorizedTransaction.accountId) return;
 
+    // Cannot continue if the transaction is still pending.
+    if (uncategorizedTransaction.isPending) return;
+
     await Account.query(trx)
       .findById(uncategorizedTransaction.accountId)
       .increment('uncategorizedTransactions', 1);

packages/server/src/subscribers/events.ts

@@ -659,6 +659,9 @@ export default {
 
     onUnexcluding: 'onBankTransactionUnexcluding',
     onUnexcluded: 'onBankTransactionUnexcluded',
+
+    onPendingRemoving: 'onBankTransactionPendingRemoving',
+    onPendingRemoved: 'onBankTransactionPendingRemoved',
   },
 
   bankAccount: {