Merge pull request #489 from bigcapitalhq/fix-plaid-syncing

fix: Plaid data available syncing

CHANGELOG.md

@@ -2,6 +2,41 @@
 
 All notable changes to Bigcapital server-side will be in this file.
 
+## [0.17.0] - 04-06-2024
+
+### New
+
+* feat: Upload and attach documents by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/461
+* feat: Export resource tables to pdf by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/460
+* feat: Build and deploy develop Docker container by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/476
+* feat: Internal docker virtual network by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/478
+
+### Fixes
+
+* fix: Skip send confirmation email if disabled by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/459
+* fix: Lemon Squeezy redirect to base url by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/479
+* fix: Organize Plaid env variables for development and sandbox envs by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/480
+* fix: Plaid syncs deposit imports as withdrawals by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/481
+* fix: Validate the s3 configures exist by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/482
+* fix: Run migrations only for initialized tenants by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/484
+
+## [0.16.16] - 
+
+* feat: handle http exceptions by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/456
+* feat: add the missing Newrelic env vars to docker-compose.prod file by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/457
+* fix: add the signup email confirmation env var by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/458
+
+## [0.16.14] - 
+
+* fix: Typo in setup wizard by @ccantrell72 in https://github.com/bigcapitalhq/bigcapital/pull/440
+* fix: Showing the real mail address on email confirmation view by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/445
+* fix: Auto-increment setting parsing by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/453
+
+## [0.16.12] - 
+
+* feat: Create a manifest list for `webapp` Docker image and push it to DockerHub. by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/436
+* feat: Combine arm64 and amd64 in one Github action runner by @abouolia in https://github.com/bigcapitalhq/bigcapital/pull/437
+
 ## [0.16.11] - 06-05-2024
 
 ### improvements

packages/server/src/api/controllers/Webhooks/Webhooks.ts

@@ -1,7 +1,6 @@
-import { Router } from 'express';
-import { PlaidApplication } from '@/services/Banking/Plaid/PlaidApplication';
-import { Request, Response } from 'express';
+import { NextFunction, Router, Request, Response } from 'express';
 import { Inject, Service } from 'typedi';
+import { PlaidApplication } from '@/services/Banking/Plaid/PlaidApplication';
 import BaseController from '../BaseController';
 import { LemonSqueezyWebhooks } from '@/services/Subscription/LemonSqueezyWebhooks';
 import { PlaidWebhookTenantBootMiddleware } from '@/services/Banking/Plaid/PlaidWebhookTenantBootMiddleware';
@@ -34,7 +33,7 @@ export class Webhooks extends BaseController {
    * @param {Response} res
    * @returns {Response}
    */
-  public async lemonWebhooks(req: Request, res: Response, next: any) {
+  public async lemonWebhooks(req: Request, res: Response, next: NextFunction) {
     const data = req.body;
     const signature = req.headers['x-signature'] ?? '';
     const rawBody = req.rawBody;
@@ -57,20 +56,25 @@ export class Webhooks extends BaseController {
    * @param {Response} res
    * @returns {Response}
    */
-  public async plaidWebhooks(req: Request, res: Response) {
+  public async plaidWebhooks(req: Request, res: Response, next: NextFunction) {
     const { tenantId } = req;
-    const {
-      webhook_type: webhookType,
-      webhook_code: webhookCode,
-      item_id: plaidItemId,
-    } = req.body;
 
-    await this.plaidApp.webhooks(
-      tenantId,
-      plaidItemId,
-      webhookType,
-      webhookCode
-    );
-    return res.status(200).send({ code: 200, message: 'ok' });
+    try {
+      const {
+        webhook_type: webhookType,
+        webhook_code: webhookCode,
+        item_id: plaidItemId,
+      } = req.body;
+
+      await this.plaidApp.webhooks(
+        tenantId,
+        plaidItemId,
+        webhookType,
+        webhookCode
+      );
+      return res.status(200).send({ code: 200, message: 'ok' });
+    } catch (error) {
+      next(error);
+    }
   }
 }

packages/server/src/database/migrations/20231108170207_create_storage_table.js

@@ -0,0 +1,14 @@
+exports.up = function (knex) {
+  return knex.schema.createTable('storage', (table) => {
+    table.increments('id').primary();
+    table.string('key').notNullable();
+    table.string('path').notNullable();
+    table.string('extension').notNullable();
+    table.integer('expire_in');
+    table.timestamps();
+  });
+};
+
+exports.down = function (knex) {
+  return knex.schema.dropTableIfExists('storage');
+};

packages/server/src/database/migrations/20240604153938_drop_storage_table.js

@@ -0,0 +1,5 @@
+exports.up = function (knex) {
+  return knex.schema.dropTableIfExists('storage');
+};
+
+exports.down = function (knex) {};

packages/server/src/interfaces/Account.ts

@@ -164,3 +164,7 @@ export enum TaxRateAction {
   DELETE = 'Delete',
   VIEW = 'View',
 }
+
+export interface CreateAccountParams {
+  ignoreUniqueName: boolean;
+}

packages/server/src/services/Accounts/CreateAccount.ts

@@ -7,6 +7,7 @@ import {
   IAccountEventCreatedPayload,
   IAccountEventCreatingPayload,
   IAccountCreateDTO,
+  CreateAccountParams,
 } from '@/interfaces';
 import events from '@/subscribers/events';
 import UnitOfWork from '@/services/UnitOfWork';
@@ -30,19 +31,22 @@ export class CreateAccount {
 
   /**
    * Authorize the account creation.
-   * @param  {number} tenantId
-   * @param  {IAccountCreateDTO} accountDTO
+   * @param {number} tenantId
+   * @param {IAccountCreateDTO} accountDTO
    */
   private authorize = async (
     tenantId: number,
     accountDTO: IAccountCreateDTO,
-    baseCurrency: string
+    baseCurrency: string,
+    params?: CreateAccountParams
   ) => {
     // Validate account name uniquiness.
-    await this.validator.validateAccountNameUniquiness(
-      tenantId,
-      accountDTO.name
-    );
+    if (!params.ignoreUniqueName) {
+      await this.validator.validateAccountNameUniquiness(
+        tenantId,
+        accountDTO.name
+      );
+    }
     // Validate the account code uniquiness.
     if (accountDTO.code) {
       await this.validator.isAccountCodeUniqueOrThrowError(
@@ -82,7 +86,7 @@ export class CreateAccount {
 
   /**
    * Transformes the create account DTO to input model.
-   * @param   {IAccountCreateDTO} createAccountDTO
+   * @param {IAccountCreateDTO} createAccountDTO
    */
   private transformDTOToModel = (
     createAccountDTO: IAccountCreateDTO,
@@ -104,7 +108,8 @@ export class CreateAccount {
   public createAccount = async (
     tenantId: number,
     accountDTO: IAccountCreateDTO,
-    trx?: Knex.Transaction
+    trx?: Knex.Transaction,
+    params: CreateAccountParams = { ignoreUniqueName: false }
   ): Promise<IAccount> => {
     const { Account } = this.tenancy.models(tenantId);
 
@@ -112,8 +117,12 @@ export class CreateAccount {
     const tenantMeta = await TenantMetadata.query().findOne({ tenantId });
 
     // Authorize the account creation.
-    await this.authorize(tenantId, accountDTO, tenantMeta.baseCurrency);
-
+    await this.authorize(
+      tenantId,
+      accountDTO,
+      tenantMeta.baseCurrency,
+      params
+    );
     // Transformes the DTO to model.
     const accountInputModel = this.transformDTOToModel(
       accountDTO,
@@ -148,3 +157,4 @@ export class CreateAccount {
     );
   };
 }
+

packages/server/src/services/Banking/Plaid/PlaidSyncDB.ts

@@ -3,7 +3,11 @@ import { Inject, Service } from 'typedi';
 import bluebird from 'bluebird';
 import { entries, groupBy } from 'lodash';
 import { CreateAccount } from '@/services/Accounts/CreateAccount';
-import { PlaidAccount, PlaidTransaction } from '@/interfaces';
+import {
+  IAccountCreateDTO,
+  PlaidAccount,
+  PlaidTransaction,
+} from '@/interfaces';
 import {
   transformPlaidAccountToCreateAccount,
   transformPlaidTrxsToCashflowCreate,
@@ -11,6 +15,7 @@ import {
 import { DeleteCashflowTransaction } from '@/services/Cashflow/DeleteCashflowTransactionService';
 import HasTenancyService from '@/services/Tenancy/TenancyService';
 import { CashflowApplication } from '@/services/Cashflow/CashflowApplication';
+import { Knex } from 'knex';
 
 const CONCURRENCY_ASYNC = 10;
 
@@ -28,6 +33,35 @@ export class PlaidSyncDb {
   @Inject()
   private deleteCashflowTransactionService: DeleteCashflowTransaction;
 
+  /**
+   * Syncs the Plaid bank account.
+   * @param {number} tenantId
+   * @param {IAccountCreateDTO} createBankAccountDTO
+   * @param {Knex.Transaction} trx
+   * @returns {Promise<void>}
+   */
+  public async syncBankAccount(
+    tenantId: number,
+    createBankAccountDTO: IAccountCreateDTO,
+    trx?: Knex.Transaction
+  ) {
+    const { Account } = this.tenancy.models(tenantId);
+    const plaidAccount = await Account.query().findOne(
+      'plaidAccountId',
+      createBankAccountDTO.plaidAccountId
+    );
+    // Can't continue if the Plaid account is already created.
+    if (plaidAccount) {
+      return;
+    }
+    await this.createAccountService.createAccount(
+      tenantId,
+      createBankAccountDTO,
+      trx,
+      { ignoreUniqueName: true }
+    );
+  }
+
   /**
    * Syncs the plaid accounts to the system accounts.
    * @param {number} tenantId Tenant ID.
@@ -37,7 +71,8 @@ export class PlaidSyncDb {
   public async syncBankAccounts(
     tenantId: number,
     plaidAccounts: PlaidAccount[],
-    institution: any
+    institution: any,
+    trx?: Knex.Transaction
   ): Promise<void> {
     const transformToPlaidAccounts =
       transformPlaidAccountToCreateAccount(institution);
@@ -47,7 +82,7 @@ export class PlaidSyncDb {
     await bluebird.map(
       accountCreateDTOs,
       (createAccountDTO: any) =>
-        this.createAccountService.createAccount(tenantId, createAccountDTO),
+        this.syncBankAccount(tenantId, createAccountDTO, trx),
       { concurrency: CONCURRENCY_ASYNC }
     );
   }
@@ -61,15 +96,16 @@ export class PlaidSyncDb {
   public async syncAccountTranactions(
     tenantId: number,
     plaidAccountId: number,
-    plaidTranasctions: PlaidTransaction[]
+    plaidTranasctions: PlaidTransaction[],
+    trx?: Knex.Transaction
   ): Promise<void> {
     const { Account } = this.tenancy.models(tenantId);
 
-    const cashflowAccount = await Account.query()
+    const cashflowAccount = await Account.query(trx)
       .findOne({ plaidAccountId })
       .throwIfNotFound();
 
-    const openingEquityBalance = await Account.query().findOne(
+    const openingEquityBalance = await Account.query(trx).findOne(
       'slug',
       'opening-balance-equity'
     );
@@ -87,7 +123,8 @@ export class PlaidSyncDb {
       (uncategoriedDTO) =>
         this.cashflowApp.createUncategorizedTransaction(
           tenantId,
-          uncategoriedDTO
+          uncategoriedDTO,
+          trx
         ),
       { concurrency: 1 }
     );
@@ -100,7 +137,8 @@ export class PlaidSyncDb {
    */
   public async syncAccountsTransactions(
     tenantId: number,
-    plaidAccountsTransactions: PlaidTransaction[]
+    plaidAccountsTransactions: PlaidTransaction[],
+    trx?: Knex.Transaction
   ): Promise<void> {
     const groupedTrnsxByAccountId = entries(
       groupBy(plaidAccountsTransactions, 'account_id')
@@ -111,7 +149,8 @@ export class PlaidSyncDb {
         return this.syncAccountTranactions(
           tenantId,
           plaidAccountId,
-          plaidTransactions
+          plaidTransactions,
+          trx
         );
       },
       { concurrency: CONCURRENCY_ASYNC }
@@ -124,11 +163,12 @@ export class PlaidSyncDb {
    */
   public async syncRemoveTransactions(
     tenantId: number,
-    plaidTransactionsIds: string[]
+    plaidTransactionsIds: string[],
+    trx?: Knex.Transaction
   ) {
     const { CashflowTransaction } = this.tenancy.models(tenantId);
 
-    const cashflowTransactions = await CashflowTransaction.query().whereIn(
+    const cashflowTransactions = await CashflowTransaction.query(trx).whereIn(
       'plaidTransactionId',
       plaidTransactionsIds
     );
@@ -140,7 +180,8 @@ export class PlaidSyncDb {
       (transactionId: number) =>
         this.deleteCashflowTransactionService.deleteCashflowTransaction(
           tenantId,
-          transactionId
+          transactionId,
+          trx
         ),
       { concurrency: CONCURRENCY_ASYNC }
     );
@@ -155,11 +196,12 @@ export class PlaidSyncDb {
   public async syncTransactionsCursor(
     tenantId: number,
     plaidItemId: string,
-    lastCursor: string
+    lastCursor: string,
+    trx?: Knex.Transaction
   ) {
     const { PlaidItem } = this.tenancy.models(tenantId);
 
-    await PlaidItem.query().findOne({ plaidItemId }).patch({ lastCursor });
+    await PlaidItem.query(trx).findOne({ plaidItemId }).patch({ lastCursor });
   }
 
   /**
@@ -169,13 +211,16 @@ export class PlaidSyncDb {
    */
   public async updateLastFeedsUpdatedAt(
     tenantId: number,
-    plaidAccountIds: string[]
+    plaidAccountIds: string[],
+    trx?: Knex.Transaction
   ) {
     const { Account } = this.tenancy.models(tenantId);
 
-    await Account.query().whereIn('plaid_account_id', plaidAccountIds).patch({
-      lastFeedsUpdatedAt: new Date(),
-    });
+    await Account.query(trx)
+      .whereIn('plaid_account_id', plaidAccountIds)
+      .patch({
+        lastFeedsUpdatedAt: new Date(),
+      });
   }
 
   /**
@@ -187,12 +232,15 @@ export class PlaidSyncDb {
   public async updateAccountsFeedsActive(
     tenantId: number,
     plaidAccountIds: string[],
-    isFeedsActive: boolean = true
+    isFeedsActive: boolean = true,
+    trx?: Knex.Transaction
   ) {
     const { Account } = this.tenancy.models(tenantId);
 
-    await Account.query().whereIn('plaid_account_id', plaidAccountIds).patch({
-      isFeedsActive,
-    });
+    await Account.query(trx)
+      .whereIn('plaid_account_id', plaidAccountIds)
+      .patch({
+        isFeedsActive,
+      });
   }
 }

packages/server/src/services/Banking/Plaid/PlaidUpdateTransactions.ts

@@ -3,6 +3,8 @@ import { Inject, Service } from 'typedi';
 import { PlaidClientWrapper } from '@/lib/Plaid/Plaid';
 import { PlaidSyncDb } from './PlaidSyncDB';
 import { PlaidFetchedTransactionsUpdates } from '@/interfaces';
+import UnitOfWork from '@/services/UnitOfWork';
+import { Knex } from 'knex';
 
 @Service()
 export class PlaidUpdateTransactions {
@@ -12,12 +14,40 @@ export class PlaidUpdateTransactions {
   @Inject()
   private plaidSync: PlaidSyncDb;
 
+  @Inject()
+  private uow: UnitOfWork;
+
   /**
-   * Handles the fetching and storing of new, modified, or removed transactions
-   * @param {number} tenantId Tenant ID.
-   * @param {string} plaidItemId the Plaid ID for the item.
+   * Handles sync the Plaid item to Bigcaptial under UOW.
+   * @param {number} tenantId
+   * @param {number} plaidItemId
+   * @returns {Promise<{  addedCount: number; modifiedCount: number; removedCount: number; }>}
    */
   public async updateTransactions(tenantId: number, plaidItemId: string) {
+    return this.uow.withTransaction(tenantId, (trx: Knex.Transaction) => {
+      return this.updateTransactionsWork(tenantId, plaidItemId, trx);
+    });
+  }
+
+  /**
+   * Handles the fetching and storing the following:
+   *  - New, modified, or removed transactions.
+   *  - New bank accounts.
+   *  - Last accounts feeds updated at.
+   *  - Turn on the accounts feed flag.
+   * @param {number} tenantId - Tenant ID.
+   * @param {string} plaidItemId - The Plaid ID for the item.
+   * @returns {Promise<{  addedCount: number; modifiedCount: number; removedCount: number; }>}
+   */
+  public async updateTransactionsWork(
+    tenantId: number,
+    plaidItemId: string,
+    trx?: Knex.Transaction
+  ): Promise<{
+    addedCount: number;
+    modifiedCount: number;
+    removedCount: number;
+  }> {
     // Fetch new transactions from plaid api.
     const { added, modified, removed, cursor, accessToken } =
       await this.fetchTransactionUpdates(tenantId, plaidItemId);
@@ -29,28 +59,42 @@ export class PlaidUpdateTransactions {
     } = await plaidInstance.accountsGet(request);
 
     const plaidAccountsIds = accounts.map((a) => a.account_id);
-
     const {
       data: { institution },
     } = await plaidInstance.institutionsGetById({
       institution_id: item.institution_id,
       country_codes: ['US', 'UK'],
     });
-    // Update the DB.
-    await this.plaidSync.syncBankAccounts(tenantId, accounts, institution);
+    // Sync bank accounts.
+    await this.plaidSync.syncBankAccounts(tenantId, accounts, institution, trx);
+    // Sync bank account transactions.
     await this.plaidSync.syncAccountsTransactions(
       tenantId,
-      added.concat(modified)
+      added.concat(modified),
+      trx
+    );
+    // Sync removed transactions.
+    await this.plaidSync.syncRemoveTransactions(tenantId, removed, trx);
+    // Sync transactions cursor.
+    await this.plaidSync.syncTransactionsCursor(
+      tenantId,
+      plaidItemId,
+      cursor,
+      trx
     );
-    await this.plaidSync.syncRemoveTransactions(tenantId, removed);
-    await this.plaidSync.syncTransactionsCursor(tenantId, plaidItemId, cursor);
-
     // Update the last feeds updated at of the updated accounts.
-    await this.plaidSync.updateLastFeedsUpdatedAt(tenantId, plaidAccountsIds);
-
+    await this.plaidSync.updateLastFeedsUpdatedAt(
+      tenantId,
+      plaidAccountsIds,
+      trx
+    );
     // Turn on the accounts feeds flag.
-    await this.plaidSync.updateAccountsFeedsActive(tenantId, plaidAccountsIds);
-
+    await this.plaidSync.updateAccountsFeedsActive(
+      tenantId,
+      plaidAccountsIds,
+      true,
+      trx
+    );
     return {
       addedCount: added.length,
       modifiedCount: modified.length,

packages/server/src/services/Cashflow/CashflowApplication.ts

@@ -1,3 +1,4 @@
+import { Knex } from 'knex';
 import { Inject, Service } from 'typedi';
 import { DeleteCashflowTransaction } from './DeleteCashflowTransactionService';
 import { UncategorizeCashflowTransaction } from './UncategorizeCashflowTransaction';
@@ -119,11 +120,13 @@ export class CashflowApplication {
    */
   public createUncategorizedTransaction(
     tenantId: number,
-    createUncategorizedTransactionDTO: CreateUncategorizedTransactionDTO
+    createUncategorizedTransactionDTO: CreateUncategorizedTransactionDTO,
+    trx?: Knex.Transaction
   ) {
     return this.createUncategorizedTransactionService.create(
       tenantId,
-      createUncategorizedTransactionDTO
+      createUncategorizedTransactionDTO,
+      trx
     );
   }
 

packages/server/src/services/Cashflow/DeleteCashflowTransactionService.ts

@@ -30,7 +30,8 @@ export class DeleteCashflowTransaction {
    */
   public deleteCashflowTransaction = async (
     tenantId: number,
-    cashflowTransactionId: number
+    cashflowTransactionId: number,
+    trx?: Knex.Transaction
   ): Promise<{ oldCashflowTransaction: ICashflowTransaction }> => {
     const { CashflowTransaction, CashflowTransactionLine } =
       this.tenancy.models(tenantId);
@@ -43,34 +44,44 @@ export class DeleteCashflowTransaction {
     this.throwErrorIfTransactionNotFound(oldCashflowTransaction);
 
     // Starting database transaction.
-    return this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => {
-      // Triggers `onCashflowTransactionDelete` event.
-      await this.eventPublisher.emitAsync(events.cashflow.onTransactionDeleting, {
-        trx,
-        tenantId,
-        oldCashflowTransaction,
-      } as ICommandCashflowDeletingPayload);
+    return this.uow.withTransaction(
+      tenantId,
+      async (trx: Knex.Transaction) => {
+        // Triggers `onCashflowTransactionDelete` event.
+        await this.eventPublisher.emitAsync(
+          events.cashflow.onTransactionDeleting,
+          {
+            trx,
+            tenantId,
+            oldCashflowTransaction,
+          } as ICommandCashflowDeletingPayload
+        );
 
-      // Delete cashflow transaction associated lines first.
-      await CashflowTransactionLine.query(trx)
-        .where('cashflow_transaction_id', cashflowTransactionId)
-        .delete();
+        // Delete cashflow transaction associated lines first.
+        await CashflowTransactionLine.query(trx)
+          .where('cashflow_transaction_id', cashflowTransactionId)
+          .delete();
 
-      // Delete cashflow transaction.
-      await CashflowTransaction.query(trx)
-        .findById(cashflowTransactionId)
-        .delete();
+        // Delete cashflow transaction.
+        await CashflowTransaction.query(trx)
+          .findById(cashflowTransactionId)
+          .delete();
 
-      // Triggers `onCashflowTransactionDeleted` event.
-      await this.eventPublisher.emitAsync(events.cashflow.onTransactionDeleted, {
-        trx,
-        tenantId,
-        cashflowTransactionId,
-        oldCashflowTransaction,
-      } as ICommandCashflowDeletedPayload);
+        // Triggers `onCashflowTransactionDeleted` event.
+        await this.eventPublisher.emitAsync(
+          events.cashflow.onTransactionDeleted,
+          {
+            trx,
+            tenantId,
+            cashflowTransactionId,
+            oldCashflowTransaction,
+          } as ICommandCashflowDeletedPayload
+        );
 
-      return { oldCashflowTransaction };
-    });
+        return { oldCashflowTransaction };
+      },
+      trx
+    );
   };
 
   /**

packages/webapp/src/containers/Setup/SetupSubscription/SubscriptionPlan.tsx

@@ -26,7 +26,7 @@ function SubscriptionPricing({
     useGetLemonSqueezyCheckout();
 
   const handleClick = () => {
-    getLemonCheckout({ variantId: '337977' })
+    getLemonCheckout({ variantId: '338516' })
       .then((res) => {
         const checkoutUrl = res.data.data.attributes.url;
         window.LemonSqueezy.Url.Open(checkoutUrl);