Merge pull request #400 from bigcapitalhq/clean-up-templ-import-files

feat: clean up the imported temp files

packages/server/.gitignore

@@ -3,3 +3,4 @@
 stdout.log
 /dist
 /build
+/public/imports

packages/server/src/api/controllers/Import/ImportController.ts

@@ -48,6 +48,7 @@ export class ImportController extends BaseController {
     router.get(
       '/sample',
       [query('resource').exists(), query('format').optional()],
+      this.validationResult,
       this.downloadImportSample.bind(this),
       this.catchServiceErrors
     );

packages/server/src/api/controllers/Import/_utils.ts

@@ -4,18 +4,29 @@ import { ServiceError } from '@/exceptions';
 export function allowSheetExtensions(req, file, cb) {
   if (
     file.mimetype !== 'text/csv' &&
-    file.mimetype !== 'application/vnd.ms-excel' && 
-    file.mimetype !== 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+    file.mimetype !== 'application/vnd.ms-excel' &&
+    file.mimetype !==
+      'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
   ) {
     cb(new ServiceError('IMPORTED_FILE_EXTENSION_INVALID'));
-
     return;
   }
   cb(null, true);
 }
 
+const storage = Multer.diskStorage({
+  destination: function (req, file, cb) {
+    cb(null, './public/imports');
+  },
+  filename: function (req, file, cb) {
+    // Add the creation timestamp to clean up temp files later.
+    const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1e9);
+    cb(null, uniqueSuffix);
+  },
+});
+
 export const uploadImportFile = Multer({
-  dest: './public/imports',
+  storage,
   limits: { fileSize: 5 * 1024 * 1024 },
   fileFilter: allowSheetExtensions,
 });

packages/server/src/loaders/jobs.ts

@@ -11,6 +11,7 @@ import { SendSaleEstimateMailJob } from '@/services/Sales/Estimates/SendSaleEsti
 import { SaleReceiptMailNotificationJob } from '@/services/Sales/Receipts/SaleReceiptMailNotificationJob';
 import { PaymentReceiveMailNotificationJob } from '@/services/Sales/PaymentReceives/PaymentReceiveMailNotificationJob';
 import { PlaidFetchTransactionsJob } from '@/services/Banking/Plaid/PlaidFetchTransactionsJob';
+import { ImportDeleteExpiredFilesJobs } from '@/services/Import/jobs/ImportDeleteExpiredFilesJob';
 
 export default ({ agenda }: { agenda: Agenda }) => {
   new ResetPasswordMailJob(agenda);
@@ -25,6 +26,9 @@ export default ({ agenda }: { agenda: Agenda }) => {
   new SaleReceiptMailNotificationJob(agenda);
   new PaymentReceiveMailNotificationJob(agenda);
   new PlaidFetchTransactionsJob(agenda);
+  new ImportDeleteExpiredFilesJobs(agenda);
 
-  agenda.start();
+  agenda.start().then(() => {
+    agenda.every('1 hours', 'delete-expired-imported-files', {});
+  });
 };

packages/server/src/loaders/tenantModels.ts

@@ -61,7 +61,6 @@ import Task from 'models/Task';
 import TaxRate from 'models/TaxRate';
 import TaxRateTransaction from 'models/TaxRateTransaction';
 import Attachment from 'models/Attachment';
-import Import from 'models/Import';
 import PlaidItem from 'models/PlaidItem';
 import UncategorizedCashflowTransaction from 'models/UncategorizedCashflowTransaction';
 
@@ -128,7 +127,6 @@ export default (knex) => {
     TaxRate,
     TaxRateTransaction,
     Attachment,
-    Import,
     PlaidItem,
     UncategorizedCashflowTransaction
   };

packages/server/src/services/Import/ImportFileCommon.ts

@@ -1,4 +1,3 @@
-import fs from 'fs/promises';
 import XLSX from 'xlsx';
 import bluebird from 'bluebird';
 import * as R from 'ramda';
@@ -17,7 +16,7 @@ import { getUniqueImportableValue, trimObject } from './_utils';
 import { ImportableResources } from './ImportableResources';
 import ResourceService from '../Resource/ResourceService';
 import HasTenancyService from '../Tenancy/TenancyService';
-import Import from '@/models/Import';
+import { Import } from '@/system/models';
 
 @Service()
 export class ImportFileCommon {
@@ -48,14 +47,6 @@ export class ImportFileCommon {
     return XLSX.utils.sheet_to_json(worksheet, {});
   }
 
-  /**
-   * Reads the import file.
-   * @param {string} filename
-   * @returns {Promise<Buffer>}
-   */
-  public readImportFile(filename: string) {
-    return fs.readFile(`public/imports/${filename}`);
-  }
 
   /**
    * Imports the given parsed data to the resource storage through registered importable service.
@@ -202,19 +193,4 @@ export class ImportFileCommon {
   public parseSheetColumns(json: unknown[]): string[] {
     return R.compose(Object.keys, trimObject, first)(json);
   }
-
-  /**
-   * Deletes the imported file from the storage and database.
-   * @param {number} tenantId
-   * @param {} importFile
-   */
-  public async deleteImportFile(tenantId: number, importFile: any) {
-    const { Import } = this.tenancy.models(tenantId);
-
-    // Deletes the import row.
-    await Import.query().findById(importFile.id).delete();
-
-    // Deletes the imported file.
-    await fs.unlink(`public/imports/${importFile.filename}`);
-  }
 }

packages/server/src/services/Import/ImportFileDataTransformer.ts

@@ -9,14 +9,11 @@ import {
   getFieldKey,
   aggregate,
   sanitizeSheetData,
+  getMapToPath,
 } from './_utils';
 import ResourceService from '../Resource/ResourceService';
 import HasTenancyService from '../Tenancy/TenancyService';
-
-const CurrencyParsingDTOs = 10;
-
-const getMapToPath = (to: string, group = '') =>
-  group ? `${group}.${to}` : to;
+import { CurrencyParsingDTOs } from './_constants';
 
 @Service()
 export class ImportFileDataTransformer {

packages/server/src/services/Import/ImportFileMapping.ts

@@ -1,6 +1,5 @@
 import { fromPairs, isUndefined } from 'lodash';
 import { Inject, Service } from 'typedi';
-import HasTenancyService from '../Tenancy/TenancyService';
 import {
   ImportDateFormats,
   ImportFileMapPOJO,
@@ -9,12 +8,10 @@ import {
 import ResourceService from '../Resource/ResourceService';
 import { ServiceError } from '@/exceptions';
 import { ERRORS } from './_utils';
+import { Import } from '@/system/models';
 
 @Service()
 export class ImportFileMapping {
-  @Inject()
-  private tenancy: HasTenancyService;
-
   @Inject()
   private resource: ResourceService;
 
@@ -29,8 +26,6 @@ export class ImportFileMapping {
     importId: number,
     maps: ImportMappingAttr[]
   ): Promise<ImportFileMapPOJO> {
-    const { Import } = this.tenancy.models(tenantId);
-
     const importFile = await Import.query()
       .findOne('filename', importId)
       .throwIfNotFound();

packages/server/src/services/Import/ImportFileMeta.ts

@@ -2,6 +2,7 @@ import { Inject, Service } from 'typedi';
 import HasTenancyService from '../Tenancy/TenancyService';
 import { TransformerInjectable } from '@/lib/Transformer/TransformerInjectable';
 import { ImportFileMetaTransformer } from './ImportFileMetaTransformer';
+import { Import } from '@/system/models';
 
 @Service()
 export class ImportFileMeta {
@@ -12,15 +13,15 @@ export class ImportFileMeta {
   private transformer: TransformerInjectable;
 
   /**
-   *
+   * Retrieves the import meta of the given import model id.
    * @param {number} tenantId
    * @param {number} importId
    * @returns {}
    */
   async getImportMeta(tenantId: number, importId: string) {
-    const { Import } = this.tenancy.models(tenantId);
-
-    const importFile = await Import.query().findOne('importId', importId);
+    const importFile = await Import.query()
+      .where('tenantId', tenantId)
+      .findOne('importId', importId);
 
     // Retrieves the transformed accounts collection.
     return this.transformer.transform(

packages/server/src/services/Import/ImportFileProcess.ts

@@ -2,19 +2,16 @@ import { Inject, Service } from 'typedi';
 import { chain } from 'lodash';
 import { Knex } from 'knex';
 import { ServiceError } from '@/exceptions';
-import { ERRORS, getSheetColumns, getUnmappedSheetColumns } from './_utils';
-import HasTenancyService from '../Tenancy/TenancyService';
+import { ERRORS, getSheetColumns, getUnmappedSheetColumns, readImportFile } from './_utils';
 import { ImportFileCommon } from './ImportFileCommon';
 import { ImportFileDataTransformer } from './ImportFileDataTransformer';
 import ResourceService from '../Resource/ResourceService';
 import UnitOfWork from '../UnitOfWork';
 import { ImportFilePreviewPOJO } from './interfaces';
+import { Import } from '@/system/models';
 
 @Service()
 export class ImportFileProcess {
-  @Inject()
-  private tenancy: HasTenancyService;
-
   @Inject()
   private resource: ResourceService;
 
@@ -38,10 +35,9 @@ export class ImportFileProcess {
     importId: number,
     trx?: Knex.Transaction
   ): Promise<ImportFilePreviewPOJO> {
-    const { Import } = this.tenancy.models(tenantId);
-
     const importFile = await Import.query()
       .findOne('importId', importId)
+      .where('tenantId', tenantId)
       .throwIfNotFound();
 
     // Throw error if the import file is not mapped yet.
@@ -49,7 +45,7 @@ export class ImportFileProcess {
       throw new ServiceError(ERRORS.IMPORT_FILE_NOT_MAPPED);
     }
     // Read the imported file.
-    const buffer = await this.importCommon.readImportFile(importFile.filename);
+    const buffer = await readImportFile(importFile.filename);
     const sheetData = this.importCommon.parseXlsxSheet(buffer);
     const header = getSheetColumns(sheetData);
 

packages/server/src/services/Import/ImportFileUpload.ts

@@ -1,7 +1,8 @@
 import { Inject, Service } from 'typedi';
-import HasTenancyService from '../Tenancy/TenancyService';
 import {
+  deleteImportFile,
   getResourceColumns,
+  readImportFile,
   sanitizeResourceName,
   validateSheetEmpty,
 } from './_utils';
@@ -9,12 +10,10 @@ import ResourceService from '../Resource/ResourceService';
 import { ImportFileCommon } from './ImportFileCommon';
 import { ImportFileDataValidator } from './ImportFileDataValidator';
 import { ImportFileUploadPOJO } from './interfaces';
+import { Import } from '@/system/models';
 
 @Service()
 export class ImportFileUploadService {
-  @Inject()
-  private tenancy: HasTenancyService;
-
   @Inject()
   private resourceService: ResourceService;
 
@@ -25,11 +24,12 @@ export class ImportFileUploadService {
   private importValidator: ImportFileDataValidator;
 
   /**
-   * Reads the imported file and stores the import file meta under unqiue id.
-   * @param {number} tenantId - Tenant id.
-   * @param {string} resource - Resource name.
-   * @param {string} filePath - File path.
-   * @param {string} fileName - File name.
+   * Imports the specified file for the given resource.
+   * Deletes the file if an error occurs during the import process.
+   * @param {number} tenantId
+   * @param {string} resourceName
+   * @param {string} filename
+   * @param {Record<string, number | string>} params
    * @returns {Promise<ImportFileUploadPOJO>}
    */
   public async import(
@@ -38,8 +38,35 @@ export class ImportFileUploadService {
     filename: string,
     params: Record<string, number | string>
   ): Promise<ImportFileUploadPOJO> {
-    const { Import } = this.tenancy.models(tenantId);
+    console.log(filename, 'filename');
 
+    try {
+      return await this.importUnhandled(
+        tenantId,
+        resourceName,
+        filename,
+        params
+      );
+    } catch (err) {
+      deleteImportFile(filename);
+      throw err;
+    }
+  }
+
+  /**
+   * Reads the imported file and stores the import file meta under unqiue id.
+   * @param {number} tenantId - Tenant id.
+   * @param {string} resource - Resource name.
+   * @param {string} filePath - File path.
+   * @param {string} fileName - File name.
+   * @returns {Promise<ImportFileUploadPOJO>}
+   */
+  public async importUnhandled(
+    tenantId: number,
+    resourceName: string,
+    filename: string,
+    params: Record<string, number | string>
+  ): Promise<ImportFileUploadPOJO> {
     const resource = sanitizeResourceName(resourceName);
     const resourceMeta = this.resourceService.getResourceMeta(
       tenantId,
@@ -49,7 +76,7 @@ export class ImportFileUploadService {
     this.importValidator.validateResourceImportable(resourceMeta);
 
     // Reads the imported file into buffer.
-    const buffer = await this.importFileCommon.readImportFile(filename);
+    const buffer = await readImportFile(filename);
 
     // Parse the buffer file to array data.
     const sheetData = this.importFileCommon.parseXlsxSheet(buffer);
@@ -76,6 +103,7 @@ export class ImportFileUploadService {
     const importFile = await Import.query().insert({
       filename,
       resource,
+      tenantId,
       importId: filename,
       columns: coumnsStringified,
       params: paramsStringified,

packages/server/src/services/Import/ImportRemoveExpiredFiles.ts

@@ -0,0 +1,34 @@
+import moment from 'moment';
+import bluebird from 'bluebird';
+import { Import } from '@/system/models';
+import { deleteImportFile } from './_utils';
+import { Service } from 'typedi';
+
+@Service()
+export class ImportDeleteExpiredFiles {
+  /**
+   * Delete expired files.
+   */
+  async deleteExpiredFiles() {
+    const yesterday = moment().subtract(1, 'hour').format('YYYY-MM-DD HH:mm');
+
+    const expiredImports = await Import.query().where(
+      'createdAt',
+      '<',
+      yesterday
+    );
+    await bluebird.map(
+      expiredImports,
+      async (expiredImport) => {
+        await deleteImportFile(expiredImport.filename);
+      },
+      { concurrency: 10 }
+    );
+    const expiredImportsIds = expiredImports.map(
+      (expiredImport) => expiredImport.id
+    );
+    if (expiredImportsIds.length > 0) {
+      await Import.query().whereIn('id', expiredImportsIds).delete();
+    }
+  }
+}

packages/server/src/services/Import/ImportSample.ts

@@ -10,9 +10,9 @@ export class ImportSampleService {
 
   /**
    * Retrieves the sample sheet of the given resource.
-   * @param {number} tenantId 
-   * @param {string} resource 
-   * @param {string} format 
+   * @param {number} tenantId
+   * @param {string} resource
+   * @param {string} format
    * @returns {Buffer | string}
    */
   public sample(

packages/server/src/services/Import/Importable.ts

@@ -69,4 +69,4 @@ export abstract class Importable {
   public transformParams(parmas: Record<string, any>) {
     return parmas;
   }
-}
+}

packages/server/src/services/Import/_constants.ts

@@ -0,0 +1,3 @@
+
+
+export const CurrencyParsingDTOs = 10;

packages/server/src/services/Import/_utils.ts

@@ -2,6 +2,7 @@ import * as Yup from 'yup';
 import moment from 'moment';
 import * as R from 'ramda';
 import { Knex } from 'knex';
+import fs from 'fs/promises';
 import {
   defaultTo,
   upperFirst,
@@ -141,9 +142,9 @@ const parseFieldName = (fieldName: string, field: IModelMetaField) => {
 
 /**
  * Retrieves the unmapped sheet columns.
- * @param columns 
- * @param mapping 
- * @returns 
+ * @param columns
+ * @param mapping
+ * @returns
  */
 export const getUnmappedSheetColumns = (columns, mapping) => {
   return columns.filter(
@@ -421,3 +422,30 @@ export function aggregate(
 export const sanitizeSheetData = (json) => {
   return R.compose(R.map(trimObject))(json);
 };
+
+/**
+ * Returns the path to map a value to based on the 'to' and 'group' parameters.
+ * @param {string} to - The target key to map the value to.
+ * @param {string} group - The group key to nest the target key under.
+ * @returns {string} - The path to map the value to.
+ */
+export const getMapToPath = (to: string, group = '') =>
+  group ? `${group}.${to}` : to;
+
+/**
+ * Deletes the imported file from the storage and database.
+ * @param {string} filename
+ */
+export const deleteImportFile = async (filename: string) => {
+  // Deletes the imported file.
+  await fs.unlink(`public/imports/${filename}`);
+};
+
+/**
+ * Reads the import file.
+ * @param {string} filename
+ * @returns {Promise<Buffer>}
+ */
+export const readImportFile = (filename: string) => {
+  return fs.readFile(`public/imports/${filename}`);
+};

packages/server/src/services/Import/jobs/ImportDeleteExpiredFilesJob.ts

@@ -1 +1,28 @@
-export class ImportDeleteExpiredFilesJobs {}
+import Container, { Service } from 'typedi';
+import { ImportDeleteExpiredFiles } from '../ImportRemoveExpiredFiles';
+
+@Service()
+export class ImportDeleteExpiredFilesJobs {
+  /**
+   * Constructor method.
+   */
+  constructor(agenda) {
+    agenda.define('delete-expired-imported-files', this.handler);
+  }
+
+  /**
+   * Triggers sending invoice mail.
+   */
+  private handler = async (job, done: Function) => {
+    const importDeleteExpiredFiles = Container.get(ImportDeleteExpiredFiles);
+
+    try {
+      console.log('Delete expired import files has started.');
+      await importDeleteExpiredFiles.deleteExpiredFiles();
+      done();
+    } catch (error) {
+      console.log(error);
+      done(error);
+    }
+  };
+}

packages/server/src/system/migrations/20231209230719_create_imports_table.js

@@ -7,6 +7,12 @@ exports.up = function (knex) {
     table.json('columns');
     table.json('mapping');
     table.json('params');
+    table
+      .bigInteger('tenant_id')
+      .unsigned()
+      .index()
+      .references('id')
+      .inTable('tenants');
     table.timestamps();
   });
 };

packages/server/src/system/models/Import.ts

@@ -1,11 +1,13 @@
-import TenantModel from 'models/TenantModel';
+import { Model, ModelObject } from 'objection';
+import SystemModel from './SystemModel';
 
-export default class Import extends TenantModel {
-  resource!: string;
+export class Import extends SystemModel {
+  resource: string;
+  tenantId: number;
   mapping!: string;
   columns!: string;
-  params!: Record<string, any>;
-  
+  params!: string;
+
   /**
    * Table name.
    */
@@ -24,14 +26,7 @@ export default class Import extends TenantModel {
    * Timestamps columns.
    */
   get timestamps() {
-    return [];
-  }
-
-  /**
-   * Relationship mapping.
-   */
-  static get relationMappings() {
-    return {};
+    return ['createdAt', 'updatedAt'];
   }
 
   /**
@@ -50,7 +45,6 @@ export default class Import extends TenantModel {
     }
   }
 
-
   public get paramsParsed() {
     try {
       return JSON.parse(this.params);
@@ -66,4 +60,27 @@ export default class Import extends TenantModel {
       return [];
     }
   }
+
+  /**
+   * Relationship mapping.
+   */
+  static get relationMappings() {
+    const Tenant = require('system/models/Tenant');
+
+    return {
+      /**
+       * System user may belongs to tenant model.
+       */
+      tenant: {
+        relation: Model.BelongsToOneRelation,
+        modelClass: Tenant.default,
+        join: {
+          from: 'imports.tenantId',
+          to: 'tenants.id',
+        },
+      },
+    };
+  }
 }
+
+export type ImportShape = ModelObject<Import>;

packages/server/src/system/models/Tenant.ts

@@ -5,6 +5,11 @@ import BaseModel from 'models/Model';
 import TenantMetadata from './TenantMetadata';
 
 export default class Tenant extends BaseModel {
+  upgradeJobId: string;
+  buildJobId: string;
+  initializedAt!: Date | null;
+  seededAt!: Date | null;
+
   /**
    * Table name.
    */
@@ -14,6 +19,7 @@ export default class Tenant extends BaseModel {
 
   /**
    * Timestamps columns.
+   * @returns {string[]}
    */
   get timestamps() {
     return ['createdAt', 'updatedAt'];
@@ -21,6 +27,7 @@ export default class Tenant extends BaseModel {
 
   /**
    * Virtual attributes.
+   * @returns {string[]}
    */
   static get virtualAttributes() {
     return ['isReady', 'isBuildRunning', 'isUpgradeRunning'];
@@ -28,6 +35,7 @@ export default class Tenant extends BaseModel {
 
   /**
    * Tenant is ready.
+   * @returns {boolean}
    */
   get isReady() {
     return !!(this.initializedAt && this.seededAt);
@@ -35,6 +43,7 @@ export default class Tenant extends BaseModel {
 
   /**
    * Detarimes the tenant whether is build currently running.
+   * @returns {boolean}
    */
   get isBuildRunning() {
     return !!this.buildJobId;
@@ -42,6 +51,7 @@ export default class Tenant extends BaseModel {
 
   /**
    * Detarmines the tenant whether is upgrade currently running.
+   * @returns {boolean}
    */
   get isUpgradeRunning() {
     return !!this.upgradeJobId;
@@ -64,6 +74,7 @@ export default class Tenant extends BaseModel {
       },
     };
   }
+
   /**
    * Creates a new tenant with random organization id.
    */

packages/server/src/system/models/index.ts

@@ -4,6 +4,7 @@ import SystemUser from './SystemUser';
 import PasswordReset from './PasswordReset';
 import Invite from './Invite';
 import SystemPlaidItem from './SystemPlaidItem';
+import { Import } from './Import';
 
 export {
   Tenant,
@@ -12,4 +13,5 @@ export {
   PasswordReset,
   Invite,
   SystemPlaidItem,
+  Import,
 };