feat: clean up the imported temp files

packages/server/src/services/Import/ImportFileCommon.ts

@@ -1,4 +1,3 @@
-import fs from 'fs/promises';
 import XLSX from 'xlsx';
 import bluebird from 'bluebird';
 import * as R from 'ramda';
@@ -17,7 +16,7 @@ import { getUniqueImportableValue, trimObject } from './_utils';
 import { ImportableResources } from './ImportableResources';
 import ResourceService from '../Resource/ResourceService';
 import HasTenancyService from '../Tenancy/TenancyService';
-import Import from '@/models/Import';
+import { Import } from '@/system/models';
 
 @Service()
 export class ImportFileCommon {
@@ -48,14 +47,6 @@ export class ImportFileCommon {
     return XLSX.utils.sheet_to_json(worksheet, {});
   }
 
-  /**
-   * Reads the import file.
-   * @param {string} filename
-   * @returns {Promise<Buffer>}
-   */
-  public readImportFile(filename: string) {
-    return fs.readFile(`public/imports/${filename}`);
-  }
 
   /**
    * Imports the given parsed data to the resource storage through registered importable service.
@@ -202,19 +193,4 @@ export class ImportFileCommon {
   public parseSheetColumns(json: unknown[]): string[] {
     return R.compose(Object.keys, trimObject, first)(json);
   }
-
-  /**
-   * Deletes the imported file from the storage and database.
-   * @param {number} tenantId
-   * @param {} importFile
-   */
-  public async deleteImportFile(tenantId: number, importFile: any) {
-    const { Import } = this.tenancy.models(tenantId);
-
-    // Deletes the import row.
-    await Import.query().findById(importFile.id).delete();
-
-    // Deletes the imported file.
-    await fs.unlink(`public/imports/${importFile.filename}`);
-  }
 }

packages/server/src/services/Import/ImportFileDataTransformer.ts

@@ -9,14 +9,11 @@ import {
   getFieldKey,
   aggregate,
   sanitizeSheetData,
+  getMapToPath,
 } from './_utils';
 import ResourceService from '../Resource/ResourceService';
 import HasTenancyService from '../Tenancy/TenancyService';
-
-const CurrencyParsingDTOs = 10;
-
-const getMapToPath = (to: string, group = '') =>
-  group ? `${group}.${to}` : to;
+import { CurrencyParsingDTOs } from './_constants';
 
 @Service()
 export class ImportFileDataTransformer {

packages/server/src/services/Import/ImportFileMapping.ts

@@ -1,6 +1,5 @@
 import { fromPairs, isUndefined } from 'lodash';
 import { Inject, Service } from 'typedi';
-import HasTenancyService from '../Tenancy/TenancyService';
 import {
   ImportDateFormats,
   ImportFileMapPOJO,
@@ -9,12 +8,10 @@ import {
 import ResourceService from '../Resource/ResourceService';
 import { ServiceError } from '@/exceptions';
 import { ERRORS } from './_utils';
+import { Import } from '@/system/models';
 
 @Service()
 export class ImportFileMapping {
-  @Inject()
-  private tenancy: HasTenancyService;
-
   @Inject()
   private resource: ResourceService;
 
@@ -29,8 +26,6 @@ export class ImportFileMapping {
     importId: number,
     maps: ImportMappingAttr[]
   ): Promise<ImportFileMapPOJO> {
-    const { Import } = this.tenancy.models(tenantId);
-
     const importFile = await Import.query()
       .findOne('filename', importId)
       .throwIfNotFound();

packages/server/src/services/Import/ImportFileMeta.ts

@@ -2,6 +2,7 @@ import { Inject, Service } from 'typedi';
 import HasTenancyService from '../Tenancy/TenancyService';
 import { TransformerInjectable } from '@/lib/Transformer/TransformerInjectable';
 import { ImportFileMetaTransformer } from './ImportFileMetaTransformer';
+import { Import } from '@/system/models';
 
 @Service()
 export class ImportFileMeta {
@@ -12,15 +13,15 @@ export class ImportFileMeta {
   private transformer: TransformerInjectable;
 
   /**
-   *
+   * Retrieves the import meta of the given import model id.
    * @param {number} tenantId
    * @param {number} importId
    * @returns {}
    */
   async getImportMeta(tenantId: number, importId: string) {
-    const { Import } = this.tenancy.models(tenantId);
-
-    const importFile = await Import.query().findOne('importId', importId);
+    const importFile = await Import.query()
+      .where('tenantId', tenantId)
+      .findOne('importId', importId);
 
     // Retrieves the transformed accounts collection.
     return this.transformer.transform(

packages/server/src/services/Import/ImportFileProcess.ts

@@ -2,19 +2,16 @@ import { Inject, Service } from 'typedi';
 import { chain } from 'lodash';
 import { Knex } from 'knex';
 import { ServiceError } from '@/exceptions';
-import { ERRORS, getSheetColumns, getUnmappedSheetColumns } from './_utils';
-import HasTenancyService from '../Tenancy/TenancyService';
+import { ERRORS, getSheetColumns, getUnmappedSheetColumns, readImportFile } from './_utils';
 import { ImportFileCommon } from './ImportFileCommon';
 import { ImportFileDataTransformer } from './ImportFileDataTransformer';
 import ResourceService from '../Resource/ResourceService';
 import UnitOfWork from '../UnitOfWork';
 import { ImportFilePreviewPOJO } from './interfaces';
+import { Import } from '@/system/models';
 
 @Service()
 export class ImportFileProcess {
-  @Inject()
-  private tenancy: HasTenancyService;
-
   @Inject()
   private resource: ResourceService;
 
@@ -38,10 +35,9 @@ export class ImportFileProcess {
     importId: number,
     trx?: Knex.Transaction
   ): Promise<ImportFilePreviewPOJO> {
-    const { Import } = this.tenancy.models(tenantId);
-
     const importFile = await Import.query()
       .findOne('importId', importId)
+      .where('tenantId', tenantId)
       .throwIfNotFound();
 
     // Throw error if the import file is not mapped yet.
@@ -49,7 +45,7 @@ export class ImportFileProcess {
       throw new ServiceError(ERRORS.IMPORT_FILE_NOT_MAPPED);
     }
     // Read the imported file.
-    const buffer = await this.importCommon.readImportFile(importFile.filename);
+    const buffer = await readImportFile(importFile.filename);
     const sheetData = this.importCommon.parseXlsxSheet(buffer);
     const header = getSheetColumns(sheetData);
 

packages/server/src/services/Import/ImportFileUpload.ts

@@ -1,7 +1,8 @@
 import { Inject, Service } from 'typedi';
-import HasTenancyService from '../Tenancy/TenancyService';
 import {
+  deleteImportFile,
   getResourceColumns,
+  readImportFile,
   sanitizeResourceName,
   validateSheetEmpty,
 } from './_utils';
@@ -9,12 +10,10 @@ import ResourceService from '../Resource/ResourceService';
 import { ImportFileCommon } from './ImportFileCommon';
 import { ImportFileDataValidator } from './ImportFileDataValidator';
 import { ImportFileUploadPOJO } from './interfaces';
+import { Import } from '@/system/models';
 
 @Service()
 export class ImportFileUploadService {
-  @Inject()
-  private tenancy: HasTenancyService;
-
   @Inject()
   private resourceService: ResourceService;
 
@@ -25,11 +24,12 @@ export class ImportFileUploadService {
   private importValidator: ImportFileDataValidator;
 
   /**
-   * Reads the imported file and stores the import file meta under unqiue id.
-   * @param {number} tenantId - Tenant id.
-   * @param {string} resource - Resource name.
-   * @param {string} filePath - File path.
-   * @param {string} fileName - File name.
+   * Imports the specified file for the given resource.
+   * Deletes the file if an error occurs during the import process.
+   * @param {number} tenantId
+   * @param {string} resourceName
+   * @param {string} filename
+   * @param {Record<string, number | string>} params
    * @returns {Promise<ImportFileUploadPOJO>}
    */
   public async import(
@@ -38,8 +38,35 @@ export class ImportFileUploadService {
     filename: string,
     params: Record<string, number | string>
   ): Promise<ImportFileUploadPOJO> {
-    const { Import } = this.tenancy.models(tenantId);
+    console.log(filename, 'filename');
 
+    try {
+      return await this.importUnhandled(
+        tenantId,
+        resourceName,
+        filename,
+        params
+      );
+    } catch (err) {
+      // deleteImportFile(filename);
+      throw err;
+    }
+  }
+
+  /**
+   * Reads the imported file and stores the import file meta under unqiue id.
+   * @param {number} tenantId - Tenant id.
+   * @param {string} resource - Resource name.
+   * @param {string} filePath - File path.
+   * @param {string} fileName - File name.
+   * @returns {Promise<ImportFileUploadPOJO>}
+   */
+  public async importUnhandled(
+    tenantId: number,
+    resourceName: string,
+    filename: string,
+    params: Record<string, number | string>
+  ): Promise<ImportFileUploadPOJO> {
     const resource = sanitizeResourceName(resourceName);
     const resourceMeta = this.resourceService.getResourceMeta(
       tenantId,
@@ -49,7 +76,7 @@ export class ImportFileUploadService {
     this.importValidator.validateResourceImportable(resourceMeta);
 
     // Reads the imported file into buffer.
-    const buffer = await this.importFileCommon.readImportFile(filename);
+    const buffer = await readImportFile(filename);
 
     // Parse the buffer file to array data.
     const sheetData = this.importFileCommon.parseXlsxSheet(buffer);
@@ -76,6 +103,7 @@ export class ImportFileUploadService {
     const importFile = await Import.query().insert({
       filename,
       resource,
+      tenantId,
       importId: filename,
       columns: coumnsStringified,
       params: paramsStringified,

packages/server/src/services/Import/ImportRemoveExpiredFiles.ts

@@ -0,0 +1,34 @@
+import moment from 'moment';
+import bluebird from 'bluebird';
+import { Import } from '@/system/models';
+import { deleteImportFile } from './_utils';
+import { Service } from 'typedi';
+
+@Service()
+export class ImportDeleteExpiredFiles {
+  /**
+   * Delete expired files.
+   */
+  async deleteExpiredFiles() {
+    const yesterday = moment().subtract(1, 'hour').format('YYYY-MM-DD HH:mm');
+
+    const expiredImports = await Import.query().where(
+      'createdAt',
+      '<',
+      yesterday
+    );
+    await bluebird.map(
+      expiredImports,
+      async (expiredImport) => {
+        await deleteImportFile(expiredImport.filename);
+      },
+      { concurrency: 10 }
+    );
+    const expiredImportsIds = expiredImports.map(
+      (expiredImport) => expiredImport.id
+    );
+    if (expiredImportsIds.length > 0) {
+      await Import.query().whereIn('id', expiredImportsIds).delete();
+    }
+  }
+}

packages/server/src/services/Import/ImportSample.ts

@@ -10,9 +10,9 @@ export class ImportSampleService {
 
   /**
    * Retrieves the sample sheet of the given resource.
-   * @param {number} tenantId 
-   * @param {string} resource 
-   * @param {string} format 
+   * @param {number} tenantId
+   * @param {string} resource
+   * @param {string} format
    * @returns {Buffer | string}
    */
   public sample(

packages/server/src/services/Import/Importable.ts

@@ -69,4 +69,4 @@ export abstract class Importable {
   public transformParams(parmas: Record<string, any>) {
     return parmas;
   }
-}
+}

packages/server/src/services/Import/_constants.ts

@@ -0,0 +1,3 @@
+
+
+export const CurrencyParsingDTOs = 10;

packages/server/src/services/Import/_utils.ts

@@ -2,6 +2,7 @@ import * as Yup from 'yup';
 import moment from 'moment';
 import * as R from 'ramda';
 import { Knex } from 'knex';
+import fs from 'fs/promises';
 import {
   defaultTo,
   upperFirst,
@@ -141,9 +142,9 @@ const parseFieldName = (fieldName: string, field: IModelMetaField) => {
 
 /**
  * Retrieves the unmapped sheet columns.
- * @param columns 
- * @param mapping 
- * @returns 
+ * @param columns
+ * @param mapping
+ * @returns
  */
 export const getUnmappedSheetColumns = (columns, mapping) => {
   return columns.filter(
@@ -421,3 +422,30 @@ export function aggregate(
 export const sanitizeSheetData = (json) => {
   return R.compose(R.map(trimObject))(json);
 };
+
+/**
+ * Returns the path to map a value to based on the 'to' and 'group' parameters.
+ * @param {string} to - The target key to map the value to.
+ * @param {string} group - The group key to nest the target key under.
+ * @returns {string} - The path to map the value to.
+ */
+export const getMapToPath = (to: string, group = '') =>
+  group ? `${group}.${to}` : to;
+
+/**
+ * Deletes the imported file from the storage and database.
+ * @param {string} filename
+ */
+export const deleteImportFile = async (filename: string) => {
+  // Deletes the imported file.
+  await fs.unlink(`public/imports/${filename}`);
+};
+
+/**
+ * Reads the import file.
+ * @param {string} filename
+ * @returns {Promise<Buffer>}
+ */
+export const readImportFile = (filename: string) => {
+  return fs.readFile(`public/imports/${filename}`);
+};

packages/server/src/services/Import/jobs/ImportDeleteExpiredFilesJob.ts

@@ -1 +1,28 @@
-export class ImportDeleteExpiredFilesJobs {}
+import Container, { Service } from 'typedi';
+import { ImportDeleteExpiredFiles } from '../ImportRemoveExpiredFiles';
+
+@Service()
+export class ImportDeleteExpiredFilesJobs {
+  /**
+   * Constructor method.
+   */
+  constructor(agenda) {
+    agenda.define('delete-expired-imported-files', this.handler);
+  }
+
+  /**
+   * Triggers sending invoice mail.
+   */
+  private handler = async (job, done: Function) => {
+    const importDeleteExpiredFiles = Container.get(ImportDeleteExpiredFiles);
+
+    try {
+      console.log('Delete expired import files has started.');
+      await importDeleteExpiredFiles.deleteExpiredFiles();
+      done();
+    } catch (error) {
+      console.log(error);
+      done(error);
+    }
+  };
+}