feat: import resource

packages/server/src/services/Import/ImportFileProcess.ts

@@ -1,18 +1,14 @@
 import { Inject, Service } from 'typedi';
-import * as R from 'ramda';
-import XLSX from 'xlsx';
-import { first, isUndefined } from 'lodash';
-import bluebird from 'bluebird';
-import fs from 'fs/promises';
+import { chain } from 'lodash';
 import { Knex } from 'knex';
-import HasTenancyService from '../Tenancy/TenancyService';
-import { ERRORS, convertFieldsToYupValidation, trimObject } from './_utils';
-import { ImportMappingAttr, ImportValidationError } from './interfaces';
-import { AccountsImportable } from './AccountsImportable';
-import UnitOfWork from '../UnitOfWork';
 import { ServiceError } from '@/exceptions';
+import { ERRORS, getSheetColumns, getUnmappedSheetColumns } from './_utils';
+import HasTenancyService from '../Tenancy/TenancyService';
+import { ImportFileCommon } from './ImportFileCommon';
+import { ImportFileDataTransformer } from './ImportFileDataTransformer';
 import ResourceService from '../Resource/ResourceService';
-
+import UnitOfWork from '../UnitOfWork';
+import { ImportFilePreviewPOJO } from './interfaces';
 
 @Service()
 export class ImportFileProcess {
@@ -20,121 +16,28 @@ export class ImportFileProcess {
   private tenancy: HasTenancyService;
 
   @Inject()
-  private importable: AccountsImportable;
+  private resource: ResourceService;
+
+  @Inject()
+  private importCommon: ImportFileCommon;
+
+  @Inject()
+  private importParser: ImportFileDataTransformer;
 
   @Inject()
   private uow: UnitOfWork;
 
-  @Inject()
-  private resourceService: ResourceService;
-
   /**
-   * Reads the import file.
-   * @param {string} filename
-   * @returns {Promise<Buffer>}
-   */
-  public readImportFile(filename: string) {
-    return fs.readFile(`public/imports/${filename}`);
-  }
-
-  /**
-   * Maps the columns of the imported data based on the provided mapping attributes.
-   * @param {Record<string, any>[]} body - The array of data objects to map.
-   * @param {ImportMappingAttr[]} map - The mapping attributes.
-   * @returns {Record<string, any>[]} - The mapped data objects.
-   */
-  public parseXlsxSheet(buffer) {
-    const workbook = XLSX.read(buffer, { type: 'buffer' });
-
-    const firstSheetName = workbook.SheetNames[0];
-    const worksheet = workbook.Sheets[firstSheetName];
-
-    return XLSX.utils.sheet_to_json(worksheet);
-  }
-
-  /**
-   * Sanitizes the data in the imported sheet by trimming object keys.
-   * @param json - The JSON data representing the imported sheet.
-   * @returns {string[][]} - The sanitized data with trimmed object keys.
-   */
-  public sanitizeSheetData(json) {
-    return R.compose(R.map(Object.keys), R.map(trimObject))(json);
-  }
-
-  /**
-   * Maps the columns of the imported data based on the provided mapping attributes.
-   * @param {Record<string, any>[]} body - The array of data objects to map.
-   * @param {ImportMappingAttr[]} map - The mapping attributes.
-   * @returns {Record<string, any>[]} - The mapped data objects.
-   */
-  private mapSheetColumns(
-    body: Record<string, any>[],
-    map: ImportMappingAttr[]
-  ): Record<string, any>[] {
-    return body.map((item) => {
-      const newItem = {};
-      map
-        .filter((mapping) => !isUndefined(item[mapping.from]))
-        .forEach((mapping) => {
-          newItem[mapping.to] = item[mapping.from];
-        });
-      return newItem;
-    });
-  }
-
-  /**
-   * Validates the given mapped DTOs and returns errors with their index.
-   * @param {Record<string, any>} mappedDTOs
-   * @returns {Promise<ImportValidationError[][]>}
-   */
-  private async validateData(
-    tenantId: number,
-    resource: string,
-    mappedDTOs: Record<string, any>
-  ): Promise<ImportValidationError[][]> {
-    const importableFields = this.resourceService.getResourceImportableFields(
-      tenantId,
-      resource
-    );
-    const YupSchema = convertFieldsToYupValidation(importableFields);
-
-    const validateData = async (data, index: number) => {
-      const _data = { ...data };
-
-      try {
-        await YupSchema.validate(_data, { abortEarly: false });
-        return { index, data: _data, errors: [] };
-      } catch (validationError) {
-        const errors = validationError.inner.map((error) => ({
-          path: error.params.path,
-          label: error.params.label,
-          message: error.errors,
-        }));
-        return { index, data: _data, errors };
-      }
-    };
-    const errors = await bluebird.map(mappedDTOs, validateData, {
-      concurrency: 20,
-    });
-    return errors.filter((error) => error !== false);
-  }
-
-  /**
-   * Transformes the mapped DTOs.
-   * @param DTOs
-   * @returns
-   */
-  private transformDTOs(DTOs) {
-    return DTOs.map((DTO) => this.importable.transform(DTO));
-  }
-
-  /**
-   * Processes the import file sheet through the resource service.
+   * Preview the imported file results before commiting the transactions.
    * @param {number} tenantId
    * @param {number} importId
-   * @returns {Promise<void>}
+   * @returns {Promise<ImportFilePreviewPOJO>}
    */
-  public async process(tenantId: number, importId: number) {
+  public async import(
+    tenantId: number,
+    importId: number,
+    trx?: Knex.Transaction
+  ): Promise<ImportFilePreviewPOJO> {
     const { Import } = this.tenancy.models(tenantId);
 
     const importFile = await Import.query()
@@ -145,55 +48,54 @@ export class ImportFileProcess {
     if (!importFile.isMapped) {
       throw new ServiceError(ERRORS.IMPORT_FILE_NOT_MAPPED);
     }
-    const buffer = await this.readImportFile(importFile.filename);
-    const jsonData = this.parseXlsxSheet(buffer);
+    // Read the imported file.
+    const buffer = await this.importCommon.readImportFile(importFile.filename);
+    const sheetData = this.importCommon.parseXlsxSheet(buffer);
+    const header = getSheetColumns(sheetData);
 
-    const data = this.sanitizeSheetData(jsonData);
-
-    const header = first(data);
-    const body = jsonData;
-
-    const mappedDTOs = this.mapSheetColumns(body, importFile.mappingParsed);
-    const transformedDTOs = this.transformDTOs(mappedDTOs);
-
-    // Validate the mapped DTOs.
-    const rowsWithErrors = await this.validateData(
+    const importableFields = this.resource.getResourceImportableFields(
       tenantId,
-      importFile.resource,
-      transformedDTOs
+      importFile.resource
     );
-    // Runs the importing under UOW envirement.
-    await this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => {
-      await bluebird.map(
-        rowsWithErrors,
-        (rowWithErrors) => {
-          if (rowWithErrors.errors.length === 0) {
-            return this.importable.importable(
-              tenantId,
-              rowWithErrors.data,
-              trx
-            );
-          }
-        },
-        { concurrency: 10 }
-      );
-    });
-    // Deletes the imported file after importing success./
-    await this.deleteImportFile(tenantId, importFile)
-  }
+    // Prases the sheet json data.
+    const parsedData = this.importParser.parseSheetData(
+      importFile,
+      importableFields,
+      sheetData
+    );
+    // Runs the importing operation with ability to return errors that will happen.
+    const [successedImport, failedImport] = await this.uow.withTransaction(
+      tenantId,
+      (trx: Knex.Transaction) =>
+        this.importCommon.import(
+          tenantId,
+          importFile.resource,
+          parsedData,
+          trx
+        ),
+      trx
+    );
+    const mapping = importFile.mappingParsed;
+    const errors = chain(failedImport)
+      .map((oper) => oper.error)
+      .flatten()
+      .value();
 
-  /**
-   * Deletes the imported file from the storage and database.
-   * @param {number} tenantId 
-   * @param {} importFile 
-   */
-  private async deleteImportFile(tenantId: number, importFile: any) {
-    const { Import } = this.tenancy.models(tenantId);
+    const unmappedColumns = getUnmappedSheetColumns(header, mapping);
+    const totalCount = parsedData.length;
 
-    // Deletes the import row.
-    await Import.query().findById(importFile.id).delete();
+    const createdCount = successedImport.length;
+    const errorsCount = failedImport.length;
+    const skippedCount = errorsCount;
 
-    // Deletes the imported file.
-    await fs.unlink(`public/imports/${importFile.filename}`);
+    return {
+      createdCount,
+      skippedCount,
+      totalCount,
+      errorsCount,
+      errors,
+      unmappedColumns: unmappedColumns,
+      unmappedColumnsCount: unmappedColumns.length,
+    };
   }
 }