feat: wip import resource

packages/server/src/api/controllers/Import/ImportController.ts

@@ -1,10 +1,9 @@
 import { Inject, Service } from 'typedi';
 import { Router, Request, Response, NextFunction } from 'express';
-import { query, body, param } from 'express-validator';
+import { body, param } from 'express-validator';
 import Multer from 'multer';
 import BaseController from '@/api/controllers/BaseController';
 import { ServiceError } from '@/exceptions';
-import { ImportResourceInjectable } from '@/services/Import/ImportResourceInjectable';
 import { ImportResourceApplication } from '@/services/Import/ImportResourceApplication';
 
 const upload = Multer({
@@ -23,11 +22,6 @@ export class ImportController extends BaseController {
   router() {
     const router = Router();
 
-    router.post(
-      '/:import_id/import',
-      this.asyncMiddleware(this.import.bind(this)),
-      this.catchServiceErrors
-    );
     router.post(
       '/file',
       upload.single('file'),
@@ -36,6 +30,11 @@ export class ImportController extends BaseController {
       this.asyncMiddleware(this.fileUpload.bind(this)),
       this.catchServiceErrors
     );
+    router.post(
+      '/:import_id/import',
+      this.asyncMiddleware(this.import.bind(this)),
+      this.catchServiceErrors
+    );
     router.post(
       '/:import_id/mapping',
       [
@@ -48,11 +47,11 @@ export class ImportController extends BaseController {
       this.asyncMiddleware(this.mapping.bind(this)),
       this.catchServiceErrors
     );
-    // router.get(
+    router.post(
-    //   '/:import_id/preview',
+      '/:import_id/preview',
-    //   this.asyncMiddleware(this.preview.bind(this)),
+      this.asyncMiddleware(this.preview.bind(this)),
-    //   this.catchServiceErrors
+      this.catchServiceErrors
-    // );
+    );
     return router;
   }
 
@@ -86,7 +85,7 @@ export class ImportController extends BaseController {
    * @param {Response} res -
    * @param {NextFunction} next -
    */
-  async fileUpload(req: Request, res: Response, next: NextFunction) {
+  private async fileUpload(req: Request, res: Response, next: NextFunction) {
     const { tenantId } = req;
 
     try {
@@ -103,10 +102,10 @@ export class ImportController extends BaseController {
   }
 
   /**
-   *
+   * Maps the columns of the imported file.
-   * @param req
+   * @param {Request} req
-   * @param res
+   * @param {Response} res
-   * @param next
+   * @param {NextFunction} next
    */
   private   async mapping(req: Request, res: Response, next: NextFunction) {
     const { tenantId } = req;
@@ -126,12 +125,23 @@ export class ImportController extends BaseController {
   }
 
   /**
-   *
+   * Preview the imported file before actual importing.
-   * @param req
+   * @param {Request} req
-   * @param res
+   * @param {Response} res
-   * @param next
+   * @param {NextFunction} next
    */
-  private async preview(req: Request, res: Response, next: NextFunction) {}
+  private async preview(req: Request, res: Response, next: NextFunction) {
+    const { tenantId } = req;
+    const { import_id: importId } = req.params;
+
+    try {
+      const  preview = await this.importResourceApp.preview(tenantId, importId);
+
+      return res.status(200).send(preview);
+    } catch (error) {
+      next(error);
+    }
+  }
 
   /**
    *

packages/server/src/interfaces/Model.ts

@@ -34,6 +34,7 @@ export interface IModelMetaFieldCommon {
   columnable?: boolean;
   fieldType: IModelColumnType;
   customQuery?: Function;
+  required?: boolean;
 }
 
 export interface IModelMetaFieldNumber {

packages/server/src/models/Account.Settings.ts

@@ -63,7 +63,7 @@ export default {
       sortable: false,
       importable: false,
     },
-    type: {
+    accountType: {
       name: 'account.field.type',
       column: 'account_type',
       fieldType: 'enumeration',

packages/server/src/services/Import/AccountsImportable.ts

@@ -1,13 +1,13 @@
-import { IAccountCreateDTO } from '@/interfaces';
-import { AccountsApplication } from '../Accounts/AccountsApplication';
-import { AccountDTOSchema } from '../Accounts/CreateAccountDTOSchema';
 import { Inject, Service } from 'typedi';
 import { Knex } from 'knex';
+import { IAccountCreateDTO } from '@/interfaces';
+import { AccountsApplication } from '../Accounts/AccountsApplication';
+import { CreateAccount } from '../Accounts/CreateAccount';
 
 @Service()
 export class AccountsImportable {
   @Inject()
-  private accountsApp: AccountsApplication;
+  private createAccountService: CreateAccount;
 
   /**
    *
@@ -20,7 +20,11 @@ export class AccountsImportable {
     createAccountDTO: IAccountCreateDTO,
     trx?: Knex.Transaction
   ) {
-    return this.accountsApp.createAccount(tenantId, createAccountDTO, trx);
+    return this.createAccountService.createAccount(
+      tenantId,
+      createAccountDTO,
+      trx
+    );
   }
 
   /**
@@ -29,12 +33,15 @@ export class AccountsImportable {
    * @returns
    */
   public transform(data) {
-    return {
+    return { ...data };
-      ...data,
-    };
   }
 
-  mapAccountType(accountType: string) {
+  /**
-    return 'Cash';
+   *
+   * @param data
+   * @returns
+   */
+  public preTransform(data) {
+    return { ...data };
   }
 }

packages/server/src/services/Import/ImportFileCommon.ts

@@ -0,0 +1,87 @@
+import fs from 'fs/promises';
+import XLSX from 'xlsx';
+import bluebird from 'bluebird';
+import { Inject, Service } from 'typedi';
+import { ImportFileDataValidator } from './ImportFileDataValidator';
+import { Knex } from 'knex';
+import { ImportInsertError } from './interfaces';
+import { AccountsImportable } from './AccountsImportable';
+import { ServiceError } from '@/exceptions';
+
+@Service()
+export class ImportFileCommon {
+  @Inject()
+  private importFileValidator: ImportFileDataValidator;
+
+  @Inject()
+  private importable: AccountsImportable;
+
+  /**
+   * Maps the columns of the imported data based on the provided mapping attributes.
+   * @param {Record<string, any>[]} body - The array of data objects to map.
+   * @param {ImportMappingAttr[]} map - The mapping attributes.
+   * @returns {Record<string, any>[]} - The mapped data objects.
+   */
+  public parseXlsxSheet(buffer) {
+    const workbook = XLSX.read(buffer, { type: 'buffer' });
+
+    const firstSheetName = workbook.SheetNames[0];
+    const worksheet = workbook.Sheets[firstSheetName];
+
+    return XLSX.utils.sheet_to_json(worksheet);
+  }
+
+  /**
+   * Reads the import file.
+   * @param {string} filename
+   * @returns {Promise<Buffer>}
+   */
+  public readImportFile(filename: string) {
+    return fs.readFile(`public/imports/${filename}`);
+  }
+
+  /**
+   *
+   * @param {number} tenantId -
+   * @param {Record<string, any>} importableFields
+   * @param {Record<string, any>} parsedData
+   * @param {Knex.Transaction} trx
+   * @returns
+   */
+  public import(
+    tenantId: number,
+    importableFields,
+    parsedData: Record<string, any>,
+    trx?: Knex.Transaction
+  ): Promise<(void | ImportInsertError[])[]> {
+    return bluebird.map(
+      parsedData,
+      async (objectDTO, index: number): Promise<true | ImportInsertError[]> => {
+        try {
+          // Validate the DTO object before passing it to the service layer.
+          await this.importFileValidator.validateData(
+            importableFields,
+            objectDTO
+          );
+          try {
+            // Run the importable function and listen to the errors.
+            await this.importable.importable(tenantId, objectDTO, trx);
+          } catch (error) {
+            if (error instanceof ServiceError) {
+              return [
+                {
+                  errorCode: 'ValidationError',
+                  errorMessage: error.message || error.errorType,
+                  rowNumber: index + 1,
+                },
+              ];
+            }
+          }
+        } catch (errors) {
+          return errors.map((er) => ({ ...er, rowNumber: index + 1 }));
+        }
+      },
+      { concurrency: 2 }
+    );
+  }
+}

packages/server/src/services/Import/ImportFileDataTransformer.ts

@@ -0,0 +1,104 @@
+import { Inject, Service } from 'typedi';
+import * as R from 'ramda';
+import { isUndefined, mapValues, get, pickBy, chain } from 'lodash';
+import { ImportMappingAttr, ResourceMetaFieldsMap } from './interfaces';
+import { parseBoolean } from '@/utils';
+import { trimObject } from './_utils';
+import ResourceService from '../Resource/ResourceService';
+
+@Service()
+export class ImportFileDataTransformer {
+  @Inject()
+  private resource: ResourceService;
+  /**
+   *
+   * @param {number} tenantId -
+   * @param {}
+   */
+  public transformSheetData(
+    importFile: any,
+    importableFields: any,
+    data: Record<string, unknown>[]
+  ) {
+    // Sanitize the sheet data.
+    const sanitizedData = this.sanitizeSheetData(data);
+
+    // Map the sheet columns key with the given map.
+    const mappedDTOs = this.mapSheetColumns(
+      sanitizedData,
+      importFile.mappingParsed
+    );
+    // Parse the mapped sheet values.
+    const parsedValues = this.parseExcelValues(importableFields, mappedDTOs);
+
+    return parsedValues;
+  }
+
+  /**
+   * Sanitizes the data in the imported sheet by trimming object keys.
+   * @param json - The JSON data representing the imported sheet.
+   * @returns {string[][]} - The sanitized data with trimmed object keys.
+   */
+  public sanitizeSheetData(json) {
+    return R.compose(R.map(trimObject))(json);
+  }
+
+  /**
+   * Maps the columns of the imported data based on the provided mapping attributes.
+   * @param {Record<string, any>[]} body - The array of data objects to map.
+   * @param {ImportMappingAttr[]} map - The mapping attributes.
+   * @returns {Record<string, any>[]} - The mapped data objects.
+   */
+  public mapSheetColumns(
+    body: Record<string, any>[],
+    map: ImportMappingAttr[]
+  ): Record<string, any>[] {
+    return body.map((item) => {
+      const newItem = {};
+      map
+        .filter((mapping) => !isUndefined(item[mapping.from]))
+        .forEach((mapping) => {
+          newItem[mapping.to] = item[mapping.from];
+        });
+      return newItem;
+    });
+  }
+
+  /**
+   * Parses sheet values before passing to the service layer.
+   * @param {ResourceMetaFieldsMap} fields -
+   * @param {Record<string, any>} valueDTOS -
+   * @returns {Record<string, any>}
+   */
+  public parseExcelValues(
+    fields: ResourceMetaFieldsMap,
+    valueDTOs: Record<string, any>[]
+  ): Record<string, any> {
+    const parser = (value, key) => {
+      let _value = value;
+
+      // Parses the boolean value.
+      if (fields[key].fieldType === 'boolean') {
+        _value = parseBoolean(value, false);
+
+        // Parses the enumeration value.
+      } else if (fields[key].fieldType === 'enumeration') {
+        const field = fields[key];
+        const option = get(field, 'options', []).find(
+          (option) => option.label === value
+        );
+        _value = get(option, 'key');
+        // Prases the numeric value.
+      } else if (fields[key].fieldType === 'number') {
+        _value = parseFloat(value);
+      }
+      return _value;
+    };
+    return valueDTOs.map((DTO) => {
+      return chain(DTO)
+        .pickBy((value, key) => !isUndefined(fields[key]))
+        .mapValues(parser)
+        .value();
+    });
+  }
+}

packages/server/src/services/Import/ImportFileDataValidator.ts

@@ -0,0 +1,29 @@
+import { Service } from 'typedi';
+import { ImportInsertError, ResourceMetaFieldsMap } from './interfaces';
+import { convertFieldsToYupValidation } from './_utils';
+
+@Service()
+export class ImportFileDataValidator {
+  /**
+   * Validates the given mapped DTOs and returns errors with their index.
+   * @param {Record<string, any>} mappedDTOs
+   * @returns {Promise<ImportValidationError[][]>}
+   */
+  public async validateData(
+    importableFields: ResourceMetaFieldsMap,
+    data: Record<string, any>
+  ): Promise<void | ImportInsertError[]> {
+    const YupSchema = convertFieldsToYupValidation(importableFields);
+    const _data = { ...data };
+
+    try {
+      await YupSchema.validate(_data, { abortEarly: false });
+    } catch (validationError) {
+      const errors = validationError.inner.map((error) => ({
+        errorCode: 'ValidationError',
+        errorMessage: error.errors,
+      }));
+      throw errors;
+    }
+  }
+}

packages/server/src/services/Import/ImportFilePreview.ts

@@ -1,11 +1,98 @@
-import { Service } from 'typedi';
+import { Inject, Service } from 'typedi';
+import { first, omit } from 'lodash';
+import { ServiceError } from '@/exceptions';
+import { ERRORS, getUnmappedSheetColumns } from './_utils';
+import HasTenancyService from '../Tenancy/TenancyService';
+import { ImportFileCommon } from './ImportFileCommon';
+import { ImportFileDataTransformer } from './ImportFileDataTransformer';
+import ResourceService from '../Resource/ResourceService';
 
 @Service()
 export class ImportFilePreview {
+  @Inject()
+  private tenancy: HasTenancyService;
+
+  @Inject()
+  private resource: ResourceService;
+
+  @Inject()
+  private importFileCommon: ImportFileCommon;
+
+  @Inject()
+  private importFileParser: ImportFileDataTransformer;
+
   /**
+   *
+   * - Returns the passed rows and will be in inserted.
+   * - Returns the passed rows will be overwritten.
+   * - Returns the rows errors from the validation.
+   * - Returns the unmapped fields.
    *
    * @param {number} tenantId
    * @param {number} importId
    */
-  public preview(tenantId: number, importId: number) {}
+  public async preview(tenantId: number, importId: number) {
+    const { Import } = this.tenancy.models(tenantId);
+
+    const importFile = await Import.query()
+      .findOne('importId', importId)
+      .throwIfNotFound();
+
+    // Throw error if the import file is not mapped yet.
+    if (!importFile.isMapped) {
+      throw new ServiceError(ERRORS.IMPORT_FILE_NOT_MAPPED);
+    }
+    const buffer = await this.importFileCommon.readImportFile(
+      importFile.filename
+    );
+    const jsonData = this.importFileCommon.parseXlsxSheet(buffer);
+
+    const importableFields = this.resource.getResourceImportableFields(
+      tenantId,
+      importFile.resource
+    );
+    // Prases the sheet json data.
+    const parsedData = this.importFileParser.transformSheetData(
+      importFile,
+      importableFields,
+      jsonData
+    );
+    const knex = this.tenancy.knex(tenantId);
+    const trx = await knex.transaction({ isolationLevel: 'read uncommitted' });
+
+    // Runs the importing operation with ability to return errors that will happen.
+    const asyncOpers = await this.importFileCommon.import(
+      tenantId,
+      importableFields,
+      parsedData,
+      trx
+    );
+    // Filter out the operations that have successed.
+    const successAsyncOpers = asyncOpers.filter((oper) => !oper);
+    const errors = asyncOpers.filter((oper) => oper);
+
+    // Rollback all the successed transactions.
+    await trx.rollback();
+
+    const header = Object.keys(first(jsonData));
+    const mapping = importFile.mappingParsed;
+
+    const unmappedColumns = getUnmappedSheetColumns(header, mapping);
+    const totalCount = parsedData.length;
+
+    const createdCount = successAsyncOpers.length;
+    const errorsCount = errors.length;
+    const skippedCount = errorsCount;
+
+    return {
+      createdCount,
+      skippedCount,
+      totalCount,
+      errorsCount,
+      errors,
+      unmappedColumns: unmappedColumns,
+      unmappedColumnsCount: unmappedColumns.length,
+    };
+  }
+
 }

packages/server/src/services/Import/ImportFileUpload.ts

@@ -8,7 +8,7 @@ import { ERRORS, trimObject } from './_utils';
 import ResourceService from '../Resource/ResourceService';
 import fs from 'fs/promises';
 import { IModelMetaField } from '@/interfaces';
-
+import { ImportFileCommon } from './ImportFileCommon';
 @Service()
 export class ImportFileUploadService {
   @Inject()
@@ -17,11 +17,15 @@ export class ImportFileUploadService {
   @Inject()
   private resourceService: ResourceService;
 
+  @Inject()
+  private importFileCommon: ImportFileCommon;
+
   /**
    * Reads the imported file and stores the import file meta under unqiue id.
-   * @param {number} tenantId -
+   * @param {number} tenantId - Tenant id.
-   * @param {string} filePath -
+   * @param {string} resource - Resource name.
-   * @param {string} fileName -
+   * @param {string} filePath - File path.
+   * @param {string} fileName - File name.
    * @returns
    */
   public async import(
@@ -40,16 +44,15 @@ export class ImportFileUploadService {
     if (!resourceMeta.importable) {
       throw new ServiceError(ERRORS.RESOURCE_NOT_IMPORTABLE);
     }
-    const buffer = await fs.readFile(filePath);
+    // Reads the imported file into buffer.
-    const workbook = XLSX.read(buffer, { type: 'buffer' });
+    const buffer = await this.importFileCommon.readImportFile(filename);
+
+    // Parse the buffer file to array data.
+    const jsonData = this.importFileCommon.parseXlsxSheet(buffer);
 
-    const firstSheetName = workbook.SheetNames[0];
-    const worksheet = workbook.Sheets[firstSheetName];
-    const jsonData = XLSX.utils.sheet_to_json(worksheet);
-    
     const columns = this.getColumns(jsonData);
     const coumnsStringified = JSON.stringify(columns);
-    
+
     // @todo validate the resource.
     const _resource = this.resourceService.resourceToModelName(resource);
 
@@ -66,7 +69,6 @@ export class ImportFileUploadService {
     const resourceColumnsTransformeed = Object.entries(resourceColumns).map(
       ([key, { name }]: [string, IModelMetaField]) => ({ key, name })
     );
-    // @todo return the resource importable columns.
     return {
       export: exportFile,
       columns,

packages/server/src/services/Import/ImportResourceRegistry.ts

@@ -0,0 +1,7 @@
+import { Service } from "typedi";
+
+
+@Service()
+export class ImportResourceRegistry {
+
+}

packages/server/src/services/Import/_utils.ts

@@ -1,4 +1,6 @@
 import * as Yup from 'yup';
+import { ResourceMetaFieldsMap } from './interfaces';
+import { IModelMetaField } from '@/interfaces';
 
 export function trimObject(obj) {
   return Object.entries(obj).reduce((acc, [key, value]) => {
@@ -13,10 +15,10 @@ export function trimObject(obj) {
   }, {});
 }
 
-export const convertFieldsToYupValidation = (fields: any) => {
+export const convertFieldsToYupValidation = (fields: ResourceMetaFieldsMap) => {
   const yupSchema = {};
-  Object.keys(fields).forEach((fieldName) => {
+  Object.keys(fields).forEach((fieldName: string) => {
-    const field = fields[fieldName];
+    const field = fields[fieldName] as IModelMetaField;
     let fieldSchema;
     fieldSchema = Yup.string().label(field.name);
 
@@ -59,3 +61,12 @@ export const ERRORS = {
   DUPLICATED_TO_MAP_ATTR: 'DUPLICATED_TO_MAP_ATTR',
   IMPORT_FILE_NOT_MAPPED: 'IMPORT_FILE_NOT_MAPPED',
 };
+
+/**
+ *
+ */
+export const getUnmappedSheetColumns = (columns, mapping) => {
+  return columns.filter(
+    (column) => !mapping.some((map) => map.from === column)
+  );
+};

packages/server/src/services/Import/interfaces.ts

@@ -1,3 +1,5 @@
+import { IModelMetaField } from '@/interfaces';
+
 export interface ImportMappingAttr {
   from: string;
   to: string;
@@ -8,3 +10,11 @@ export interface ImportValidationError {
   property: string;
   constraints: Record<string, string>;
 }
+
+export type ResourceMetaFieldsMap = { [key: string]: IModelMetaField };
+
+export interface ImportInsertError {
+  rowNumber: number;
+  errorCode: string;
+  errorMessage: string;
+}

packages/server/src/services/UnitOfWork/index.ts

@@ -50,11 +50,15 @@ export default class UnitOfWork {
     }
     try {
       const result = await work(_trx);
-      _trx.commit();
 
+      if (!trx) {
+        _trx.commit();
+      }
       return result;
     } catch (error) {
-      _trx.rollback();
+      if (!trx) {
+        _trx.rollback();
+      }
       throw error;
     }
   };