feat(server): wip import resources

packages/server/src/services/Import/AccountsImportable.ts

@@ -23,14 +23,6 @@ export class AccountsImportable {
     return this.accountsApp.createAccount(tenantId, createAccountDTO, trx);
   }
 
-  /**
-   *
-   * @returns {}
-   */
-  public validation() {
-    return AccountDTOSchema;
-  }
-
   /**
    *
    * @param data
@@ -39,7 +31,6 @@ export class AccountsImportable {
   public transform(data) {
     return {
       ...data,
-      accountType: this.mapAccountType(data.accounType),
     };
   }
 

packages/server/src/services/Import/ImportFileMapping.ts

@@ -1,12 +1,19 @@
 import { Inject, Service } from 'typedi';
 import HasTenancyService from '../Tenancy/TenancyService';
 import { ImportMappingAttr } from './interfaces';
+import ResourceService from '../Resource/ResourceService';
+import { ServiceError } from '@/exceptions';
+import { ERRORS } from './_utils';
+import { fromPairs } from 'lodash';
 
 @Service()
 export class ImportFileMapping {
   @Inject()
   private tenancy: HasTenancyService;
 
+  @Inject()
+  private resource: ResourceService;
+
   /**
    * Mapping the excel sheet columns with resource columns.
    * @param {number} tenantId
@@ -24,8 +31,11 @@ export class ImportFileMapping {
       .findOne('filename', importId)
       .throwIfNotFound();
 
-    // @todo validate the resource columns.
-    // @todo validate the sheet columns.
+    // Invalidate the from/to map attributes.
+    this.validateMapsAttrs(tenantId, importFile, maps);
+
+    // Validate the diplicated relations of map attrs.
+    this.validateDuplicatedMapAttrs(maps);
 
     const mappingStringified = JSON.stringify(maps);
 
@@ -33,4 +43,60 @@ export class ImportFileMapping {
       mapping: mappingStringified,
     });
   }
+
+  /**
+   * Validate the mapping attributes.
+   * @param {number} tenantId -
+   * @param {} importFile -
+   * @param {ImportMappingAttr[]} maps
+   * @throws {ServiceError(ERRORS.INVALID_MAP_ATTRS)}
+   */
+  private validateMapsAttrs(
+    tenantId: number,
+    importFile: any,
+    maps: ImportMappingAttr[]
+  ) {
+    const fields = this.resource.getResourceImportableFields(
+      tenantId,
+      importFile.resource
+    );
+    const columnsMap = fromPairs(
+      importFile.columnsParsed.map((field) => [field, ''])
+    );
+    const invalid = [];
+
+    maps.forEach((map) => {
+      if (
+        'undefined' === typeof fields[map.to] ||
+        'undefined' === typeof columnsMap[map.from]
+      ) {
+        invalid.push(map);
+      }
+    });
+    if (invalid.length > 0) {
+      throw new ServiceError(ERRORS.INVALID_MAP_ATTRS);
+    }
+  }
+
+  /**
+   * Validate the map attrs relation should be one-to-one relation only.
+   * @param {ImportMappingAttr[]} maps 
+   */
+  private validateDuplicatedMapAttrs(maps: ImportMappingAttr[]) {
+    const fromMap = {};
+    const toMap = {};
+
+    maps.forEach((map) => {
+      if (fromMap[map.from]) {
+        throw new ServiceError(ERRORS.DUPLICATED_FROM_MAP_ATTR);
+      } else {
+        fromMap[map.from] = true;
+      }
+      if (toMap[map.to]) {
+        throw new ServiceError(ERRORS.DUPLICATED_TO_MAP_ATTR);
+      } else {
+        toMap[map.to] = true;
+      }
+    });
+  }
 }

packages/server/src/services/Import/ImportFileProcess.ts

@@ -3,15 +3,16 @@ import * as R from 'ramda';
 import XLSX from 'xlsx';
 import { first, isUndefined } from 'lodash';
 import bluebird from 'bluebird';
+import fs from 'fs/promises';
+import { Knex } from 'knex';
 import HasTenancyService from '../Tenancy/TenancyService';
-import { trimObject } from './_utils';
+import { ERRORS, convertFieldsToYupValidation, trimObject } from './_utils';
 import { ImportMappingAttr, ImportValidationError } from './interfaces';
 import { AccountsImportable } from './AccountsImportable';
-import { plainToInstance } from 'class-transformer';
-import { validate } from 'class-validator';
 import UnitOfWork from '../UnitOfWork';
-import { Knex } from 'knex';
-const fs = require('fs').promises;
+import { ServiceError } from '@/exceptions';
+import ResourceService from '../Resource/ResourceService';
+
 
 @Service()
 export class ImportFileProcess {
@@ -24,6 +25,9 @@ export class ImportFileProcess {
   @Inject()
   private uow: UnitOfWork;
 
+  @Inject()
+  private resourceService: ResourceService;
+
   /**
    * Reads the import file.
    * @param {string} filename
@@ -84,24 +88,30 @@ export class ImportFileProcess {
    * @returns {Promise<ImportValidationError[][]>}
    */
   private async validateData(
+    tenantId: number,
+    resource: string,
     mappedDTOs: Record<string, any>
   ): Promise<ImportValidationError[][]> {
-    const validateData = async (data, index: number) => {
-      const account = { ...data };
-      const accountClass = plainToInstance(
-        this.importable.validation(),
-        account
-      );
-      const errors = await validate(accountClass);
+    const importableFields = this.resourceService.getResourceImportableFields(
+      tenantId,
+      resource
+    );
+    const YupSchema = convertFieldsToYupValidation(importableFields);
 
-      if (errors?.length > 0) {
-        return errors.map((error) => ({
-          index,
-          property: error.property,
-          constraints: error.constraints,
+    const validateData = async (data, index: number) => {
+      const _data = { ...data };
+
+      try {
+        await YupSchema.validate(_data, { abortEarly: false });
+        return { index, data: _data, errors: [] };
+      } catch (validationError) {
+        const errors = validationError.inner.map((error) => ({
+          path: error.params.path,
+          label: error.params.label,
+          message: error.errors,
         }));
+        return { index, data: _data, errors };
       }
-      return false;
     };
     const errors = await bluebird.map(mappedDTOs, validateData, {
       concurrency: 20,
@@ -110,7 +120,7 @@ export class ImportFileProcess {
   }
 
   /**
-   * Transfomees the mapped DTOs.
+   * Transformes the mapped DTOs.
    * @param DTOs
    * @returns
    */
@@ -119,21 +129,22 @@ export class ImportFileProcess {
   }
 
   /**
-   * Process
+   * Processes the import file sheet through the resource service.
    * @param {number} tenantId
    * @param {number} importId
+   * @returns {Promise<void>}
    */
-  public async process(
-    tenantId: number,
-    importId: number,
-    settings = { skipErrors: true }
-  ) {
+  public async process(tenantId: number, importId: number) {
     const { Import } = this.tenancy.models(tenantId);
 
     const importFile = await Import.query()
       .findOne('importId', importId)
       .throwIfNotFound();
 
+    // Throw error if the import file is not mapped yet.
+    if (!importFile.isMapped) {
+      throw new ServiceError(ERRORS.IMPORT_FILE_NOT_MAPPED);
+    }
     const buffer = await this.readImportFile(importFile.filename);
     const jsonData = this.parseXlsxSheet(buffer);
 
@@ -146,15 +157,43 @@ export class ImportFileProcess {
     const transformedDTOs = this.transformDTOs(mappedDTOs);
 
     // Validate the mapped DTOs.
-    const errors = await this.validateData(transformedDTOs);
-
-    return this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => {
+    const rowsWithErrors = await this.validateData(
+      tenantId,
+      importFile.resource,
+      transformedDTOs
+    );
+    // Runs the importing under UOW envirement.
+    await this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => {
       await bluebird.map(
-        transformedDTOs,
-        (transformedDTO) =>
-          this.importable.importable(tenantId, transformedDTO, trx),
+        rowsWithErrors,
+        (rowWithErrors) => {
+          if (rowWithErrors.errors.length === 0) {
+            return this.importable.importable(
+              tenantId,
+              rowWithErrors.data,
+              trx
+            );
+          }
+        },
         { concurrency: 10 }
       );
     });
+    // Deletes the imported file after importing success./
+    await this.deleteImportFile(tenantId, importFile)
+  }
+
+  /**
+   * Deletes the imported file from the storage and database.
+   * @param {number} tenantId 
+   * @param {} importFile 
+   */
+  private async deleteImportFile(tenantId: number, importFile: any) {
+    const { Import } = this.tenancy.models(tenantId);
+
+    // Deletes the import row.
+    await Import.query().findById(importFile.id).delete();
+
+    // Deletes the imported file.
+    await fs.unlink(`public/imports/${importFile.filename}`);
   }
 }

packages/server/src/services/Import/ImportFileUpload.ts

@@ -1,17 +1,22 @@
-import { first } from 'lodash';
+import { first, values } from 'lodash';
 import { Inject, Service } from 'typedi';
-import { snakeCase, upperFirst } from 'lodash';
+import { ServiceError } from '@/exceptions';
 import XLSX from 'xlsx';
 import * as R from 'ramda';
 import HasTenancyService from '../Tenancy/TenancyService';
-import { trimObject } from './_utils';
-const fs = require('fs').promises;
+import { ERRORS, trimObject } from './_utils';
+import ResourceService from '../Resource/ResourceService';
+import fs from 'fs/promises';
+import { IModelMetaField } from '@/interfaces';
 
 @Service()
 export class ImportFileUploadService {
   @Inject()
   private tenancy: HasTenancyService;
 
+  @Inject()
+  private resourceService: ResourceService;
+
   /**
    * Reads the imported file and stores the import file meta under unqiue id.
    * @param {number} tenantId -
@@ -27,31 +32,54 @@ export class ImportFileUploadService {
   ) {
     const { Import } = this.tenancy.models(tenantId);
 
+    const resourceMeta = this.resourceService.getResourceMeta(
+      tenantId,
+      resource
+    );
+    // Throw service error if the resource does not support importing.
+    if (!resourceMeta.importable) {
+      throw new ServiceError(ERRORS.RESOURCE_NOT_IMPORTABLE);
+    }
     const buffer = await fs.readFile(filePath);
     const workbook = XLSX.read(buffer, { type: 'buffer' });
 
     const firstSheetName = workbook.SheetNames[0];
     const worksheet = workbook.Sheets[firstSheetName];
     const jsonData = XLSX.utils.sheet_to_json(worksheet);
-
+    
+    const columns = this.getColumns(jsonData);
+    const coumnsStringified = JSON.stringify(columns);
+    
     // @todo validate the resource.
-    const _resource = upperFirst(snakeCase(resource));
+    const _resource = this.resourceService.resourceToModelName(resource);
 
     const exportFile = await Import.query().insert({
       filename,
       importId: filename,
       resource: _resource,
+      columns: coumnsStringified,
     });
-    const columns = this.getColumns(jsonData);
-
+    const resourceColumns = this.resourceService.getResourceImportableFields(
+      tenantId,
+      resource
+    );
+    const resourceColumnsTransformeed = Object.entries(resourceColumns).map(
+      ([key, { name }]: [string, IModelMetaField]) => ({ key, name })
+    );
     // @todo return the resource importable columns.
     return {
       export: exportFile,
       columns,
+      resourceColumns: resourceColumnsTransformeed,
     };
   }
 
-  private getColumns(json: unknown[]) {
+  /**
+   * Retrieves the sheet columns from the given sheet data.
+   * @param {unknown[]} json
+   * @returns {string[]}
+   */
+  private getColumns(json: unknown[]): string[] {
     return R.compose(Object.keys, trimObject, first)(json);
   }
 }

packages/server/src/services/Import/ImportResourceApplication.ts

@@ -65,10 +65,10 @@ export class ImportResourceApplication {
   }
 
   /**
-   *
+   * Process the import file sheet through service for creating entities.
    * @param {number} tenantId
    * @param {number} importId
-   * @returns
+   * @returns {Promise<void>}
    */
   public async process(tenantId: number, importId: number) {
     return this.importProcessService.process(tenantId, importId);

packages/server/src/services/Import/_utils.ts

@@ -1,3 +1,5 @@
+import * as Yup from 'yup';
+
 export function trimObject(obj) {
   return Object.entries(obj).reduce((acc, [key, value]) => {
     // Trim the key
@@ -10,3 +12,50 @@ export function trimObject(obj) {
     return { ...acc, [trimmedKey]: trimmedValue };
   }, {});
 }
+
+export const convertFieldsToYupValidation = (fields: any) => {
+  const yupSchema = {};
+  Object.keys(fields).forEach((fieldName) => {
+    const field = fields[fieldName];
+    let fieldSchema;
+    fieldSchema = Yup.string().label(field.name);
+
+    if (field.fieldType === 'text') {
+      if (field.minLength) {
+        fieldSchema = fieldSchema.min(
+          field.minLength,
+          `Minimum length is ${field.minLength} characters`
+        );
+      }
+      if (field.maxLength) {
+        fieldSchema = fieldSchema.max(
+          field.maxLength,
+          `Maximum length is ${field.maxLength} characters`
+        );
+      }
+    } else if (field.fieldType === 'number') {
+      fieldSchema = Yup.number().label(field.name);
+    } else if (field.fieldType === 'boolean') {
+      fieldSchema = Yup.boolean().label(field.name);
+    } else if (field.fieldType === 'enumeration') {
+      const options = field.options.reduce((acc, option) => {
+        acc[option.key] = option.label;
+        return acc;
+      }, {});
+      fieldSchema = Yup.string().oneOf(Object.keys(options)).label(field.name);
+    }
+    if (field.required) {
+      fieldSchema = fieldSchema.required();
+    }
+    yupSchema[fieldName] = fieldSchema;
+  });
+  return Yup.object().shape(yupSchema);
+};
+
+export const ERRORS = {
+  RESOURCE_NOT_IMPORTABLE: 'RESOURCE_NOT_IMPORTABLE',
+  INVALID_MAP_ATTRS: 'INVALID_MAP_ATTRS',
+  DUPLICATED_FROM_MAP_ATTR: 'DUPLICATED_FROM_MAP_ATTR',
+  DUPLICATED_TO_MAP_ATTR: 'DUPLICATED_TO_MAP_ATTR',
+  IMPORT_FILE_NOT_MAPPED: 'IMPORT_FILE_NOT_MAPPED',
+};