feat(server): wip import resources

packages/server/package.json

@@ -25,6 +25,7 @@
     "@types/i18n": "^0.8.7",
     "@types/knex": "^0.16.1",
     "@types/mathjs": "^6.0.12",
+    "@types/yup": "^0.29.13",
     "accepts": "^1.3.7",
     "accounting": "^0.4.1",
     "agenda": "^4.2.1",
@@ -108,7 +109,8 @@
     "typedi": "^0.8.0",
     "uniqid": "^5.2.0",
     "winston": "^3.2.1",
-    "xlsx": "^0.18.5"
+    "xlsx": "^0.18.5",
+    "yup": "^0.28.1"
   },
   "devDependencies": {
     "@types/lodash": "^4.14.158",

packages/server/src/api/controllers/Import/ImportController.ts

@@ -169,6 +169,21 @@ export class ImportController extends BaseController {
     next: NextFunction
   ) {
     if (error instanceof ServiceError) {
+      if (error.errorType === 'INVALID_MAP_ATTRS') {
+        return res.status(400).send({
+          errors: [{ type: 'INVALID_MAP_ATTRS' }]
+        });
+      }
+      if (error.errorType === 'DUPLICATED_FROM_MAP_ATTR') {
+        return res.status(400).send({
+          errors: [{ type: 'DUPLICATED_FROM_MAP_ATTR' }],
+        });
+      };
+      if (error.errorType === 'DUPLICATED_TO_MAP_ATTR') {
+        return res.status(400).send({
+          errors: [{ type: 'DUPLICATED_TO_MAP_ATTR' }],
+        })
+      }
     }
     next(error);
   }

packages/server/src/database/migrations/20231209230719_create_imports_table.js

@@ -4,6 +4,7 @@ exports.up = function (knex) {
     table.string('filename');
     table.string('import_id');
     table.string('resource');
+    table.json('columns');
     table.json('mapping');
     table.timestamps();
   });

packages/server/src/interfaces/Model.ts

@@ -77,5 +77,6 @@ export type IModelMetaRelationField = IModelMetaRelationFieldCommon & (
 export interface IModelMeta {
   defaultFilterField: string;
   defaultSort: IModelMetaDefaultSort;
+  importable?: boolean;
   fields: { [key: string]: IModelMetaField };
 }

packages/server/src/models/Account.Settings.ts

@@ -6,16 +6,21 @@ export default {
     sortOrder: 'DESC',
     sortField: 'name',
   },
+  importable: true,
   fields: {
     name: {
       name: 'account.field.name',
       column: 'name',
       fieldType: 'text',
+      unique: true,
+      required: true,
+      importable: true,
     },
     description: {
       name: 'account.field.description',
       column: 'description',
       fieldType: 'text',
+      importable: true,
     },
     slug: {
       name: 'account.field.slug',
@@ -23,13 +28,17 @@ export default {
       fieldType: 'text',
       columnable: false,
       filterable: false,
+      importable: false,
     },
     code: {
       name: 'account.field.code',
       column: 'code',
       fieldType: 'text',
+      importable: true,
+      minLength: 3,
+      maxLength: 6,
     },
-    root_type: {
+    rootType: {
       name: 'account.field.root_type',
       fieldType: 'enumeration',
       options: [
@@ -41,6 +50,7 @@ export default {
       ],
       filterCustomQuery: RootTypeFieldFilterQuery,
       sortable: false,
+      importable: false,
     },
     normal: {
       name: 'account.field.normal',
@@ -51,6 +61,7 @@ export default {
       ],
       filterCustomQuery: NormalTypeFieldFilterQuery,
       sortable: false,
+      importable: false,
     },
     type: {
       name: 'account.field.type',
@@ -58,30 +69,42 @@ export default {
       fieldType: 'enumeration',
       options: ACCOUNT_TYPES.map((accountType) => ({
         label: accountType.label,
-        key: accountType.key
+        key: accountType.key,
       })),
+      required: true,
+      importable: true,
     },
     active: {
       name: 'account.field.active',
       column: 'active',
       fieldType: 'boolean',
       filterable: false,
+      importable: true,
     },
-    balance: {
+    openingBalance: {
       name: 'account.field.balance',
       column: 'amount',
       fieldType: 'number',
+      importable: true,
     },
-    currency: {
+    currencyCode: {
       name: 'account.field.currency',
       column: 'currency_code',
       fieldType: 'text',
       filterable: false,
+      importable: true,
     },
-    created_at: {
+    parentAccount: {
+      name: 'account.field.parent_account',
+      column: 'parent_account_id',
+      fieldType: 'relation',
+      to: { model: 'Account', to: 'id' },
+    },
+    createdAt: {
       name: 'account.field.created_at',
       column: 'created_at',
       fieldType: 'date',
+      importable: false,
     },
   },
 };

packages/server/src/models/Import.ts

@@ -31,6 +31,18 @@ export default class Import extends TenantModel {
     return {};
   }
 
+  public get isMapped() {
+    return Boolean(this.mapping);
+  }
+
+  public get columnsParsed() {
+    try {
+      return JSON.parse(this.columns);
+    } catch {
+      return [];
+    }
+  }
+
   public get mappingParsed() {
     try {
       return JSON.parse(this.mapping);

packages/server/src/services/Import/AccountsImportable.ts

@@ -23,14 +23,6 @@ export class AccountsImportable {
     return this.accountsApp.createAccount(tenantId, createAccountDTO, trx);
   }
 
-  /**
-   *
-   * @returns {}
-   */
-  public validation() {
-    return AccountDTOSchema;
-  }
-
   /**
    *
    * @param data
@@ -39,7 +31,6 @@ export class AccountsImportable {
   public transform(data) {
     return {
       ...data,
-      accountType: this.mapAccountType(data.accounType),
     };
   }
 

packages/server/src/services/Import/ImportFileMapping.ts

@@ -1,12 +1,19 @@
 import { Inject, Service } from 'typedi';
 import HasTenancyService from '../Tenancy/TenancyService';
 import { ImportMappingAttr } from './interfaces';
+import ResourceService from '../Resource/ResourceService';
+import { ServiceError } from '@/exceptions';
+import { ERRORS } from './_utils';
+import { fromPairs } from 'lodash';
 
 @Service()
 export class ImportFileMapping {
   @Inject()
   private tenancy: HasTenancyService;
 
+  @Inject()
+  private resource: ResourceService;
+
   /**
    * Mapping the excel sheet columns with resource columns.
    * @param {number} tenantId
@@ -24,8 +31,11 @@ export class ImportFileMapping {
       .findOne('filename', importId)
       .throwIfNotFound();
 
-    // @todo validate the resource columns.
+    // Invalidate the from/to map attributes.
-    // @todo validate the sheet columns.
+    this.validateMapsAttrs(tenantId, importFile, maps);
+
+    // Validate the diplicated relations of map attrs.
+    this.validateDuplicatedMapAttrs(maps);
 
     const mappingStringified = JSON.stringify(maps);
 
@@ -33,4 +43,60 @@ export class ImportFileMapping {
       mapping: mappingStringified,
     });
   }
+
+  /**
+   * Validate the mapping attributes.
+   * @param {number} tenantId -
+   * @param {} importFile -
+   * @param {ImportMappingAttr[]} maps
+   * @throws {ServiceError(ERRORS.INVALID_MAP_ATTRS)}
+   */
+  private validateMapsAttrs(
+    tenantId: number,
+    importFile: any,
+    maps: ImportMappingAttr[]
+  ) {
+    const fields = this.resource.getResourceImportableFields(
+      tenantId,
+      importFile.resource
+    );
+    const columnsMap = fromPairs(
+      importFile.columnsParsed.map((field) => [field, ''])
+    );
+    const invalid = [];
+
+    maps.forEach((map) => {
+      if (
+        'undefined' === typeof fields[map.to] ||
+        'undefined' === typeof columnsMap[map.from]
+      ) {
+        invalid.push(map);
+      }
+    });
+    if (invalid.length > 0) {
+      throw new ServiceError(ERRORS.INVALID_MAP_ATTRS);
+    }
+  }
+
+  /**
+   * Validate the map attrs relation should be one-to-one relation only.
+   * @param {ImportMappingAttr[]} maps 
+   */
+  private validateDuplicatedMapAttrs(maps: ImportMappingAttr[]) {
+    const fromMap = {};
+    const toMap = {};
+
+    maps.forEach((map) => {
+      if (fromMap[map.from]) {
+        throw new ServiceError(ERRORS.DUPLICATED_FROM_MAP_ATTR);
+      } else {
+        fromMap[map.from] = true;
+      }
+      if (toMap[map.to]) {
+        throw new ServiceError(ERRORS.DUPLICATED_TO_MAP_ATTR);
+      } else {
+        toMap[map.to] = true;
+      }
+    });
+  }
 }

packages/server/src/services/Import/ImportFileProcess.ts

@@ -3,15 +3,16 @@ import * as R from 'ramda';
 import XLSX from 'xlsx';
 import { first, isUndefined } from 'lodash';
 import bluebird from 'bluebird';
+import fs from 'fs/promises';
+import { Knex } from 'knex';
 import HasTenancyService from '../Tenancy/TenancyService';
-import { trimObject } from './_utils';
+import { ERRORS, convertFieldsToYupValidation, trimObject } from './_utils';
 import { ImportMappingAttr, ImportValidationError } from './interfaces';
 import { AccountsImportable } from './AccountsImportable';
-import { plainToInstance } from 'class-transformer';
-import { validate } from 'class-validator';
 import UnitOfWork from '../UnitOfWork';
-import { Knex } from 'knex';
+import { ServiceError } from '@/exceptions';
-const fs = require('fs').promises;
+import ResourceService from '../Resource/ResourceService';
+
 
 @Service()
 export class ImportFileProcess {
@@ -24,6 +25,9 @@ export class ImportFileProcess {
   @Inject()
   private uow: UnitOfWork;
 
+  @Inject()
+  private resourceService: ResourceService;
+
   /**
    * Reads the import file.
    * @param {string} filename
@@ -84,24 +88,30 @@ export class ImportFileProcess {
    * @returns {Promise<ImportValidationError[][]>}
    */
   private async validateData(
+    tenantId: number,
+    resource: string,
     mappedDTOs: Record<string, any>
   ): Promise<ImportValidationError[][]> {
-    const validateData = async (data, index: number) => {
+    const importableFields = this.resourceService.getResourceImportableFields(
-      const account = { ...data };
+      tenantId,
-      const accountClass = plainToInstance(
+      resource
-        this.importable.validation(),
-        account
     );
-      const errors = await validate(accountClass);
+    const YupSchema = convertFieldsToYupValidation(importableFields);
 
-      if (errors?.length > 0) {
+    const validateData = async (data, index: number) => {
-        return errors.map((error) => ({
+      const _data = { ...data };
-          index,
+
-          property: error.property,
+      try {
-          constraints: error.constraints,
+        await YupSchema.validate(_data, { abortEarly: false });
+        return { index, data: _data, errors: [] };
+      } catch (validationError) {
+        const errors = validationError.inner.map((error) => ({
+          path: error.params.path,
+          label: error.params.label,
+          message: error.errors,
         }));
+        return { index, data: _data, errors };
       }
-      return false;
     };
     const errors = await bluebird.map(mappedDTOs, validateData, {
       concurrency: 20,
@@ -110,7 +120,7 @@ export class ImportFileProcess {
   }
 
   /**
-   * Transfomees the mapped DTOs.
+   * Transformes the mapped DTOs.
    * @param DTOs
    * @returns
    */
@@ -119,21 +129,22 @@ export class ImportFileProcess {
   }
 
   /**
-   * Process
+   * Processes the import file sheet through the resource service.
    * @param {number} tenantId
    * @param {number} importId
+   * @returns {Promise<void>}
    */
-  public async process(
+  public async process(tenantId: number, importId: number) {
-    tenantId: number,
-    importId: number,
-    settings = { skipErrors: true }
-  ) {
     const { Import } = this.tenancy.models(tenantId);
 
     const importFile = await Import.query()
       .findOne('importId', importId)
       .throwIfNotFound();
 
+    // Throw error if the import file is not mapped yet.
+    if (!importFile.isMapped) {
+      throw new ServiceError(ERRORS.IMPORT_FILE_NOT_MAPPED);
+    }
     const buffer = await this.readImportFile(importFile.filename);
     const jsonData = this.parseXlsxSheet(buffer);
 
@@ -146,15 +157,43 @@ export class ImportFileProcess {
     const transformedDTOs = this.transformDTOs(mappedDTOs);
 
     // Validate the mapped DTOs.
-    const errors = await this.validateData(transformedDTOs);
+    const rowsWithErrors = await this.validateData(
-
+      tenantId,
-    return this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => {
+      importFile.resource,
+      transformedDTOs
+    );
+    // Runs the importing under UOW envirement.
+    await this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => {
       await bluebird.map(
-        transformedDTOs,
+        rowsWithErrors,
-        (transformedDTO) =>
+        (rowWithErrors) => {
-          this.importable.importable(tenantId, transformedDTO, trx),
+          if (rowWithErrors.errors.length === 0) {
+            return this.importable.importable(
+              tenantId,
+              rowWithErrors.data,
+              trx
+            );
+          }
+        },
         { concurrency: 10 }
       );
     });
+    // Deletes the imported file after importing success./
+    await this.deleteImportFile(tenantId, importFile)
+  }
+
+  /**
+   * Deletes the imported file from the storage and database.
+   * @param {number} tenantId 
+   * @param {} importFile 
+   */
+  private async deleteImportFile(tenantId: number, importFile: any) {
+    const { Import } = this.tenancy.models(tenantId);
+
+    // Deletes the import row.
+    await Import.query().findById(importFile.id).delete();
+
+    // Deletes the imported file.
+    await fs.unlink(`public/imports/${importFile.filename}`);
   }
 }

packages/server/src/services/Import/ImportFileUpload.ts

@@ -1,17 +1,22 @@
-import { first } from 'lodash';
+import { first, values } from 'lodash';
 import { Inject, Service } from 'typedi';
-import { snakeCase, upperFirst } from 'lodash';
+import { ServiceError } from '@/exceptions';
 import XLSX from 'xlsx';
 import * as R from 'ramda';
 import HasTenancyService from '../Tenancy/TenancyService';
-import { trimObject } from './_utils';
+import { ERRORS, trimObject } from './_utils';
-const fs = require('fs').promises;
+import ResourceService from '../Resource/ResourceService';
+import fs from 'fs/promises';
+import { IModelMetaField } from '@/interfaces';
 
 @Service()
 export class ImportFileUploadService {
   @Inject()
   private tenancy: HasTenancyService;
 
+  @Inject()
+  private resourceService: ResourceService;
+
   /**
    * Reads the imported file and stores the import file meta under unqiue id.
    * @param {number} tenantId -
@@ -27,6 +32,14 @@ export class ImportFileUploadService {
   ) {
     const { Import } = this.tenancy.models(tenantId);
 
+    const resourceMeta = this.resourceService.getResourceMeta(
+      tenantId,
+      resource
+    );
+    // Throw service error if the resource does not support importing.
+    if (!resourceMeta.importable) {
+      throw new ServiceError(ERRORS.RESOURCE_NOT_IMPORTABLE);
+    }
     const buffer = await fs.readFile(filePath);
     const workbook = XLSX.read(buffer, { type: 'buffer' });
 
@@ -34,24 +47,39 @@ export class ImportFileUploadService {
     const worksheet = workbook.Sheets[firstSheetName];
     const jsonData = XLSX.utils.sheet_to_json(worksheet);
     
+    const columns = this.getColumns(jsonData);
+    const coumnsStringified = JSON.stringify(columns);
+    
     // @todo validate the resource.
-    const _resource = upperFirst(snakeCase(resource));
+    const _resource = this.resourceService.resourceToModelName(resource);
 
     const exportFile = await Import.query().insert({
       filename,
       importId: filename,
       resource: _resource,
+      columns: coumnsStringified,
     });
-    const columns = this.getColumns(jsonData);
+    const resourceColumns = this.resourceService.getResourceImportableFields(
-
+      tenantId,
+      resource
+    );
+    const resourceColumnsTransformeed = Object.entries(resourceColumns).map(
+      ([key, { name }]: [string, IModelMetaField]) => ({ key, name })
+    );
     // @todo return the resource importable columns.
     return {
       export: exportFile,
       columns,
+      resourceColumns: resourceColumnsTransformeed,
     };
   }
 
-  private getColumns(json: unknown[]) {
+  /**
+   * Retrieves the sheet columns from the given sheet data.
+   * @param {unknown[]} json
+   * @returns {string[]}
+   */
+  private getColumns(json: unknown[]): string[] {
     return R.compose(Object.keys, trimObject, first)(json);
   }
 }

packages/server/src/services/Import/ImportResourceApplication.ts

@@ -65,10 +65,10 @@ export class ImportResourceApplication {
   }
 
   /**
-   *
+   * Process the import file sheet through service for creating entities.
    * @param {number} tenantId
    * @param {number} importId
-   * @returns
+   * @returns {Promise<void>}
    */
   public async process(tenantId: number, importId: number) {
     return this.importProcessService.process(tenantId, importId);

packages/server/src/services/Import/_utils.ts

@@ -1,3 +1,5 @@
+import * as Yup from 'yup';
+
 export function trimObject(obj) {
   return Object.entries(obj).reduce((acc, [key, value]) => {
     // Trim the key
@@ -10,3 +12,50 @@ export function trimObject(obj) {
     return { ...acc, [trimmedKey]: trimmedValue };
   }, {});
 }
+
+export const convertFieldsToYupValidation = (fields: any) => {
+  const yupSchema = {};
+  Object.keys(fields).forEach((fieldName) => {
+    const field = fields[fieldName];
+    let fieldSchema;
+    fieldSchema = Yup.string().label(field.name);
+
+    if (field.fieldType === 'text') {
+      if (field.minLength) {
+        fieldSchema = fieldSchema.min(
+          field.minLength,
+          `Minimum length is ${field.minLength} characters`
+        );
+      }
+      if (field.maxLength) {
+        fieldSchema = fieldSchema.max(
+          field.maxLength,
+          `Maximum length is ${field.maxLength} characters`
+        );
+      }
+    } else if (field.fieldType === 'number') {
+      fieldSchema = Yup.number().label(field.name);
+    } else if (field.fieldType === 'boolean') {
+      fieldSchema = Yup.boolean().label(field.name);
+    } else if (field.fieldType === 'enumeration') {
+      const options = field.options.reduce((acc, option) => {
+        acc[option.key] = option.label;
+        return acc;
+      }, {});
+      fieldSchema = Yup.string().oneOf(Object.keys(options)).label(field.name);
+    }
+    if (field.required) {
+      fieldSchema = fieldSchema.required();
+    }
+    yupSchema[fieldName] = fieldSchema;
+  });
+  return Yup.object().shape(yupSchema);
+};
+
+export const ERRORS = {
+  RESOURCE_NOT_IMPORTABLE: 'RESOURCE_NOT_IMPORTABLE',
+  INVALID_MAP_ATTRS: 'INVALID_MAP_ATTRS',
+  DUPLICATED_FROM_MAP_ATTR: 'DUPLICATED_FROM_MAP_ATTR',
+  DUPLICATED_TO_MAP_ATTR: 'DUPLICATED_TO_MAP_ATTR',
+  IMPORT_FILE_NOT_MAPPED: 'IMPORT_FILE_NOT_MAPPED',
+};

packages/server/src/services/Organization/OrganizationService.ts

@@ -21,25 +21,19 @@ import { ERRORS } from './constants';
 @Service()
 export default class OrganizationService {
   @Inject()
-  eventPublisher: EventPublisher;
+  private eventPublisher: EventPublisher;
-
-  @Inject('logger')
-  logger: any;
-
-  @Inject('repositories')
-  sysRepositories: any;
 
   @Inject()
-  tenantsManager: TenantsManager;
+  private tenantsManager: TenantsManager;
 
   @Inject('agenda')
-  agenda: any;
+  private agenda: any;
 
   @Inject()
-  baseCurrencyMutateLocking: OrganizationBaseCurrencyLocking;
+  private baseCurrencyMutateLocking: OrganizationBaseCurrencyLocking;
 
   @Inject()
-  tenancy: HasTenancyService;
+  private tenancy: HasTenancyService;
 
   /**
    * Builds the database schema and seed data of the given organization id.

packages/server/src/services/Resource/ResourceService.ts

@@ -1,12 +1,11 @@
 import { Service, Inject } from 'typedi';
-import { camelCase, upperFirst } from 'lodash';
+import { camelCase, upperFirst, pickBy } from 'lodash';
 import * as qim from 'qim';
 import pluralize from 'pluralize';
-import { IModelMeta } from '@/interfaces';
+import { IModelMeta, IModelMetaField } from '@/interfaces';
 import TenancyService from '@/services/Tenancy/TenancyService';
 import { ServiceError } from '@/exceptions';
 import I18nService from '@/services/I18n/I18nService';
-import { tenantKnexConfig } from 'config/knexConfig';
 
 const ERRORS = {
   RESOURCE_MODEL_NOT_FOUND: 'RESOURCE_MODEL_NOT_FOUND',
@@ -24,7 +23,7 @@ export default class ResourceService {
    * Transform resource to model name.
    * @param {string} resourceName
    */
-  private resourceToModelName(resourceName: string): string {
+  public resourceToModelName(resourceName: string): string {
     return upperFirst(camelCase(pluralize.singular(resourceName)));
   }
 
@@ -63,6 +62,33 @@ export default class ResourceService {
     return this.getResourceMetaLocalized(resourceMeta, tenantId);
   }
 
+  /**
+   *
+   */
+  public getResourceFields(
+    tenantId: number,
+    modelName: string
+  ): { [key: string]: IModelMetaField } {
+    const meta = this.getResourceMeta(tenantId, modelName);
+
+    return meta.fields;
+  }
+
+  /**
+   * 
+   * @param {number} tenantId 
+   * @param {string} modelName 
+   * @returns 
+   */
+  public getResourceImportableFields(
+    tenantId: number,
+    modelName: string
+  ): { [key: string]: IModelMetaField } {
+    const fields = this.getResourceFields(tenantId, modelName);
+
+    return pickBy(fields, (field) => field.importable);
+  }
+
   /**
    * Retrieve the resource meta localized based on the current user language.
    */

packages/server/src/services/UnitOfWork/index.ts

@@ -45,7 +45,7 @@ export default class UnitOfWork {
     const knex = this.tenancy.knex(tenantId);
     let _trx = trx;
 
-    if (_trx) {
+    if (!_trx) {
       _trx = await knex.transaction({ isolationLevel });
     }
     try {

pnpm-lock.yaml

@@ -47,6 +47,9 @@ importers:
       '@types/mathjs':
         specifier: ^6.0.12
         version: 6.0.12
+      '@types/yup':
+        specifier: ^0.29.13
+        version: 0.29.14
       accepts:
         specifier: ^1.3.7
         version: 1.3.8
@@ -299,6 +302,9 @@ importers:
       xlsx:
         specifier: ^0.18.5
         version: 0.18.5
+      yup:
+        specifier: ^0.28.1
+        version: 0.28.5
     devDependencies:
       '@types/lodash':
         specifier: ^4.14.158