diff --git a/packages/server/package.json b/packages/server/package.json index ef260cd19..a117eb5f6 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -25,6 +25,7 @@ "@types/i18n": "^0.8.7", "@types/knex": "^0.16.1", "@types/mathjs": "^6.0.12", + "@types/yup": "^0.29.13", "accepts": "^1.3.7", "accounting": "^0.4.1", "agenda": "^4.2.1", @@ -108,7 +109,8 @@ "typedi": "^0.8.0", "uniqid": "^5.2.0", "winston": "^3.2.1", - "xlsx": "^0.18.5" + "xlsx": "^0.18.5", + "yup": "^0.28.1" }, "devDependencies": { "@types/lodash": "^4.14.158", diff --git a/packages/server/src/api/controllers/Import/ImportController.ts b/packages/server/src/api/controllers/Import/ImportController.ts index 2d7e10dd3..4461f16d8 100644 --- a/packages/server/src/api/controllers/Import/ImportController.ts +++ b/packages/server/src/api/controllers/Import/ImportController.ts @@ -169,6 +169,21 @@ export class ImportController extends BaseController { next: NextFunction ) { if (error instanceof ServiceError) { + if (error.errorType === 'INVALID_MAP_ATTRS') { + return res.status(400).send({ + errors: [{ type: 'INVALID_MAP_ATTRS' }] + }); + } + if (error.errorType === 'DUPLICATED_FROM_MAP_ATTR') { + return res.status(400).send({ + errors: [{ type: 'DUPLICATED_FROM_MAP_ATTR' }], + }); + }; + if (error.errorType === 'DUPLICATED_TO_MAP_ATTR') { + return res.status(400).send({ + errors: [{ type: 'DUPLICATED_TO_MAP_ATTR' }], + }) + } } next(error); } diff --git a/packages/server/src/database/migrations/20231209230719_create_imports_table.js b/packages/server/src/database/migrations/20231209230719_create_imports_table.js index 69d0a0f7e..60fd5a83d 100644 --- a/packages/server/src/database/migrations/20231209230719_create_imports_table.js +++ b/packages/server/src/database/migrations/20231209230719_create_imports_table.js @@ -4,6 +4,7 @@ exports.up = function (knex) { table.string('filename'); table.string('import_id'); table.string('resource'); + table.json('columns'); table.json('mapping'); table.timestamps(); }); diff --git a/packages/server/src/interfaces/Model.ts b/packages/server/src/interfaces/Model.ts index 67b90e872..2bf8d6916 100644 --- a/packages/server/src/interfaces/Model.ts +++ b/packages/server/src/interfaces/Model.ts @@ -77,5 +77,6 @@ export type IModelMetaRelationField = IModelMetaRelationFieldCommon & ( export interface IModelMeta { defaultFilterField: string; defaultSort: IModelMetaDefaultSort; + importable?: boolean; fields: { [key: string]: IModelMetaField }; } diff --git a/packages/server/src/models/Account.Settings.ts b/packages/server/src/models/Account.Settings.ts index 3d0698d0d..7cba91bfd 100644 --- a/packages/server/src/models/Account.Settings.ts +++ b/packages/server/src/models/Account.Settings.ts @@ -6,16 +6,21 @@ export default { sortOrder: 'DESC', sortField: 'name', }, + importable: true, fields: { name: { name: 'account.field.name', column: 'name', fieldType: 'text', + unique: true, + required: true, + importable: true, }, description: { name: 'account.field.description', column: 'description', fieldType: 'text', + importable: true, }, slug: { name: 'account.field.slug', @@ -23,13 +28,17 @@ export default { fieldType: 'text', columnable: false, filterable: false, + importable: false, }, code: { name: 'account.field.code', column: 'code', fieldType: 'text', + importable: true, + minLength: 3, + maxLength: 6, }, - root_type: { + rootType: { name: 'account.field.root_type', fieldType: 'enumeration', options: [ @@ -41,6 +50,7 @@ export default { ], filterCustomQuery: RootTypeFieldFilterQuery, sortable: false, + importable: false, }, normal: { name: 'account.field.normal', @@ -51,6 +61,7 @@ export default { ], filterCustomQuery: NormalTypeFieldFilterQuery, sortable: false, + importable: false, }, type: { name: 'account.field.type', @@ -58,30 +69,42 @@ export default { fieldType: 'enumeration', options: ACCOUNT_TYPES.map((accountType) => ({ label: accountType.label, - key: accountType.key + key: accountType.key, })), + required: true, + importable: true, }, active: { name: 'account.field.active', column: 'active', fieldType: 'boolean', filterable: false, + importable: true, }, - balance: { + openingBalance: { name: 'account.field.balance', column: 'amount', fieldType: 'number', + importable: true, }, - currency: { + currencyCode: { name: 'account.field.currency', column: 'currency_code', fieldType: 'text', filterable: false, + importable: true, }, - created_at: { + parentAccount: { + name: 'account.field.parent_account', + column: 'parent_account_id', + fieldType: 'relation', + to: { model: 'Account', to: 'id' }, + }, + createdAt: { name: 'account.field.created_at', column: 'created_at', fieldType: 'date', + importable: false, }, }, }; diff --git a/packages/server/src/models/Import.ts b/packages/server/src/models/Import.ts index 7b1e9a736..1181071af 100644 --- a/packages/server/src/models/Import.ts +++ b/packages/server/src/models/Import.ts @@ -31,6 +31,18 @@ export default class Import extends TenantModel { return {}; } + public get isMapped() { + return Boolean(this.mapping); + } + + public get columnsParsed() { + try { + return JSON.parse(this.columns); + } catch { + return []; + } + } + public get mappingParsed() { try { return JSON.parse(this.mapping); diff --git a/packages/server/src/services/Import/AccountsImportable.ts b/packages/server/src/services/Import/AccountsImportable.ts index fe4bfdcf0..c02eb2950 100644 --- a/packages/server/src/services/Import/AccountsImportable.ts +++ b/packages/server/src/services/Import/AccountsImportable.ts @@ -23,14 +23,6 @@ export class AccountsImportable { return this.accountsApp.createAccount(tenantId, createAccountDTO, trx); } - /** - * - * @returns {} - */ - public validation() { - return AccountDTOSchema; - } - /** * * @param data @@ -39,7 +31,6 @@ export class AccountsImportable { public transform(data) { return { ...data, - accountType: this.mapAccountType(data.accounType), }; } diff --git a/packages/server/src/services/Import/ImportFileMapping.ts b/packages/server/src/services/Import/ImportFileMapping.ts index 0408f7435..285aacbd5 100644 --- a/packages/server/src/services/Import/ImportFileMapping.ts +++ b/packages/server/src/services/Import/ImportFileMapping.ts @@ -1,12 +1,19 @@ import { Inject, Service } from 'typedi'; import HasTenancyService from '../Tenancy/TenancyService'; import { ImportMappingAttr } from './interfaces'; +import ResourceService from '../Resource/ResourceService'; +import { ServiceError } from '@/exceptions'; +import { ERRORS } from './_utils'; +import { fromPairs } from 'lodash'; @Service() export class ImportFileMapping { @Inject() private tenancy: HasTenancyService; + @Inject() + private resource: ResourceService; + /** * Mapping the excel sheet columns with resource columns. * @param {number} tenantId @@ -24,8 +31,11 @@ export class ImportFileMapping { .findOne('filename', importId) .throwIfNotFound(); - // @todo validate the resource columns. - // @todo validate the sheet columns. + // Invalidate the from/to map attributes. + this.validateMapsAttrs(tenantId, importFile, maps); + + // Validate the diplicated relations of map attrs. + this.validateDuplicatedMapAttrs(maps); const mappingStringified = JSON.stringify(maps); @@ -33,4 +43,60 @@ export class ImportFileMapping { mapping: mappingStringified, }); } + + /** + * Validate the mapping attributes. + * @param {number} tenantId - + * @param {} importFile - + * @param {ImportMappingAttr[]} maps + * @throws {ServiceError(ERRORS.INVALID_MAP_ATTRS)} + */ + private validateMapsAttrs( + tenantId: number, + importFile: any, + maps: ImportMappingAttr[] + ) { + const fields = this.resource.getResourceImportableFields( + tenantId, + importFile.resource + ); + const columnsMap = fromPairs( + importFile.columnsParsed.map((field) => [field, '']) + ); + const invalid = []; + + maps.forEach((map) => { + if ( + 'undefined' === typeof fields[map.to] || + 'undefined' === typeof columnsMap[map.from] + ) { + invalid.push(map); + } + }); + if (invalid.length > 0) { + throw new ServiceError(ERRORS.INVALID_MAP_ATTRS); + } + } + + /** + * Validate the map attrs relation should be one-to-one relation only. + * @param {ImportMappingAttr[]} maps + */ + private validateDuplicatedMapAttrs(maps: ImportMappingAttr[]) { + const fromMap = {}; + const toMap = {}; + + maps.forEach((map) => { + if (fromMap[map.from]) { + throw new ServiceError(ERRORS.DUPLICATED_FROM_MAP_ATTR); + } else { + fromMap[map.from] = true; + } + if (toMap[map.to]) { + throw new ServiceError(ERRORS.DUPLICATED_TO_MAP_ATTR); + } else { + toMap[map.to] = true; + } + }); + } } diff --git a/packages/server/src/services/Import/ImportFileProcess.ts b/packages/server/src/services/Import/ImportFileProcess.ts index cad785869..23b040c6e 100644 --- a/packages/server/src/services/Import/ImportFileProcess.ts +++ b/packages/server/src/services/Import/ImportFileProcess.ts @@ -3,15 +3,16 @@ import * as R from 'ramda'; import XLSX from 'xlsx'; import { first, isUndefined } from 'lodash'; import bluebird from 'bluebird'; +import fs from 'fs/promises'; +import { Knex } from 'knex'; import HasTenancyService from '../Tenancy/TenancyService'; -import { trimObject } from './_utils'; +import { ERRORS, convertFieldsToYupValidation, trimObject } from './_utils'; import { ImportMappingAttr, ImportValidationError } from './interfaces'; import { AccountsImportable } from './AccountsImportable'; -import { plainToInstance } from 'class-transformer'; -import { validate } from 'class-validator'; import UnitOfWork from '../UnitOfWork'; -import { Knex } from 'knex'; -const fs = require('fs').promises; +import { ServiceError } from '@/exceptions'; +import ResourceService from '../Resource/ResourceService'; + @Service() export class ImportFileProcess { @@ -24,6 +25,9 @@ export class ImportFileProcess { @Inject() private uow: UnitOfWork; + @Inject() + private resourceService: ResourceService; + /** * Reads the import file. * @param {string} filename @@ -84,24 +88,30 @@ export class ImportFileProcess { * @returns {Promise} */ private async validateData( + tenantId: number, + resource: string, mappedDTOs: Record ): Promise { - const validateData = async (data, index: number) => { - const account = { ...data }; - const accountClass = plainToInstance( - this.importable.validation(), - account - ); - const errors = await validate(accountClass); + const importableFields = this.resourceService.getResourceImportableFields( + tenantId, + resource + ); + const YupSchema = convertFieldsToYupValidation(importableFields); - if (errors?.length > 0) { - return errors.map((error) => ({ - index, - property: error.property, - constraints: error.constraints, + const validateData = async (data, index: number) => { + const _data = { ...data }; + + try { + await YupSchema.validate(_data, { abortEarly: false }); + return { index, data: _data, errors: [] }; + } catch (validationError) { + const errors = validationError.inner.map((error) => ({ + path: error.params.path, + label: error.params.label, + message: error.errors, })); + return { index, data: _data, errors }; } - return false; }; const errors = await bluebird.map(mappedDTOs, validateData, { concurrency: 20, @@ -110,7 +120,7 @@ export class ImportFileProcess { } /** - * Transfomees the mapped DTOs. + * Transformes the mapped DTOs. * @param DTOs * @returns */ @@ -119,21 +129,22 @@ export class ImportFileProcess { } /** - * Process + * Processes the import file sheet through the resource service. * @param {number} tenantId * @param {number} importId + * @returns {Promise} */ - public async process( - tenantId: number, - importId: number, - settings = { skipErrors: true } - ) { + public async process(tenantId: number, importId: number) { const { Import } = this.tenancy.models(tenantId); const importFile = await Import.query() .findOne('importId', importId) .throwIfNotFound(); + // Throw error if the import file is not mapped yet. + if (!importFile.isMapped) { + throw new ServiceError(ERRORS.IMPORT_FILE_NOT_MAPPED); + } const buffer = await this.readImportFile(importFile.filename); const jsonData = this.parseXlsxSheet(buffer); @@ -146,15 +157,43 @@ export class ImportFileProcess { const transformedDTOs = this.transformDTOs(mappedDTOs); // Validate the mapped DTOs. - const errors = await this.validateData(transformedDTOs); - - return this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => { + const rowsWithErrors = await this.validateData( + tenantId, + importFile.resource, + transformedDTOs + ); + // Runs the importing under UOW envirement. + await this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => { await bluebird.map( - transformedDTOs, - (transformedDTO) => - this.importable.importable(tenantId, transformedDTO, trx), + rowsWithErrors, + (rowWithErrors) => { + if (rowWithErrors.errors.length === 0) { + return this.importable.importable( + tenantId, + rowWithErrors.data, + trx + ); + } + }, { concurrency: 10 } ); }); + // Deletes the imported file after importing success./ + await this.deleteImportFile(tenantId, importFile) + } + + /** + * Deletes the imported file from the storage and database. + * @param {number} tenantId + * @param {} importFile + */ + private async deleteImportFile(tenantId: number, importFile: any) { + const { Import } = this.tenancy.models(tenantId); + + // Deletes the import row. + await Import.query().findById(importFile.id).delete(); + + // Deletes the imported file. + await fs.unlink(`public/imports/${importFile.filename}`); } } diff --git a/packages/server/src/services/Import/ImportFileUpload.ts b/packages/server/src/services/Import/ImportFileUpload.ts index cdd1e14ac..6e476a616 100644 --- a/packages/server/src/services/Import/ImportFileUpload.ts +++ b/packages/server/src/services/Import/ImportFileUpload.ts @@ -1,17 +1,22 @@ -import { first } from 'lodash'; +import { first, values } from 'lodash'; import { Inject, Service } from 'typedi'; -import { snakeCase, upperFirst } from 'lodash'; +import { ServiceError } from '@/exceptions'; import XLSX from 'xlsx'; import * as R from 'ramda'; import HasTenancyService from '../Tenancy/TenancyService'; -import { trimObject } from './_utils'; -const fs = require('fs').promises; +import { ERRORS, trimObject } from './_utils'; +import ResourceService from '../Resource/ResourceService'; +import fs from 'fs/promises'; +import { IModelMetaField } from '@/interfaces'; @Service() export class ImportFileUploadService { @Inject() private tenancy: HasTenancyService; + @Inject() + private resourceService: ResourceService; + /** * Reads the imported file and stores the import file meta under unqiue id. * @param {number} tenantId - @@ -27,31 +32,54 @@ export class ImportFileUploadService { ) { const { Import } = this.tenancy.models(tenantId); + const resourceMeta = this.resourceService.getResourceMeta( + tenantId, + resource + ); + // Throw service error if the resource does not support importing. + if (!resourceMeta.importable) { + throw new ServiceError(ERRORS.RESOURCE_NOT_IMPORTABLE); + } const buffer = await fs.readFile(filePath); const workbook = XLSX.read(buffer, { type: 'buffer' }); const firstSheetName = workbook.SheetNames[0]; const worksheet = workbook.Sheets[firstSheetName]; const jsonData = XLSX.utils.sheet_to_json(worksheet); - + + const columns = this.getColumns(jsonData); + const coumnsStringified = JSON.stringify(columns); + // @todo validate the resource. - const _resource = upperFirst(snakeCase(resource)); + const _resource = this.resourceService.resourceToModelName(resource); const exportFile = await Import.query().insert({ filename, importId: filename, resource: _resource, + columns: coumnsStringified, }); - const columns = this.getColumns(jsonData); - + const resourceColumns = this.resourceService.getResourceImportableFields( + tenantId, + resource + ); + const resourceColumnsTransformeed = Object.entries(resourceColumns).map( + ([key, { name }]: [string, IModelMetaField]) => ({ key, name }) + ); // @todo return the resource importable columns. return { export: exportFile, columns, + resourceColumns: resourceColumnsTransformeed, }; } - private getColumns(json: unknown[]) { + /** + * Retrieves the sheet columns from the given sheet data. + * @param {unknown[]} json + * @returns {string[]} + */ + private getColumns(json: unknown[]): string[] { return R.compose(Object.keys, trimObject, first)(json); } } diff --git a/packages/server/src/services/Import/ImportResourceApplication.ts b/packages/server/src/services/Import/ImportResourceApplication.ts index db464b6ec..604818a29 100644 --- a/packages/server/src/services/Import/ImportResourceApplication.ts +++ b/packages/server/src/services/Import/ImportResourceApplication.ts @@ -65,10 +65,10 @@ export class ImportResourceApplication { } /** - * + * Process the import file sheet through service for creating entities. * @param {number} tenantId * @param {number} importId - * @returns + * @returns {Promise} */ public async process(tenantId: number, importId: number) { return this.importProcessService.process(tenantId, importId); diff --git a/packages/server/src/services/Import/_utils.ts b/packages/server/src/services/Import/_utils.ts index 3700a313e..404f0a8ab 100644 --- a/packages/server/src/services/Import/_utils.ts +++ b/packages/server/src/services/Import/_utils.ts @@ -1,3 +1,5 @@ +import * as Yup from 'yup'; + export function trimObject(obj) { return Object.entries(obj).reduce((acc, [key, value]) => { // Trim the key @@ -10,3 +12,50 @@ export function trimObject(obj) { return { ...acc, [trimmedKey]: trimmedValue }; }, {}); } + +export const convertFieldsToYupValidation = (fields: any) => { + const yupSchema = {}; + Object.keys(fields).forEach((fieldName) => { + const field = fields[fieldName]; + let fieldSchema; + fieldSchema = Yup.string().label(field.name); + + if (field.fieldType === 'text') { + if (field.minLength) { + fieldSchema = fieldSchema.min( + field.minLength, + `Minimum length is ${field.minLength} characters` + ); + } + if (field.maxLength) { + fieldSchema = fieldSchema.max( + field.maxLength, + `Maximum length is ${field.maxLength} characters` + ); + } + } else if (field.fieldType === 'number') { + fieldSchema = Yup.number().label(field.name); + } else if (field.fieldType === 'boolean') { + fieldSchema = Yup.boolean().label(field.name); + } else if (field.fieldType === 'enumeration') { + const options = field.options.reduce((acc, option) => { + acc[option.key] = option.label; + return acc; + }, {}); + fieldSchema = Yup.string().oneOf(Object.keys(options)).label(field.name); + } + if (field.required) { + fieldSchema = fieldSchema.required(); + } + yupSchema[fieldName] = fieldSchema; + }); + return Yup.object().shape(yupSchema); +}; + +export const ERRORS = { + RESOURCE_NOT_IMPORTABLE: 'RESOURCE_NOT_IMPORTABLE', + INVALID_MAP_ATTRS: 'INVALID_MAP_ATTRS', + DUPLICATED_FROM_MAP_ATTR: 'DUPLICATED_FROM_MAP_ATTR', + DUPLICATED_TO_MAP_ATTR: 'DUPLICATED_TO_MAP_ATTR', + IMPORT_FILE_NOT_MAPPED: 'IMPORT_FILE_NOT_MAPPED', +}; diff --git a/packages/server/src/services/Organization/OrganizationService.ts b/packages/server/src/services/Organization/OrganizationService.ts index 32431a066..453b6522c 100644 --- a/packages/server/src/services/Organization/OrganizationService.ts +++ b/packages/server/src/services/Organization/OrganizationService.ts @@ -21,25 +21,19 @@ import { ERRORS } from './constants'; @Service() export default class OrganizationService { @Inject() - eventPublisher: EventPublisher; - - @Inject('logger') - logger: any; - - @Inject('repositories') - sysRepositories: any; + private eventPublisher: EventPublisher; @Inject() - tenantsManager: TenantsManager; + private tenantsManager: TenantsManager; @Inject('agenda') - agenda: any; + private agenda: any; @Inject() - baseCurrencyMutateLocking: OrganizationBaseCurrencyLocking; + private baseCurrencyMutateLocking: OrganizationBaseCurrencyLocking; @Inject() - tenancy: HasTenancyService; + private tenancy: HasTenancyService; /** * Builds the database schema and seed data of the given organization id. diff --git a/packages/server/src/services/Resource/ResourceService.ts b/packages/server/src/services/Resource/ResourceService.ts index 79887851a..529448f8d 100644 --- a/packages/server/src/services/Resource/ResourceService.ts +++ b/packages/server/src/services/Resource/ResourceService.ts @@ -1,12 +1,11 @@ import { Service, Inject } from 'typedi'; -import { camelCase, upperFirst } from 'lodash'; +import { camelCase, upperFirst, pickBy } from 'lodash'; import * as qim from 'qim'; import pluralize from 'pluralize'; -import { IModelMeta } from '@/interfaces'; +import { IModelMeta, IModelMetaField } from '@/interfaces'; import TenancyService from '@/services/Tenancy/TenancyService'; import { ServiceError } from '@/exceptions'; import I18nService from '@/services/I18n/I18nService'; -import { tenantKnexConfig } from 'config/knexConfig'; const ERRORS = { RESOURCE_MODEL_NOT_FOUND: 'RESOURCE_MODEL_NOT_FOUND', @@ -24,7 +23,7 @@ export default class ResourceService { * Transform resource to model name. * @param {string} resourceName */ - private resourceToModelName(resourceName: string): string { + public resourceToModelName(resourceName: string): string { return upperFirst(camelCase(pluralize.singular(resourceName))); } @@ -63,6 +62,33 @@ export default class ResourceService { return this.getResourceMetaLocalized(resourceMeta, tenantId); } + /** + * + */ + public getResourceFields( + tenantId: number, + modelName: string + ): { [key: string]: IModelMetaField } { + const meta = this.getResourceMeta(tenantId, modelName); + + return meta.fields; + } + + /** + * + * @param {number} tenantId + * @param {string} modelName + * @returns + */ + public getResourceImportableFields( + tenantId: number, + modelName: string + ): { [key: string]: IModelMetaField } { + const fields = this.getResourceFields(tenantId, modelName); + + return pickBy(fields, (field) => field.importable); + } + /** * Retrieve the resource meta localized based on the current user language. */ diff --git a/packages/server/src/services/UnitOfWork/index.ts b/packages/server/src/services/UnitOfWork/index.ts index d5f9d0cda..024d330b7 100644 --- a/packages/server/src/services/UnitOfWork/index.ts +++ b/packages/server/src/services/UnitOfWork/index.ts @@ -45,7 +45,7 @@ export default class UnitOfWork { const knex = this.tenancy.knex(tenantId); let _trx = trx; - if (_trx) { + if (!_trx) { _trx = await knex.transaction({ isolationLevel }); } try { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0d7f044e8..b62633b19 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -47,6 +47,9 @@ importers: '@types/mathjs': specifier: ^6.0.12 version: 6.0.12 + '@types/yup': + specifier: ^0.29.13 + version: 0.29.14 accepts: specifier: ^1.3.7 version: 1.3.8 @@ -299,6 +302,9 @@ importers: xlsx: specifier: ^0.18.5 version: 0.18.5 + yup: + specifier: ^0.28.1 + version: 0.28.5 devDependencies: '@types/lodash': specifier: ^4.14.158