From daa1e3a6bd7b2644ec7807ac2dbce1e2f15bd7d6 Mon Sep 17 00:00:00 2001 From: Ahmed Bouhuolia Date: Wed, 13 Mar 2024 02:14:25 +0200 Subject: [PATCH] feat: wip import resource --- .../controllers/Import/ImportController.ts | 54 +++++---- packages/server/src/interfaces/Model.ts | 1 + .../server/src/models/Account.Settings.ts | 2 +- .../src/services/Import/AccountsImportable.ts | 27 +++-- .../src/services/Import/ImportFileCommon.ts | 87 +++++++++++++++ .../Import/ImportFileDataTransformer.ts | 104 ++++++++++++++++++ .../Import/ImportFileDataValidator.ts | 29 +++++ .../src/services/Import/ImportFilePreview.ts | 91 ++++++++++++++- .../src/services/Import/ImportFileUpload.ts | 26 +++-- .../services/Import/ImportResourceRegistry.ts | 7 ++ packages/server/src/services/Import/_utils.ts | 17 ++- .../server/src/services/Import/interfaces.ts | 10 ++ .../server/src/services/UnitOfWork/index.ts | 8 +- 13 files changed, 411 insertions(+), 52 deletions(-) create mode 100644 packages/server/src/services/Import/ImportFileCommon.ts create mode 100644 packages/server/src/services/Import/ImportFileDataTransformer.ts create mode 100644 packages/server/src/services/Import/ImportFileDataValidator.ts create mode 100644 packages/server/src/services/Import/ImportResourceRegistry.ts diff --git a/packages/server/src/api/controllers/Import/ImportController.ts b/packages/server/src/api/controllers/Import/ImportController.ts index 4461f16d8..0d4efbf93 100644 --- a/packages/server/src/api/controllers/Import/ImportController.ts +++ b/packages/server/src/api/controllers/Import/ImportController.ts @@ -1,10 +1,9 @@ import { Inject, Service } from 'typedi'; import { Router, Request, Response, NextFunction } from 'express'; -import { query, body, param } from 'express-validator'; +import { body, param } from 'express-validator'; import Multer from 'multer'; import BaseController from '@/api/controllers/BaseController'; import { ServiceError } from '@/exceptions'; -import { ImportResourceInjectable } from '@/services/Import/ImportResourceInjectable'; import { ImportResourceApplication } from '@/services/Import/ImportResourceApplication'; const upload = Multer({ @@ -23,11 +22,6 @@ export class ImportController extends BaseController { router() { const router = Router(); - router.post( - '/:import_id/import', - this.asyncMiddleware(this.import.bind(this)), - this.catchServiceErrors - ); router.post( '/file', upload.single('file'), @@ -36,6 +30,11 @@ export class ImportController extends BaseController { this.asyncMiddleware(this.fileUpload.bind(this)), this.catchServiceErrors ); + router.post( + '/:import_id/import', + this.asyncMiddleware(this.import.bind(this)), + this.catchServiceErrors + ); router.post( '/:import_id/mapping', [ @@ -48,11 +47,11 @@ export class ImportController extends BaseController { this.asyncMiddleware(this.mapping.bind(this)), this.catchServiceErrors ); - // router.get( - // '/:import_id/preview', - // this.asyncMiddleware(this.preview.bind(this)), - // this.catchServiceErrors - // ); + router.post( + '/:import_id/preview', + this.asyncMiddleware(this.preview.bind(this)), + this.catchServiceErrors + ); return router; } @@ -86,7 +85,7 @@ export class ImportController extends BaseController { * @param {Response} res - * @param {NextFunction} next - */ - async fileUpload(req: Request, res: Response, next: NextFunction) { + private async fileUpload(req: Request, res: Response, next: NextFunction) { const { tenantId } = req; try { @@ -103,10 +102,10 @@ export class ImportController extends BaseController { } /** - * - * @param req - * @param res - * @param next + * Maps the columns of the imported file. + * @param {Request} req + * @param {Response} res + * @param {NextFunction} next */ private async mapping(req: Request, res: Response, next: NextFunction) { const { tenantId } = req; @@ -126,12 +125,23 @@ export class ImportController extends BaseController { } /** - * - * @param req - * @param res - * @param next + * Preview the imported file before actual importing. + * @param {Request} req + * @param {Response} res + * @param {NextFunction} next */ - private async preview(req: Request, res: Response, next: NextFunction) {} + private async preview(req: Request, res: Response, next: NextFunction) { + const { tenantId } = req; + const { import_id: importId } = req.params; + + try { + const preview = await this.importResourceApp.preview(tenantId, importId); + + return res.status(200).send(preview); + } catch (error) { + next(error); + } + } /** * diff --git a/packages/server/src/interfaces/Model.ts b/packages/server/src/interfaces/Model.ts index 2bf8d6916..385bce172 100644 --- a/packages/server/src/interfaces/Model.ts +++ b/packages/server/src/interfaces/Model.ts @@ -34,6 +34,7 @@ export interface IModelMetaFieldCommon { columnable?: boolean; fieldType: IModelColumnType; customQuery?: Function; + required?: boolean; } export interface IModelMetaFieldNumber { diff --git a/packages/server/src/models/Account.Settings.ts b/packages/server/src/models/Account.Settings.ts index 7cba91bfd..ec67885c4 100644 --- a/packages/server/src/models/Account.Settings.ts +++ b/packages/server/src/models/Account.Settings.ts @@ -63,7 +63,7 @@ export default { sortable: false, importable: false, }, - type: { + accountType: { name: 'account.field.type', column: 'account_type', fieldType: 'enumeration', diff --git a/packages/server/src/services/Import/AccountsImportable.ts b/packages/server/src/services/Import/AccountsImportable.ts index c02eb2950..66b6c4570 100644 --- a/packages/server/src/services/Import/AccountsImportable.ts +++ b/packages/server/src/services/Import/AccountsImportable.ts @@ -1,13 +1,13 @@ -import { IAccountCreateDTO } from '@/interfaces'; -import { AccountsApplication } from '../Accounts/AccountsApplication'; -import { AccountDTOSchema } from '../Accounts/CreateAccountDTOSchema'; import { Inject, Service } from 'typedi'; import { Knex } from 'knex'; +import { IAccountCreateDTO } from '@/interfaces'; +import { AccountsApplication } from '../Accounts/AccountsApplication'; +import { CreateAccount } from '../Accounts/CreateAccount'; @Service() export class AccountsImportable { @Inject() - private accountsApp: AccountsApplication; + private createAccountService: CreateAccount; /** * @@ -20,7 +20,11 @@ export class AccountsImportable { createAccountDTO: IAccountCreateDTO, trx?: Knex.Transaction ) { - return this.accountsApp.createAccount(tenantId, createAccountDTO, trx); + return this.createAccountService.createAccount( + tenantId, + createAccountDTO, + trx + ); } /** @@ -29,12 +33,15 @@ export class AccountsImportable { * @returns */ public transform(data) { - return { - ...data, - }; + return { ...data }; } - mapAccountType(accountType: string) { - return 'Cash'; + /** + * + * @param data + * @returns + */ + public preTransform(data) { + return { ...data }; } } diff --git a/packages/server/src/services/Import/ImportFileCommon.ts b/packages/server/src/services/Import/ImportFileCommon.ts new file mode 100644 index 000000000..6719280fb --- /dev/null +++ b/packages/server/src/services/Import/ImportFileCommon.ts @@ -0,0 +1,87 @@ +import fs from 'fs/promises'; +import XLSX from 'xlsx'; +import bluebird from 'bluebird'; +import { Inject, Service } from 'typedi'; +import { ImportFileDataValidator } from './ImportFileDataValidator'; +import { Knex } from 'knex'; +import { ImportInsertError } from './interfaces'; +import { AccountsImportable } from './AccountsImportable'; +import { ServiceError } from '@/exceptions'; + +@Service() +export class ImportFileCommon { + @Inject() + private importFileValidator: ImportFileDataValidator; + + @Inject() + private importable: AccountsImportable; + + /** + * Maps the columns of the imported data based on the provided mapping attributes. + * @param {Record[]} body - The array of data objects to map. + * @param {ImportMappingAttr[]} map - The mapping attributes. + * @returns {Record[]} - The mapped data objects. + */ + public parseXlsxSheet(buffer) { + const workbook = XLSX.read(buffer, { type: 'buffer' }); + + const firstSheetName = workbook.SheetNames[0]; + const worksheet = workbook.Sheets[firstSheetName]; + + return XLSX.utils.sheet_to_json(worksheet); + } + + /** + * Reads the import file. + * @param {string} filename + * @returns {Promise} + */ + public readImportFile(filename: string) { + return fs.readFile(`public/imports/${filename}`); + } + + /** + * + * @param {number} tenantId - + * @param {Record} importableFields + * @param {Record} parsedData + * @param {Knex.Transaction} trx + * @returns + */ + public import( + tenantId: number, + importableFields, + parsedData: Record, + trx?: Knex.Transaction + ): Promise<(void | ImportInsertError[])[]> { + return bluebird.map( + parsedData, + async (objectDTO, index: number): Promise => { + try { + // Validate the DTO object before passing it to the service layer. + await this.importFileValidator.validateData( + importableFields, + objectDTO + ); + try { + // Run the importable function and listen to the errors. + await this.importable.importable(tenantId, objectDTO, trx); + } catch (error) { + if (error instanceof ServiceError) { + return [ + { + errorCode: 'ValidationError', + errorMessage: error.message || error.errorType, + rowNumber: index + 1, + }, + ]; + } + } + } catch (errors) { + return errors.map((er) => ({ ...er, rowNumber: index + 1 })); + } + }, + { concurrency: 2 } + ); + } +} diff --git a/packages/server/src/services/Import/ImportFileDataTransformer.ts b/packages/server/src/services/Import/ImportFileDataTransformer.ts new file mode 100644 index 000000000..859a16c25 --- /dev/null +++ b/packages/server/src/services/Import/ImportFileDataTransformer.ts @@ -0,0 +1,104 @@ +import { Inject, Service } from 'typedi'; +import * as R from 'ramda'; +import { isUndefined, mapValues, get, pickBy, chain } from 'lodash'; +import { ImportMappingAttr, ResourceMetaFieldsMap } from './interfaces'; +import { parseBoolean } from '@/utils'; +import { trimObject } from './_utils'; +import ResourceService from '../Resource/ResourceService'; + +@Service() +export class ImportFileDataTransformer { + @Inject() + private resource: ResourceService; + /** + * + * @param {number} tenantId - + * @param {} + */ + public transformSheetData( + importFile: any, + importableFields: any, + data: Record[] + ) { + // Sanitize the sheet data. + const sanitizedData = this.sanitizeSheetData(data); + + // Map the sheet columns key with the given map. + const mappedDTOs = this.mapSheetColumns( + sanitizedData, + importFile.mappingParsed + ); + // Parse the mapped sheet values. + const parsedValues = this.parseExcelValues(importableFields, mappedDTOs); + + return parsedValues; + } + + /** + * Sanitizes the data in the imported sheet by trimming object keys. + * @param json - The JSON data representing the imported sheet. + * @returns {string[][]} - The sanitized data with trimmed object keys. + */ + public sanitizeSheetData(json) { + return R.compose(R.map(trimObject))(json); + } + + /** + * Maps the columns of the imported data based on the provided mapping attributes. + * @param {Record[]} body - The array of data objects to map. + * @param {ImportMappingAttr[]} map - The mapping attributes. + * @returns {Record[]} - The mapped data objects. + */ + public mapSheetColumns( + body: Record[], + map: ImportMappingAttr[] + ): Record[] { + return body.map((item) => { + const newItem = {}; + map + .filter((mapping) => !isUndefined(item[mapping.from])) + .forEach((mapping) => { + newItem[mapping.to] = item[mapping.from]; + }); + return newItem; + }); + } + + /** + * Parses sheet values before passing to the service layer. + * @param {ResourceMetaFieldsMap} fields - + * @param {Record} valueDTOS - + * @returns {Record} + */ + public parseExcelValues( + fields: ResourceMetaFieldsMap, + valueDTOs: Record[] + ): Record { + const parser = (value, key) => { + let _value = value; + + // Parses the boolean value. + if (fields[key].fieldType === 'boolean') { + _value = parseBoolean(value, false); + + // Parses the enumeration value. + } else if (fields[key].fieldType === 'enumeration') { + const field = fields[key]; + const option = get(field, 'options', []).find( + (option) => option.label === value + ); + _value = get(option, 'key'); + // Prases the numeric value. + } else if (fields[key].fieldType === 'number') { + _value = parseFloat(value); + } + return _value; + }; + return valueDTOs.map((DTO) => { + return chain(DTO) + .pickBy((value, key) => !isUndefined(fields[key])) + .mapValues(parser) + .value(); + }); + } +} diff --git a/packages/server/src/services/Import/ImportFileDataValidator.ts b/packages/server/src/services/Import/ImportFileDataValidator.ts new file mode 100644 index 000000000..371117694 --- /dev/null +++ b/packages/server/src/services/Import/ImportFileDataValidator.ts @@ -0,0 +1,29 @@ +import { Service } from 'typedi'; +import { ImportInsertError, ResourceMetaFieldsMap } from './interfaces'; +import { convertFieldsToYupValidation } from './_utils'; + +@Service() +export class ImportFileDataValidator { + /** + * Validates the given mapped DTOs and returns errors with their index. + * @param {Record} mappedDTOs + * @returns {Promise} + */ + public async validateData( + importableFields: ResourceMetaFieldsMap, + data: Record + ): Promise { + const YupSchema = convertFieldsToYupValidation(importableFields); + const _data = { ...data }; + + try { + await YupSchema.validate(_data, { abortEarly: false }); + } catch (validationError) { + const errors = validationError.inner.map((error) => ({ + errorCode: 'ValidationError', + errorMessage: error.errors, + })); + throw errors; + } + } +} diff --git a/packages/server/src/services/Import/ImportFilePreview.ts b/packages/server/src/services/Import/ImportFilePreview.ts index eb20bbcf9..1d0cf1442 100644 --- a/packages/server/src/services/Import/ImportFilePreview.ts +++ b/packages/server/src/services/Import/ImportFilePreview.ts @@ -1,11 +1,98 @@ -import { Service } from 'typedi'; +import { Inject, Service } from 'typedi'; +import { first, omit } from 'lodash'; +import { ServiceError } from '@/exceptions'; +import { ERRORS, getUnmappedSheetColumns } from './_utils'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { ImportFileCommon } from './ImportFileCommon'; +import { ImportFileDataTransformer } from './ImportFileDataTransformer'; +import ResourceService from '../Resource/ResourceService'; @Service() export class ImportFilePreview { + @Inject() + private tenancy: HasTenancyService; + + @Inject() + private resource: ResourceService; + + @Inject() + private importFileCommon: ImportFileCommon; + + @Inject() + private importFileParser: ImportFileDataTransformer; + /** + * + * - Returns the passed rows and will be in inserted. + * - Returns the passed rows will be overwritten. + * - Returns the rows errors from the validation. + * - Returns the unmapped fields. * * @param {number} tenantId * @param {number} importId */ - public preview(tenantId: number, importId: number) {} + public async preview(tenantId: number, importId: number) { + const { Import } = this.tenancy.models(tenantId); + + const importFile = await Import.query() + .findOne('importId', importId) + .throwIfNotFound(); + + // Throw error if the import file is not mapped yet. + if (!importFile.isMapped) { + throw new ServiceError(ERRORS.IMPORT_FILE_NOT_MAPPED); + } + const buffer = await this.importFileCommon.readImportFile( + importFile.filename + ); + const jsonData = this.importFileCommon.parseXlsxSheet(buffer); + + const importableFields = this.resource.getResourceImportableFields( + tenantId, + importFile.resource + ); + // Prases the sheet json data. + const parsedData = this.importFileParser.transformSheetData( + importFile, + importableFields, + jsonData + ); + const knex = this.tenancy.knex(tenantId); + const trx = await knex.transaction({ isolationLevel: 'read uncommitted' }); + + // Runs the importing operation with ability to return errors that will happen. + const asyncOpers = await this.importFileCommon.import( + tenantId, + importableFields, + parsedData, + trx + ); + // Filter out the operations that have successed. + const successAsyncOpers = asyncOpers.filter((oper) => !oper); + const errors = asyncOpers.filter((oper) => oper); + + // Rollback all the successed transactions. + await trx.rollback(); + + const header = Object.keys(first(jsonData)); + const mapping = importFile.mappingParsed; + + const unmappedColumns = getUnmappedSheetColumns(header, mapping); + const totalCount = parsedData.length; + + const createdCount = successAsyncOpers.length; + const errorsCount = errors.length; + const skippedCount = errorsCount; + + return { + createdCount, + skippedCount, + totalCount, + errorsCount, + errors, + unmappedColumns: unmappedColumns, + unmappedColumnsCount: unmappedColumns.length, + }; + } + } diff --git a/packages/server/src/services/Import/ImportFileUpload.ts b/packages/server/src/services/Import/ImportFileUpload.ts index 6e476a616..42d61e021 100644 --- a/packages/server/src/services/Import/ImportFileUpload.ts +++ b/packages/server/src/services/Import/ImportFileUpload.ts @@ -8,7 +8,7 @@ import { ERRORS, trimObject } from './_utils'; import ResourceService from '../Resource/ResourceService'; import fs from 'fs/promises'; import { IModelMetaField } from '@/interfaces'; - +import { ImportFileCommon } from './ImportFileCommon'; @Service() export class ImportFileUploadService { @Inject() @@ -17,11 +17,15 @@ export class ImportFileUploadService { @Inject() private resourceService: ResourceService; + @Inject() + private importFileCommon: ImportFileCommon; + /** * Reads the imported file and stores the import file meta under unqiue id. - * @param {number} tenantId - - * @param {string} filePath - - * @param {string} fileName - + * @param {number} tenantId - Tenant id. + * @param {string} resource - Resource name. + * @param {string} filePath - File path. + * @param {string} fileName - File name. * @returns */ public async import( @@ -40,16 +44,15 @@ export class ImportFileUploadService { if (!resourceMeta.importable) { throw new ServiceError(ERRORS.RESOURCE_NOT_IMPORTABLE); } - const buffer = await fs.readFile(filePath); - const workbook = XLSX.read(buffer, { type: 'buffer' }); + // Reads the imported file into buffer. + const buffer = await this.importFileCommon.readImportFile(filename); + + // Parse the buffer file to array data. + const jsonData = this.importFileCommon.parseXlsxSheet(buffer); - const firstSheetName = workbook.SheetNames[0]; - const worksheet = workbook.Sheets[firstSheetName]; - const jsonData = XLSX.utils.sheet_to_json(worksheet); - const columns = this.getColumns(jsonData); const coumnsStringified = JSON.stringify(columns); - + // @todo validate the resource. const _resource = this.resourceService.resourceToModelName(resource); @@ -66,7 +69,6 @@ export class ImportFileUploadService { const resourceColumnsTransformeed = Object.entries(resourceColumns).map( ([key, { name }]: [string, IModelMetaField]) => ({ key, name }) ); - // @todo return the resource importable columns. return { export: exportFile, columns, diff --git a/packages/server/src/services/Import/ImportResourceRegistry.ts b/packages/server/src/services/Import/ImportResourceRegistry.ts new file mode 100644 index 000000000..ddd101a15 --- /dev/null +++ b/packages/server/src/services/Import/ImportResourceRegistry.ts @@ -0,0 +1,7 @@ +import { Service } from "typedi"; + + +@Service() +export class ImportResourceRegistry { + +} \ No newline at end of file diff --git a/packages/server/src/services/Import/_utils.ts b/packages/server/src/services/Import/_utils.ts index 404f0a8ab..843db14cc 100644 --- a/packages/server/src/services/Import/_utils.ts +++ b/packages/server/src/services/Import/_utils.ts @@ -1,4 +1,6 @@ import * as Yup from 'yup'; +import { ResourceMetaFieldsMap } from './interfaces'; +import { IModelMetaField } from '@/interfaces'; export function trimObject(obj) { return Object.entries(obj).reduce((acc, [key, value]) => { @@ -13,10 +15,10 @@ export function trimObject(obj) { }, {}); } -export const convertFieldsToYupValidation = (fields: any) => { +export const convertFieldsToYupValidation = (fields: ResourceMetaFieldsMap) => { const yupSchema = {}; - Object.keys(fields).forEach((fieldName) => { - const field = fields[fieldName]; + Object.keys(fields).forEach((fieldName: string) => { + const field = fields[fieldName] as IModelMetaField; let fieldSchema; fieldSchema = Yup.string().label(field.name); @@ -59,3 +61,12 @@ export const ERRORS = { DUPLICATED_TO_MAP_ATTR: 'DUPLICATED_TO_MAP_ATTR', IMPORT_FILE_NOT_MAPPED: 'IMPORT_FILE_NOT_MAPPED', }; + +/** + * + */ +export const getUnmappedSheetColumns = (columns, mapping) => { + return columns.filter( + (column) => !mapping.some((map) => map.from === column) + ); +}; diff --git a/packages/server/src/services/Import/interfaces.ts b/packages/server/src/services/Import/interfaces.ts index 1baed3a9b..0fb5c96f1 100644 --- a/packages/server/src/services/Import/interfaces.ts +++ b/packages/server/src/services/Import/interfaces.ts @@ -1,3 +1,5 @@ +import { IModelMetaField } from '@/interfaces'; + export interface ImportMappingAttr { from: string; to: string; @@ -8,3 +10,11 @@ export interface ImportValidationError { property: string; constraints: Record; } + +export type ResourceMetaFieldsMap = { [key: string]: IModelMetaField }; + +export interface ImportInsertError { + rowNumber: number; + errorCode: string; + errorMessage: string; +} diff --git a/packages/server/src/services/UnitOfWork/index.ts b/packages/server/src/services/UnitOfWork/index.ts index 024d330b7..a13be95c6 100644 --- a/packages/server/src/services/UnitOfWork/index.ts +++ b/packages/server/src/services/UnitOfWork/index.ts @@ -50,11 +50,15 @@ export default class UnitOfWork { } try { const result = await work(_trx); - _trx.commit(); + if (!trx) { + _trx.commit(); + } return result; } catch (error) { - _trx.rollback(); + if (!trx) { + _trx.rollback(); + } throw error; } };