From b1d5390bfce8918ea5f08fb34ebb52e196344f04 Mon Sep 17 00:00:00 2001 From: Ahmed Bouhuolia Date: Sun, 10 Mar 2024 14:53:10 +0200 Subject: [PATCH 1/7] WIP --- packages/server/package.json | 3 + .../controllers/Import/ImportController.ts | 149 ++++++++++++++++++ packages/server/src/api/index.ts | 4 + .../20231209230719_create_imports_table.js | 14 ++ packages/server/src/loaders/tenantModels.ts | 4 +- .../Accounts/CreateAccountDTOSchema.ts | 32 ++++ .../src/services/Import/ImportFileUpload.ts | 54 +++++++ .../Import/ImportResourceApplication.ts | 29 ++++ .../Import/ImportResourceInjectable.ts | 120 ++++++++++++++ packages/server/src/services/Import/_utils.ts | 12 ++ pnpm-lock.yaml | 42 +++++ 11 files changed, 462 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/api/controllers/Import/ImportController.ts create mode 100644 packages/server/src/database/migrations/20231209230719_create_imports_table.js create mode 100644 packages/server/src/services/Accounts/CreateAccountDTOSchema.ts create mode 100644 packages/server/src/services/Import/ImportFileUpload.ts create mode 100644 packages/server/src/services/Import/ImportResourceApplication.ts create mode 100644 packages/server/src/services/Import/ImportResourceInjectable.ts create mode 100644 packages/server/src/services/Import/_utils.ts diff --git a/packages/server/package.json b/packages/server/package.json index d359376de..39bc8a217 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -36,6 +36,8 @@ "bcryptjs": "^2.4.3", "bluebird": "^3.7.2", "body-parser": "^1.20.2", + "class-transformer": "^0.5.1", + "class-validator": "^0.14.0", "compression": "^1.7.4", "country-codes-list": "^1.6.8", "cpy": "^8.1.2", @@ -77,6 +79,7 @@ "moment-timezone": "^0.5.43", "mongodb": "^6.1.0", "mongoose": "^5.10.0", + "multer": "1.4.5-lts.1", "mustache": "^3.0.3", "mysql": "^2.17.1", "mysql2": "^1.6.5", diff --git a/packages/server/src/api/controllers/Import/ImportController.ts b/packages/server/src/api/controllers/Import/ImportController.ts new file mode 100644 index 000000000..cfdcb3dc6 --- /dev/null +++ b/packages/server/src/api/controllers/Import/ImportController.ts @@ -0,0 +1,149 @@ +import { Inject, Service } from 'typedi'; +import { Router, Request, Response, NextFunction } from 'express'; +import { query } from 'express-validator'; +import Multer from 'multer'; +import BaseController from '@/api/controllers/BaseController'; +import { ServiceError } from '@/exceptions'; +import { ImportResourceInjectable } from '@/services/Import/ImportResourceInjectable'; +import { ImportResourceApplication } from '@/services/Import/ImportResourceApplication'; + +const upload = Multer({ + dest: './public/imports', + limits: { fileSize: 5 * 1024 * 1024 }, +}); + +@Service() +export class ImportController extends BaseController { + @Inject() + private importResource: ImportResourceInjectable; + + @Inject() + private importResourceApp: ImportResourceApplication; + + /** + * Router constructor method. + */ + router() { + const router = Router(); + + // router.post( + // '/:import_id/import', + // this.asyncMiddleware(this.import.bind(this)), + // this.catchServiceErrors + // ); + router.post( + '/file', + // [...this.importValidationSchema], + upload.single('file'), + this.asyncMiddleware(this.fileUpload.bind(this)) + // this.catchServiceErrors + ); + // router.post( + // '/:import_id/mapping', + // this.asyncMiddleware(this.mapping.bind(this)), + // this.catchServiceErrors + // ); + // router.get( + // '/:import_id/preview', + // this.asyncMiddleware(this.preview.bind(this)), + // this.catchServiceErrors + // ); + return router; + } + + /** + * Import validation schema. + * @returns {ValidationSchema[]} + */ + get importValidationSchema() { + return [query('resource').exists().isString().toString()]; + } + + /** + * Imports xlsx/csv to the given resource type. + * + * - Save the xlsx/csv file and give it a random name. + * - Save the file metadata on the DB storage. + * - + * + * @param {Request} req - + * @param {Response} res - + * @param {NextFunction} next - + */ + async fileUpload(req: Request, res: Response, next: NextFunction) { + const { tenantId } = req; + + try { + const data = await this.importResourceApp.import( + tenantId, + req.body.resource, + req.file.path, + req.file.filename + ); + return res.status(200).send(data); + } catch (error) { + next(error); + } + } + + /** + * + * @param req + * @param res + * @param next + */ + async mapping(req: Request, res: Response, next: NextFunction) { + const { tenantId } = req; + + try { + await this.importResource.mapping(tenantId); + } catch (error) { + next(error); + } + } + + /** + * + * @param req + * @param res + * @param next + */ + async preview(req: Request, res: Response, next: NextFunction) {} + + /** + * + * @param req + * @param res + * @param next + */ + async import(req: Request, res: Response, next: NextFunction) { + const { tenantId } = req; + const { import_id: importId } = req.params; + + try { + await this.importResource.importFile(tenantId, importId); + + return res.status(200).send({}); + } catch (error) { + next(error); + } + } + + /** + * Transforms service errors to response. + * @param {Error} + * @param {Request} req + * @param {Response} res + * @param {ServiceError} error + */ + private catchServiceErrors( + error, + req: Request, + res: Response, + next: NextFunction + ) { + if (error instanceof ServiceError) { + } + next(error); + } +} diff --git a/packages/server/src/api/index.ts b/packages/server/src/api/index.ts index 6a41c8304..1a3ff17c6 100644 --- a/packages/server/src/api/index.ts +++ b/packages/server/src/api/index.ts @@ -56,6 +56,7 @@ import { ProjectsController } from './controllers/Projects/Projects'; import { ProjectTasksController } from './controllers/Projects/Tasks'; import { ProjectTimesController } from './controllers/Projects/Times'; import { TaxRatesController } from './controllers/TaxRates/TaxRates'; +import { ImportController } from './controllers/Import/ImportController'; export default () => { const app = Router(); @@ -131,6 +132,9 @@ export default () => { dashboard.use('/warehouses', Container.get(WarehousesController).router()); dashboard.use('/projects', Container.get(ProjectsController).router()); dashboard.use('/tax-rates', Container.get(TaxRatesController).router()); + + dashboard.use('/import', Container.get(ImportController).router()); + dashboard.use('/', Container.get(ProjectTasksController).router()); dashboard.use('/', Container.get(ProjectTimesController).router()); diff --git a/packages/server/src/database/migrations/20231209230719_create_imports_table.js b/packages/server/src/database/migrations/20231209230719_create_imports_table.js new file mode 100644 index 000000000..69d0a0f7e --- /dev/null +++ b/packages/server/src/database/migrations/20231209230719_create_imports_table.js @@ -0,0 +1,14 @@ +exports.up = function (knex) { + return knex.schema.createTable('imports', (table) => { + table.increments(); + table.string('filename'); + table.string('import_id'); + table.string('resource'); + table.json('mapping'); + table.timestamps(); + }); +}; + +exports.down = function (knex) { + return knex.schema.dropTableIfExists('imports'); +}; diff --git a/packages/server/src/loaders/tenantModels.ts b/packages/server/src/loaders/tenantModels.ts index fcf1936be..8640819d4 100644 --- a/packages/server/src/loaders/tenantModels.ts +++ b/packages/server/src/loaders/tenantModels.ts @@ -61,6 +61,7 @@ import Task from 'models/Task'; import TaxRate from 'models/TaxRate'; import TaxRateTransaction from 'models/TaxRateTransaction'; import Attachment from 'models/Attachment'; +import Import from 'models/Import'; export default (knex) => { const models = { @@ -124,7 +125,8 @@ export default (knex) => { Task, TaxRate, TaxRateTransaction, - Attachment + Attachment, + Import }; return mapValues(models, (model) => model.bindKnex(knex)); }; diff --git a/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts b/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts new file mode 100644 index 000000000..a4f9eab2e --- /dev/null +++ b/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts @@ -0,0 +1,32 @@ +import { DATATYPES_LENGTH } from '@/data/DataTypes'; +import { IsInt, IsOptional, IsString, Length, Min, Max } from 'class-validator'; + +export class AccountDTOSchema { + @IsString() + @Length(3, DATATYPES_LENGTH.STRING) + name: string; + + // @IsString() + // @IsInt() + @IsOptional() + // @Length(3, 6) + code?: string; + + @IsOptional() + currencyCode?: string; + + @IsString() + @Length(3, DATATYPES_LENGTH.STRING) + accountType: string; + + @IsString() + @IsOptional() + @Length(0, DATATYPES_LENGTH.TEXT) + description?: string; + + @IsInt() + @IsOptional() + @Min(0) + @Max(DATATYPES_LENGTH.INT_10) + parentAccountId?: number; +} diff --git a/packages/server/src/services/Import/ImportFileUpload.ts b/packages/server/src/services/Import/ImportFileUpload.ts new file mode 100644 index 000000000..fd537640d --- /dev/null +++ b/packages/server/src/services/Import/ImportFileUpload.ts @@ -0,0 +1,54 @@ +import { first } from 'lodash'; +import { Inject, Service } from 'typedi'; +import { snakeCase, upperFirst } from 'lodash'; +import XLSX from 'xlsx'; +import * as R from 'ramda'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { trimObject } from './_utils'; +const fs = require('fs').promises; + +@Service() +export class ImportFileUploadService { + @Inject() + private tenancy: HasTenancyService; + + /** + * Reads the imported file and stores the import file meta under unqiue id. + * @param {number} tenantId - + * @param {string} filePath - + * @param {string} fileName - + * @returns + */ + public async import( + tenantId: number, + resource: string, + filePath: string, + filename: string + ) { + const { Import } = this.tenancy.models(tenantId); + const buffer = await fs.readFile(filePath); + const workbook = XLSX.read(buffer, { type: 'buffer' }); + + const firstSheetName = workbook.SheetNames[0]; + const worksheet = workbook.Sheets[firstSheetName]; + const jsonData = XLSX.utils.sheet_to_json(worksheet); + + const _resource = upperFirst(snakeCase(resource)); + + const exportFile = await Import.query().insert({ + filename, + importId: filename, + resource: _resource, + }); + const columns = this.getColumns(jsonData); + + return { + ...exportFile, + columns, + }; + } + + private getColumns(json: unknown[]) { + return R.compose(Object.keys, trimObject, first)(json); + } +} diff --git a/packages/server/src/services/Import/ImportResourceApplication.ts b/packages/server/src/services/Import/ImportResourceApplication.ts new file mode 100644 index 000000000..f17d1fa32 --- /dev/null +++ b/packages/server/src/services/Import/ImportResourceApplication.ts @@ -0,0 +1,29 @@ +import { Inject } from 'typedi'; +import { ImportFileUploadService } from './ImportFileUpload'; + +@Inject() +export class ImportResourceApplication { + @Inject() + private importFileService: ImportFileUploadService; + + /** + * Reads the imported file and stores the import file meta under unqiue id. + * @param {number} tenantId - + * @param {string} filePath - + * @param {string} fileName - + * @returns + */ + public async import( + tenantId: number, + resource: string, + filePath: string, + filename: string + ) { + return this.importFileService.import( + tenantId, + resource, + filePath, + filename + ); + } +} diff --git a/packages/server/src/services/Import/ImportResourceInjectable.ts b/packages/server/src/services/Import/ImportResourceInjectable.ts new file mode 100644 index 000000000..0bfc184a5 --- /dev/null +++ b/packages/server/src/services/Import/ImportResourceInjectable.ts @@ -0,0 +1,120 @@ +import XLSX, { readFile } from 'xlsx'; +import * as R from 'ramda'; +import async from 'async'; +import { camelCase, snakeCase, upperFirst } from 'lodash'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { Inject, Service } from 'typedi'; +import { first } from 'lodash'; +import { ServiceError } from '@/exceptions'; +import { validate } from 'class-validator'; +import { AccountDTOSchema } from '../Accounts/CreateAccountDTOSchema'; +import { AccountsApplication } from '../Accounts/AccountsApplication'; +import { plainToClass, plainToInstance } from 'class-transformer'; +const fs = require('fs').promises; + +const ERRORS = { + IMPORT_ID_NOT_FOUND: 'IMPORT_ID_NOT_FOUND', +}; + + +@Service() +export class ImportResourceInjectable { + @Inject() + private tenancy: HasTenancyService; + + @Inject() + private accountsApplication: AccountsApplication; + + public async mapping( + tenantId: number, + importId: number, + maps: { from: string; to: string }[] + ) { + const { Import } = this.tenancy.models(tenantId); + + const importFile = await Import.query().find('filename', importId); + + if (!importFile) { + throw new ServiceError(ERRORS.IMPORT_ID_NOT_FOUND); + } + // + await Import.query() + .findById(importFile.id) + .update({ + maps: JSON.stringify(maps), + }); + // - Validate the to columns. + // - Store the mapping in the import table. + // - + } + + public async preview(tenantId: number, importId: string) {} + + /** + * + * @param tenantId + * @param importId + */ + public async importFile(tenantId: number, importId: string) { + const { Import } = this.tenancy.models(tenantId); + + const importFile = await Import.query().where('importId', importId).first(); + + if (!importFile) { + throw new ServiceError(ERRORS.IMPORT_ID_NOT_FOUND); + } + const buffer = await fs.readFile(`public/imports/${importFile.filename}`); + const workbook = XLSX.read(buffer, { type: 'buffer' }); + + const firstSheetName = workbook.SheetNames[0]; + const worksheet = workbook.Sheets[firstSheetName]; + const jsonData = XLSX.utils.sheet_to_json(worksheet); + + const data = R.compose(R.map(Object.keys), R.map(trimObject))(jsonData); + + const header = first(data); + const body = jsonData; + + const mapping = JSON.parse(importFile.mapping) || []; + const newData = []; + + const findToAttr = (from: string) => { + const found = mapping.find((item) => { + return item.from === from; + }); + return found?.to; + }; + + body.forEach((row) => { + const obj = {}; + + header.forEach((key, index) => { + const toIndex = camelCase(findToAttr(key)); + obj[toIndex] = row[key]; + }); + newData.push(obj); + }); + + const saveJob = async (data) => { + const account = {}; + + Object.keys(data).map((key) => { + account[key] = data[key]; + }); + const accountClass = plainToInstance(AccountDTOSchema, account); + const errors = await validate(accountClass); + + if (errors.length > 0) { + console.log('validation failed. errors: ', errors); + } else { + return this.accountsApplication.createAccount(tenantId, account); + } + }; + const saveDataQueue = async.queue(saveJob, 10); + + newData.forEach((data) => { + saveDataQueue.push(data); + }); + await saveDataQueue.drain(); + } +} diff --git a/packages/server/src/services/Import/_utils.ts b/packages/server/src/services/Import/_utils.ts new file mode 100644 index 000000000..3700a313e --- /dev/null +++ b/packages/server/src/services/Import/_utils.ts @@ -0,0 +1,12 @@ +export function trimObject(obj) { + return Object.entries(obj).reduce((acc, [key, value]) => { + // Trim the key + const trimmedKey = key.trim(); + + // Trim the value if it's a string, otherwise leave it as is + const trimmedValue = typeof value === 'string' ? value.trim() : value; + + // Assign the trimmed key and value to the accumulator object + return { ...acc, [trimmedKey]: trimmedValue }; + }, {}); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9482a23bf..25a16e3de 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -80,6 +80,12 @@ importers: body-parser: specifier: ^1.20.2 version: 1.20.2 + class-transformer: + specifier: ^0.5.1 + version: 0.5.1 + class-validator: + specifier: ^0.14.0 + version: 0.14.0 compression: specifier: ^1.7.4 version: 1.7.4 @@ -203,6 +209,9 @@ importers: mongoose: specifier: ^5.10.0 version: 5.13.20 + multer: + specifier: 1.4.5-lts.1 + version: 1.4.5-lts.1 mustache: specifier: ^3.0.3 version: 3.2.1 @@ -6481,6 +6490,10 @@ packages: resolution: {integrity: sha512-IDaobHimLQhjwsQ/NMwRVfa/yL7L/wriQPMhw1ZJall0KX6E1oxk29XMDeilW5qTIg5aoiqf5Udy8U/51aNoQQ==} dev: false + /@types/validator@13.11.7: + resolution: {integrity: sha512-q0JomTsJ2I5Mv7dhHhQLGjMvX0JJm5dyZ1DXQySIUzU1UlwzB8bt+R6+LODUbz0UDIOvEzGc28tk27gBJw2N8Q==} + dev: false + /@types/webidl-conversions@7.0.0: resolution: {integrity: sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==} dev: false @@ -7458,6 +7471,10 @@ packages: buffer-equal: 1.0.1 dev: false + /append-field@1.0.0: + resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==} + dev: false + /append-transform@1.0.0: resolution: {integrity: sha512-P009oYkeHyU742iSZJzZZywj4QRJdnTWffaKuJQLablCZ1uz6/cW4yaRgcDaoQ+uwOxxnt0gRUcwfsNP2ri0gw==} engines: {node: '>=4'} @@ -9026,6 +9043,10 @@ packages: resolution: {integrity: sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==} dev: false + /class-transformer@0.5.1: + resolution: {integrity: sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==} + dev: false + /class-utils@0.3.6: resolution: {integrity: sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==} engines: {node: '>=0.10.0'} @@ -9036,6 +9057,14 @@ packages: static-extend: 0.1.2 dev: false + /class-validator@0.14.0: + resolution: {integrity: sha512-ct3ltplN8I9fOwUd8GrP8UQixwff129BkEtuWDKL5W45cQuLd19xqmTLu5ge78YDm/fdje6FMt0hGOhl0lii3A==} + dependencies: + '@types/validator': 13.11.7 + libphonenumber-js: 1.10.19 + validator: 13.9.0 + dev: false + /classnames@2.3.2: resolution: {integrity: sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==} dev: false @@ -17307,6 +17336,19 @@ packages: /ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + /multer@1.4.5-lts.1: + resolution: {integrity: sha512-ywPWvcDMeH+z9gQq5qYHCCy+ethsk4goepZ45GLD63fOu0YcNecQxi64nDs3qluZB+murG3/D4dJ7+dGctcCQQ==} + engines: {node: '>= 6.0.0'} + dependencies: + append-field: 1.0.0 + busboy: 1.6.0 + concat-stream: 1.6.2 + mkdirp: 0.5.6 + object-assign: 4.1.1 + type-is: 1.6.18 + xtend: 4.0.2 + dev: false + /multicast-dns@7.2.5: resolution: {integrity: sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==} hasBin: true From 90b4f3ef6d88a82f01e87dcb2458dbde5ceb9121 Mon Sep 17 00:00:00 2001 From: Ahmed Bouhuolia Date: Mon, 11 Mar 2024 00:21:36 +0200 Subject: [PATCH 2/7] feat: import resources from csv/xlsx --- .../controllers/Import/ImportController.ts | 76 ++++++--- packages/server/src/loaders/express.ts | 10 +- packages/server/src/models/Import.ts | 41 +++++ .../services/Accounts/AccountsApplication.ts | 6 +- .../src/services/Accounts/CreateAccount.ts | 51 +++--- .../Accounts/CreateAccountDTOSchema.ts | 9 +- .../src/services/Import/AccountsImportable.ts | 49 ++++++ .../src/services/Import/ImportFileMapping.ts | 36 ++++ .../src/services/Import/ImportFilePreview.ts | 11 ++ .../src/services/Import/ImportFileProcess.ts | 160 ++++++++++++++++++ .../src/services/Import/ImportFileUpload.ts | 5 +- .../Import/ImportResourceApplication.ts | 47 +++++ .../Import/ImportResourceInjectable.ts | 120 ------------- .../server/src/services/Import/Importable.ts | 7 + .../server/src/services/Import/interfaces.ts | 10 ++ .../server/src/services/UnitOfWork/index.ts | 13 +- 16 files changed, 467 insertions(+), 184 deletions(-) create mode 100644 packages/server/src/models/Import.ts create mode 100644 packages/server/src/services/Import/AccountsImportable.ts create mode 100644 packages/server/src/services/Import/ImportFileMapping.ts create mode 100644 packages/server/src/services/Import/ImportFilePreview.ts create mode 100644 packages/server/src/services/Import/ImportFileProcess.ts delete mode 100644 packages/server/src/services/Import/ImportResourceInjectable.ts create mode 100644 packages/server/src/services/Import/Importable.ts create mode 100644 packages/server/src/services/Import/interfaces.ts diff --git a/packages/server/src/api/controllers/Import/ImportController.ts b/packages/server/src/api/controllers/Import/ImportController.ts index cfdcb3dc6..2d7e10dd3 100644 --- a/packages/server/src/api/controllers/Import/ImportController.ts +++ b/packages/server/src/api/controllers/Import/ImportController.ts @@ -1,6 +1,6 @@ import { Inject, Service } from 'typedi'; import { Router, Request, Response, NextFunction } from 'express'; -import { query } from 'express-validator'; +import { query, body, param } from 'express-validator'; import Multer from 'multer'; import BaseController from '@/api/controllers/BaseController'; import { ServiceError } from '@/exceptions'; @@ -14,9 +14,6 @@ const upload = Multer({ @Service() export class ImportController extends BaseController { - @Inject() - private importResource: ImportResourceInjectable; - @Inject() private importResourceApp: ImportResourceApplication; @@ -26,23 +23,31 @@ export class ImportController extends BaseController { router() { const router = Router(); - // router.post( - // '/:import_id/import', - // this.asyncMiddleware(this.import.bind(this)), - // this.catchServiceErrors - // ); + router.post( + '/:import_id/import', + this.asyncMiddleware(this.import.bind(this)), + this.catchServiceErrors + ); router.post( '/file', - // [...this.importValidationSchema], upload.single('file'), - this.asyncMiddleware(this.fileUpload.bind(this)) - // this.catchServiceErrors + this.importValidationSchema, + this.validationResult, + this.asyncMiddleware(this.fileUpload.bind(this)), + this.catchServiceErrors + ); + router.post( + '/:import_id/mapping', + [ + param('import_id').exists().isString(), + body('mapping').exists().isArray({ min: 1 }), + body('mapping.*.from').exists(), + body('mapping.*.to').exists(), + ], + this.validationResult, + this.asyncMiddleware(this.mapping.bind(this)), + this.catchServiceErrors ); - // router.post( - // '/:import_id/mapping', - // this.asyncMiddleware(this.mapping.bind(this)), - // this.catchServiceErrors - // ); // router.get( // '/:import_id/preview', // this.asyncMiddleware(this.preview.bind(this)), @@ -55,8 +60,19 @@ export class ImportController extends BaseController { * Import validation schema. * @returns {ValidationSchema[]} */ - get importValidationSchema() { - return [query('resource').exists().isString().toString()]; + private get importValidationSchema() { + return [ + body('resource').exists(), + // body('file').custom((value, { req }) => { + // if (!value) { + // throw new Error('File is required'); + // } + // if (!['xlsx', 'csv'].includes(value.split('.').pop())) { + // throw new Error('File must be in xlsx or csv format'); + // } + // return true; + // }), + // ]; } /** @@ -92,11 +108,18 @@ export class ImportController extends BaseController { * @param res * @param next */ - async mapping(req: Request, res: Response, next: NextFunction) { + private async mapping(req: Request, res: Response, next: NextFunction) { const { tenantId } = req; + const { import_id: importId } = req.params; + const body = this.matchedBodyData(req); try { - await this.importResource.mapping(tenantId); + await this.importResourceApp.mapping(tenantId, importId, body?.mapping); + + return res.status(200).send({ + id: importId, + message: 'The given import sheet has mapped successfully.' + }) } catch (error) { next(error); } @@ -108,7 +131,7 @@ export class ImportController extends BaseController { * @param res * @param next */ - async preview(req: Request, res: Response, next: NextFunction) {} + private async preview(req: Request, res: Response, next: NextFunction) {} /** * @@ -116,14 +139,17 @@ export class ImportController extends BaseController { * @param res * @param next */ - async import(req: Request, res: Response, next: NextFunction) { + private async import(req: Request, res: Response, next: NextFunction) { const { tenantId } = req; const { import_id: importId } = req.params; try { - await this.importResource.importFile(tenantId, importId); + await this.importResourceApp.process(tenantId, importId); - return res.status(200).send({}); + return res.status(200).send({ + id: importId, + message: 'Importing the uploaded file is importing.' + }); } catch (error) { next(error); } diff --git a/packages/server/src/loaders/express.ts b/packages/server/src/loaders/express.ts index 694de8574..2f769e1b4 100644 --- a/packages/server/src/loaders/express.ts +++ b/packages/server/src/loaders/express.ts @@ -48,11 +48,11 @@ export default ({ app }) => { app.use('/public', express.static(path.join(global.__storage_dir))); // Handle multi-media requests. - app.use( - fileUpload({ - createParentPath: true, - }) - ); + // app.use( + // fileUpload({ + // createParentPath: true, + // }) + // ); // Logger middleware. app.use(LoggerMiddleware); diff --git a/packages/server/src/models/Import.ts b/packages/server/src/models/Import.ts new file mode 100644 index 000000000..7b1e9a736 --- /dev/null +++ b/packages/server/src/models/Import.ts @@ -0,0 +1,41 @@ +import TenantModel from 'models/TenantModel'; + +export default class Import extends TenantModel { + mapping!: string; + + /** + * Table name. + */ + static get tableName() { + return 'imports'; + } + + /** + * Virtual attributes. + */ + static get virtualAttributes() { + return ['mappingParsed']; + } + + /** + * Timestamps columns. + */ + get timestamps() { + return []; + } + + /** + * Relationship mapping. + */ + static get relationMappings() { + return {}; + } + + public get mappingParsed() { + try { + return JSON.parse(this.mapping); + } catch { + return []; + } + } +} diff --git a/packages/server/src/services/Accounts/AccountsApplication.ts b/packages/server/src/services/Accounts/AccountsApplication.ts index 8182b3058..b90eb37e9 100644 --- a/packages/server/src/services/Accounts/AccountsApplication.ts +++ b/packages/server/src/services/Accounts/AccountsApplication.ts @@ -16,6 +16,7 @@ import { ActivateAccount } from './ActivateAccount'; import { GetAccounts } from './GetAccounts'; import { GetAccount } from './GetAccount'; import { GetAccountTransactions } from './GetAccountTransactions'; +import { Knex } from 'knex'; @Service() export class AccountsApplication { @@ -48,9 +49,10 @@ export class AccountsApplication { */ public createAccount = ( tenantId: number, - accountDTO: IAccountCreateDTO + accountDTO: IAccountCreateDTO, + trx?: Knex.Transaction ): Promise => { - return this.createAccountService.createAccount(tenantId, accountDTO); + return this.createAccountService.createAccount(tenantId, accountDTO, trx); }; /** diff --git a/packages/server/src/services/Accounts/CreateAccount.ts b/packages/server/src/services/Accounts/CreateAccount.ts index b621d9104..c0eff3a55 100644 --- a/packages/server/src/services/Accounts/CreateAccount.ts +++ b/packages/server/src/services/Accounts/CreateAccount.ts @@ -97,13 +97,14 @@ export class CreateAccount { /** * Creates a new account on the storage. - * @param {number} tenantId - * @param {IAccountCreateDTO} accountDTO + * @param {number} tenantId + * @param {IAccountCreateDTO} accountDTO * @returns {Promise} */ public createAccount = async ( tenantId: number, - accountDTO: IAccountCreateDTO + accountDTO: IAccountCreateDTO, + trx?: Knex.Transaction ): Promise => { const { Account } = this.tenancy.models(tenantId); @@ -119,27 +120,31 @@ export class CreateAccount { tenantMeta.baseCurrency ); // Creates a new account with associated transactions under unit-of-work envirement. - return this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => { - // Triggers `onAccountCreating` event. - await this.eventPublisher.emitAsync(events.accounts.onCreating, { - tenantId, - accountDTO, - trx, - } as IAccountEventCreatingPayload); + return this.uow.withTransaction( + tenantId, + async (trx: Knex.Transaction) => { + // Triggers `onAccountCreating` event. + await this.eventPublisher.emitAsync(events.accounts.onCreating, { + tenantId, + accountDTO, + trx, + } as IAccountEventCreatingPayload); - // Inserts account to the storage. - const account = await Account.query(trx).insertAndFetch({ - ...accountInputModel, - }); - // Triggers `onAccountCreated` event. - await this.eventPublisher.emitAsync(events.accounts.onCreated, { - tenantId, - account, - accountId: account.id, - trx, - } as IAccountEventCreatedPayload); + // Inserts account to the storage. + const account = await Account.query(trx).insertAndFetch({ + ...accountInputModel, + }); + // Triggers `onAccountCreated` event. + await this.eventPublisher.emitAsync(events.accounts.onCreated, { + tenantId, + account, + accountId: account.id, + trx, + } as IAccountEventCreatedPayload); - return account; - }); + return account; + }, + trx + ); }; } diff --git a/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts b/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts index a4f9eab2e..d05f3b81a 100644 --- a/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts +++ b/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts @@ -1,15 +1,15 @@ import { DATATYPES_LENGTH } from '@/data/DataTypes'; -import { IsInt, IsOptional, IsString, Length, Min, Max } from 'class-validator'; +import { IsInt, IsOptional, IsString, Length, Min, Max, IsNotEmpty } from 'class-validator'; export class AccountDTOSchema { @IsString() @Length(3, DATATYPES_LENGTH.STRING) + @IsNotEmpty() name: string; - // @IsString() - // @IsInt() + @IsString() @IsOptional() - // @Length(3, 6) + @Length(3, 6) code?: string; @IsOptional() @@ -17,6 +17,7 @@ export class AccountDTOSchema { @IsString() @Length(3, DATATYPES_LENGTH.STRING) + @IsNotEmpty() accountType: string; @IsString() diff --git a/packages/server/src/services/Import/AccountsImportable.ts b/packages/server/src/services/Import/AccountsImportable.ts new file mode 100644 index 000000000..fe4bfdcf0 --- /dev/null +++ b/packages/server/src/services/Import/AccountsImportable.ts @@ -0,0 +1,49 @@ +import { IAccountCreateDTO } from '@/interfaces'; +import { AccountsApplication } from '../Accounts/AccountsApplication'; +import { AccountDTOSchema } from '../Accounts/CreateAccountDTOSchema'; +import { Inject, Service } from 'typedi'; +import { Knex } from 'knex'; + +@Service() +export class AccountsImportable { + @Inject() + private accountsApp: AccountsApplication; + + /** + * + * @param {number} tenantId + * @param {IAccountCreateDTO} createAccountDTO + * @returns + */ + public importable( + tenantId: number, + createAccountDTO: IAccountCreateDTO, + trx?: Knex.Transaction + ) { + return this.accountsApp.createAccount(tenantId, createAccountDTO, trx); + } + + /** + * + * @returns {} + */ + public validation() { + return AccountDTOSchema; + } + + /** + * + * @param data + * @returns + */ + public transform(data) { + return { + ...data, + accountType: this.mapAccountType(data.accounType), + }; + } + + mapAccountType(accountType: string) { + return 'Cash'; + } +} diff --git a/packages/server/src/services/Import/ImportFileMapping.ts b/packages/server/src/services/Import/ImportFileMapping.ts new file mode 100644 index 000000000..0408f7435 --- /dev/null +++ b/packages/server/src/services/Import/ImportFileMapping.ts @@ -0,0 +1,36 @@ +import { Inject, Service } from 'typedi'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { ImportMappingAttr } from './interfaces'; + +@Service() +export class ImportFileMapping { + @Inject() + private tenancy: HasTenancyService; + + /** + * Mapping the excel sheet columns with resource columns. + * @param {number} tenantId + * @param {number} importId + * @param {ImportMappingAttr} maps + */ + public async mapping( + tenantId: number, + importId: number, + maps: ImportMappingAttr[] + ) { + const { Import } = this.tenancy.models(tenantId); + + const importFile = await Import.query() + .findOne('filename', importId) + .throwIfNotFound(); + + // @todo validate the resource columns. + // @todo validate the sheet columns. + + const mappingStringified = JSON.stringify(maps); + + await Import.query().findById(importFile.id).patch({ + mapping: mappingStringified, + }); + } +} diff --git a/packages/server/src/services/Import/ImportFilePreview.ts b/packages/server/src/services/Import/ImportFilePreview.ts new file mode 100644 index 000000000..eb20bbcf9 --- /dev/null +++ b/packages/server/src/services/Import/ImportFilePreview.ts @@ -0,0 +1,11 @@ +import { Service } from 'typedi'; + +@Service() +export class ImportFilePreview { + /** + * + * @param {number} tenantId + * @param {number} importId + */ + public preview(tenantId: number, importId: number) {} +} diff --git a/packages/server/src/services/Import/ImportFileProcess.ts b/packages/server/src/services/Import/ImportFileProcess.ts new file mode 100644 index 000000000..cad785869 --- /dev/null +++ b/packages/server/src/services/Import/ImportFileProcess.ts @@ -0,0 +1,160 @@ +import { Inject, Service } from 'typedi'; +import * as R from 'ramda'; +import XLSX from 'xlsx'; +import { first, isUndefined } from 'lodash'; +import bluebird from 'bluebird'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { trimObject } from './_utils'; +import { ImportMappingAttr, ImportValidationError } from './interfaces'; +import { AccountsImportable } from './AccountsImportable'; +import { plainToInstance } from 'class-transformer'; +import { validate } from 'class-validator'; +import UnitOfWork from '../UnitOfWork'; +import { Knex } from 'knex'; +const fs = require('fs').promises; + +@Service() +export class ImportFileProcess { + @Inject() + private tenancy: HasTenancyService; + + @Inject() + private importable: AccountsImportable; + + @Inject() + private uow: UnitOfWork; + + /** + * Reads the import file. + * @param {string} filename + * @returns {Promise} + */ + public readImportFile(filename: string) { + return fs.readFile(`public/imports/${filename}`); + } + + /** + * Maps the columns of the imported data based on the provided mapping attributes. + * @param {Record[]} body - The array of data objects to map. + * @param {ImportMappingAttr[]} map - The mapping attributes. + * @returns {Record[]} - The mapped data objects. + */ + public parseXlsxSheet(buffer) { + const workbook = XLSX.read(buffer, { type: 'buffer' }); + + const firstSheetName = workbook.SheetNames[0]; + const worksheet = workbook.Sheets[firstSheetName]; + + return XLSX.utils.sheet_to_json(worksheet); + } + + /** + * Sanitizes the data in the imported sheet by trimming object keys. + * @param json - The JSON data representing the imported sheet. + * @returns {string[][]} - The sanitized data with trimmed object keys. + */ + public sanitizeSheetData(json) { + return R.compose(R.map(Object.keys), R.map(trimObject))(json); + } + + /** + * Maps the columns of the imported data based on the provided mapping attributes. + * @param {Record[]} body - The array of data objects to map. + * @param {ImportMappingAttr[]} map - The mapping attributes. + * @returns {Record[]} - The mapped data objects. + */ + private mapSheetColumns( + body: Record[], + map: ImportMappingAttr[] + ): Record[] { + return body.map((item) => { + const newItem = {}; + map + .filter((mapping) => !isUndefined(item[mapping.from])) + .forEach((mapping) => { + newItem[mapping.to] = item[mapping.from]; + }); + return newItem; + }); + } + + /** + * Validates the given mapped DTOs and returns errors with their index. + * @param {Record} mappedDTOs + * @returns {Promise} + */ + private async validateData( + mappedDTOs: Record + ): Promise { + const validateData = async (data, index: number) => { + const account = { ...data }; + const accountClass = plainToInstance( + this.importable.validation(), + account + ); + const errors = await validate(accountClass); + + if (errors?.length > 0) { + return errors.map((error) => ({ + index, + property: error.property, + constraints: error.constraints, + })); + } + return false; + }; + const errors = await bluebird.map(mappedDTOs, validateData, { + concurrency: 20, + }); + return errors.filter((error) => error !== false); + } + + /** + * Transfomees the mapped DTOs. + * @param DTOs + * @returns + */ + private transformDTOs(DTOs) { + return DTOs.map((DTO) => this.importable.transform(DTO)); + } + + /** + * Process + * @param {number} tenantId + * @param {number} importId + */ + public async process( + tenantId: number, + importId: number, + settings = { skipErrors: true } + ) { + const { Import } = this.tenancy.models(tenantId); + + const importFile = await Import.query() + .findOne('importId', importId) + .throwIfNotFound(); + + const buffer = await this.readImportFile(importFile.filename); + const jsonData = this.parseXlsxSheet(buffer); + + const data = this.sanitizeSheetData(jsonData); + + const header = first(data); + const body = jsonData; + + const mappedDTOs = this.mapSheetColumns(body, importFile.mappingParsed); + const transformedDTOs = this.transformDTOs(mappedDTOs); + + // Validate the mapped DTOs. + const errors = await this.validateData(transformedDTOs); + + return this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => { + await bluebird.map( + transformedDTOs, + (transformedDTO) => + this.importable.importable(tenantId, transformedDTO, trx), + { concurrency: 10 } + ); + }); + } +} diff --git a/packages/server/src/services/Import/ImportFileUpload.ts b/packages/server/src/services/Import/ImportFileUpload.ts index fd537640d..cdd1e14ac 100644 --- a/packages/server/src/services/Import/ImportFileUpload.ts +++ b/packages/server/src/services/Import/ImportFileUpload.ts @@ -26,6 +26,7 @@ export class ImportFileUploadService { filename: string ) { const { Import } = this.tenancy.models(tenantId); + const buffer = await fs.readFile(filePath); const workbook = XLSX.read(buffer, { type: 'buffer' }); @@ -33,6 +34,7 @@ export class ImportFileUploadService { const worksheet = workbook.Sheets[firstSheetName]; const jsonData = XLSX.utils.sheet_to_json(worksheet); + // @todo validate the resource. const _resource = upperFirst(snakeCase(resource)); const exportFile = await Import.query().insert({ @@ -42,8 +44,9 @@ export class ImportFileUploadService { }); const columns = this.getColumns(jsonData); + // @todo return the resource importable columns. return { - ...exportFile, + export: exportFile, columns, }; } diff --git a/packages/server/src/services/Import/ImportResourceApplication.ts b/packages/server/src/services/Import/ImportResourceApplication.ts index f17d1fa32..db464b6ec 100644 --- a/packages/server/src/services/Import/ImportResourceApplication.ts +++ b/packages/server/src/services/Import/ImportResourceApplication.ts @@ -1,11 +1,24 @@ import { Inject } from 'typedi'; import { ImportFileUploadService } from './ImportFileUpload'; +import { ImportFileMapping } from './ImportFileMapping'; +import { ImportMappingAttr } from './interfaces'; +import { ImportFileProcess } from './ImportFileProcess'; +import { ImportFilePreview } from './ImportFilePreview'; @Inject() export class ImportResourceApplication { @Inject() private importFileService: ImportFileUploadService; + @Inject() + private importMappingService: ImportFileMapping; + + @Inject() + private importProcessService: ImportFileProcess; + + @Inject() + private ImportFilePreviewService: ImportFilePreview; + /** * Reads the imported file and stores the import file meta under unqiue id. * @param {number} tenantId - @@ -26,4 +39,38 @@ export class ImportResourceApplication { filename ); } + + /** + * Mapping the excel sheet columns with resource columns. + * @param {number} tenantId + * @param {number} importId + * @param {ImportMappingAttr} maps + */ + public async mapping( + tenantId: number, + importId: number, + maps: ImportMappingAttr[] + ) { + return this.importMappingService.mapping(tenantId, importId, maps); + } + + /** + * Preview the mapped results before process importing. + * @param {number} tenantId + * @param {number} importId + * @returns {} + */ + public async preview(tenantId: number, importId: number) { + return this.ImportFilePreviewService.preview(tenantId, importId); + } + + /** + * + * @param {number} tenantId + * @param {number} importId + * @returns + */ + public async process(tenantId: number, importId: number) { + return this.importProcessService.process(tenantId, importId); + } } diff --git a/packages/server/src/services/Import/ImportResourceInjectable.ts b/packages/server/src/services/Import/ImportResourceInjectable.ts deleted file mode 100644 index 0bfc184a5..000000000 --- a/packages/server/src/services/Import/ImportResourceInjectable.ts +++ /dev/null @@ -1,120 +0,0 @@ -import XLSX, { readFile } from 'xlsx'; -import * as R from 'ramda'; -import async from 'async'; -import { camelCase, snakeCase, upperFirst } from 'lodash'; -import HasTenancyService from '../Tenancy/TenancyService'; -import { Inject, Service } from 'typedi'; -import { first } from 'lodash'; -import { ServiceError } from '@/exceptions'; -import { validate } from 'class-validator'; -import { AccountDTOSchema } from '../Accounts/CreateAccountDTOSchema'; -import { AccountsApplication } from '../Accounts/AccountsApplication'; -import { plainToClass, plainToInstance } from 'class-transformer'; -const fs = require('fs').promises; - -const ERRORS = { - IMPORT_ID_NOT_FOUND: 'IMPORT_ID_NOT_FOUND', -}; - - -@Service() -export class ImportResourceInjectable { - @Inject() - private tenancy: HasTenancyService; - - @Inject() - private accountsApplication: AccountsApplication; - - public async mapping( - tenantId: number, - importId: number, - maps: { from: string; to: string }[] - ) { - const { Import } = this.tenancy.models(tenantId); - - const importFile = await Import.query().find('filename', importId); - - if (!importFile) { - throw new ServiceError(ERRORS.IMPORT_ID_NOT_FOUND); - } - // - await Import.query() - .findById(importFile.id) - .update({ - maps: JSON.stringify(maps), - }); - // - Validate the to columns. - // - Store the mapping in the import table. - // - - } - - public async preview(tenantId: number, importId: string) {} - - /** - * - * @param tenantId - * @param importId - */ - public async importFile(tenantId: number, importId: string) { - const { Import } = this.tenancy.models(tenantId); - - const importFile = await Import.query().where('importId', importId).first(); - - if (!importFile) { - throw new ServiceError(ERRORS.IMPORT_ID_NOT_FOUND); - } - const buffer = await fs.readFile(`public/imports/${importFile.filename}`); - const workbook = XLSX.read(buffer, { type: 'buffer' }); - - const firstSheetName = workbook.SheetNames[0]; - const worksheet = workbook.Sheets[firstSheetName]; - const jsonData = XLSX.utils.sheet_to_json(worksheet); - - const data = R.compose(R.map(Object.keys), R.map(trimObject))(jsonData); - - const header = first(data); - const body = jsonData; - - const mapping = JSON.parse(importFile.mapping) || []; - const newData = []; - - const findToAttr = (from: string) => { - const found = mapping.find((item) => { - return item.from === from; - }); - return found?.to; - }; - - body.forEach((row) => { - const obj = {}; - - header.forEach((key, index) => { - const toIndex = camelCase(findToAttr(key)); - obj[toIndex] = row[key]; - }); - newData.push(obj); - }); - - const saveJob = async (data) => { - const account = {}; - - Object.keys(data).map((key) => { - account[key] = data[key]; - }); - const accountClass = plainToInstance(AccountDTOSchema, account); - const errors = await validate(accountClass); - - if (errors.length > 0) { - console.log('validation failed. errors: ', errors); - } else { - return this.accountsApplication.createAccount(tenantId, account); - } - }; - const saveDataQueue = async.queue(saveJob, 10); - - newData.forEach((data) => { - saveDataQueue.push(data); - }); - await saveDataQueue.drain(); - } -} diff --git a/packages/server/src/services/Import/Importable.ts b/packages/server/src/services/Import/Importable.ts new file mode 100644 index 000000000..b3019f34f --- /dev/null +++ b/packages/server/src/services/Import/Importable.ts @@ -0,0 +1,7 @@ + + +abstract class importable { + + + +} \ No newline at end of file diff --git a/packages/server/src/services/Import/interfaces.ts b/packages/server/src/services/Import/interfaces.ts new file mode 100644 index 000000000..1baed3a9b --- /dev/null +++ b/packages/server/src/services/Import/interfaces.ts @@ -0,0 +1,10 @@ +export interface ImportMappingAttr { + from: string; + to: string; +} + +export interface ImportValidationError { + index: number; + property: string; + constraints: Record; +} diff --git a/packages/server/src/services/UnitOfWork/index.ts b/packages/server/src/services/UnitOfWork/index.ts index c4c0c0dec..d5f9d0cda 100644 --- a/packages/server/src/services/UnitOfWork/index.ts +++ b/packages/server/src/services/UnitOfWork/index.ts @@ -1,5 +1,6 @@ import { Service, Inject } from 'typedi'; import TenancyService from '@/services/Tenancy/TenancyService'; +import { Transaction } from 'objection'; /** * Enumeration that represents transaction isolation levels for use with the {@link Transactional} annotation @@ -38,18 +39,22 @@ export default class UnitOfWork { public withTransaction = async ( tenantId: number, work, + trx?: Transaction, isolationLevel: IsolationLevel = IsolationLevel.READ_UNCOMMITTED ) => { const knex = this.tenancy.knex(tenantId); - const trx = await knex.transaction({ isolationLevel }); + let _trx = trx; + if (_trx) { + _trx = await knex.transaction({ isolationLevel }); + } try { - const result = await work(trx); - trx.commit(); + const result = await work(_trx); + _trx.commit(); return result; } catch (error) { - trx.rollback(); + _trx.rollback(); throw error; } }; From 4270d66928eef1f26b1395f385ae0ed49103e4a9 Mon Sep 17 00:00:00 2001 From: Ahmed Bouhuolia Date: Mon, 11 Mar 2024 20:05:12 +0200 Subject: [PATCH 3/7] feat(server): wip import resources --- packages/server/package.json | 4 +- .../controllers/Import/ImportController.ts | 15 +++ .../20231209230719_create_imports_table.js | 1 + packages/server/src/interfaces/Model.ts | 1 + .../server/src/models/Account.Settings.ts | 33 +++++- packages/server/src/models/Import.ts | 12 +++ .../src/services/Import/AccountsImportable.ts | 9 -- .../src/services/Import/ImportFileMapping.ts | 70 +++++++++++- .../src/services/Import/ImportFileProcess.ts | 101 ++++++++++++------ .../src/services/Import/ImportFileUpload.ts | 46 ++++++-- .../Import/ImportResourceApplication.ts | 4 +- packages/server/src/services/Import/_utils.ts | 49 +++++++++ .../Organization/OrganizationService.ts | 16 +-- .../src/services/Resource/ResourceService.ts | 34 +++++- .../server/src/services/UnitOfWork/index.ts | 2 +- pnpm-lock.yaml | 6 ++ 16 files changed, 328 insertions(+), 75 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index ef260cd19..a117eb5f6 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -25,6 +25,7 @@ "@types/i18n": "^0.8.7", "@types/knex": "^0.16.1", "@types/mathjs": "^6.0.12", + "@types/yup": "^0.29.13", "accepts": "^1.3.7", "accounting": "^0.4.1", "agenda": "^4.2.1", @@ -108,7 +109,8 @@ "typedi": "^0.8.0", "uniqid": "^5.2.0", "winston": "^3.2.1", - "xlsx": "^0.18.5" + "xlsx": "^0.18.5", + "yup": "^0.28.1" }, "devDependencies": { "@types/lodash": "^4.14.158", diff --git a/packages/server/src/api/controllers/Import/ImportController.ts b/packages/server/src/api/controllers/Import/ImportController.ts index 2d7e10dd3..4461f16d8 100644 --- a/packages/server/src/api/controllers/Import/ImportController.ts +++ b/packages/server/src/api/controllers/Import/ImportController.ts @@ -169,6 +169,21 @@ export class ImportController extends BaseController { next: NextFunction ) { if (error instanceof ServiceError) { + if (error.errorType === 'INVALID_MAP_ATTRS') { + return res.status(400).send({ + errors: [{ type: 'INVALID_MAP_ATTRS' }] + }); + } + if (error.errorType === 'DUPLICATED_FROM_MAP_ATTR') { + return res.status(400).send({ + errors: [{ type: 'DUPLICATED_FROM_MAP_ATTR' }], + }); + }; + if (error.errorType === 'DUPLICATED_TO_MAP_ATTR') { + return res.status(400).send({ + errors: [{ type: 'DUPLICATED_TO_MAP_ATTR' }], + }) + } } next(error); } diff --git a/packages/server/src/database/migrations/20231209230719_create_imports_table.js b/packages/server/src/database/migrations/20231209230719_create_imports_table.js index 69d0a0f7e..60fd5a83d 100644 --- a/packages/server/src/database/migrations/20231209230719_create_imports_table.js +++ b/packages/server/src/database/migrations/20231209230719_create_imports_table.js @@ -4,6 +4,7 @@ exports.up = function (knex) { table.string('filename'); table.string('import_id'); table.string('resource'); + table.json('columns'); table.json('mapping'); table.timestamps(); }); diff --git a/packages/server/src/interfaces/Model.ts b/packages/server/src/interfaces/Model.ts index 67b90e872..2bf8d6916 100644 --- a/packages/server/src/interfaces/Model.ts +++ b/packages/server/src/interfaces/Model.ts @@ -77,5 +77,6 @@ export type IModelMetaRelationField = IModelMetaRelationFieldCommon & ( export interface IModelMeta { defaultFilterField: string; defaultSort: IModelMetaDefaultSort; + importable?: boolean; fields: { [key: string]: IModelMetaField }; } diff --git a/packages/server/src/models/Account.Settings.ts b/packages/server/src/models/Account.Settings.ts index 3d0698d0d..7cba91bfd 100644 --- a/packages/server/src/models/Account.Settings.ts +++ b/packages/server/src/models/Account.Settings.ts @@ -6,16 +6,21 @@ export default { sortOrder: 'DESC', sortField: 'name', }, + importable: true, fields: { name: { name: 'account.field.name', column: 'name', fieldType: 'text', + unique: true, + required: true, + importable: true, }, description: { name: 'account.field.description', column: 'description', fieldType: 'text', + importable: true, }, slug: { name: 'account.field.slug', @@ -23,13 +28,17 @@ export default { fieldType: 'text', columnable: false, filterable: false, + importable: false, }, code: { name: 'account.field.code', column: 'code', fieldType: 'text', + importable: true, + minLength: 3, + maxLength: 6, }, - root_type: { + rootType: { name: 'account.field.root_type', fieldType: 'enumeration', options: [ @@ -41,6 +50,7 @@ export default { ], filterCustomQuery: RootTypeFieldFilterQuery, sortable: false, + importable: false, }, normal: { name: 'account.field.normal', @@ -51,6 +61,7 @@ export default { ], filterCustomQuery: NormalTypeFieldFilterQuery, sortable: false, + importable: false, }, type: { name: 'account.field.type', @@ -58,30 +69,42 @@ export default { fieldType: 'enumeration', options: ACCOUNT_TYPES.map((accountType) => ({ label: accountType.label, - key: accountType.key + key: accountType.key, })), + required: true, + importable: true, }, active: { name: 'account.field.active', column: 'active', fieldType: 'boolean', filterable: false, + importable: true, }, - balance: { + openingBalance: { name: 'account.field.balance', column: 'amount', fieldType: 'number', + importable: true, }, - currency: { + currencyCode: { name: 'account.field.currency', column: 'currency_code', fieldType: 'text', filterable: false, + importable: true, }, - created_at: { + parentAccount: { + name: 'account.field.parent_account', + column: 'parent_account_id', + fieldType: 'relation', + to: { model: 'Account', to: 'id' }, + }, + createdAt: { name: 'account.field.created_at', column: 'created_at', fieldType: 'date', + importable: false, }, }, }; diff --git a/packages/server/src/models/Import.ts b/packages/server/src/models/Import.ts index 7b1e9a736..1181071af 100644 --- a/packages/server/src/models/Import.ts +++ b/packages/server/src/models/Import.ts @@ -31,6 +31,18 @@ export default class Import extends TenantModel { return {}; } + public get isMapped() { + return Boolean(this.mapping); + } + + public get columnsParsed() { + try { + return JSON.parse(this.columns); + } catch { + return []; + } + } + public get mappingParsed() { try { return JSON.parse(this.mapping); diff --git a/packages/server/src/services/Import/AccountsImportable.ts b/packages/server/src/services/Import/AccountsImportable.ts index fe4bfdcf0..c02eb2950 100644 --- a/packages/server/src/services/Import/AccountsImportable.ts +++ b/packages/server/src/services/Import/AccountsImportable.ts @@ -23,14 +23,6 @@ export class AccountsImportable { return this.accountsApp.createAccount(tenantId, createAccountDTO, trx); } - /** - * - * @returns {} - */ - public validation() { - return AccountDTOSchema; - } - /** * * @param data @@ -39,7 +31,6 @@ export class AccountsImportable { public transform(data) { return { ...data, - accountType: this.mapAccountType(data.accounType), }; } diff --git a/packages/server/src/services/Import/ImportFileMapping.ts b/packages/server/src/services/Import/ImportFileMapping.ts index 0408f7435..285aacbd5 100644 --- a/packages/server/src/services/Import/ImportFileMapping.ts +++ b/packages/server/src/services/Import/ImportFileMapping.ts @@ -1,12 +1,19 @@ import { Inject, Service } from 'typedi'; import HasTenancyService from '../Tenancy/TenancyService'; import { ImportMappingAttr } from './interfaces'; +import ResourceService from '../Resource/ResourceService'; +import { ServiceError } from '@/exceptions'; +import { ERRORS } from './_utils'; +import { fromPairs } from 'lodash'; @Service() export class ImportFileMapping { @Inject() private tenancy: HasTenancyService; + @Inject() + private resource: ResourceService; + /** * Mapping the excel sheet columns with resource columns. * @param {number} tenantId @@ -24,8 +31,11 @@ export class ImportFileMapping { .findOne('filename', importId) .throwIfNotFound(); - // @todo validate the resource columns. - // @todo validate the sheet columns. + // Invalidate the from/to map attributes. + this.validateMapsAttrs(tenantId, importFile, maps); + + // Validate the diplicated relations of map attrs. + this.validateDuplicatedMapAttrs(maps); const mappingStringified = JSON.stringify(maps); @@ -33,4 +43,60 @@ export class ImportFileMapping { mapping: mappingStringified, }); } + + /** + * Validate the mapping attributes. + * @param {number} tenantId - + * @param {} importFile - + * @param {ImportMappingAttr[]} maps + * @throws {ServiceError(ERRORS.INVALID_MAP_ATTRS)} + */ + private validateMapsAttrs( + tenantId: number, + importFile: any, + maps: ImportMappingAttr[] + ) { + const fields = this.resource.getResourceImportableFields( + tenantId, + importFile.resource + ); + const columnsMap = fromPairs( + importFile.columnsParsed.map((field) => [field, '']) + ); + const invalid = []; + + maps.forEach((map) => { + if ( + 'undefined' === typeof fields[map.to] || + 'undefined' === typeof columnsMap[map.from] + ) { + invalid.push(map); + } + }); + if (invalid.length > 0) { + throw new ServiceError(ERRORS.INVALID_MAP_ATTRS); + } + } + + /** + * Validate the map attrs relation should be one-to-one relation only. + * @param {ImportMappingAttr[]} maps + */ + private validateDuplicatedMapAttrs(maps: ImportMappingAttr[]) { + const fromMap = {}; + const toMap = {}; + + maps.forEach((map) => { + if (fromMap[map.from]) { + throw new ServiceError(ERRORS.DUPLICATED_FROM_MAP_ATTR); + } else { + fromMap[map.from] = true; + } + if (toMap[map.to]) { + throw new ServiceError(ERRORS.DUPLICATED_TO_MAP_ATTR); + } else { + toMap[map.to] = true; + } + }); + } } diff --git a/packages/server/src/services/Import/ImportFileProcess.ts b/packages/server/src/services/Import/ImportFileProcess.ts index cad785869..23b040c6e 100644 --- a/packages/server/src/services/Import/ImportFileProcess.ts +++ b/packages/server/src/services/Import/ImportFileProcess.ts @@ -3,15 +3,16 @@ import * as R from 'ramda'; import XLSX from 'xlsx'; import { first, isUndefined } from 'lodash'; import bluebird from 'bluebird'; +import fs from 'fs/promises'; +import { Knex } from 'knex'; import HasTenancyService from '../Tenancy/TenancyService'; -import { trimObject } from './_utils'; +import { ERRORS, convertFieldsToYupValidation, trimObject } from './_utils'; import { ImportMappingAttr, ImportValidationError } from './interfaces'; import { AccountsImportable } from './AccountsImportable'; -import { plainToInstance } from 'class-transformer'; -import { validate } from 'class-validator'; import UnitOfWork from '../UnitOfWork'; -import { Knex } from 'knex'; -const fs = require('fs').promises; +import { ServiceError } from '@/exceptions'; +import ResourceService from '../Resource/ResourceService'; + @Service() export class ImportFileProcess { @@ -24,6 +25,9 @@ export class ImportFileProcess { @Inject() private uow: UnitOfWork; + @Inject() + private resourceService: ResourceService; + /** * Reads the import file. * @param {string} filename @@ -84,24 +88,30 @@ export class ImportFileProcess { * @returns {Promise} */ private async validateData( + tenantId: number, + resource: string, mappedDTOs: Record ): Promise { - const validateData = async (data, index: number) => { - const account = { ...data }; - const accountClass = plainToInstance( - this.importable.validation(), - account - ); - const errors = await validate(accountClass); + const importableFields = this.resourceService.getResourceImportableFields( + tenantId, + resource + ); + const YupSchema = convertFieldsToYupValidation(importableFields); - if (errors?.length > 0) { - return errors.map((error) => ({ - index, - property: error.property, - constraints: error.constraints, + const validateData = async (data, index: number) => { + const _data = { ...data }; + + try { + await YupSchema.validate(_data, { abortEarly: false }); + return { index, data: _data, errors: [] }; + } catch (validationError) { + const errors = validationError.inner.map((error) => ({ + path: error.params.path, + label: error.params.label, + message: error.errors, })); + return { index, data: _data, errors }; } - return false; }; const errors = await bluebird.map(mappedDTOs, validateData, { concurrency: 20, @@ -110,7 +120,7 @@ export class ImportFileProcess { } /** - * Transfomees the mapped DTOs. + * Transformes the mapped DTOs. * @param DTOs * @returns */ @@ -119,21 +129,22 @@ export class ImportFileProcess { } /** - * Process + * Processes the import file sheet through the resource service. * @param {number} tenantId * @param {number} importId + * @returns {Promise} */ - public async process( - tenantId: number, - importId: number, - settings = { skipErrors: true } - ) { + public async process(tenantId: number, importId: number) { const { Import } = this.tenancy.models(tenantId); const importFile = await Import.query() .findOne('importId', importId) .throwIfNotFound(); + // Throw error if the import file is not mapped yet. + if (!importFile.isMapped) { + throw new ServiceError(ERRORS.IMPORT_FILE_NOT_MAPPED); + } const buffer = await this.readImportFile(importFile.filename); const jsonData = this.parseXlsxSheet(buffer); @@ -146,15 +157,43 @@ export class ImportFileProcess { const transformedDTOs = this.transformDTOs(mappedDTOs); // Validate the mapped DTOs. - const errors = await this.validateData(transformedDTOs); - - return this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => { + const rowsWithErrors = await this.validateData( + tenantId, + importFile.resource, + transformedDTOs + ); + // Runs the importing under UOW envirement. + await this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => { await bluebird.map( - transformedDTOs, - (transformedDTO) => - this.importable.importable(tenantId, transformedDTO, trx), + rowsWithErrors, + (rowWithErrors) => { + if (rowWithErrors.errors.length === 0) { + return this.importable.importable( + tenantId, + rowWithErrors.data, + trx + ); + } + }, { concurrency: 10 } ); }); + // Deletes the imported file after importing success./ + await this.deleteImportFile(tenantId, importFile) + } + + /** + * Deletes the imported file from the storage and database. + * @param {number} tenantId + * @param {} importFile + */ + private async deleteImportFile(tenantId: number, importFile: any) { + const { Import } = this.tenancy.models(tenantId); + + // Deletes the import row. + await Import.query().findById(importFile.id).delete(); + + // Deletes the imported file. + await fs.unlink(`public/imports/${importFile.filename}`); } } diff --git a/packages/server/src/services/Import/ImportFileUpload.ts b/packages/server/src/services/Import/ImportFileUpload.ts index cdd1e14ac..6e476a616 100644 --- a/packages/server/src/services/Import/ImportFileUpload.ts +++ b/packages/server/src/services/Import/ImportFileUpload.ts @@ -1,17 +1,22 @@ -import { first } from 'lodash'; +import { first, values } from 'lodash'; import { Inject, Service } from 'typedi'; -import { snakeCase, upperFirst } from 'lodash'; +import { ServiceError } from '@/exceptions'; import XLSX from 'xlsx'; import * as R from 'ramda'; import HasTenancyService from '../Tenancy/TenancyService'; -import { trimObject } from './_utils'; -const fs = require('fs').promises; +import { ERRORS, trimObject } from './_utils'; +import ResourceService from '../Resource/ResourceService'; +import fs from 'fs/promises'; +import { IModelMetaField } from '@/interfaces'; @Service() export class ImportFileUploadService { @Inject() private tenancy: HasTenancyService; + @Inject() + private resourceService: ResourceService; + /** * Reads the imported file and stores the import file meta under unqiue id. * @param {number} tenantId - @@ -27,31 +32,54 @@ export class ImportFileUploadService { ) { const { Import } = this.tenancy.models(tenantId); + const resourceMeta = this.resourceService.getResourceMeta( + tenantId, + resource + ); + // Throw service error if the resource does not support importing. + if (!resourceMeta.importable) { + throw new ServiceError(ERRORS.RESOURCE_NOT_IMPORTABLE); + } const buffer = await fs.readFile(filePath); const workbook = XLSX.read(buffer, { type: 'buffer' }); const firstSheetName = workbook.SheetNames[0]; const worksheet = workbook.Sheets[firstSheetName]; const jsonData = XLSX.utils.sheet_to_json(worksheet); - + + const columns = this.getColumns(jsonData); + const coumnsStringified = JSON.stringify(columns); + // @todo validate the resource. - const _resource = upperFirst(snakeCase(resource)); + const _resource = this.resourceService.resourceToModelName(resource); const exportFile = await Import.query().insert({ filename, importId: filename, resource: _resource, + columns: coumnsStringified, }); - const columns = this.getColumns(jsonData); - + const resourceColumns = this.resourceService.getResourceImportableFields( + tenantId, + resource + ); + const resourceColumnsTransformeed = Object.entries(resourceColumns).map( + ([key, { name }]: [string, IModelMetaField]) => ({ key, name }) + ); // @todo return the resource importable columns. return { export: exportFile, columns, + resourceColumns: resourceColumnsTransformeed, }; } - private getColumns(json: unknown[]) { + /** + * Retrieves the sheet columns from the given sheet data. + * @param {unknown[]} json + * @returns {string[]} + */ + private getColumns(json: unknown[]): string[] { return R.compose(Object.keys, trimObject, first)(json); } } diff --git a/packages/server/src/services/Import/ImportResourceApplication.ts b/packages/server/src/services/Import/ImportResourceApplication.ts index db464b6ec..604818a29 100644 --- a/packages/server/src/services/Import/ImportResourceApplication.ts +++ b/packages/server/src/services/Import/ImportResourceApplication.ts @@ -65,10 +65,10 @@ export class ImportResourceApplication { } /** - * + * Process the import file sheet through service for creating entities. * @param {number} tenantId * @param {number} importId - * @returns + * @returns {Promise} */ public async process(tenantId: number, importId: number) { return this.importProcessService.process(tenantId, importId); diff --git a/packages/server/src/services/Import/_utils.ts b/packages/server/src/services/Import/_utils.ts index 3700a313e..404f0a8ab 100644 --- a/packages/server/src/services/Import/_utils.ts +++ b/packages/server/src/services/Import/_utils.ts @@ -1,3 +1,5 @@ +import * as Yup from 'yup'; + export function trimObject(obj) { return Object.entries(obj).reduce((acc, [key, value]) => { // Trim the key @@ -10,3 +12,50 @@ export function trimObject(obj) { return { ...acc, [trimmedKey]: trimmedValue }; }, {}); } + +export const convertFieldsToYupValidation = (fields: any) => { + const yupSchema = {}; + Object.keys(fields).forEach((fieldName) => { + const field = fields[fieldName]; + let fieldSchema; + fieldSchema = Yup.string().label(field.name); + + if (field.fieldType === 'text') { + if (field.minLength) { + fieldSchema = fieldSchema.min( + field.minLength, + `Minimum length is ${field.minLength} characters` + ); + } + if (field.maxLength) { + fieldSchema = fieldSchema.max( + field.maxLength, + `Maximum length is ${field.maxLength} characters` + ); + } + } else if (field.fieldType === 'number') { + fieldSchema = Yup.number().label(field.name); + } else if (field.fieldType === 'boolean') { + fieldSchema = Yup.boolean().label(field.name); + } else if (field.fieldType === 'enumeration') { + const options = field.options.reduce((acc, option) => { + acc[option.key] = option.label; + return acc; + }, {}); + fieldSchema = Yup.string().oneOf(Object.keys(options)).label(field.name); + } + if (field.required) { + fieldSchema = fieldSchema.required(); + } + yupSchema[fieldName] = fieldSchema; + }); + return Yup.object().shape(yupSchema); +}; + +export const ERRORS = { + RESOURCE_NOT_IMPORTABLE: 'RESOURCE_NOT_IMPORTABLE', + INVALID_MAP_ATTRS: 'INVALID_MAP_ATTRS', + DUPLICATED_FROM_MAP_ATTR: 'DUPLICATED_FROM_MAP_ATTR', + DUPLICATED_TO_MAP_ATTR: 'DUPLICATED_TO_MAP_ATTR', + IMPORT_FILE_NOT_MAPPED: 'IMPORT_FILE_NOT_MAPPED', +}; diff --git a/packages/server/src/services/Organization/OrganizationService.ts b/packages/server/src/services/Organization/OrganizationService.ts index 32431a066..453b6522c 100644 --- a/packages/server/src/services/Organization/OrganizationService.ts +++ b/packages/server/src/services/Organization/OrganizationService.ts @@ -21,25 +21,19 @@ import { ERRORS } from './constants'; @Service() export default class OrganizationService { @Inject() - eventPublisher: EventPublisher; - - @Inject('logger') - logger: any; - - @Inject('repositories') - sysRepositories: any; + private eventPublisher: EventPublisher; @Inject() - tenantsManager: TenantsManager; + private tenantsManager: TenantsManager; @Inject('agenda') - agenda: any; + private agenda: any; @Inject() - baseCurrencyMutateLocking: OrganizationBaseCurrencyLocking; + private baseCurrencyMutateLocking: OrganizationBaseCurrencyLocking; @Inject() - tenancy: HasTenancyService; + private tenancy: HasTenancyService; /** * Builds the database schema and seed data of the given organization id. diff --git a/packages/server/src/services/Resource/ResourceService.ts b/packages/server/src/services/Resource/ResourceService.ts index 79887851a..529448f8d 100644 --- a/packages/server/src/services/Resource/ResourceService.ts +++ b/packages/server/src/services/Resource/ResourceService.ts @@ -1,12 +1,11 @@ import { Service, Inject } from 'typedi'; -import { camelCase, upperFirst } from 'lodash'; +import { camelCase, upperFirst, pickBy } from 'lodash'; import * as qim from 'qim'; import pluralize from 'pluralize'; -import { IModelMeta } from '@/interfaces'; +import { IModelMeta, IModelMetaField } from '@/interfaces'; import TenancyService from '@/services/Tenancy/TenancyService'; import { ServiceError } from '@/exceptions'; import I18nService from '@/services/I18n/I18nService'; -import { tenantKnexConfig } from 'config/knexConfig'; const ERRORS = { RESOURCE_MODEL_NOT_FOUND: 'RESOURCE_MODEL_NOT_FOUND', @@ -24,7 +23,7 @@ export default class ResourceService { * Transform resource to model name. * @param {string} resourceName */ - private resourceToModelName(resourceName: string): string { + public resourceToModelName(resourceName: string): string { return upperFirst(camelCase(pluralize.singular(resourceName))); } @@ -63,6 +62,33 @@ export default class ResourceService { return this.getResourceMetaLocalized(resourceMeta, tenantId); } + /** + * + */ + public getResourceFields( + tenantId: number, + modelName: string + ): { [key: string]: IModelMetaField } { + const meta = this.getResourceMeta(tenantId, modelName); + + return meta.fields; + } + + /** + * + * @param {number} tenantId + * @param {string} modelName + * @returns + */ + public getResourceImportableFields( + tenantId: number, + modelName: string + ): { [key: string]: IModelMetaField } { + const fields = this.getResourceFields(tenantId, modelName); + + return pickBy(fields, (field) => field.importable); + } + /** * Retrieve the resource meta localized based on the current user language. */ diff --git a/packages/server/src/services/UnitOfWork/index.ts b/packages/server/src/services/UnitOfWork/index.ts index d5f9d0cda..024d330b7 100644 --- a/packages/server/src/services/UnitOfWork/index.ts +++ b/packages/server/src/services/UnitOfWork/index.ts @@ -45,7 +45,7 @@ export default class UnitOfWork { const knex = this.tenancy.knex(tenantId); let _trx = trx; - if (_trx) { + if (!_trx) { _trx = await knex.transaction({ isolationLevel }); } try { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0d7f044e8..b62633b19 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -47,6 +47,9 @@ importers: '@types/mathjs': specifier: ^6.0.12 version: 6.0.12 + '@types/yup': + specifier: ^0.29.13 + version: 0.29.14 accepts: specifier: ^1.3.7 version: 1.3.8 @@ -299,6 +302,9 @@ importers: xlsx: specifier: ^0.18.5 version: 0.18.5 + yup: + specifier: ^0.28.1 + version: 0.28.5 devDependencies: '@types/lodash': specifier: ^4.14.158 From daa1e3a6bd7b2644ec7807ac2dbce1e2f15bd7d6 Mon Sep 17 00:00:00 2001 From: Ahmed Bouhuolia Date: Wed, 13 Mar 2024 02:14:25 +0200 Subject: [PATCH 4/7] feat: wip import resource --- .../controllers/Import/ImportController.ts | 54 +++++---- packages/server/src/interfaces/Model.ts | 1 + .../server/src/models/Account.Settings.ts | 2 +- .../src/services/Import/AccountsImportable.ts | 27 +++-- .../src/services/Import/ImportFileCommon.ts | 87 +++++++++++++++ .../Import/ImportFileDataTransformer.ts | 104 ++++++++++++++++++ .../Import/ImportFileDataValidator.ts | 29 +++++ .../src/services/Import/ImportFilePreview.ts | 91 ++++++++++++++- .../src/services/Import/ImportFileUpload.ts | 26 +++-- .../services/Import/ImportResourceRegistry.ts | 7 ++ packages/server/src/services/Import/_utils.ts | 17 ++- .../server/src/services/Import/interfaces.ts | 10 ++ .../server/src/services/UnitOfWork/index.ts | 8 +- 13 files changed, 411 insertions(+), 52 deletions(-) create mode 100644 packages/server/src/services/Import/ImportFileCommon.ts create mode 100644 packages/server/src/services/Import/ImportFileDataTransformer.ts create mode 100644 packages/server/src/services/Import/ImportFileDataValidator.ts create mode 100644 packages/server/src/services/Import/ImportResourceRegistry.ts diff --git a/packages/server/src/api/controllers/Import/ImportController.ts b/packages/server/src/api/controllers/Import/ImportController.ts index 4461f16d8..0d4efbf93 100644 --- a/packages/server/src/api/controllers/Import/ImportController.ts +++ b/packages/server/src/api/controllers/Import/ImportController.ts @@ -1,10 +1,9 @@ import { Inject, Service } from 'typedi'; import { Router, Request, Response, NextFunction } from 'express'; -import { query, body, param } from 'express-validator'; +import { body, param } from 'express-validator'; import Multer from 'multer'; import BaseController from '@/api/controllers/BaseController'; import { ServiceError } from '@/exceptions'; -import { ImportResourceInjectable } from '@/services/Import/ImportResourceInjectable'; import { ImportResourceApplication } from '@/services/Import/ImportResourceApplication'; const upload = Multer({ @@ -23,11 +22,6 @@ export class ImportController extends BaseController { router() { const router = Router(); - router.post( - '/:import_id/import', - this.asyncMiddleware(this.import.bind(this)), - this.catchServiceErrors - ); router.post( '/file', upload.single('file'), @@ -36,6 +30,11 @@ export class ImportController extends BaseController { this.asyncMiddleware(this.fileUpload.bind(this)), this.catchServiceErrors ); + router.post( + '/:import_id/import', + this.asyncMiddleware(this.import.bind(this)), + this.catchServiceErrors + ); router.post( '/:import_id/mapping', [ @@ -48,11 +47,11 @@ export class ImportController extends BaseController { this.asyncMiddleware(this.mapping.bind(this)), this.catchServiceErrors ); - // router.get( - // '/:import_id/preview', - // this.asyncMiddleware(this.preview.bind(this)), - // this.catchServiceErrors - // ); + router.post( + '/:import_id/preview', + this.asyncMiddleware(this.preview.bind(this)), + this.catchServiceErrors + ); return router; } @@ -86,7 +85,7 @@ export class ImportController extends BaseController { * @param {Response} res - * @param {NextFunction} next - */ - async fileUpload(req: Request, res: Response, next: NextFunction) { + private async fileUpload(req: Request, res: Response, next: NextFunction) { const { tenantId } = req; try { @@ -103,10 +102,10 @@ export class ImportController extends BaseController { } /** - * - * @param req - * @param res - * @param next + * Maps the columns of the imported file. + * @param {Request} req + * @param {Response} res + * @param {NextFunction} next */ private async mapping(req: Request, res: Response, next: NextFunction) { const { tenantId } = req; @@ -126,12 +125,23 @@ export class ImportController extends BaseController { } /** - * - * @param req - * @param res - * @param next + * Preview the imported file before actual importing. + * @param {Request} req + * @param {Response} res + * @param {NextFunction} next */ - private async preview(req: Request, res: Response, next: NextFunction) {} + private async preview(req: Request, res: Response, next: NextFunction) { + const { tenantId } = req; + const { import_id: importId } = req.params; + + try { + const preview = await this.importResourceApp.preview(tenantId, importId); + + return res.status(200).send(preview); + } catch (error) { + next(error); + } + } /** * diff --git a/packages/server/src/interfaces/Model.ts b/packages/server/src/interfaces/Model.ts index 2bf8d6916..385bce172 100644 --- a/packages/server/src/interfaces/Model.ts +++ b/packages/server/src/interfaces/Model.ts @@ -34,6 +34,7 @@ export interface IModelMetaFieldCommon { columnable?: boolean; fieldType: IModelColumnType; customQuery?: Function; + required?: boolean; } export interface IModelMetaFieldNumber { diff --git a/packages/server/src/models/Account.Settings.ts b/packages/server/src/models/Account.Settings.ts index 7cba91bfd..ec67885c4 100644 --- a/packages/server/src/models/Account.Settings.ts +++ b/packages/server/src/models/Account.Settings.ts @@ -63,7 +63,7 @@ export default { sortable: false, importable: false, }, - type: { + accountType: { name: 'account.field.type', column: 'account_type', fieldType: 'enumeration', diff --git a/packages/server/src/services/Import/AccountsImportable.ts b/packages/server/src/services/Import/AccountsImportable.ts index c02eb2950..66b6c4570 100644 --- a/packages/server/src/services/Import/AccountsImportable.ts +++ b/packages/server/src/services/Import/AccountsImportable.ts @@ -1,13 +1,13 @@ -import { IAccountCreateDTO } from '@/interfaces'; -import { AccountsApplication } from '../Accounts/AccountsApplication'; -import { AccountDTOSchema } from '../Accounts/CreateAccountDTOSchema'; import { Inject, Service } from 'typedi'; import { Knex } from 'knex'; +import { IAccountCreateDTO } from '@/interfaces'; +import { AccountsApplication } from '../Accounts/AccountsApplication'; +import { CreateAccount } from '../Accounts/CreateAccount'; @Service() export class AccountsImportable { @Inject() - private accountsApp: AccountsApplication; + private createAccountService: CreateAccount; /** * @@ -20,7 +20,11 @@ export class AccountsImportable { createAccountDTO: IAccountCreateDTO, trx?: Knex.Transaction ) { - return this.accountsApp.createAccount(tenantId, createAccountDTO, trx); + return this.createAccountService.createAccount( + tenantId, + createAccountDTO, + trx + ); } /** @@ -29,12 +33,15 @@ export class AccountsImportable { * @returns */ public transform(data) { - return { - ...data, - }; + return { ...data }; } - mapAccountType(accountType: string) { - return 'Cash'; + /** + * + * @param data + * @returns + */ + public preTransform(data) { + return { ...data }; } } diff --git a/packages/server/src/services/Import/ImportFileCommon.ts b/packages/server/src/services/Import/ImportFileCommon.ts new file mode 100644 index 000000000..6719280fb --- /dev/null +++ b/packages/server/src/services/Import/ImportFileCommon.ts @@ -0,0 +1,87 @@ +import fs from 'fs/promises'; +import XLSX from 'xlsx'; +import bluebird from 'bluebird'; +import { Inject, Service } from 'typedi'; +import { ImportFileDataValidator } from './ImportFileDataValidator'; +import { Knex } from 'knex'; +import { ImportInsertError } from './interfaces'; +import { AccountsImportable } from './AccountsImportable'; +import { ServiceError } from '@/exceptions'; + +@Service() +export class ImportFileCommon { + @Inject() + private importFileValidator: ImportFileDataValidator; + + @Inject() + private importable: AccountsImportable; + + /** + * Maps the columns of the imported data based on the provided mapping attributes. + * @param {Record[]} body - The array of data objects to map. + * @param {ImportMappingAttr[]} map - The mapping attributes. + * @returns {Record[]} - The mapped data objects. + */ + public parseXlsxSheet(buffer) { + const workbook = XLSX.read(buffer, { type: 'buffer' }); + + const firstSheetName = workbook.SheetNames[0]; + const worksheet = workbook.Sheets[firstSheetName]; + + return XLSX.utils.sheet_to_json(worksheet); + } + + /** + * Reads the import file. + * @param {string} filename + * @returns {Promise} + */ + public readImportFile(filename: string) { + return fs.readFile(`public/imports/${filename}`); + } + + /** + * + * @param {number} tenantId - + * @param {Record} importableFields + * @param {Record} parsedData + * @param {Knex.Transaction} trx + * @returns + */ + public import( + tenantId: number, + importableFields, + parsedData: Record, + trx?: Knex.Transaction + ): Promise<(void | ImportInsertError[])[]> { + return bluebird.map( + parsedData, + async (objectDTO, index: number): Promise => { + try { + // Validate the DTO object before passing it to the service layer. + await this.importFileValidator.validateData( + importableFields, + objectDTO + ); + try { + // Run the importable function and listen to the errors. + await this.importable.importable(tenantId, objectDTO, trx); + } catch (error) { + if (error instanceof ServiceError) { + return [ + { + errorCode: 'ValidationError', + errorMessage: error.message || error.errorType, + rowNumber: index + 1, + }, + ]; + } + } + } catch (errors) { + return errors.map((er) => ({ ...er, rowNumber: index + 1 })); + } + }, + { concurrency: 2 } + ); + } +} diff --git a/packages/server/src/services/Import/ImportFileDataTransformer.ts b/packages/server/src/services/Import/ImportFileDataTransformer.ts new file mode 100644 index 000000000..859a16c25 --- /dev/null +++ b/packages/server/src/services/Import/ImportFileDataTransformer.ts @@ -0,0 +1,104 @@ +import { Inject, Service } from 'typedi'; +import * as R from 'ramda'; +import { isUndefined, mapValues, get, pickBy, chain } from 'lodash'; +import { ImportMappingAttr, ResourceMetaFieldsMap } from './interfaces'; +import { parseBoolean } from '@/utils'; +import { trimObject } from './_utils'; +import ResourceService from '../Resource/ResourceService'; + +@Service() +export class ImportFileDataTransformer { + @Inject() + private resource: ResourceService; + /** + * + * @param {number} tenantId - + * @param {} + */ + public transformSheetData( + importFile: any, + importableFields: any, + data: Record[] + ) { + // Sanitize the sheet data. + const sanitizedData = this.sanitizeSheetData(data); + + // Map the sheet columns key with the given map. + const mappedDTOs = this.mapSheetColumns( + sanitizedData, + importFile.mappingParsed + ); + // Parse the mapped sheet values. + const parsedValues = this.parseExcelValues(importableFields, mappedDTOs); + + return parsedValues; + } + + /** + * Sanitizes the data in the imported sheet by trimming object keys. + * @param json - The JSON data representing the imported sheet. + * @returns {string[][]} - The sanitized data with trimmed object keys. + */ + public sanitizeSheetData(json) { + return R.compose(R.map(trimObject))(json); + } + + /** + * Maps the columns of the imported data based on the provided mapping attributes. + * @param {Record[]} body - The array of data objects to map. + * @param {ImportMappingAttr[]} map - The mapping attributes. + * @returns {Record[]} - The mapped data objects. + */ + public mapSheetColumns( + body: Record[], + map: ImportMappingAttr[] + ): Record[] { + return body.map((item) => { + const newItem = {}; + map + .filter((mapping) => !isUndefined(item[mapping.from])) + .forEach((mapping) => { + newItem[mapping.to] = item[mapping.from]; + }); + return newItem; + }); + } + + /** + * Parses sheet values before passing to the service layer. + * @param {ResourceMetaFieldsMap} fields - + * @param {Record} valueDTOS - + * @returns {Record} + */ + public parseExcelValues( + fields: ResourceMetaFieldsMap, + valueDTOs: Record[] + ): Record { + const parser = (value, key) => { + let _value = value; + + // Parses the boolean value. + if (fields[key].fieldType === 'boolean') { + _value = parseBoolean(value, false); + + // Parses the enumeration value. + } else if (fields[key].fieldType === 'enumeration') { + const field = fields[key]; + const option = get(field, 'options', []).find( + (option) => option.label === value + ); + _value = get(option, 'key'); + // Prases the numeric value. + } else if (fields[key].fieldType === 'number') { + _value = parseFloat(value); + } + return _value; + }; + return valueDTOs.map((DTO) => { + return chain(DTO) + .pickBy((value, key) => !isUndefined(fields[key])) + .mapValues(parser) + .value(); + }); + } +} diff --git a/packages/server/src/services/Import/ImportFileDataValidator.ts b/packages/server/src/services/Import/ImportFileDataValidator.ts new file mode 100644 index 000000000..371117694 --- /dev/null +++ b/packages/server/src/services/Import/ImportFileDataValidator.ts @@ -0,0 +1,29 @@ +import { Service } from 'typedi'; +import { ImportInsertError, ResourceMetaFieldsMap } from './interfaces'; +import { convertFieldsToYupValidation } from './_utils'; + +@Service() +export class ImportFileDataValidator { + /** + * Validates the given mapped DTOs and returns errors with their index. + * @param {Record} mappedDTOs + * @returns {Promise} + */ + public async validateData( + importableFields: ResourceMetaFieldsMap, + data: Record + ): Promise { + const YupSchema = convertFieldsToYupValidation(importableFields); + const _data = { ...data }; + + try { + await YupSchema.validate(_data, { abortEarly: false }); + } catch (validationError) { + const errors = validationError.inner.map((error) => ({ + errorCode: 'ValidationError', + errorMessage: error.errors, + })); + throw errors; + } + } +} diff --git a/packages/server/src/services/Import/ImportFilePreview.ts b/packages/server/src/services/Import/ImportFilePreview.ts index eb20bbcf9..1d0cf1442 100644 --- a/packages/server/src/services/Import/ImportFilePreview.ts +++ b/packages/server/src/services/Import/ImportFilePreview.ts @@ -1,11 +1,98 @@ -import { Service } from 'typedi'; +import { Inject, Service } from 'typedi'; +import { first, omit } from 'lodash'; +import { ServiceError } from '@/exceptions'; +import { ERRORS, getUnmappedSheetColumns } from './_utils'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { ImportFileCommon } from './ImportFileCommon'; +import { ImportFileDataTransformer } from './ImportFileDataTransformer'; +import ResourceService from '../Resource/ResourceService'; @Service() export class ImportFilePreview { + @Inject() + private tenancy: HasTenancyService; + + @Inject() + private resource: ResourceService; + + @Inject() + private importFileCommon: ImportFileCommon; + + @Inject() + private importFileParser: ImportFileDataTransformer; + /** + * + * - Returns the passed rows and will be in inserted. + * - Returns the passed rows will be overwritten. + * - Returns the rows errors from the validation. + * - Returns the unmapped fields. * * @param {number} tenantId * @param {number} importId */ - public preview(tenantId: number, importId: number) {} + public async preview(tenantId: number, importId: number) { + const { Import } = this.tenancy.models(tenantId); + + const importFile = await Import.query() + .findOne('importId', importId) + .throwIfNotFound(); + + // Throw error if the import file is not mapped yet. + if (!importFile.isMapped) { + throw new ServiceError(ERRORS.IMPORT_FILE_NOT_MAPPED); + } + const buffer = await this.importFileCommon.readImportFile( + importFile.filename + ); + const jsonData = this.importFileCommon.parseXlsxSheet(buffer); + + const importableFields = this.resource.getResourceImportableFields( + tenantId, + importFile.resource + ); + // Prases the sheet json data. + const parsedData = this.importFileParser.transformSheetData( + importFile, + importableFields, + jsonData + ); + const knex = this.tenancy.knex(tenantId); + const trx = await knex.transaction({ isolationLevel: 'read uncommitted' }); + + // Runs the importing operation with ability to return errors that will happen. + const asyncOpers = await this.importFileCommon.import( + tenantId, + importableFields, + parsedData, + trx + ); + // Filter out the operations that have successed. + const successAsyncOpers = asyncOpers.filter((oper) => !oper); + const errors = asyncOpers.filter((oper) => oper); + + // Rollback all the successed transactions. + await trx.rollback(); + + const header = Object.keys(first(jsonData)); + const mapping = importFile.mappingParsed; + + const unmappedColumns = getUnmappedSheetColumns(header, mapping); + const totalCount = parsedData.length; + + const createdCount = successAsyncOpers.length; + const errorsCount = errors.length; + const skippedCount = errorsCount; + + return { + createdCount, + skippedCount, + totalCount, + errorsCount, + errors, + unmappedColumns: unmappedColumns, + unmappedColumnsCount: unmappedColumns.length, + }; + } + } diff --git a/packages/server/src/services/Import/ImportFileUpload.ts b/packages/server/src/services/Import/ImportFileUpload.ts index 6e476a616..42d61e021 100644 --- a/packages/server/src/services/Import/ImportFileUpload.ts +++ b/packages/server/src/services/Import/ImportFileUpload.ts @@ -8,7 +8,7 @@ import { ERRORS, trimObject } from './_utils'; import ResourceService from '../Resource/ResourceService'; import fs from 'fs/promises'; import { IModelMetaField } from '@/interfaces'; - +import { ImportFileCommon } from './ImportFileCommon'; @Service() export class ImportFileUploadService { @Inject() @@ -17,11 +17,15 @@ export class ImportFileUploadService { @Inject() private resourceService: ResourceService; + @Inject() + private importFileCommon: ImportFileCommon; + /** * Reads the imported file and stores the import file meta under unqiue id. - * @param {number} tenantId - - * @param {string} filePath - - * @param {string} fileName - + * @param {number} tenantId - Tenant id. + * @param {string} resource - Resource name. + * @param {string} filePath - File path. + * @param {string} fileName - File name. * @returns */ public async import( @@ -40,16 +44,15 @@ export class ImportFileUploadService { if (!resourceMeta.importable) { throw new ServiceError(ERRORS.RESOURCE_NOT_IMPORTABLE); } - const buffer = await fs.readFile(filePath); - const workbook = XLSX.read(buffer, { type: 'buffer' }); + // Reads the imported file into buffer. + const buffer = await this.importFileCommon.readImportFile(filename); + + // Parse the buffer file to array data. + const jsonData = this.importFileCommon.parseXlsxSheet(buffer); - const firstSheetName = workbook.SheetNames[0]; - const worksheet = workbook.Sheets[firstSheetName]; - const jsonData = XLSX.utils.sheet_to_json(worksheet); - const columns = this.getColumns(jsonData); const coumnsStringified = JSON.stringify(columns); - + // @todo validate the resource. const _resource = this.resourceService.resourceToModelName(resource); @@ -66,7 +69,6 @@ export class ImportFileUploadService { const resourceColumnsTransformeed = Object.entries(resourceColumns).map( ([key, { name }]: [string, IModelMetaField]) => ({ key, name }) ); - // @todo return the resource importable columns. return { export: exportFile, columns, diff --git a/packages/server/src/services/Import/ImportResourceRegistry.ts b/packages/server/src/services/Import/ImportResourceRegistry.ts new file mode 100644 index 000000000..ddd101a15 --- /dev/null +++ b/packages/server/src/services/Import/ImportResourceRegistry.ts @@ -0,0 +1,7 @@ +import { Service } from "typedi"; + + +@Service() +export class ImportResourceRegistry { + +} \ No newline at end of file diff --git a/packages/server/src/services/Import/_utils.ts b/packages/server/src/services/Import/_utils.ts index 404f0a8ab..843db14cc 100644 --- a/packages/server/src/services/Import/_utils.ts +++ b/packages/server/src/services/Import/_utils.ts @@ -1,4 +1,6 @@ import * as Yup from 'yup'; +import { ResourceMetaFieldsMap } from './interfaces'; +import { IModelMetaField } from '@/interfaces'; export function trimObject(obj) { return Object.entries(obj).reduce((acc, [key, value]) => { @@ -13,10 +15,10 @@ export function trimObject(obj) { }, {}); } -export const convertFieldsToYupValidation = (fields: any) => { +export const convertFieldsToYupValidation = (fields: ResourceMetaFieldsMap) => { const yupSchema = {}; - Object.keys(fields).forEach((fieldName) => { - const field = fields[fieldName]; + Object.keys(fields).forEach((fieldName: string) => { + const field = fields[fieldName] as IModelMetaField; let fieldSchema; fieldSchema = Yup.string().label(field.name); @@ -59,3 +61,12 @@ export const ERRORS = { DUPLICATED_TO_MAP_ATTR: 'DUPLICATED_TO_MAP_ATTR', IMPORT_FILE_NOT_MAPPED: 'IMPORT_FILE_NOT_MAPPED', }; + +/** + * + */ +export const getUnmappedSheetColumns = (columns, mapping) => { + return columns.filter( + (column) => !mapping.some((map) => map.from === column) + ); +}; diff --git a/packages/server/src/services/Import/interfaces.ts b/packages/server/src/services/Import/interfaces.ts index 1baed3a9b..0fb5c96f1 100644 --- a/packages/server/src/services/Import/interfaces.ts +++ b/packages/server/src/services/Import/interfaces.ts @@ -1,3 +1,5 @@ +import { IModelMetaField } from '@/interfaces'; + export interface ImportMappingAttr { from: string; to: string; @@ -8,3 +10,11 @@ export interface ImportValidationError { property: string; constraints: Record; } + +export type ResourceMetaFieldsMap = { [key: string]: IModelMetaField }; + +export interface ImportInsertError { + rowNumber: number; + errorCode: string; + errorMessage: string; +} diff --git a/packages/server/src/services/UnitOfWork/index.ts b/packages/server/src/services/UnitOfWork/index.ts index 024d330b7..a13be95c6 100644 --- a/packages/server/src/services/UnitOfWork/index.ts +++ b/packages/server/src/services/UnitOfWork/index.ts @@ -50,11 +50,15 @@ export default class UnitOfWork { } try { const result = await work(_trx); - _trx.commit(); + if (!trx) { + _trx.commit(); + } return result; } catch (error) { - _trx.rollback(); + if (!trx) { + _trx.rollback(); + } throw error; } }; From 084d9d3d10ed0f694dc7a2e42b9508317c50215a Mon Sep 17 00:00:00 2001 From: Ahmed Bouhuolia Date: Thu, 14 Mar 2024 22:18:12 +0200 Subject: [PATCH 5/7] feat: import resource --- .../controllers/Import/ImportController.ts | 44 ++-- .../src/services/Import/AccountsImportable.ts | 1 - .../src/services/Import/ImportFileCommon.ts | 126 +++++++--- .../Import/ImportFileDataTransformer.ts | 4 +- .../Import/ImportFileDataValidator.ts | 17 +- .../src/services/Import/ImportFileMapping.ts | 15 +- .../src/services/Import/ImportFilePreview.ts | 88 +------ .../src/services/Import/ImportFileProcess.ts | 226 +++++------------- .../src/services/Import/ImportFileUpload.ts | 57 ++--- .../Import/ImportResourceApplication.ts | 14 +- .../src/services/Import/ImportableRegistry.ts | 31 +++ .../services/Import/ImportableResources.ts | 39 +++ packages/server/src/services/Import/_utils.ts | 13 +- .../server/src/services/Import/interfaces.ts | 37 +++ 14 files changed, 360 insertions(+), 352 deletions(-) create mode 100644 packages/server/src/services/Import/ImportableRegistry.ts create mode 100644 packages/server/src/services/Import/ImportableResources.ts diff --git a/packages/server/src/api/controllers/Import/ImportController.ts b/packages/server/src/api/controllers/Import/ImportController.ts index 0d4efbf93..e459389c8 100644 --- a/packages/server/src/api/controllers/Import/ImportController.ts +++ b/packages/server/src/api/controllers/Import/ImportController.ts @@ -62,16 +62,16 @@ export class ImportController extends BaseController { private get importValidationSchema() { return [ body('resource').exists(), - // body('file').custom((value, { req }) => { - // if (!value) { - // throw new Error('File is required'); - // } - // if (!['xlsx', 'csv'].includes(value.split('.').pop())) { - // throw new Error('File must be in xlsx or csv format'); - // } - // return true; - // }), - // ]; + // body('file').custom((value, { req }) => { + // if (!value) { + // throw new Error('File is required'); + // } + // if (!['xlsx', 'csv'].includes(value.split('.').pop())) { + // throw new Error('File must be in xlsx or csv format'); + // } + // return true; + // }), + ]; } /** @@ -92,7 +92,6 @@ export class ImportController extends BaseController { const data = await this.importResourceApp.import( tenantId, req.body.resource, - req.file.path, req.file.filename ); return res.status(200).send(data); @@ -107,18 +106,19 @@ export class ImportController extends BaseController { * @param {Response} res * @param {NextFunction} next */ - private async mapping(req: Request, res: Response, next: NextFunction) { + private async mapping(req: Request, res: Response, next: NextFunction) { const { tenantId } = req; const { import_id: importId } = req.params; const body = this.matchedBodyData(req); try { - await this.importResourceApp.mapping(tenantId, importId, body?.mapping); + const mapping = await this.importResourceApp.mapping( + tenantId, + importId, + body?.mapping + ); - return res.status(200).send({ - id: importId, - message: 'The given import sheet has mapped successfully.' - }) + return res.status(200).send(mapping); } catch (error) { next(error); } @@ -135,7 +135,7 @@ export class ImportController extends BaseController { const { import_id: importId } = req.params; try { - const preview = await this.importResourceApp.preview(tenantId, importId); + const preview = await this.importResourceApp.preview(tenantId, importId); return res.status(200).send(preview); } catch (error) { @@ -158,7 +158,7 @@ export class ImportController extends BaseController { return res.status(200).send({ id: importId, - message: 'Importing the uploaded file is importing.' + message: 'Importing the uploaded file is importing.', }); } catch (error) { next(error); @@ -181,18 +181,18 @@ export class ImportController extends BaseController { if (error instanceof ServiceError) { if (error.errorType === 'INVALID_MAP_ATTRS') { return res.status(400).send({ - errors: [{ type: 'INVALID_MAP_ATTRS' }] + errors: [{ type: 'INVALID_MAP_ATTRS' }], }); } if (error.errorType === 'DUPLICATED_FROM_MAP_ATTR') { return res.status(400).send({ errors: [{ type: 'DUPLICATED_FROM_MAP_ATTR' }], }); - }; + } if (error.errorType === 'DUPLICATED_TO_MAP_ATTR') { return res.status(400).send({ errors: [{ type: 'DUPLICATED_TO_MAP_ATTR' }], - }) + }); } } next(error); diff --git a/packages/server/src/services/Import/AccountsImportable.ts b/packages/server/src/services/Import/AccountsImportable.ts index 66b6c4570..12b67a742 100644 --- a/packages/server/src/services/Import/AccountsImportable.ts +++ b/packages/server/src/services/Import/AccountsImportable.ts @@ -1,7 +1,6 @@ import { Inject, Service } from 'typedi'; import { Knex } from 'knex'; import { IAccountCreateDTO } from '@/interfaces'; -import { AccountsApplication } from '../Accounts/AccountsApplication'; import { CreateAccount } from '../Accounts/CreateAccount'; @Service() diff --git a/packages/server/src/services/Import/ImportFileCommon.ts b/packages/server/src/services/Import/ImportFileCommon.ts index 6719280fb..eaafe5b13 100644 --- a/packages/server/src/services/Import/ImportFileCommon.ts +++ b/packages/server/src/services/Import/ImportFileCommon.ts @@ -1,20 +1,36 @@ import fs from 'fs/promises'; import XLSX from 'xlsx'; import bluebird from 'bluebird'; +import * as R from 'ramda'; import { Inject, Service } from 'typedi'; +import { first } from 'lodash'; import { ImportFileDataValidator } from './ImportFileDataValidator'; import { Knex } from 'knex'; -import { ImportInsertError } from './interfaces'; +import { + ImportInsertError, + ImportOperError, + ImportOperSuccess, +} from './interfaces'; import { AccountsImportable } from './AccountsImportable'; import { ServiceError } from '@/exceptions'; +import { trimObject } from './_utils'; +import { ImportableResources } from './ImportableResources'; +import ResourceService from '../Resource/ResourceService'; +import HasTenancyService from '../Tenancy/TenancyService'; @Service() export class ImportFileCommon { + @Inject() + private tenancy: HasTenancyService; + @Inject() private importFileValidator: ImportFileDataValidator; @Inject() - private importable: AccountsImportable; + private importable: ImportableResources; + + @Inject() + private resource: ResourceService; /** * Maps the columns of the imported data based on the provided mapping attributes. @@ -22,7 +38,7 @@ export class ImportFileCommon { * @param {ImportMappingAttr[]} map - The mapping attributes. * @returns {Record[]} - The mapped data objects. */ - public parseXlsxSheet(buffer) { + public parseXlsxSheet(buffer: Buffer): Record[] { const workbook = XLSX.read(buffer, { type: 'buffer' }); const firstSheetName = workbook.SheetNames[0]; @@ -43,45 +59,81 @@ export class ImportFileCommon { /** * * @param {number} tenantId - - * @param {Record} importableFields - * @param {Record} parsedData + * @param {string} resourceName - Resource name. + * @param {Record} parsedData - * @param {Knex.Transaction} trx * @returns */ - public import( + public async import( tenantId: number, - importableFields, - parsedData: Record, + resourceName: string, + parsedData: Record[], trx?: Knex.Transaction - ): Promise<(void | ImportInsertError[])[]> { - return bluebird.map( - parsedData, - async (objectDTO, index: number): Promise => { - try { - // Validate the DTO object before passing it to the service layer. - await this.importFileValidator.validateData( - importableFields, - objectDTO - ); - try { - // Run the importable function and listen to the errors. - await this.importable.importable(tenantId, objectDTO, trx); - } catch (error) { - if (error instanceof ServiceError) { - return [ - { - errorCode: 'ValidationError', - errorMessage: error.message || error.errorType, - rowNumber: index + 1, - }, - ]; - } - } - } catch (errors) { - return errors.map((er) => ({ ...er, rowNumber: index + 1 })); - } - }, - { concurrency: 2 } + ): Promise<[ImportOperSuccess[], ImportOperError[]]> { + const importableFields = this.resource.getResourceImportableFields( + tenantId, + resourceName ); + const ImportableRegistry = this.importable.registry; + const importable = ImportableRegistry.getImportable(resourceName); + + const success: ImportOperSuccess[] = []; + const failed: ImportOperError[] = []; + + const importAsync = async (objectDTO, index: number): Promise => { + try { + // Validate the DTO object before passing it to the service layer. + await this.importFileValidator.validateData( + importableFields, + objectDTO + ); + try { + // Run the importable function and listen to the errors. + const data = await importable.importable(tenantId, objectDTO, trx); + success.push({ index, data }); + } catch (err) { + if (err instanceof ServiceError) { + const error = [ + { + errorCode: 'ValidationError', + errorMessage: err.message || err.errorType, + rowNumber: index + 1, + }, + ]; + failed.push({ index, error }); + } + } + } catch (errors) { + const error = errors.map((er) => ({ ...er, rowNumber: index + 1 })); + failed.push({ index, error }); + } + }; + await bluebird.map(parsedData, importAsync, { concurrency: 2 }); + + return [success, failed]; + } + + /** + * Retrieves the sheet columns from the given sheet data. + * @param {unknown[]} json + * @returns {string[]} + */ + public parseSheetColumns(json: unknown[]): string[] { + return R.compose(Object.keys, trimObject, first)(json); + } + + /** + * Deletes the imported file from the storage and database. + * @param {number} tenantId + * @param {} importFile + */ + private async deleteImportFile(tenantId: number, importFile: any) { + const { Import } = this.tenancy.models(tenantId); + + // Deletes the import row. + await Import.query().findById(importFile.id).delete(); + + // Deletes the imported file. + await fs.unlink(`public/imports/${importFile.filename}`); } } diff --git a/packages/server/src/services/Import/ImportFileDataTransformer.ts b/packages/server/src/services/Import/ImportFileDataTransformer.ts index 859a16c25..dc4ecb3ad 100644 --- a/packages/server/src/services/Import/ImportFileDataTransformer.ts +++ b/packages/server/src/services/Import/ImportFileDataTransformer.ts @@ -8,14 +8,12 @@ import ResourceService from '../Resource/ResourceService'; @Service() export class ImportFileDataTransformer { - @Inject() - private resource: ResourceService; /** * * @param {number} tenantId - * @param {} */ - public transformSheetData( + public parseSheetData( importFile: any, importableFields: any, data: Record[] diff --git a/packages/server/src/services/Import/ImportFileDataValidator.ts b/packages/server/src/services/Import/ImportFileDataValidator.ts index 371117694..0f41f2bb4 100644 --- a/packages/server/src/services/Import/ImportFileDataValidator.ts +++ b/packages/server/src/services/Import/ImportFileDataValidator.ts @@ -1,13 +1,26 @@ import { Service } from 'typedi'; import { ImportInsertError, ResourceMetaFieldsMap } from './interfaces'; -import { convertFieldsToYupValidation } from './_utils'; +import { ERRORS, convertFieldsToYupValidation } from './_utils'; +import { IModelMeta } from '@/interfaces'; +import { ServiceError } from '@/exceptions'; @Service() export class ImportFileDataValidator { + /** + * Validates the given resource is importable. + * @param {IModelMeta} resourceMeta + */ + public validateResourceImportable(resourceMeta: IModelMeta) { + // Throw service error if the resource does not support importing. + if (!resourceMeta.importable) { + throw new ServiceError(ERRORS.RESOURCE_NOT_IMPORTABLE); + } + } + /** * Validates the given mapped DTOs and returns errors with their index. * @param {Record} mappedDTOs - * @returns {Promise} + * @returns {Promise} */ public async validateData( importableFields: ResourceMetaFieldsMap, diff --git a/packages/server/src/services/Import/ImportFileMapping.ts b/packages/server/src/services/Import/ImportFileMapping.ts index 285aacbd5..1229c84f5 100644 --- a/packages/server/src/services/Import/ImportFileMapping.ts +++ b/packages/server/src/services/Import/ImportFileMapping.ts @@ -1,10 +1,10 @@ +import { fromPairs } from 'lodash'; import { Inject, Service } from 'typedi'; import HasTenancyService from '../Tenancy/TenancyService'; -import { ImportMappingAttr } from './interfaces'; +import { ImportFileMapPOJO, ImportMappingAttr } from './interfaces'; import ResourceService from '../Resource/ResourceService'; import { ServiceError } from '@/exceptions'; import { ERRORS } from './_utils'; -import { fromPairs } from 'lodash'; @Service() export class ImportFileMapping { @@ -24,7 +24,7 @@ export class ImportFileMapping { tenantId: number, importId: number, maps: ImportMappingAttr[] - ) { + ): Promise { const { Import } = this.tenancy.models(tenantId); const importFile = await Import.query() @@ -42,6 +42,13 @@ export class ImportFileMapping { await Import.query().findById(importFile.id).patch({ mapping: mappingStringified, }); + + return { + import: { + importId: importFile.importId, + resource: importFile.resource, + }, + }; } /** @@ -80,7 +87,7 @@ export class ImportFileMapping { /** * Validate the map attrs relation should be one-to-one relation only. - * @param {ImportMappingAttr[]} maps + * @param {ImportMappingAttr[]} maps */ private validateDuplicatedMapAttrs(maps: ImportMappingAttr[]) { const fromMap = {}; diff --git a/packages/server/src/services/Import/ImportFilePreview.ts b/packages/server/src/services/Import/ImportFilePreview.ts index 1d0cf1442..56a1df09e 100644 --- a/packages/server/src/services/Import/ImportFilePreview.ts +++ b/packages/server/src/services/Import/ImportFilePreview.ts @@ -1,11 +1,7 @@ import { Inject, Service } from 'typedi'; -import { first, omit } from 'lodash'; -import { ServiceError } from '@/exceptions'; -import { ERRORS, getUnmappedSheetColumns } from './_utils'; import HasTenancyService from '../Tenancy/TenancyService'; -import { ImportFileCommon } from './ImportFileCommon'; -import { ImportFileDataTransformer } from './ImportFileDataTransformer'; -import ResourceService from '../Resource/ResourceService'; +import { ImportFilePreviewPOJO } from './interfaces'; +import { ImportFileProcess } from './ImportFileProcess'; @Service() export class ImportFilePreview { @@ -13,86 +9,26 @@ export class ImportFilePreview { private tenancy: HasTenancyService; @Inject() - private resource: ResourceService; - - @Inject() - private importFileCommon: ImportFileCommon; - - @Inject() - private importFileParser: ImportFileDataTransformer; + private importFile: ImportFileProcess; /** - * - * - Returns the passed rows and will be in inserted. - * - Returns the passed rows will be overwritten. - * - Returns the rows errors from the validation. - * - Returns the unmapped fields. - * + * Preview the imported file results before commiting the transactions. * @param {number} tenantId * @param {number} importId + * @returns {Promise} */ - public async preview(tenantId: number, importId: number) { - const { Import } = this.tenancy.models(tenantId); - - const importFile = await Import.query() - .findOne('importId', importId) - .throwIfNotFound(); - - // Throw error if the import file is not mapped yet. - if (!importFile.isMapped) { - throw new ServiceError(ERRORS.IMPORT_FILE_NOT_MAPPED); - } - const buffer = await this.importFileCommon.readImportFile( - importFile.filename - ); - const jsonData = this.importFileCommon.parseXlsxSheet(buffer); - - const importableFields = this.resource.getResourceImportableFields( - tenantId, - importFile.resource - ); - // Prases the sheet json data. - const parsedData = this.importFileParser.transformSheetData( - importFile, - importableFields, - jsonData - ); + public async preview( + tenantId: number, + importId: number + ): Promise { const knex = this.tenancy.knex(tenantId); const trx = await knex.transaction({ isolationLevel: 'read uncommitted' }); - // Runs the importing operation with ability to return errors that will happen. - const asyncOpers = await this.importFileCommon.import( - tenantId, - importableFields, - parsedData, - trx - ); - // Filter out the operations that have successed. - const successAsyncOpers = asyncOpers.filter((oper) => !oper); - const errors = asyncOpers.filter((oper) => oper); + const meta = await this.importFile.import(tenantId, importId, trx); - // Rollback all the successed transactions. + // Rollback the successed transaction. await trx.rollback(); - const header = Object.keys(first(jsonData)); - const mapping = importFile.mappingParsed; - - const unmappedColumns = getUnmappedSheetColumns(header, mapping); - const totalCount = parsedData.length; - - const createdCount = successAsyncOpers.length; - const errorsCount = errors.length; - const skippedCount = errorsCount; - - return { - createdCount, - skippedCount, - totalCount, - errorsCount, - errors, - unmappedColumns: unmappedColumns, - unmappedColumnsCount: unmappedColumns.length, - }; + return meta; } - } diff --git a/packages/server/src/services/Import/ImportFileProcess.ts b/packages/server/src/services/Import/ImportFileProcess.ts index 23b040c6e..ff95f35d6 100644 --- a/packages/server/src/services/Import/ImportFileProcess.ts +++ b/packages/server/src/services/Import/ImportFileProcess.ts @@ -1,18 +1,14 @@ import { Inject, Service } from 'typedi'; -import * as R from 'ramda'; -import XLSX from 'xlsx'; -import { first, isUndefined } from 'lodash'; -import bluebird from 'bluebird'; -import fs from 'fs/promises'; +import { chain } from 'lodash'; import { Knex } from 'knex'; -import HasTenancyService from '../Tenancy/TenancyService'; -import { ERRORS, convertFieldsToYupValidation, trimObject } from './_utils'; -import { ImportMappingAttr, ImportValidationError } from './interfaces'; -import { AccountsImportable } from './AccountsImportable'; -import UnitOfWork from '../UnitOfWork'; import { ServiceError } from '@/exceptions'; +import { ERRORS, getSheetColumns, getUnmappedSheetColumns } from './_utils'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { ImportFileCommon } from './ImportFileCommon'; +import { ImportFileDataTransformer } from './ImportFileDataTransformer'; import ResourceService from '../Resource/ResourceService'; - +import UnitOfWork from '../UnitOfWork'; +import { ImportFilePreviewPOJO } from './interfaces'; @Service() export class ImportFileProcess { @@ -20,121 +16,28 @@ export class ImportFileProcess { private tenancy: HasTenancyService; @Inject() - private importable: AccountsImportable; + private resource: ResourceService; + + @Inject() + private importCommon: ImportFileCommon; + + @Inject() + private importParser: ImportFileDataTransformer; @Inject() private uow: UnitOfWork; - @Inject() - private resourceService: ResourceService; - /** - * Reads the import file. - * @param {string} filename - * @returns {Promise} - */ - public readImportFile(filename: string) { - return fs.readFile(`public/imports/${filename}`); - } - - /** - * Maps the columns of the imported data based on the provided mapping attributes. - * @param {Record[]} body - The array of data objects to map. - * @param {ImportMappingAttr[]} map - The mapping attributes. - * @returns {Record[]} - The mapped data objects. - */ - public parseXlsxSheet(buffer) { - const workbook = XLSX.read(buffer, { type: 'buffer' }); - - const firstSheetName = workbook.SheetNames[0]; - const worksheet = workbook.Sheets[firstSheetName]; - - return XLSX.utils.sheet_to_json(worksheet); - } - - /** - * Sanitizes the data in the imported sheet by trimming object keys. - * @param json - The JSON data representing the imported sheet. - * @returns {string[][]} - The sanitized data with trimmed object keys. - */ - public sanitizeSheetData(json) { - return R.compose(R.map(Object.keys), R.map(trimObject))(json); - } - - /** - * Maps the columns of the imported data based on the provided mapping attributes. - * @param {Record[]} body - The array of data objects to map. - * @param {ImportMappingAttr[]} map - The mapping attributes. - * @returns {Record[]} - The mapped data objects. - */ - private mapSheetColumns( - body: Record[], - map: ImportMappingAttr[] - ): Record[] { - return body.map((item) => { - const newItem = {}; - map - .filter((mapping) => !isUndefined(item[mapping.from])) - .forEach((mapping) => { - newItem[mapping.to] = item[mapping.from]; - }); - return newItem; - }); - } - - /** - * Validates the given mapped DTOs and returns errors with their index. - * @param {Record} mappedDTOs - * @returns {Promise} - */ - private async validateData( - tenantId: number, - resource: string, - mappedDTOs: Record - ): Promise { - const importableFields = this.resourceService.getResourceImportableFields( - tenantId, - resource - ); - const YupSchema = convertFieldsToYupValidation(importableFields); - - const validateData = async (data, index: number) => { - const _data = { ...data }; - - try { - await YupSchema.validate(_data, { abortEarly: false }); - return { index, data: _data, errors: [] }; - } catch (validationError) { - const errors = validationError.inner.map((error) => ({ - path: error.params.path, - label: error.params.label, - message: error.errors, - })); - return { index, data: _data, errors }; - } - }; - const errors = await bluebird.map(mappedDTOs, validateData, { - concurrency: 20, - }); - return errors.filter((error) => error !== false); - } - - /** - * Transformes the mapped DTOs. - * @param DTOs - * @returns - */ - private transformDTOs(DTOs) { - return DTOs.map((DTO) => this.importable.transform(DTO)); - } - - /** - * Processes the import file sheet through the resource service. + * Preview the imported file results before commiting the transactions. * @param {number} tenantId * @param {number} importId - * @returns {Promise} + * @returns {Promise} */ - public async process(tenantId: number, importId: number) { + public async import( + tenantId: number, + importId: number, + trx?: Knex.Transaction + ): Promise { const { Import } = this.tenancy.models(tenantId); const importFile = await Import.query() @@ -145,55 +48,54 @@ export class ImportFileProcess { if (!importFile.isMapped) { throw new ServiceError(ERRORS.IMPORT_FILE_NOT_MAPPED); } - const buffer = await this.readImportFile(importFile.filename); - const jsonData = this.parseXlsxSheet(buffer); + // Read the imported file. + const buffer = await this.importCommon.readImportFile(importFile.filename); + const sheetData = this.importCommon.parseXlsxSheet(buffer); + const header = getSheetColumns(sheetData); - const data = this.sanitizeSheetData(jsonData); - - const header = first(data); - const body = jsonData; - - const mappedDTOs = this.mapSheetColumns(body, importFile.mappingParsed); - const transformedDTOs = this.transformDTOs(mappedDTOs); - - // Validate the mapped DTOs. - const rowsWithErrors = await this.validateData( + const importableFields = this.resource.getResourceImportableFields( tenantId, - importFile.resource, - transformedDTOs + importFile.resource ); - // Runs the importing under UOW envirement. - await this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => { - await bluebird.map( - rowsWithErrors, - (rowWithErrors) => { - if (rowWithErrors.errors.length === 0) { - return this.importable.importable( - tenantId, - rowWithErrors.data, - trx - ); - } - }, - { concurrency: 10 } - ); - }); - // Deletes the imported file after importing success./ - await this.deleteImportFile(tenantId, importFile) - } + // Prases the sheet json data. + const parsedData = this.importParser.parseSheetData( + importFile, + importableFields, + sheetData + ); + // Runs the importing operation with ability to return errors that will happen. + const [successedImport, failedImport] = await this.uow.withTransaction( + tenantId, + (trx: Knex.Transaction) => + this.importCommon.import( + tenantId, + importFile.resource, + parsedData, + trx + ), + trx + ); + const mapping = importFile.mappingParsed; + const errors = chain(failedImport) + .map((oper) => oper.error) + .flatten() + .value(); - /** - * Deletes the imported file from the storage and database. - * @param {number} tenantId - * @param {} importFile - */ - private async deleteImportFile(tenantId: number, importFile: any) { - const { Import } = this.tenancy.models(tenantId); + const unmappedColumns = getUnmappedSheetColumns(header, mapping); + const totalCount = parsedData.length; - // Deletes the import row. - await Import.query().findById(importFile.id).delete(); + const createdCount = successedImport.length; + const errorsCount = failedImport.length; + const skippedCount = errorsCount; - // Deletes the imported file. - await fs.unlink(`public/imports/${importFile.filename}`); + return { + createdCount, + skippedCount, + totalCount, + errorsCount, + errors, + unmappedColumns: unmappedColumns, + unmappedColumnsCount: unmappedColumns.length, + }; } } diff --git a/packages/server/src/services/Import/ImportFileUpload.ts b/packages/server/src/services/Import/ImportFileUpload.ts index 42d61e021..fff7dac7f 100644 --- a/packages/server/src/services/Import/ImportFileUpload.ts +++ b/packages/server/src/services/Import/ImportFileUpload.ts @@ -1,14 +1,12 @@ -import { first, values } from 'lodash'; import { Inject, Service } from 'typedi'; -import { ServiceError } from '@/exceptions'; -import XLSX from 'xlsx'; -import * as R from 'ramda'; import HasTenancyService from '../Tenancy/TenancyService'; -import { ERRORS, trimObject } from './_utils'; +import { sanitizeResourceName } from './_utils'; import ResourceService from '../Resource/ResourceService'; -import fs from 'fs/promises'; import { IModelMetaField } from '@/interfaces'; import { ImportFileCommon } from './ImportFileCommon'; +import { ImportFileDataValidator } from './ImportFileDataValidator'; +import { ImportFileUploadPOJO } from './interfaces'; + @Service() export class ImportFileUploadService { @Inject() @@ -20,6 +18,9 @@ export class ImportFileUploadService { @Inject() private importFileCommon: ImportFileCommon; + @Inject() + private importValidator: ImportFileDataValidator; + /** * Reads the imported file and stores the import file meta under unqiue id. * @param {number} tenantId - Tenant id. @@ -30,58 +31,50 @@ export class ImportFileUploadService { */ public async import( tenantId: number, - resource: string, - filePath: string, + resourceName: string, filename: string - ) { + ): Promise { const { Import } = this.tenancy.models(tenantId); const resourceMeta = this.resourceService.getResourceMeta( tenantId, - resource + resourceName ); // Throw service error if the resource does not support importing. - if (!resourceMeta.importable) { - throw new ServiceError(ERRORS.RESOURCE_NOT_IMPORTABLE); - } + this.importValidator.validateResourceImportable(resourceMeta); + // Reads the imported file into buffer. const buffer = await this.importFileCommon.readImportFile(filename); // Parse the buffer file to array data. - const jsonData = this.importFileCommon.parseXlsxSheet(buffer); + const sheetData = this.importFileCommon.parseXlsxSheet(buffer); - const columns = this.getColumns(jsonData); - const coumnsStringified = JSON.stringify(columns); + const sheetColumns = this.importFileCommon.parseSheetColumns(sheetData); + const coumnsStringified = JSON.stringify(sheetColumns); - // @todo validate the resource. - const _resource = this.resourceService.resourceToModelName(resource); + const _resourceName = sanitizeResourceName(resourceName); - const exportFile = await Import.query().insert({ + // Store the import model with related metadata. + const importFile = await Import.query().insert({ filename, importId: filename, - resource: _resource, + resource: _resourceName, columns: coumnsStringified, }); const resourceColumns = this.resourceService.getResourceImportableFields( tenantId, - resource + _resourceName ); const resourceColumnsTransformeed = Object.entries(resourceColumns).map( ([key, { name }]: [string, IModelMetaField]) => ({ key, name }) ); return { - export: exportFile, - columns, + import: { + importId: importFile.importId, + resource: importFile.resource, + }, + sheetColumns, resourceColumns: resourceColumnsTransformeed, }; } - - /** - * Retrieves the sheet columns from the given sheet data. - * @param {unknown[]} json - * @returns {string[]} - */ - private getColumns(json: unknown[]): string[] { - return R.compose(Object.keys, trimObject, first)(json); - } } diff --git a/packages/server/src/services/Import/ImportResourceApplication.ts b/packages/server/src/services/Import/ImportResourceApplication.ts index 604818a29..07a9a7ea5 100644 --- a/packages/server/src/services/Import/ImportResourceApplication.ts +++ b/packages/server/src/services/Import/ImportResourceApplication.ts @@ -22,22 +22,16 @@ export class ImportResourceApplication { /** * Reads the imported file and stores the import file meta under unqiue id. * @param {number} tenantId - - * @param {string} filePath - + * @param {string} resource - * @param {string} fileName - - * @returns + * @returns {Promise} */ public async import( tenantId: number, resource: string, - filePath: string, filename: string ) { - return this.importFileService.import( - tenantId, - resource, - filePath, - filename - ); + return this.importFileService.import(tenantId, resource, filename); } /** @@ -71,6 +65,6 @@ export class ImportResourceApplication { * @returns {Promise} */ public async process(tenantId: number, importId: number) { - return this.importProcessService.process(tenantId, importId); + return this.importProcessService.import(tenantId, importId); } } diff --git a/packages/server/src/services/Import/ImportableRegistry.ts b/packages/server/src/services/Import/ImportableRegistry.ts new file mode 100644 index 000000000..c7b0120f2 --- /dev/null +++ b/packages/server/src/services/Import/ImportableRegistry.ts @@ -0,0 +1,31 @@ +import { camelCase, upperFirst } from 'lodash'; + +export class ImportableRegistry { + private static instance: ImportableRegistry; + private importables: Record; + + private constructor() { + this.importables = {}; + } + + public static getInstance(): ImportableRegistry { + if (!ImportableRegistry.instance) { + ImportableRegistry.instance = new ImportableRegistry(); + } + return ImportableRegistry.instance; + } + + public registerImportable(resource: string, importable: any): void { + const _resource = this.sanitizeResourceName(resource); + this.importables[_resource] = importable; + } + + public getImportable(name: string): any { + const _name = this.sanitizeResourceName(name); + return this.importables[_name]; + } + + private sanitizeResourceName(resource: string) { + return upperFirst(camelCase(resource)); + } +} diff --git a/packages/server/src/services/Import/ImportableResources.ts b/packages/server/src/services/Import/ImportableResources.ts new file mode 100644 index 000000000..f0da491f6 --- /dev/null +++ b/packages/server/src/services/Import/ImportableResources.ts @@ -0,0 +1,39 @@ +import Container, { Service } from 'typedi'; +import { AccountsImportable } from './AccountsImportable'; +import { ImportableRegistry } from './ImportableRegistry'; + +@Service() +export class ImportableResources { + private static registry: ImportableRegistry; + + + constructor() { + this.boot(); + } + + /** + * Importable instances. + */ + private importables = [ + { resource: 'Account', importable: AccountsImportable }, + ]; + + public get registry() { + return ImportableResources.registry; + } + + /** + * Boots all the registered importables. + */ + public boot() { + if (!ImportableResources.registry) { + const instance = ImportableRegistry.getInstance(); + + this.importables.forEach((importable) => { + const importableInstance = Container.get(importable.importable); + instance.registerImportable(importable.resource, importableInstance); + }); + ImportableResources.registry = instance; + } + } +} diff --git a/packages/server/src/services/Import/_utils.ts b/packages/server/src/services/Import/_utils.ts index 843db14cc..08146c367 100644 --- a/packages/server/src/services/Import/_utils.ts +++ b/packages/server/src/services/Import/_utils.ts @@ -1,4 +1,6 @@ import * as Yup from 'yup'; +import { upperFirst, camelCase, first } from 'lodash'; +import pluralize from 'pluralize'; import { ResourceMetaFieldsMap } from './interfaces'; import { IModelMetaField } from '@/interfaces'; @@ -62,11 +64,16 @@ export const ERRORS = { IMPORT_FILE_NOT_MAPPED: 'IMPORT_FILE_NOT_MAPPED', }; -/** - * - */ export const getUnmappedSheetColumns = (columns, mapping) => { return columns.filter( (column) => !mapping.some((map) => map.from === column) ); }; + +export const sanitizeResourceName = (resourceName: string) => { + return upperFirst(camelCase(pluralize.singular(resourceName))); +}; + +export const getSheetColumns = (sheetData: unknown[]) => { + return Object.keys(first(sheetData)); +}; diff --git a/packages/server/src/services/Import/interfaces.ts b/packages/server/src/services/Import/interfaces.ts index 0fb5c96f1..3eca339f7 100644 --- a/packages/server/src/services/Import/interfaces.ts +++ b/packages/server/src/services/Import/interfaces.ts @@ -18,3 +18,40 @@ export interface ImportInsertError { errorCode: string; errorMessage: string; } + +export interface ImportFileUploadPOJO { + import: { + importId: string; + resource: string; + }; + sheetColumns: string[]; + resourceColumns: { key: string; name: string }[]; +} + +export interface ImportFileMapPOJO { + import: { + importId: string; + resource: string; + }; +} + +export interface ImportFilePreviewPOJO { + createdCount: number; + skippedCount: number; + totalCount: number; + errorsCount: number; + errors: ImportInsertError[]; + unmappedColumns: string[]; + unmappedColumnsCount: number; +} + + +export interface ImportOperSuccess { + data: unknown; + index: number; +} + +export interface ImportOperError { + error: ImportInsertError; + index: number; +} \ No newline at end of file From ab4c0ab7a758b259f9a4848220f95a1e1af39121 Mon Sep 17 00:00:00 2001 From: Ahmed Bouhuolia Date: Fri, 15 Mar 2024 00:18:41 +0200 Subject: [PATCH 6/7] feat: wip import resources --- packages/server/package.json | 3 -- .../controllers/Import/ImportController.ts | 48 ++++++------------- .../src/api/controllers/Import/_utils.ts | 20 ++++++++ packages/server/src/loaders/express.ts | 8 ---- .../AccountsImportable.ts | 25 ++++------ .../Accounts/CreateAccountDTOSchema.ts | 33 ------------- .../src/services/Import/ImportFileCommon.ts | 16 ++++--- .../Import/ImportFileDataTransformer.ts | 5 +- .../src/services/Import/ImportFileUpload.ts | 2 +- .../Import/ImportResourceApplication.ts | 4 +- .../services/Import/ImportResourceRegistry.ts | 7 --- .../server/src/services/Import/Importable.ts | 26 ++++++++-- .../src/services/Import/ImportableRegistry.ts | 21 ++++++-- .../services/Import/ImportableResources.ts | 3 +- 14 files changed, 96 insertions(+), 125 deletions(-) create mode 100644 packages/server/src/api/controllers/Import/_utils.ts rename packages/server/src/services/{Import => Accounts}/AccountsImportable.ts (63%) delete mode 100644 packages/server/src/services/Accounts/CreateAccountDTOSchema.ts delete mode 100644 packages/server/src/services/Import/ImportResourceRegistry.ts diff --git a/packages/server/package.json b/packages/server/package.json index a117eb5f6..0fea6033c 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -37,8 +37,6 @@ "bcryptjs": "^2.4.3", "bluebird": "^3.7.2", "body-parser": "^1.20.2", - "class-transformer": "^0.5.1", - "class-validator": "^0.14.0", "compression": "^1.7.4", "country-codes-list": "^1.6.8", "cpy": "^8.1.2", @@ -56,7 +54,6 @@ "express": "^4.17.1", "express-basic-auth": "^1.2.0", "express-boom": "^3.0.0", - "express-fileupload": "^1.1.7-alpha.3", "express-oauth-server": "^2.0.0", "express-validator": "^6.12.2", "form-data": "^4.0.0", diff --git a/packages/server/src/api/controllers/Import/ImportController.ts b/packages/server/src/api/controllers/Import/ImportController.ts index e459389c8..c2d90cbe2 100644 --- a/packages/server/src/api/controllers/Import/ImportController.ts +++ b/packages/server/src/api/controllers/Import/ImportController.ts @@ -1,15 +1,10 @@ import { Inject, Service } from 'typedi'; import { Router, Request, Response, NextFunction } from 'express'; import { body, param } from 'express-validator'; -import Multer from 'multer'; import BaseController from '@/api/controllers/BaseController'; import { ServiceError } from '@/exceptions'; import { ImportResourceApplication } from '@/services/Import/ImportResourceApplication'; - -const upload = Multer({ - dest: './public/imports', - limits: { fileSize: 5 * 1024 * 1024 }, -}); +import { uploadImportFile } from './_utils'; @Service() export class ImportController extends BaseController { @@ -24,7 +19,7 @@ export class ImportController extends BaseController { router.post( '/file', - upload.single('file'), + uploadImportFile.single('file'), this.importValidationSchema, this.validationResult, this.asyncMiddleware(this.fileUpload.bind(this)), @@ -60,27 +55,11 @@ export class ImportController extends BaseController { * @returns {ValidationSchema[]} */ private get importValidationSchema() { - return [ - body('resource').exists(), - // body('file').custom((value, { req }) => { - // if (!value) { - // throw new Error('File is required'); - // } - // if (!['xlsx', 'csv'].includes(value.split('.').pop())) { - // throw new Error('File must be in xlsx or csv format'); - // } - // return true; - // }), - ]; + return [body('resource').exists()]; } /** * Imports xlsx/csv to the given resource type. - * - * - Save the xlsx/csv file and give it a random name. - * - Save the file metadata on the DB storage. - * - - * * @param {Request} req - * @param {Response} res - * @param {NextFunction} next - @@ -117,7 +96,6 @@ export class ImportController extends BaseController { importId, body?.mapping ); - return res.status(200).send(mapping); } catch (error) { next(error); @@ -144,22 +122,19 @@ export class ImportController extends BaseController { } /** - * - * @param req - * @param res - * @param next + * Importing the imported file to the application storage. + * @param {Request} req + * @param {Response} res + * @param {NextFunction} next */ private async import(req: Request, res: Response, next: NextFunction) { const { tenantId } = req; const { import_id: importId } = req.params; try { - await this.importResourceApp.process(tenantId, importId); + const result = await this.importResourceApp.process(tenantId, importId); - return res.status(200).send({ - id: importId, - message: 'Importing the uploaded file is importing.', - }); + return res.status(200).send(result); } catch (error) { next(error); } @@ -194,6 +169,11 @@ export class ImportController extends BaseController { errors: [{ type: 'DUPLICATED_TO_MAP_ATTR' }], }); } + if (error.errorType === 'IMPORTED_FILE_EXTENSION_INVALID') { + return res.status(400).send({ + errors: [{ type: 'IMPORTED_FILE_EXTENSION_INVALID' }], + }); + } } next(error); } diff --git a/packages/server/src/api/controllers/Import/_utils.ts b/packages/server/src/api/controllers/Import/_utils.ts new file mode 100644 index 000000000..333e280fe --- /dev/null +++ b/packages/server/src/api/controllers/Import/_utils.ts @@ -0,0 +1,20 @@ +import Multer from 'multer'; +import { ServiceError } from '@/exceptions'; + +export function allowSheetExtensions(req, file, cb) { + if ( + file.mimetype !== 'text/csv' && + file.mimetype !== 'application/vnd.ms-excel' + ) { + cb(new ServiceError('IMPORTED_FILE_EXTENSION_INVALID')); + + return; + } + cb(null, true); +} + +export const uploadImportFile = Multer({ + dest: './public/imports', + limits: { fileSize: 5 * 1024 * 1024 }, + fileFilter: allowSheetExtensions, +}); diff --git a/packages/server/src/loaders/express.ts b/packages/server/src/loaders/express.ts index 2f769e1b4..70b4fd309 100644 --- a/packages/server/src/loaders/express.ts +++ b/packages/server/src/loaders/express.ts @@ -4,7 +4,6 @@ import helmet from 'helmet'; import boom from 'express-boom'; import errorHandler from 'errorhandler'; import bodyParser from 'body-parser'; -import fileUpload from 'express-fileupload'; import { Server } from 'socket.io'; import Container from 'typedi'; import routes from 'api'; @@ -47,13 +46,6 @@ export default ({ app }) => { app.use('/public', express.static(path.join(global.__storage_dir))); - // Handle multi-media requests. - // app.use( - // fileUpload({ - // createParentPath: true, - // }) - // ); - // Logger middleware. app.use(LoggerMiddleware); diff --git a/packages/server/src/services/Import/AccountsImportable.ts b/packages/server/src/services/Accounts/AccountsImportable.ts similarity index 63% rename from packages/server/src/services/Import/AccountsImportable.ts rename to packages/server/src/services/Accounts/AccountsImportable.ts index 12b67a742..85429a751 100644 --- a/packages/server/src/services/Import/AccountsImportable.ts +++ b/packages/server/src/services/Accounts/AccountsImportable.ts @@ -1,15 +1,16 @@ import { Inject, Service } from 'typedi'; import { Knex } from 'knex'; import { IAccountCreateDTO } from '@/interfaces'; -import { CreateAccount } from '../Accounts/CreateAccount'; +import { CreateAccount } from './CreateAccount'; +import { Importable } from '../Import/Importable'; @Service() -export class AccountsImportable { +export class AccountsImportable extends Importable { @Inject() private createAccountService: CreateAccount; /** - * + * Importing to account service. * @param {number} tenantId * @param {IAccountCreateDTO} createAccountDTO * @returns @@ -27,20 +28,10 @@ export class AccountsImportable { } /** - * - * @param data - * @returns + * Concurrrency controlling of the importing process. + * @returns {number} */ - public transform(data) { - return { ...data }; - } - - /** - * - * @param data - * @returns - */ - public preTransform(data) { - return { ...data }; + public get concurrency() { + return 1; } } diff --git a/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts b/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts deleted file mode 100644 index d05f3b81a..000000000 --- a/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { DATATYPES_LENGTH } from '@/data/DataTypes'; -import { IsInt, IsOptional, IsString, Length, Min, Max, IsNotEmpty } from 'class-validator'; - -export class AccountDTOSchema { - @IsString() - @Length(3, DATATYPES_LENGTH.STRING) - @IsNotEmpty() - name: string; - - @IsString() - @IsOptional() - @Length(3, 6) - code?: string; - - @IsOptional() - currencyCode?: string; - - @IsString() - @Length(3, DATATYPES_LENGTH.STRING) - @IsNotEmpty() - accountType: string; - - @IsString() - @IsOptional() - @Length(0, DATATYPES_LENGTH.TEXT) - description?: string; - - @IsInt() - @IsOptional() - @Min(0) - @Max(DATATYPES_LENGTH.INT_10) - parentAccountId?: number; -} diff --git a/packages/server/src/services/Import/ImportFileCommon.ts b/packages/server/src/services/Import/ImportFileCommon.ts index eaafe5b13..86bfadf6c 100644 --- a/packages/server/src/services/Import/ImportFileCommon.ts +++ b/packages/server/src/services/Import/ImportFileCommon.ts @@ -11,7 +11,7 @@ import { ImportOperError, ImportOperSuccess, } from './interfaces'; -import { AccountsImportable } from './AccountsImportable'; +import { AccountsImportable } from '../Accounts/AccountsImportable'; import { ServiceError } from '@/exceptions'; import { trimObject } from './_utils'; import { ImportableResources } from './ImportableResources'; @@ -57,12 +57,12 @@ export class ImportFileCommon { } /** - * + * Imports the given parsed data to the resource storage through registered importable service. * @param {number} tenantId - * @param {string} resourceName - Resource name. - * @param {Record} parsedData - - * @param {Knex.Transaction} trx - * @returns + * @param {Record} parsedData - Parsed data. + * @param {Knex.Transaction} trx - Knex transaction. + * @returns {Promise<[ImportOperSuccess[], ImportOperError[]]>} */ public async import( tenantId: number, @@ -77,6 +77,8 @@ export class ImportFileCommon { const ImportableRegistry = this.importable.registry; const importable = ImportableRegistry.getImportable(resourceName); + const concurrency = importable.concurrency || 10; + const success: ImportOperSuccess[] = []; const failed: ImportOperError[] = []; @@ -108,7 +110,7 @@ export class ImportFileCommon { failed.push({ index, error }); } }; - await bluebird.map(parsedData, importAsync, { concurrency: 2 }); + await bluebird.map(parsedData, importAsync, { concurrency }); return [success, failed]; } @@ -127,7 +129,7 @@ export class ImportFileCommon { * @param {number} tenantId * @param {} importFile */ - private async deleteImportFile(tenantId: number, importFile: any) { + public async deleteImportFile(tenantId: number, importFile: any) { const { Import } = this.tenancy.models(tenantId); // Deletes the import row. diff --git a/packages/server/src/services/Import/ImportFileDataTransformer.ts b/packages/server/src/services/Import/ImportFileDataTransformer.ts index dc4ecb3ad..aa331d678 100644 --- a/packages/server/src/services/Import/ImportFileDataTransformer.ts +++ b/packages/server/src/services/Import/ImportFileDataTransformer.ts @@ -1,10 +1,9 @@ -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import * as R from 'ramda'; -import { isUndefined, mapValues, get, pickBy, chain } from 'lodash'; +import { isUndefined, get, chain } from 'lodash'; import { ImportMappingAttr, ResourceMetaFieldsMap } from './interfaces'; import { parseBoolean } from '@/utils'; import { trimObject } from './_utils'; -import ResourceService from '../Resource/ResourceService'; @Service() export class ImportFileDataTransformer { diff --git a/packages/server/src/services/Import/ImportFileUpload.ts b/packages/server/src/services/Import/ImportFileUpload.ts index fff7dac7f..19135d94a 100644 --- a/packages/server/src/services/Import/ImportFileUpload.ts +++ b/packages/server/src/services/Import/ImportFileUpload.ts @@ -27,7 +27,7 @@ export class ImportFileUploadService { * @param {string} resource - Resource name. * @param {string} filePath - File path. * @param {string} fileName - File name. - * @returns + * @returns {Promise} */ public async import( tenantId: number, diff --git a/packages/server/src/services/Import/ImportResourceApplication.ts b/packages/server/src/services/Import/ImportResourceApplication.ts index 07a9a7ea5..8577a4b79 100644 --- a/packages/server/src/services/Import/ImportResourceApplication.ts +++ b/packages/server/src/services/Import/ImportResourceApplication.ts @@ -52,7 +52,7 @@ export class ImportResourceApplication { * Preview the mapped results before process importing. * @param {number} tenantId * @param {number} importId - * @returns {} + * @returns {Promise} */ public async preview(tenantId: number, importId: number) { return this.ImportFilePreviewService.preview(tenantId, importId); @@ -62,7 +62,7 @@ export class ImportResourceApplication { * Process the import file sheet through service for creating entities. * @param {number} tenantId * @param {number} importId - * @returns {Promise} + * @returns {Promise} */ public async process(tenantId: number, importId: number) { return this.importProcessService.import(tenantId, importId); diff --git a/packages/server/src/services/Import/ImportResourceRegistry.ts b/packages/server/src/services/Import/ImportResourceRegistry.ts deleted file mode 100644 index ddd101a15..000000000 --- a/packages/server/src/services/Import/ImportResourceRegistry.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Service } from "typedi"; - - -@Service() -export class ImportResourceRegistry { - -} \ No newline at end of file diff --git a/packages/server/src/services/Import/Importable.ts b/packages/server/src/services/Import/Importable.ts index b3019f34f..8130910f5 100644 --- a/packages/server/src/services/Import/Importable.ts +++ b/packages/server/src/services/Import/Importable.ts @@ -1,7 +1,23 @@ +import { Knex } from 'knex'; +export abstract class Importable { + /** + * + * @param {number} tenantId + * @param {any} createDTO + * @param {Knex.Transaction} trx + */ + public importable(tenantId: number, createDTO: any, trx?: Knex.Transaction) { + throw new Error( + 'The `importable` function is not defined in service importable.' + ); + } -abstract class importable { - - - -} \ No newline at end of file + /** + * Concurrency controlling of the importing process. + * @returns {number} + */ + public get concurrency() { + return 10; + } +} diff --git a/packages/server/src/services/Import/ImportableRegistry.ts b/packages/server/src/services/Import/ImportableRegistry.ts index c7b0120f2..c260c5bd5 100644 --- a/packages/server/src/services/Import/ImportableRegistry.ts +++ b/packages/server/src/services/Import/ImportableRegistry.ts @@ -1,13 +1,18 @@ import { camelCase, upperFirst } from 'lodash'; +import { Importable } from './Importable'; export class ImportableRegistry { private static instance: ImportableRegistry; - private importables: Record; + private importables: Record; private constructor() { this.importables = {}; } + /** + * Gets singleton instance of registry. + * @returns {ImportableRegistry} + */ public static getInstance(): ImportableRegistry { if (!ImportableRegistry.instance) { ImportableRegistry.instance = new ImportableRegistry(); @@ -15,12 +20,22 @@ export class ImportableRegistry { return ImportableRegistry.instance; } - public registerImportable(resource: string, importable: any): void { + /** + * Registers the given importable service. + * @param {string} resource + * @param {Importable} importable + */ + public registerImportable(resource: string, importable: Importable): void { const _resource = this.sanitizeResourceName(resource); this.importables[_resource] = importable; } - public getImportable(name: string): any { + /** + * Retrieves the importable service instance of the given resource name. + * @param {string} name + * @returns {Importable} + */ + public getImportable(name: string): Importable { const _name = this.sanitizeResourceName(name); return this.importables[_name]; } diff --git a/packages/server/src/services/Import/ImportableResources.ts b/packages/server/src/services/Import/ImportableResources.ts index f0da491f6..3f4297075 100644 --- a/packages/server/src/services/Import/ImportableResources.ts +++ b/packages/server/src/services/Import/ImportableResources.ts @@ -1,12 +1,11 @@ import Container, { Service } from 'typedi'; -import { AccountsImportable } from './AccountsImportable'; +import { AccountsImportable } from '../Accounts/AccountsImportable'; import { ImportableRegistry } from './ImportableRegistry'; @Service() export class ImportableResources { private static registry: ImportableRegistry; - constructor() { this.boot(); } From c5063fc5b56509c160cd10b3543688596aa5f379 Mon Sep 17 00:00:00 2001 From: Ahmed Bouhuolia Date: Fri, 15 Mar 2024 00:26:23 +0200 Subject: [PATCH 7/7] chore: update the frozen pnpm-lock.yaml file --- packages/server/src/models/Import.ts | 8 ++++++- pnpm-lock.yaml | 32 ---------------------------- 2 files changed, 7 insertions(+), 33 deletions(-) diff --git a/packages/server/src/models/Import.ts b/packages/server/src/models/Import.ts index 1181071af..b0c558203 100644 --- a/packages/server/src/models/Import.ts +++ b/packages/server/src/models/Import.ts @@ -2,7 +2,8 @@ import TenantModel from 'models/TenantModel'; export default class Import extends TenantModel { mapping!: string; - + columns!: string; + /** * Table name. */ @@ -31,6 +32,10 @@ export default class Import extends TenantModel { return {}; } + /** + * Detarmines whether the import is mapped. + * @returns {boolean} + */ public get isMapped() { return Boolean(this.mapping); } @@ -43,6 +48,7 @@ export default class Import extends TenantModel { } } + public get mappingParsed() { try { return JSON.parse(this.mapping); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b62633b19..63855f63d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -83,12 +83,6 @@ importers: body-parser: specifier: ^1.20.2 version: 1.20.2 - class-transformer: - specifier: ^0.5.1 - version: 0.5.1 - class-validator: - specifier: ^0.14.0 - version: 0.14.0 compression: specifier: ^1.7.4 version: 1.7.4 @@ -140,9 +134,6 @@ importers: express-boom: specifier: ^3.0.0 version: 3.0.0 - express-fileupload: - specifier: ^1.1.7-alpha.3 - version: 1.4.0 express-oauth-server: specifier: ^2.0.0 version: 2.0.0 @@ -6931,10 +6922,6 @@ packages: resolution: {integrity: sha512-IDaobHimLQhjwsQ/NMwRVfa/yL7L/wriQPMhw1ZJall0KX6E1oxk29XMDeilW5qTIg5aoiqf5Udy8U/51aNoQQ==} dev: false - /@types/validator@13.11.7: - resolution: {integrity: sha512-q0JomTsJ2I5Mv7dhHhQLGjMvX0JJm5dyZ1DXQySIUzU1UlwzB8bt+R6+LODUbz0UDIOvEzGc28tk27gBJw2N8Q==} - dev: false - /@types/webidl-conversions@7.0.0: resolution: {integrity: sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==} dev: false @@ -9538,10 +9525,6 @@ packages: resolution: {integrity: sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==} dev: false - /class-transformer@0.5.1: - resolution: {integrity: sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==} - dev: false - /class-utils@0.3.6: resolution: {integrity: sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==} engines: {node: '>=0.10.0'} @@ -9552,14 +9535,6 @@ packages: static-extend: 0.1.2 dev: false - /class-validator@0.14.0: - resolution: {integrity: sha512-ct3ltplN8I9fOwUd8GrP8UQixwff129BkEtuWDKL5W45cQuLd19xqmTLu5ge78YDm/fdje6FMt0hGOhl0lii3A==} - dependencies: - '@types/validator': 13.11.7 - libphonenumber-js: 1.10.19 - validator: 13.9.0 - dev: false - /classnames@2.3.2: resolution: {integrity: sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==} dev: false @@ -12508,13 +12483,6 @@ packages: boom: 7.3.0 dev: false - /express-fileupload@1.4.0: - resolution: {integrity: sha512-RjzLCHxkv3umDeZKeFeMg8w7qe0V09w3B7oGZprr/oO2H/ISCgNzuqzn7gV3HRWb37GjRk429CCpSLS2KNTqMQ==} - engines: {node: '>=12.0.0'} - dependencies: - busboy: 1.6.0 - dev: false - /express-oauth-server@2.0.0: resolution: {integrity: sha512-+UrTbvU7u3LVnoUavzO7QJgSqiEZREKprCZYrDEVoSszrO4t8f/BBPbY3hQOuuatoS0PgDFLaDKQsGNtAgPm5w==} engines: {node: '>=0.11'}