diff --git a/packages/server/package.json b/packages/server/package.json index b47058471..0fea6033c 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -25,6 +25,7 @@ "@types/i18n": "^0.8.7", "@types/knex": "^0.16.1", "@types/mathjs": "^6.0.12", + "@types/yup": "^0.29.13", "accepts": "^1.3.7", "accounting": "^0.4.1", "agenda": "^4.2.1", @@ -53,7 +54,6 @@ "express": "^4.17.1", "express-basic-auth": "^1.2.0", "express-boom": "^3.0.0", - "express-fileupload": "^1.1.7-alpha.3", "express-oauth-server": "^2.0.0", "express-validator": "^6.12.2", "form-data": "^4.0.0", @@ -77,6 +77,7 @@ "moment-timezone": "^0.5.43", "mongodb": "^6.1.0", "mongoose": "^5.10.0", + "multer": "1.4.5-lts.1", "mustache": "^3.0.3", "mysql": "^2.17.1", "mysql2": "^1.6.5", @@ -105,7 +106,8 @@ "typedi": "^0.8.0", "uniqid": "^5.2.0", "winston": "^3.2.1", - "xlsx": "^0.18.5" + "xlsx": "^0.18.5", + "yup": "^0.28.1" }, "devDependencies": { "@types/lodash": "^4.14.158", diff --git a/packages/server/src/api/controllers/Import/ImportController.ts b/packages/server/src/api/controllers/Import/ImportController.ts new file mode 100644 index 000000000..c2d90cbe2 --- /dev/null +++ b/packages/server/src/api/controllers/Import/ImportController.ts @@ -0,0 +1,180 @@ +import { Inject, Service } from 'typedi'; +import { Router, Request, Response, NextFunction } from 'express'; +import { body, param } from 'express-validator'; +import BaseController from '@/api/controllers/BaseController'; +import { ServiceError } from '@/exceptions'; +import { ImportResourceApplication } from '@/services/Import/ImportResourceApplication'; +import { uploadImportFile } from './_utils'; + +@Service() +export class ImportController extends BaseController { + @Inject() + private importResourceApp: ImportResourceApplication; + + /** + * Router constructor method. + */ + router() { + const router = Router(); + + router.post( + '/file', + uploadImportFile.single('file'), + this.importValidationSchema, + this.validationResult, + this.asyncMiddleware(this.fileUpload.bind(this)), + this.catchServiceErrors + ); + router.post( + '/:import_id/import', + this.asyncMiddleware(this.import.bind(this)), + this.catchServiceErrors + ); + router.post( + '/:import_id/mapping', + [ + param('import_id').exists().isString(), + body('mapping').exists().isArray({ min: 1 }), + body('mapping.*.from').exists(), + body('mapping.*.to').exists(), + ], + this.validationResult, + this.asyncMiddleware(this.mapping.bind(this)), + this.catchServiceErrors + ); + router.post( + '/:import_id/preview', + this.asyncMiddleware(this.preview.bind(this)), + this.catchServiceErrors + ); + return router; + } + + /** + * Import validation schema. + * @returns {ValidationSchema[]} + */ + private get importValidationSchema() { + return [body('resource').exists()]; + } + + /** + * Imports xlsx/csv to the given resource type. + * @param {Request} req - + * @param {Response} res - + * @param {NextFunction} next - + */ + private async fileUpload(req: Request, res: Response, next: NextFunction) { + const { tenantId } = req; + + try { + const data = await this.importResourceApp.import( + tenantId, + req.body.resource, + req.file.filename + ); + return res.status(200).send(data); + } catch (error) { + next(error); + } + } + + /** + * Maps the columns of the imported file. + * @param {Request} req + * @param {Response} res + * @param {NextFunction} next + */ + private async mapping(req: Request, res: Response, next: NextFunction) { + const { tenantId } = req; + const { import_id: importId } = req.params; + const body = this.matchedBodyData(req); + + try { + const mapping = await this.importResourceApp.mapping( + tenantId, + importId, + body?.mapping + ); + return res.status(200).send(mapping); + } catch (error) { + next(error); + } + } + + /** + * Preview the imported file before actual importing. + * @param {Request} req + * @param {Response} res + * @param {NextFunction} next + */ + private async preview(req: Request, res: Response, next: NextFunction) { + const { tenantId } = req; + const { import_id: importId } = req.params; + + try { + const preview = await this.importResourceApp.preview(tenantId, importId); + + return res.status(200).send(preview); + } catch (error) { + next(error); + } + } + + /** + * Importing the imported file to the application storage. + * @param {Request} req + * @param {Response} res + * @param {NextFunction} next + */ + private async import(req: Request, res: Response, next: NextFunction) { + const { tenantId } = req; + const { import_id: importId } = req.params; + + try { + const result = await this.importResourceApp.process(tenantId, importId); + + return res.status(200).send(result); + } catch (error) { + next(error); + } + } + + /** + * Transforms service errors to response. + * @param {Error} + * @param {Request} req + * @param {Response} res + * @param {ServiceError} error + */ + private catchServiceErrors( + error, + req: Request, + res: Response, + next: NextFunction + ) { + if (error instanceof ServiceError) { + if (error.errorType === 'INVALID_MAP_ATTRS') { + return res.status(400).send({ + errors: [{ type: 'INVALID_MAP_ATTRS' }], + }); + } + if (error.errorType === 'DUPLICATED_FROM_MAP_ATTR') { + return res.status(400).send({ + errors: [{ type: 'DUPLICATED_FROM_MAP_ATTR' }], + }); + } + if (error.errorType === 'DUPLICATED_TO_MAP_ATTR') { + return res.status(400).send({ + errors: [{ type: 'DUPLICATED_TO_MAP_ATTR' }], + }); + } + if (error.errorType === 'IMPORTED_FILE_EXTENSION_INVALID') { + return res.status(400).send({ + errors: [{ type: 'IMPORTED_FILE_EXTENSION_INVALID' }], + }); + } + } + next(error); + } +} diff --git a/packages/server/src/api/controllers/Import/_utils.ts b/packages/server/src/api/controllers/Import/_utils.ts new file mode 100644 index 000000000..333e280fe --- /dev/null +++ b/packages/server/src/api/controllers/Import/_utils.ts @@ -0,0 +1,20 @@ +import Multer from 'multer'; +import { ServiceError } from '@/exceptions'; + +export function allowSheetExtensions(req, file, cb) { + if ( + file.mimetype !== 'text/csv' && + file.mimetype !== 'application/vnd.ms-excel' + ) { + cb(new ServiceError('IMPORTED_FILE_EXTENSION_INVALID')); + + return; + } + cb(null, true); +} + +export const uploadImportFile = Multer({ + dest: './public/imports', + limits: { fileSize: 5 * 1024 * 1024 }, + fileFilter: allowSheetExtensions, +}); diff --git a/packages/server/src/api/index.ts b/packages/server/src/api/index.ts index e8ef89c10..cc1206aee 100644 --- a/packages/server/src/api/index.ts +++ b/packages/server/src/api/index.ts @@ -56,6 +56,7 @@ import { ProjectsController } from './controllers/Projects/Projects'; import { ProjectTasksController } from './controllers/Projects/Tasks'; import { ProjectTimesController } from './controllers/Projects/Times'; import { TaxRatesController } from './controllers/TaxRates/TaxRates'; +import { ImportController } from './controllers/Import/ImportController'; import { BankingController } from './controllers/Banking/BankingController'; import { Webhooks } from './controllers/Webhooks/Webhooks'; @@ -135,6 +136,9 @@ export default () => { dashboard.use('/warehouses', Container.get(WarehousesController).router()); dashboard.use('/projects', Container.get(ProjectsController).router()); dashboard.use('/tax-rates', Container.get(TaxRatesController).router()); + + dashboard.use('/import', Container.get(ImportController).router()); + dashboard.use('/', Container.get(ProjectTasksController).router()); dashboard.use('/', Container.get(ProjectTimesController).router()); diff --git a/packages/server/src/database/migrations/20231209230719_create_imports_table.js b/packages/server/src/database/migrations/20231209230719_create_imports_table.js new file mode 100644 index 000000000..60fd5a83d --- /dev/null +++ b/packages/server/src/database/migrations/20231209230719_create_imports_table.js @@ -0,0 +1,15 @@ +exports.up = function (knex) { + return knex.schema.createTable('imports', (table) => { + table.increments(); + table.string('filename'); + table.string('import_id'); + table.string('resource'); + table.json('columns'); + table.json('mapping'); + table.timestamps(); + }); +}; + +exports.down = function (knex) { + return knex.schema.dropTableIfExists('imports'); +}; diff --git a/packages/server/src/interfaces/Model.ts b/packages/server/src/interfaces/Model.ts index 67b90e872..385bce172 100644 --- a/packages/server/src/interfaces/Model.ts +++ b/packages/server/src/interfaces/Model.ts @@ -34,6 +34,7 @@ export interface IModelMetaFieldCommon { columnable?: boolean; fieldType: IModelColumnType; customQuery?: Function; + required?: boolean; } export interface IModelMetaFieldNumber { @@ -77,5 +78,6 @@ export type IModelMetaRelationField = IModelMetaRelationFieldCommon & ( export interface IModelMeta { defaultFilterField: string; defaultSort: IModelMetaDefaultSort; + importable?: boolean; fields: { [key: string]: IModelMetaField }; } diff --git a/packages/server/src/loaders/express.ts b/packages/server/src/loaders/express.ts index 694de8574..70b4fd309 100644 --- a/packages/server/src/loaders/express.ts +++ b/packages/server/src/loaders/express.ts @@ -4,7 +4,6 @@ import helmet from 'helmet'; import boom from 'express-boom'; import errorHandler from 'errorhandler'; import bodyParser from 'body-parser'; -import fileUpload from 'express-fileupload'; import { Server } from 'socket.io'; import Container from 'typedi'; import routes from 'api'; @@ -47,13 +46,6 @@ export default ({ app }) => { app.use('/public', express.static(path.join(global.__storage_dir))); - // Handle multi-media requests. - app.use( - fileUpload({ - createParentPath: true, - }) - ); - // Logger middleware. app.use(LoggerMiddleware); diff --git a/packages/server/src/loaders/tenantModels.ts b/packages/server/src/loaders/tenantModels.ts index c3d08ab6f..0608d5e94 100644 --- a/packages/server/src/loaders/tenantModels.ts +++ b/packages/server/src/loaders/tenantModels.ts @@ -61,6 +61,7 @@ import Task from 'models/Task'; import TaxRate from 'models/TaxRate'; import TaxRateTransaction from 'models/TaxRateTransaction'; import Attachment from 'models/Attachment'; +import Import from 'models/Import'; import PlaidItem from 'models/PlaidItem'; import UncategorizedCashflowTransaction from 'models/UncategorizedCashflowTransaction'; @@ -127,6 +128,7 @@ export default (knex) => { TaxRate, TaxRateTransaction, Attachment, + Import, PlaidItem, UncategorizedCashflowTransaction }; diff --git a/packages/server/src/models/Account.Settings.ts b/packages/server/src/models/Account.Settings.ts index 3d0698d0d..ec67885c4 100644 --- a/packages/server/src/models/Account.Settings.ts +++ b/packages/server/src/models/Account.Settings.ts @@ -6,16 +6,21 @@ export default { sortOrder: 'DESC', sortField: 'name', }, + importable: true, fields: { name: { name: 'account.field.name', column: 'name', fieldType: 'text', + unique: true, + required: true, + importable: true, }, description: { name: 'account.field.description', column: 'description', fieldType: 'text', + importable: true, }, slug: { name: 'account.field.slug', @@ -23,13 +28,17 @@ export default { fieldType: 'text', columnable: false, filterable: false, + importable: false, }, code: { name: 'account.field.code', column: 'code', fieldType: 'text', + importable: true, + minLength: 3, + maxLength: 6, }, - root_type: { + rootType: { name: 'account.field.root_type', fieldType: 'enumeration', options: [ @@ -41,6 +50,7 @@ export default { ], filterCustomQuery: RootTypeFieldFilterQuery, sortable: false, + importable: false, }, normal: { name: 'account.field.normal', @@ -51,37 +61,50 @@ export default { ], filterCustomQuery: NormalTypeFieldFilterQuery, sortable: false, + importable: false, }, - type: { + accountType: { name: 'account.field.type', column: 'account_type', fieldType: 'enumeration', options: ACCOUNT_TYPES.map((accountType) => ({ label: accountType.label, - key: accountType.key + key: accountType.key, })), + required: true, + importable: true, }, active: { name: 'account.field.active', column: 'active', fieldType: 'boolean', filterable: false, + importable: true, }, - balance: { + openingBalance: { name: 'account.field.balance', column: 'amount', fieldType: 'number', + importable: true, }, - currency: { + currencyCode: { name: 'account.field.currency', column: 'currency_code', fieldType: 'text', filterable: false, + importable: true, }, - created_at: { + parentAccount: { + name: 'account.field.parent_account', + column: 'parent_account_id', + fieldType: 'relation', + to: { model: 'Account', to: 'id' }, + }, + createdAt: { name: 'account.field.created_at', column: 'created_at', fieldType: 'date', + importable: false, }, }, }; diff --git a/packages/server/src/models/Import.ts b/packages/server/src/models/Import.ts new file mode 100644 index 000000000..b0c558203 --- /dev/null +++ b/packages/server/src/models/Import.ts @@ -0,0 +1,59 @@ +import TenantModel from 'models/TenantModel'; + +export default class Import extends TenantModel { + mapping!: string; + columns!: string; + + /** + * Table name. + */ + static get tableName() { + return 'imports'; + } + + /** + * Virtual attributes. + */ + static get virtualAttributes() { + return ['mappingParsed']; + } + + /** + * Timestamps columns. + */ + get timestamps() { + return []; + } + + /** + * Relationship mapping. + */ + static get relationMappings() { + return {}; + } + + /** + * Detarmines whether the import is mapped. + * @returns {boolean} + */ + public get isMapped() { + return Boolean(this.mapping); + } + + public get columnsParsed() { + try { + return JSON.parse(this.columns); + } catch { + return []; + } + } + + + public get mappingParsed() { + try { + return JSON.parse(this.mapping); + } catch { + return []; + } + } +} diff --git a/packages/server/src/services/Accounts/AccountsApplication.ts b/packages/server/src/services/Accounts/AccountsApplication.ts index 8182b3058..b90eb37e9 100644 --- a/packages/server/src/services/Accounts/AccountsApplication.ts +++ b/packages/server/src/services/Accounts/AccountsApplication.ts @@ -16,6 +16,7 @@ import { ActivateAccount } from './ActivateAccount'; import { GetAccounts } from './GetAccounts'; import { GetAccount } from './GetAccount'; import { GetAccountTransactions } from './GetAccountTransactions'; +import { Knex } from 'knex'; @Service() export class AccountsApplication { @@ -48,9 +49,10 @@ export class AccountsApplication { */ public createAccount = ( tenantId: number, - accountDTO: IAccountCreateDTO + accountDTO: IAccountCreateDTO, + trx?: Knex.Transaction ): Promise => { - return this.createAccountService.createAccount(tenantId, accountDTO); + return this.createAccountService.createAccount(tenantId, accountDTO, trx); }; /** diff --git a/packages/server/src/services/Accounts/AccountsImportable.ts b/packages/server/src/services/Accounts/AccountsImportable.ts new file mode 100644 index 000000000..85429a751 --- /dev/null +++ b/packages/server/src/services/Accounts/AccountsImportable.ts @@ -0,0 +1,37 @@ +import { Inject, Service } from 'typedi'; +import { Knex } from 'knex'; +import { IAccountCreateDTO } from '@/interfaces'; +import { CreateAccount } from './CreateAccount'; +import { Importable } from '../Import/Importable'; + +@Service() +export class AccountsImportable extends Importable { + @Inject() + private createAccountService: CreateAccount; + + /** + * Importing to account service. + * @param {number} tenantId + * @param {IAccountCreateDTO} createAccountDTO + * @returns + */ + public importable( + tenantId: number, + createAccountDTO: IAccountCreateDTO, + trx?: Knex.Transaction + ) { + return this.createAccountService.createAccount( + tenantId, + createAccountDTO, + trx + ); + } + + /** + * Concurrrency controlling of the importing process. + * @returns {number} + */ + public get concurrency() { + return 1; + } +} diff --git a/packages/server/src/services/Accounts/CreateAccount.ts b/packages/server/src/services/Accounts/CreateAccount.ts index b621d9104..c0eff3a55 100644 --- a/packages/server/src/services/Accounts/CreateAccount.ts +++ b/packages/server/src/services/Accounts/CreateAccount.ts @@ -97,13 +97,14 @@ export class CreateAccount { /** * Creates a new account on the storage. - * @param {number} tenantId - * @param {IAccountCreateDTO} accountDTO + * @param {number} tenantId + * @param {IAccountCreateDTO} accountDTO * @returns {Promise} */ public createAccount = async ( tenantId: number, - accountDTO: IAccountCreateDTO + accountDTO: IAccountCreateDTO, + trx?: Knex.Transaction ): Promise => { const { Account } = this.tenancy.models(tenantId); @@ -119,27 +120,31 @@ export class CreateAccount { tenantMeta.baseCurrency ); // Creates a new account with associated transactions under unit-of-work envirement. - return this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => { - // Triggers `onAccountCreating` event. - await this.eventPublisher.emitAsync(events.accounts.onCreating, { - tenantId, - accountDTO, - trx, - } as IAccountEventCreatingPayload); + return this.uow.withTransaction( + tenantId, + async (trx: Knex.Transaction) => { + // Triggers `onAccountCreating` event. + await this.eventPublisher.emitAsync(events.accounts.onCreating, { + tenantId, + accountDTO, + trx, + } as IAccountEventCreatingPayload); - // Inserts account to the storage. - const account = await Account.query(trx).insertAndFetch({ - ...accountInputModel, - }); - // Triggers `onAccountCreated` event. - await this.eventPublisher.emitAsync(events.accounts.onCreated, { - tenantId, - account, - accountId: account.id, - trx, - } as IAccountEventCreatedPayload); + // Inserts account to the storage. + const account = await Account.query(trx).insertAndFetch({ + ...accountInputModel, + }); + // Triggers `onAccountCreated` event. + await this.eventPublisher.emitAsync(events.accounts.onCreated, { + tenantId, + account, + accountId: account.id, + trx, + } as IAccountEventCreatedPayload); - return account; - }); + return account; + }, + trx + ); }; } diff --git a/packages/server/src/services/Import/ImportFileCommon.ts b/packages/server/src/services/Import/ImportFileCommon.ts new file mode 100644 index 000000000..86bfadf6c --- /dev/null +++ b/packages/server/src/services/Import/ImportFileCommon.ts @@ -0,0 +1,141 @@ +import fs from 'fs/promises'; +import XLSX from 'xlsx'; +import bluebird from 'bluebird'; +import * as R from 'ramda'; +import { Inject, Service } from 'typedi'; +import { first } from 'lodash'; +import { ImportFileDataValidator } from './ImportFileDataValidator'; +import { Knex } from 'knex'; +import { + ImportInsertError, + ImportOperError, + ImportOperSuccess, +} from './interfaces'; +import { AccountsImportable } from '../Accounts/AccountsImportable'; +import { ServiceError } from '@/exceptions'; +import { trimObject } from './_utils'; +import { ImportableResources } from './ImportableResources'; +import ResourceService from '../Resource/ResourceService'; +import HasTenancyService from '../Tenancy/TenancyService'; + +@Service() +export class ImportFileCommon { + @Inject() + private tenancy: HasTenancyService; + + @Inject() + private importFileValidator: ImportFileDataValidator; + + @Inject() + private importable: ImportableResources; + + @Inject() + private resource: ResourceService; + + /** + * Maps the columns of the imported data based on the provided mapping attributes. + * @param {Record[]} body - The array of data objects to map. + * @param {ImportMappingAttr[]} map - The mapping attributes. + * @returns {Record[]} - The mapped data objects. + */ + public parseXlsxSheet(buffer: Buffer): Record[] { + const workbook = XLSX.read(buffer, { type: 'buffer' }); + + const firstSheetName = workbook.SheetNames[0]; + const worksheet = workbook.Sheets[firstSheetName]; + + return XLSX.utils.sheet_to_json(worksheet); + } + + /** + * Reads the import file. + * @param {string} filename + * @returns {Promise} + */ + public readImportFile(filename: string) { + return fs.readFile(`public/imports/${filename}`); + } + + /** + * Imports the given parsed data to the resource storage through registered importable service. + * @param {number} tenantId - + * @param {string} resourceName - Resource name. + * @param {Record} parsedData - Parsed data. + * @param {Knex.Transaction} trx - Knex transaction. + * @returns {Promise<[ImportOperSuccess[], ImportOperError[]]>} + */ + public async import( + tenantId: number, + resourceName: string, + parsedData: Record[], + trx?: Knex.Transaction + ): Promise<[ImportOperSuccess[], ImportOperError[]]> { + const importableFields = this.resource.getResourceImportableFields( + tenantId, + resourceName + ); + const ImportableRegistry = this.importable.registry; + const importable = ImportableRegistry.getImportable(resourceName); + + const concurrency = importable.concurrency || 10; + + const success: ImportOperSuccess[] = []; + const failed: ImportOperError[] = []; + + const importAsync = async (objectDTO, index: number): Promise => { + try { + // Validate the DTO object before passing it to the service layer. + await this.importFileValidator.validateData( + importableFields, + objectDTO + ); + try { + // Run the importable function and listen to the errors. + const data = await importable.importable(tenantId, objectDTO, trx); + success.push({ index, data }); + } catch (err) { + if (err instanceof ServiceError) { + const error = [ + { + errorCode: 'ValidationError', + errorMessage: err.message || err.errorType, + rowNumber: index + 1, + }, + ]; + failed.push({ index, error }); + } + } + } catch (errors) { + const error = errors.map((er) => ({ ...er, rowNumber: index + 1 })); + failed.push({ index, error }); + } + }; + await bluebird.map(parsedData, importAsync, { concurrency }); + + return [success, failed]; + } + + /** + * Retrieves the sheet columns from the given sheet data. + * @param {unknown[]} json + * @returns {string[]} + */ + public parseSheetColumns(json: unknown[]): string[] { + return R.compose(Object.keys, trimObject, first)(json); + } + + /** + * Deletes the imported file from the storage and database. + * @param {number} tenantId + * @param {} importFile + */ + public async deleteImportFile(tenantId: number, importFile: any) { + const { Import } = this.tenancy.models(tenantId); + + // Deletes the import row. + await Import.query().findById(importFile.id).delete(); + + // Deletes the imported file. + await fs.unlink(`public/imports/${importFile.filename}`); + } +} diff --git a/packages/server/src/services/Import/ImportFileDataTransformer.ts b/packages/server/src/services/Import/ImportFileDataTransformer.ts new file mode 100644 index 000000000..aa331d678 --- /dev/null +++ b/packages/server/src/services/Import/ImportFileDataTransformer.ts @@ -0,0 +1,101 @@ +import { Service } from 'typedi'; +import * as R from 'ramda'; +import { isUndefined, get, chain } from 'lodash'; +import { ImportMappingAttr, ResourceMetaFieldsMap } from './interfaces'; +import { parseBoolean } from '@/utils'; +import { trimObject } from './_utils'; + +@Service() +export class ImportFileDataTransformer { + /** + * + * @param {number} tenantId - + * @param {} + */ + public parseSheetData( + importFile: any, + importableFields: any, + data: Record[] + ) { + // Sanitize the sheet data. + const sanitizedData = this.sanitizeSheetData(data); + + // Map the sheet columns key with the given map. + const mappedDTOs = this.mapSheetColumns( + sanitizedData, + importFile.mappingParsed + ); + // Parse the mapped sheet values. + const parsedValues = this.parseExcelValues(importableFields, mappedDTOs); + + return parsedValues; + } + + /** + * Sanitizes the data in the imported sheet by trimming object keys. + * @param json - The JSON data representing the imported sheet. + * @returns {string[][]} - The sanitized data with trimmed object keys. + */ + public sanitizeSheetData(json) { + return R.compose(R.map(trimObject))(json); + } + + /** + * Maps the columns of the imported data based on the provided mapping attributes. + * @param {Record[]} body - The array of data objects to map. + * @param {ImportMappingAttr[]} map - The mapping attributes. + * @returns {Record[]} - The mapped data objects. + */ + public mapSheetColumns( + body: Record[], + map: ImportMappingAttr[] + ): Record[] { + return body.map((item) => { + const newItem = {}; + map + .filter((mapping) => !isUndefined(item[mapping.from])) + .forEach((mapping) => { + newItem[mapping.to] = item[mapping.from]; + }); + return newItem; + }); + } + + /** + * Parses sheet values before passing to the service layer. + * @param {ResourceMetaFieldsMap} fields - + * @param {Record} valueDTOS - + * @returns {Record} + */ + public parseExcelValues( + fields: ResourceMetaFieldsMap, + valueDTOs: Record[] + ): Record { + const parser = (value, key) => { + let _value = value; + + // Parses the boolean value. + if (fields[key].fieldType === 'boolean') { + _value = parseBoolean(value, false); + + // Parses the enumeration value. + } else if (fields[key].fieldType === 'enumeration') { + const field = fields[key]; + const option = get(field, 'options', []).find( + (option) => option.label === value + ); + _value = get(option, 'key'); + // Prases the numeric value. + } else if (fields[key].fieldType === 'number') { + _value = parseFloat(value); + } + return _value; + }; + return valueDTOs.map((DTO) => { + return chain(DTO) + .pickBy((value, key) => !isUndefined(fields[key])) + .mapValues(parser) + .value(); + }); + } +} diff --git a/packages/server/src/services/Import/ImportFileDataValidator.ts b/packages/server/src/services/Import/ImportFileDataValidator.ts new file mode 100644 index 000000000..0f41f2bb4 --- /dev/null +++ b/packages/server/src/services/Import/ImportFileDataValidator.ts @@ -0,0 +1,42 @@ +import { Service } from 'typedi'; +import { ImportInsertError, ResourceMetaFieldsMap } from './interfaces'; +import { ERRORS, convertFieldsToYupValidation } from './_utils'; +import { IModelMeta } from '@/interfaces'; +import { ServiceError } from '@/exceptions'; + +@Service() +export class ImportFileDataValidator { + /** + * Validates the given resource is importable. + * @param {IModelMeta} resourceMeta + */ + public validateResourceImportable(resourceMeta: IModelMeta) { + // Throw service error if the resource does not support importing. + if (!resourceMeta.importable) { + throw new ServiceError(ERRORS.RESOURCE_NOT_IMPORTABLE); + } + } + + /** + * Validates the given mapped DTOs and returns errors with their index. + * @param {Record} mappedDTOs + * @returns {Promise} + */ + public async validateData( + importableFields: ResourceMetaFieldsMap, + data: Record + ): Promise { + const YupSchema = convertFieldsToYupValidation(importableFields); + const _data = { ...data }; + + try { + await YupSchema.validate(_data, { abortEarly: false }); + } catch (validationError) { + const errors = validationError.inner.map((error) => ({ + errorCode: 'ValidationError', + errorMessage: error.errors, + })); + throw errors; + } + } +} diff --git a/packages/server/src/services/Import/ImportFileMapping.ts b/packages/server/src/services/Import/ImportFileMapping.ts new file mode 100644 index 000000000..1229c84f5 --- /dev/null +++ b/packages/server/src/services/Import/ImportFileMapping.ts @@ -0,0 +1,109 @@ +import { fromPairs } from 'lodash'; +import { Inject, Service } from 'typedi'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { ImportFileMapPOJO, ImportMappingAttr } from './interfaces'; +import ResourceService from '../Resource/ResourceService'; +import { ServiceError } from '@/exceptions'; +import { ERRORS } from './_utils'; + +@Service() +export class ImportFileMapping { + @Inject() + private tenancy: HasTenancyService; + + @Inject() + private resource: ResourceService; + + /** + * Mapping the excel sheet columns with resource columns. + * @param {number} tenantId + * @param {number} importId + * @param {ImportMappingAttr} maps + */ + public async mapping( + tenantId: number, + importId: number, + maps: ImportMappingAttr[] + ): Promise { + const { Import } = this.tenancy.models(tenantId); + + const importFile = await Import.query() + .findOne('filename', importId) + .throwIfNotFound(); + + // Invalidate the from/to map attributes. + this.validateMapsAttrs(tenantId, importFile, maps); + + // Validate the diplicated relations of map attrs. + this.validateDuplicatedMapAttrs(maps); + + const mappingStringified = JSON.stringify(maps); + + await Import.query().findById(importFile.id).patch({ + mapping: mappingStringified, + }); + + return { + import: { + importId: importFile.importId, + resource: importFile.resource, + }, + }; + } + + /** + * Validate the mapping attributes. + * @param {number} tenantId - + * @param {} importFile - + * @param {ImportMappingAttr[]} maps + * @throws {ServiceError(ERRORS.INVALID_MAP_ATTRS)} + */ + private validateMapsAttrs( + tenantId: number, + importFile: any, + maps: ImportMappingAttr[] + ) { + const fields = this.resource.getResourceImportableFields( + tenantId, + importFile.resource + ); + const columnsMap = fromPairs( + importFile.columnsParsed.map((field) => [field, '']) + ); + const invalid = []; + + maps.forEach((map) => { + if ( + 'undefined' === typeof fields[map.to] || + 'undefined' === typeof columnsMap[map.from] + ) { + invalid.push(map); + } + }); + if (invalid.length > 0) { + throw new ServiceError(ERRORS.INVALID_MAP_ATTRS); + } + } + + /** + * Validate the map attrs relation should be one-to-one relation only. + * @param {ImportMappingAttr[]} maps + */ + private validateDuplicatedMapAttrs(maps: ImportMappingAttr[]) { + const fromMap = {}; + const toMap = {}; + + maps.forEach((map) => { + if (fromMap[map.from]) { + throw new ServiceError(ERRORS.DUPLICATED_FROM_MAP_ATTR); + } else { + fromMap[map.from] = true; + } + if (toMap[map.to]) { + throw new ServiceError(ERRORS.DUPLICATED_TO_MAP_ATTR); + } else { + toMap[map.to] = true; + } + }); + } +} diff --git a/packages/server/src/services/Import/ImportFilePreview.ts b/packages/server/src/services/Import/ImportFilePreview.ts new file mode 100644 index 000000000..56a1df09e --- /dev/null +++ b/packages/server/src/services/Import/ImportFilePreview.ts @@ -0,0 +1,34 @@ +import { Inject, Service } from 'typedi'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { ImportFilePreviewPOJO } from './interfaces'; +import { ImportFileProcess } from './ImportFileProcess'; + +@Service() +export class ImportFilePreview { + @Inject() + private tenancy: HasTenancyService; + + @Inject() + private importFile: ImportFileProcess; + + /** + * Preview the imported file results before commiting the transactions. + * @param {number} tenantId + * @param {number} importId + * @returns {Promise} + */ + public async preview( + tenantId: number, + importId: number + ): Promise { + const knex = this.tenancy.knex(tenantId); + const trx = await knex.transaction({ isolationLevel: 'read uncommitted' }); + + const meta = await this.importFile.import(tenantId, importId, trx); + + // Rollback the successed transaction. + await trx.rollback(); + + return meta; + } +} diff --git a/packages/server/src/services/Import/ImportFileProcess.ts b/packages/server/src/services/Import/ImportFileProcess.ts new file mode 100644 index 000000000..ff95f35d6 --- /dev/null +++ b/packages/server/src/services/Import/ImportFileProcess.ts @@ -0,0 +1,101 @@ +import { Inject, Service } from 'typedi'; +import { chain } from 'lodash'; +import { Knex } from 'knex'; +import { ServiceError } from '@/exceptions'; +import { ERRORS, getSheetColumns, getUnmappedSheetColumns } from './_utils'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { ImportFileCommon } from './ImportFileCommon'; +import { ImportFileDataTransformer } from './ImportFileDataTransformer'; +import ResourceService from '../Resource/ResourceService'; +import UnitOfWork from '../UnitOfWork'; +import { ImportFilePreviewPOJO } from './interfaces'; + +@Service() +export class ImportFileProcess { + @Inject() + private tenancy: HasTenancyService; + + @Inject() + private resource: ResourceService; + + @Inject() + private importCommon: ImportFileCommon; + + @Inject() + private importParser: ImportFileDataTransformer; + + @Inject() + private uow: UnitOfWork; + + /** + * Preview the imported file results before commiting the transactions. + * @param {number} tenantId + * @param {number} importId + * @returns {Promise} + */ + public async import( + tenantId: number, + importId: number, + trx?: Knex.Transaction + ): Promise { + const { Import } = this.tenancy.models(tenantId); + + const importFile = await Import.query() + .findOne('importId', importId) + .throwIfNotFound(); + + // Throw error if the import file is not mapped yet. + if (!importFile.isMapped) { + throw new ServiceError(ERRORS.IMPORT_FILE_NOT_MAPPED); + } + // Read the imported file. + const buffer = await this.importCommon.readImportFile(importFile.filename); + const sheetData = this.importCommon.parseXlsxSheet(buffer); + const header = getSheetColumns(sheetData); + + const importableFields = this.resource.getResourceImportableFields( + tenantId, + importFile.resource + ); + // Prases the sheet json data. + const parsedData = this.importParser.parseSheetData( + importFile, + importableFields, + sheetData + ); + // Runs the importing operation with ability to return errors that will happen. + const [successedImport, failedImport] = await this.uow.withTransaction( + tenantId, + (trx: Knex.Transaction) => + this.importCommon.import( + tenantId, + importFile.resource, + parsedData, + trx + ), + trx + ); + const mapping = importFile.mappingParsed; + const errors = chain(failedImport) + .map((oper) => oper.error) + .flatten() + .value(); + + const unmappedColumns = getUnmappedSheetColumns(header, mapping); + const totalCount = parsedData.length; + + const createdCount = successedImport.length; + const errorsCount = failedImport.length; + const skippedCount = errorsCount; + + return { + createdCount, + skippedCount, + totalCount, + errorsCount, + errors, + unmappedColumns: unmappedColumns, + unmappedColumnsCount: unmappedColumns.length, + }; + } +} diff --git a/packages/server/src/services/Import/ImportFileUpload.ts b/packages/server/src/services/Import/ImportFileUpload.ts new file mode 100644 index 000000000..19135d94a --- /dev/null +++ b/packages/server/src/services/Import/ImportFileUpload.ts @@ -0,0 +1,80 @@ +import { Inject, Service } from 'typedi'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { sanitizeResourceName } from './_utils'; +import ResourceService from '../Resource/ResourceService'; +import { IModelMetaField } from '@/interfaces'; +import { ImportFileCommon } from './ImportFileCommon'; +import { ImportFileDataValidator } from './ImportFileDataValidator'; +import { ImportFileUploadPOJO } from './interfaces'; + +@Service() +export class ImportFileUploadService { + @Inject() + private tenancy: HasTenancyService; + + @Inject() + private resourceService: ResourceService; + + @Inject() + private importFileCommon: ImportFileCommon; + + @Inject() + private importValidator: ImportFileDataValidator; + + /** + * Reads the imported file and stores the import file meta under unqiue id. + * @param {number} tenantId - Tenant id. + * @param {string} resource - Resource name. + * @param {string} filePath - File path. + * @param {string} fileName - File name. + * @returns {Promise} + */ + public async import( + tenantId: number, + resourceName: string, + filename: string + ): Promise { + const { Import } = this.tenancy.models(tenantId); + + const resourceMeta = this.resourceService.getResourceMeta( + tenantId, + resourceName + ); + // Throw service error if the resource does not support importing. + this.importValidator.validateResourceImportable(resourceMeta); + + // Reads the imported file into buffer. + const buffer = await this.importFileCommon.readImportFile(filename); + + // Parse the buffer file to array data. + const sheetData = this.importFileCommon.parseXlsxSheet(buffer); + + const sheetColumns = this.importFileCommon.parseSheetColumns(sheetData); + const coumnsStringified = JSON.stringify(sheetColumns); + + const _resourceName = sanitizeResourceName(resourceName); + + // Store the import model with related metadata. + const importFile = await Import.query().insert({ + filename, + importId: filename, + resource: _resourceName, + columns: coumnsStringified, + }); + const resourceColumns = this.resourceService.getResourceImportableFields( + tenantId, + _resourceName + ); + const resourceColumnsTransformeed = Object.entries(resourceColumns).map( + ([key, { name }]: [string, IModelMetaField]) => ({ key, name }) + ); + return { + import: { + importId: importFile.importId, + resource: importFile.resource, + }, + sheetColumns, + resourceColumns: resourceColumnsTransformeed, + }; + } +} diff --git a/packages/server/src/services/Import/ImportResourceApplication.ts b/packages/server/src/services/Import/ImportResourceApplication.ts new file mode 100644 index 000000000..8577a4b79 --- /dev/null +++ b/packages/server/src/services/Import/ImportResourceApplication.ts @@ -0,0 +1,70 @@ +import { Inject } from 'typedi'; +import { ImportFileUploadService } from './ImportFileUpload'; +import { ImportFileMapping } from './ImportFileMapping'; +import { ImportMappingAttr } from './interfaces'; +import { ImportFileProcess } from './ImportFileProcess'; +import { ImportFilePreview } from './ImportFilePreview'; + +@Inject() +export class ImportResourceApplication { + @Inject() + private importFileService: ImportFileUploadService; + + @Inject() + private importMappingService: ImportFileMapping; + + @Inject() + private importProcessService: ImportFileProcess; + + @Inject() + private ImportFilePreviewService: ImportFilePreview; + + /** + * Reads the imported file and stores the import file meta under unqiue id. + * @param {number} tenantId - + * @param {string} resource - + * @param {string} fileName - + * @returns {Promise} + */ + public async import( + tenantId: number, + resource: string, + filename: string + ) { + return this.importFileService.import(tenantId, resource, filename); + } + + /** + * Mapping the excel sheet columns with resource columns. + * @param {number} tenantId + * @param {number} importId + * @param {ImportMappingAttr} maps + */ + public async mapping( + tenantId: number, + importId: number, + maps: ImportMappingAttr[] + ) { + return this.importMappingService.mapping(tenantId, importId, maps); + } + + /** + * Preview the mapped results before process importing. + * @param {number} tenantId + * @param {number} importId + * @returns {Promise} + */ + public async preview(tenantId: number, importId: number) { + return this.ImportFilePreviewService.preview(tenantId, importId); + } + + /** + * Process the import file sheet through service for creating entities. + * @param {number} tenantId + * @param {number} importId + * @returns {Promise} + */ + public async process(tenantId: number, importId: number) { + return this.importProcessService.import(tenantId, importId); + } +} diff --git a/packages/server/src/services/Import/Importable.ts b/packages/server/src/services/Import/Importable.ts new file mode 100644 index 000000000..8130910f5 --- /dev/null +++ b/packages/server/src/services/Import/Importable.ts @@ -0,0 +1,23 @@ +import { Knex } from 'knex'; + +export abstract class Importable { + /** + * + * @param {number} tenantId + * @param {any} createDTO + * @param {Knex.Transaction} trx + */ + public importable(tenantId: number, createDTO: any, trx?: Knex.Transaction) { + throw new Error( + 'The `importable` function is not defined in service importable.' + ); + } + + /** + * Concurrency controlling of the importing process. + * @returns {number} + */ + public get concurrency() { + return 10; + } +} diff --git a/packages/server/src/services/Import/ImportableRegistry.ts b/packages/server/src/services/Import/ImportableRegistry.ts new file mode 100644 index 000000000..c260c5bd5 --- /dev/null +++ b/packages/server/src/services/Import/ImportableRegistry.ts @@ -0,0 +1,46 @@ +import { camelCase, upperFirst } from 'lodash'; +import { Importable } from './Importable'; + +export class ImportableRegistry { + private static instance: ImportableRegistry; + private importables: Record; + + private constructor() { + this.importables = {}; + } + + /** + * Gets singleton instance of registry. + * @returns {ImportableRegistry} + */ + public static getInstance(): ImportableRegistry { + if (!ImportableRegistry.instance) { + ImportableRegistry.instance = new ImportableRegistry(); + } + return ImportableRegistry.instance; + } + + /** + * Registers the given importable service. + * @param {string} resource + * @param {Importable} importable + */ + public registerImportable(resource: string, importable: Importable): void { + const _resource = this.sanitizeResourceName(resource); + this.importables[_resource] = importable; + } + + /** + * Retrieves the importable service instance of the given resource name. + * @param {string} name + * @returns {Importable} + */ + public getImportable(name: string): Importable { + const _name = this.sanitizeResourceName(name); + return this.importables[_name]; + } + + private sanitizeResourceName(resource: string) { + return upperFirst(camelCase(resource)); + } +} diff --git a/packages/server/src/services/Import/ImportableResources.ts b/packages/server/src/services/Import/ImportableResources.ts new file mode 100644 index 000000000..3f4297075 --- /dev/null +++ b/packages/server/src/services/Import/ImportableResources.ts @@ -0,0 +1,38 @@ +import Container, { Service } from 'typedi'; +import { AccountsImportable } from '../Accounts/AccountsImportable'; +import { ImportableRegistry } from './ImportableRegistry'; + +@Service() +export class ImportableResources { + private static registry: ImportableRegistry; + + constructor() { + this.boot(); + } + + /** + * Importable instances. + */ + private importables = [ + { resource: 'Account', importable: AccountsImportable }, + ]; + + public get registry() { + return ImportableResources.registry; + } + + /** + * Boots all the registered importables. + */ + public boot() { + if (!ImportableResources.registry) { + const instance = ImportableRegistry.getInstance(); + + this.importables.forEach((importable) => { + const importableInstance = Container.get(importable.importable); + instance.registerImportable(importable.resource, importableInstance); + }); + ImportableResources.registry = instance; + } + } +} diff --git a/packages/server/src/services/Import/_utils.ts b/packages/server/src/services/Import/_utils.ts new file mode 100644 index 000000000..08146c367 --- /dev/null +++ b/packages/server/src/services/Import/_utils.ts @@ -0,0 +1,79 @@ +import * as Yup from 'yup'; +import { upperFirst, camelCase, first } from 'lodash'; +import pluralize from 'pluralize'; +import { ResourceMetaFieldsMap } from './interfaces'; +import { IModelMetaField } from '@/interfaces'; + +export function trimObject(obj) { + return Object.entries(obj).reduce((acc, [key, value]) => { + // Trim the key + const trimmedKey = key.trim(); + + // Trim the value if it's a string, otherwise leave it as is + const trimmedValue = typeof value === 'string' ? value.trim() : value; + + // Assign the trimmed key and value to the accumulator object + return { ...acc, [trimmedKey]: trimmedValue }; + }, {}); +} + +export const convertFieldsToYupValidation = (fields: ResourceMetaFieldsMap) => { + const yupSchema = {}; + Object.keys(fields).forEach((fieldName: string) => { + const field = fields[fieldName] as IModelMetaField; + let fieldSchema; + fieldSchema = Yup.string().label(field.name); + + if (field.fieldType === 'text') { + if (field.minLength) { + fieldSchema = fieldSchema.min( + field.minLength, + `Minimum length is ${field.minLength} characters` + ); + } + if (field.maxLength) { + fieldSchema = fieldSchema.max( + field.maxLength, + `Maximum length is ${field.maxLength} characters` + ); + } + } else if (field.fieldType === 'number') { + fieldSchema = Yup.number().label(field.name); + } else if (field.fieldType === 'boolean') { + fieldSchema = Yup.boolean().label(field.name); + } else if (field.fieldType === 'enumeration') { + const options = field.options.reduce((acc, option) => { + acc[option.key] = option.label; + return acc; + }, {}); + fieldSchema = Yup.string().oneOf(Object.keys(options)).label(field.name); + } + if (field.required) { + fieldSchema = fieldSchema.required(); + } + yupSchema[fieldName] = fieldSchema; + }); + return Yup.object().shape(yupSchema); +}; + +export const ERRORS = { + RESOURCE_NOT_IMPORTABLE: 'RESOURCE_NOT_IMPORTABLE', + INVALID_MAP_ATTRS: 'INVALID_MAP_ATTRS', + DUPLICATED_FROM_MAP_ATTR: 'DUPLICATED_FROM_MAP_ATTR', + DUPLICATED_TO_MAP_ATTR: 'DUPLICATED_TO_MAP_ATTR', + IMPORT_FILE_NOT_MAPPED: 'IMPORT_FILE_NOT_MAPPED', +}; + +export const getUnmappedSheetColumns = (columns, mapping) => { + return columns.filter( + (column) => !mapping.some((map) => map.from === column) + ); +}; + +export const sanitizeResourceName = (resourceName: string) => { + return upperFirst(camelCase(pluralize.singular(resourceName))); +}; + +export const getSheetColumns = (sheetData: unknown[]) => { + return Object.keys(first(sheetData)); +}; diff --git a/packages/server/src/services/Import/interfaces.ts b/packages/server/src/services/Import/interfaces.ts new file mode 100644 index 000000000..3eca339f7 --- /dev/null +++ b/packages/server/src/services/Import/interfaces.ts @@ -0,0 +1,57 @@ +import { IModelMetaField } from '@/interfaces'; + +export interface ImportMappingAttr { + from: string; + to: string; +} + +export interface ImportValidationError { + index: number; + property: string; + constraints: Record; +} + +export type ResourceMetaFieldsMap = { [key: string]: IModelMetaField }; + +export interface ImportInsertError { + rowNumber: number; + errorCode: string; + errorMessage: string; +} + +export interface ImportFileUploadPOJO { + import: { + importId: string; + resource: string; + }; + sheetColumns: string[]; + resourceColumns: { key: string; name: string }[]; +} + +export interface ImportFileMapPOJO { + import: { + importId: string; + resource: string; + }; +} + +export interface ImportFilePreviewPOJO { + createdCount: number; + skippedCount: number; + totalCount: number; + errorsCount: number; + errors: ImportInsertError[]; + unmappedColumns: string[]; + unmappedColumnsCount: number; +} + + +export interface ImportOperSuccess { + data: unknown; + index: number; +} + +export interface ImportOperError { + error: ImportInsertError; + index: number; +} \ No newline at end of file diff --git a/packages/server/src/services/Organization/OrganizationService.ts b/packages/server/src/services/Organization/OrganizationService.ts index 32431a066..453b6522c 100644 --- a/packages/server/src/services/Organization/OrganizationService.ts +++ b/packages/server/src/services/Organization/OrganizationService.ts @@ -21,25 +21,19 @@ import { ERRORS } from './constants'; @Service() export default class OrganizationService { @Inject() - eventPublisher: EventPublisher; - - @Inject('logger') - logger: any; - - @Inject('repositories') - sysRepositories: any; + private eventPublisher: EventPublisher; @Inject() - tenantsManager: TenantsManager; + private tenantsManager: TenantsManager; @Inject('agenda') - agenda: any; + private agenda: any; @Inject() - baseCurrencyMutateLocking: OrganizationBaseCurrencyLocking; + private baseCurrencyMutateLocking: OrganizationBaseCurrencyLocking; @Inject() - tenancy: HasTenancyService; + private tenancy: HasTenancyService; /** * Builds the database schema and seed data of the given organization id. diff --git a/packages/server/src/services/Resource/ResourceService.ts b/packages/server/src/services/Resource/ResourceService.ts index 79887851a..529448f8d 100644 --- a/packages/server/src/services/Resource/ResourceService.ts +++ b/packages/server/src/services/Resource/ResourceService.ts @@ -1,12 +1,11 @@ import { Service, Inject } from 'typedi'; -import { camelCase, upperFirst } from 'lodash'; +import { camelCase, upperFirst, pickBy } from 'lodash'; import * as qim from 'qim'; import pluralize from 'pluralize'; -import { IModelMeta } from '@/interfaces'; +import { IModelMeta, IModelMetaField } from '@/interfaces'; import TenancyService from '@/services/Tenancy/TenancyService'; import { ServiceError } from '@/exceptions'; import I18nService from '@/services/I18n/I18nService'; -import { tenantKnexConfig } from 'config/knexConfig'; const ERRORS = { RESOURCE_MODEL_NOT_FOUND: 'RESOURCE_MODEL_NOT_FOUND', @@ -24,7 +23,7 @@ export default class ResourceService { * Transform resource to model name. * @param {string} resourceName */ - private resourceToModelName(resourceName: string): string { + public resourceToModelName(resourceName: string): string { return upperFirst(camelCase(pluralize.singular(resourceName))); } @@ -63,6 +62,33 @@ export default class ResourceService { return this.getResourceMetaLocalized(resourceMeta, tenantId); } + /** + * + */ + public getResourceFields( + tenantId: number, + modelName: string + ): { [key: string]: IModelMetaField } { + const meta = this.getResourceMeta(tenantId, modelName); + + return meta.fields; + } + + /** + * + * @param {number} tenantId + * @param {string} modelName + * @returns + */ + public getResourceImportableFields( + tenantId: number, + modelName: string + ): { [key: string]: IModelMetaField } { + const fields = this.getResourceFields(tenantId, modelName); + + return pickBy(fields, (field) => field.importable); + } + /** * Retrieve the resource meta localized based on the current user language. */ diff --git a/packages/server/src/services/UnitOfWork/index.ts b/packages/server/src/services/UnitOfWork/index.ts index c4c0c0dec..a13be95c6 100644 --- a/packages/server/src/services/UnitOfWork/index.ts +++ b/packages/server/src/services/UnitOfWork/index.ts @@ -1,5 +1,6 @@ import { Service, Inject } from 'typedi'; import TenancyService from '@/services/Tenancy/TenancyService'; +import { Transaction } from 'objection'; /** * Enumeration that represents transaction isolation levels for use with the {@link Transactional} annotation @@ -38,18 +39,26 @@ export default class UnitOfWork { public withTransaction = async ( tenantId: number, work, + trx?: Transaction, isolationLevel: IsolationLevel = IsolationLevel.READ_UNCOMMITTED ) => { const knex = this.tenancy.knex(tenantId); - const trx = await knex.transaction({ isolationLevel }); + let _trx = trx; + if (!_trx) { + _trx = await knex.transaction({ isolationLevel }); + } try { - const result = await work(trx); - trx.commit(); + const result = await work(_trx); + if (!trx) { + _trx.commit(); + } return result; } catch (error) { - trx.rollback(); + if (!trx) { + _trx.rollback(); + } throw error; } }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cb8bc2771..63855f63d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -47,6 +47,9 @@ importers: '@types/mathjs': specifier: ^6.0.12 version: 6.0.12 + '@types/yup': + specifier: ^0.29.13 + version: 0.29.14 accepts: specifier: ^1.3.7 version: 1.3.8 @@ -131,9 +134,6 @@ importers: express-boom: specifier: ^3.0.0 version: 3.0.0 - express-fileupload: - specifier: ^1.1.7-alpha.3 - version: 1.4.0 express-oauth-server: specifier: ^2.0.0 version: 2.0.0 @@ -203,6 +203,9 @@ importers: mongoose: specifier: ^5.10.0 version: 5.13.20 + multer: + specifier: 1.4.5-lts.1 + version: 1.4.5-lts.1 mustache: specifier: ^3.0.3 version: 3.2.1 @@ -290,6 +293,9 @@ importers: xlsx: specifier: ^0.18.5 version: 0.18.5 + yup: + specifier: ^0.28.1 + version: 0.28.5 devDependencies: '@types/lodash': specifier: ^4.14.158 @@ -7899,6 +7905,10 @@ packages: buffer-equal: 1.0.1 dev: false + /append-field@1.0.0: + resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==} + dev: false + /append-transform@1.0.0: resolution: {integrity: sha512-P009oYkeHyU742iSZJzZZywj4QRJdnTWffaKuJQLablCZ1uz6/cW4yaRgcDaoQ+uwOxxnt0gRUcwfsNP2ri0gw==} engines: {node: '>=4'} @@ -12473,13 +12483,6 @@ packages: boom: 7.3.0 dev: false - /express-fileupload@1.4.0: - resolution: {integrity: sha512-RjzLCHxkv3umDeZKeFeMg8w7qe0V09w3B7oGZprr/oO2H/ISCgNzuqzn7gV3HRWb37GjRk429CCpSLS2KNTqMQ==} - engines: {node: '>=12.0.0'} - dependencies: - busboy: 1.6.0 - dev: false - /express-oauth-server@2.0.0: resolution: {integrity: sha512-+UrTbvU7u3LVnoUavzO7QJgSqiEZREKprCZYrDEVoSszrO4t8f/BBPbY3hQOuuatoS0PgDFLaDKQsGNtAgPm5w==} engines: {node: '>=0.11'} @@ -17921,6 +17924,19 @@ packages: /ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + /multer@1.4.5-lts.1: + resolution: {integrity: sha512-ywPWvcDMeH+z9gQq5qYHCCy+ethsk4goepZ45GLD63fOu0YcNecQxi64nDs3qluZB+murG3/D4dJ7+dGctcCQQ==} + engines: {node: '>= 6.0.0'} + dependencies: + append-field: 1.0.0 + busboy: 1.6.0 + concat-stream: 1.6.2 + mkdirp: 0.5.6 + object-assign: 4.1.1 + type-is: 1.6.18 + xtend: 4.0.2 + dev: false + /multicast-dns@7.2.5: resolution: {integrity: sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==} hasBin: true