diff --git a/packages/server/package.json b/packages/server/package.json index a117eb5f6..0fea6033c 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -37,8 +37,6 @@ "bcryptjs": "^2.4.3", "bluebird": "^3.7.2", "body-parser": "^1.20.2", - "class-transformer": "^0.5.1", - "class-validator": "^0.14.0", "compression": "^1.7.4", "country-codes-list": "^1.6.8", "cpy": "^8.1.2", @@ -56,7 +54,6 @@ "express": "^4.17.1", "express-basic-auth": "^1.2.0", "express-boom": "^3.0.0", - "express-fileupload": "^1.1.7-alpha.3", "express-oauth-server": "^2.0.0", "express-validator": "^6.12.2", "form-data": "^4.0.0", diff --git a/packages/server/src/api/controllers/Import/ImportController.ts b/packages/server/src/api/controllers/Import/ImportController.ts index e459389c8..c2d90cbe2 100644 --- a/packages/server/src/api/controllers/Import/ImportController.ts +++ b/packages/server/src/api/controllers/Import/ImportController.ts @@ -1,15 +1,10 @@ import { Inject, Service } from 'typedi'; import { Router, Request, Response, NextFunction } from 'express'; import { body, param } from 'express-validator'; -import Multer from 'multer'; import BaseController from '@/api/controllers/BaseController'; import { ServiceError } from '@/exceptions'; import { ImportResourceApplication } from '@/services/Import/ImportResourceApplication'; - -const upload = Multer({ - dest: './public/imports', - limits: { fileSize: 5 * 1024 * 1024 }, -}); +import { uploadImportFile } from './_utils'; @Service() export class ImportController extends BaseController { @@ -24,7 +19,7 @@ export class ImportController extends BaseController { router.post( '/file', - upload.single('file'), + uploadImportFile.single('file'), this.importValidationSchema, this.validationResult, this.asyncMiddleware(this.fileUpload.bind(this)), @@ -60,27 +55,11 @@ export class ImportController extends BaseController { * @returns {ValidationSchema[]} */ private get importValidationSchema() { - return [ - body('resource').exists(), - // body('file').custom((value, { req }) => { - // if (!value) { - // throw new Error('File is required'); - // } - // if (!['xlsx', 'csv'].includes(value.split('.').pop())) { - // throw new Error('File must be in xlsx or csv format'); - // } - // return true; - // }), - ]; + return [body('resource').exists()]; } /** * Imports xlsx/csv to the given resource type. - * - * - Save the xlsx/csv file and give it a random name. - * - Save the file metadata on the DB storage. - * - - * * @param {Request} req - * @param {Response} res - * @param {NextFunction} next - @@ -117,7 +96,6 @@ export class ImportController extends BaseController { importId, body?.mapping ); - return res.status(200).send(mapping); } catch (error) { next(error); @@ -144,22 +122,19 @@ export class ImportController extends BaseController { } /** - * - * @param req - * @param res - * @param next + * Importing the imported file to the application storage. + * @param {Request} req + * @param {Response} res + * @param {NextFunction} next */ private async import(req: Request, res: Response, next: NextFunction) { const { tenantId } = req; const { import_id: importId } = req.params; try { - await this.importResourceApp.process(tenantId, importId); + const result = await this.importResourceApp.process(tenantId, importId); - return res.status(200).send({ - id: importId, - message: 'Importing the uploaded file is importing.', - }); + return res.status(200).send(result); } catch (error) { next(error); } @@ -194,6 +169,11 @@ export class ImportController extends BaseController { errors: [{ type: 'DUPLICATED_TO_MAP_ATTR' }], }); } + if (error.errorType === 'IMPORTED_FILE_EXTENSION_INVALID') { + return res.status(400).send({ + errors: [{ type: 'IMPORTED_FILE_EXTENSION_INVALID' }], + }); + } } next(error); } diff --git a/packages/server/src/api/controllers/Import/_utils.ts b/packages/server/src/api/controllers/Import/_utils.ts new file mode 100644 index 000000000..333e280fe --- /dev/null +++ b/packages/server/src/api/controllers/Import/_utils.ts @@ -0,0 +1,20 @@ +import Multer from 'multer'; +import { ServiceError } from '@/exceptions'; + +export function allowSheetExtensions(req, file, cb) { + if ( + file.mimetype !== 'text/csv' && + file.mimetype !== 'application/vnd.ms-excel' + ) { + cb(new ServiceError('IMPORTED_FILE_EXTENSION_INVALID')); + + return; + } + cb(null, true); +} + +export const uploadImportFile = Multer({ + dest: './public/imports', + limits: { fileSize: 5 * 1024 * 1024 }, + fileFilter: allowSheetExtensions, +}); diff --git a/packages/server/src/loaders/express.ts b/packages/server/src/loaders/express.ts index 2f769e1b4..70b4fd309 100644 --- a/packages/server/src/loaders/express.ts +++ b/packages/server/src/loaders/express.ts @@ -4,7 +4,6 @@ import helmet from 'helmet'; import boom from 'express-boom'; import errorHandler from 'errorhandler'; import bodyParser from 'body-parser'; -import fileUpload from 'express-fileupload'; import { Server } from 'socket.io'; import Container from 'typedi'; import routes from 'api'; @@ -47,13 +46,6 @@ export default ({ app }) => { app.use('/public', express.static(path.join(global.__storage_dir))); - // Handle multi-media requests. - // app.use( - // fileUpload({ - // createParentPath: true, - // }) - // ); - // Logger middleware. app.use(LoggerMiddleware); diff --git a/packages/server/src/services/Import/AccountsImportable.ts b/packages/server/src/services/Accounts/AccountsImportable.ts similarity index 63% rename from packages/server/src/services/Import/AccountsImportable.ts rename to packages/server/src/services/Accounts/AccountsImportable.ts index 12b67a742..85429a751 100644 --- a/packages/server/src/services/Import/AccountsImportable.ts +++ b/packages/server/src/services/Accounts/AccountsImportable.ts @@ -1,15 +1,16 @@ import { Inject, Service } from 'typedi'; import { Knex } from 'knex'; import { IAccountCreateDTO } from '@/interfaces'; -import { CreateAccount } from '../Accounts/CreateAccount'; +import { CreateAccount } from './CreateAccount'; +import { Importable } from '../Import/Importable'; @Service() -export class AccountsImportable { +export class AccountsImportable extends Importable { @Inject() private createAccountService: CreateAccount; /** - * + * Importing to account service. * @param {number} tenantId * @param {IAccountCreateDTO} createAccountDTO * @returns @@ -27,20 +28,10 @@ export class AccountsImportable { } /** - * - * @param data - * @returns + * Concurrrency controlling of the importing process. + * @returns {number} */ - public transform(data) { - return { ...data }; - } - - /** - * - * @param data - * @returns - */ - public preTransform(data) { - return { ...data }; + public get concurrency() { + return 1; } } diff --git a/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts b/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts deleted file mode 100644 index d05f3b81a..000000000 --- a/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { DATATYPES_LENGTH } from '@/data/DataTypes'; -import { IsInt, IsOptional, IsString, Length, Min, Max, IsNotEmpty } from 'class-validator'; - -export class AccountDTOSchema { - @IsString() - @Length(3, DATATYPES_LENGTH.STRING) - @IsNotEmpty() - name: string; - - @IsString() - @IsOptional() - @Length(3, 6) - code?: string; - - @IsOptional() - currencyCode?: string; - - @IsString() - @Length(3, DATATYPES_LENGTH.STRING) - @IsNotEmpty() - accountType: string; - - @IsString() - @IsOptional() - @Length(0, DATATYPES_LENGTH.TEXT) - description?: string; - - @IsInt() - @IsOptional() - @Min(0) - @Max(DATATYPES_LENGTH.INT_10) - parentAccountId?: number; -} diff --git a/packages/server/src/services/Import/ImportFileCommon.ts b/packages/server/src/services/Import/ImportFileCommon.ts index eaafe5b13..86bfadf6c 100644 --- a/packages/server/src/services/Import/ImportFileCommon.ts +++ b/packages/server/src/services/Import/ImportFileCommon.ts @@ -11,7 +11,7 @@ import { ImportOperError, ImportOperSuccess, } from './interfaces'; -import { AccountsImportable } from './AccountsImportable'; +import { AccountsImportable } from '../Accounts/AccountsImportable'; import { ServiceError } from '@/exceptions'; import { trimObject } from './_utils'; import { ImportableResources } from './ImportableResources'; @@ -57,12 +57,12 @@ export class ImportFileCommon { } /** - * + * Imports the given parsed data to the resource storage through registered importable service. * @param {number} tenantId - * @param {string} resourceName - Resource name. - * @param {Record} parsedData - - * @param {Knex.Transaction} trx - * @returns + * @param {Record} parsedData - Parsed data. + * @param {Knex.Transaction} trx - Knex transaction. + * @returns {Promise<[ImportOperSuccess[], ImportOperError[]]>} */ public async import( tenantId: number, @@ -77,6 +77,8 @@ export class ImportFileCommon { const ImportableRegistry = this.importable.registry; const importable = ImportableRegistry.getImportable(resourceName); + const concurrency = importable.concurrency || 10; + const success: ImportOperSuccess[] = []; const failed: ImportOperError[] = []; @@ -108,7 +110,7 @@ export class ImportFileCommon { failed.push({ index, error }); } }; - await bluebird.map(parsedData, importAsync, { concurrency: 2 }); + await bluebird.map(parsedData, importAsync, { concurrency }); return [success, failed]; } @@ -127,7 +129,7 @@ export class ImportFileCommon { * @param {number} tenantId * @param {} importFile */ - private async deleteImportFile(tenantId: number, importFile: any) { + public async deleteImportFile(tenantId: number, importFile: any) { const { Import } = this.tenancy.models(tenantId); // Deletes the import row. diff --git a/packages/server/src/services/Import/ImportFileDataTransformer.ts b/packages/server/src/services/Import/ImportFileDataTransformer.ts index dc4ecb3ad..aa331d678 100644 --- a/packages/server/src/services/Import/ImportFileDataTransformer.ts +++ b/packages/server/src/services/Import/ImportFileDataTransformer.ts @@ -1,10 +1,9 @@ -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import * as R from 'ramda'; -import { isUndefined, mapValues, get, pickBy, chain } from 'lodash'; +import { isUndefined, get, chain } from 'lodash'; import { ImportMappingAttr, ResourceMetaFieldsMap } from './interfaces'; import { parseBoolean } from '@/utils'; import { trimObject } from './_utils'; -import ResourceService from '../Resource/ResourceService'; @Service() export class ImportFileDataTransformer { diff --git a/packages/server/src/services/Import/ImportFileUpload.ts b/packages/server/src/services/Import/ImportFileUpload.ts index fff7dac7f..19135d94a 100644 --- a/packages/server/src/services/Import/ImportFileUpload.ts +++ b/packages/server/src/services/Import/ImportFileUpload.ts @@ -27,7 +27,7 @@ export class ImportFileUploadService { * @param {string} resource - Resource name. * @param {string} filePath - File path. * @param {string} fileName - File name. - * @returns + * @returns {Promise} */ public async import( tenantId: number, diff --git a/packages/server/src/services/Import/ImportResourceApplication.ts b/packages/server/src/services/Import/ImportResourceApplication.ts index 07a9a7ea5..8577a4b79 100644 --- a/packages/server/src/services/Import/ImportResourceApplication.ts +++ b/packages/server/src/services/Import/ImportResourceApplication.ts @@ -52,7 +52,7 @@ export class ImportResourceApplication { * Preview the mapped results before process importing. * @param {number} tenantId * @param {number} importId - * @returns {} + * @returns {Promise} */ public async preview(tenantId: number, importId: number) { return this.ImportFilePreviewService.preview(tenantId, importId); @@ -62,7 +62,7 @@ export class ImportResourceApplication { * Process the import file sheet through service for creating entities. * @param {number} tenantId * @param {number} importId - * @returns {Promise} + * @returns {Promise} */ public async process(tenantId: number, importId: number) { return this.importProcessService.import(tenantId, importId); diff --git a/packages/server/src/services/Import/ImportResourceRegistry.ts b/packages/server/src/services/Import/ImportResourceRegistry.ts deleted file mode 100644 index ddd101a15..000000000 --- a/packages/server/src/services/Import/ImportResourceRegistry.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Service } from "typedi"; - - -@Service() -export class ImportResourceRegistry { - -} \ No newline at end of file diff --git a/packages/server/src/services/Import/Importable.ts b/packages/server/src/services/Import/Importable.ts index b3019f34f..8130910f5 100644 --- a/packages/server/src/services/Import/Importable.ts +++ b/packages/server/src/services/Import/Importable.ts @@ -1,7 +1,23 @@ +import { Knex } from 'knex'; +export abstract class Importable { + /** + * + * @param {number} tenantId + * @param {any} createDTO + * @param {Knex.Transaction} trx + */ + public importable(tenantId: number, createDTO: any, trx?: Knex.Transaction) { + throw new Error( + 'The `importable` function is not defined in service importable.' + ); + } -abstract class importable { - - - -} \ No newline at end of file + /** + * Concurrency controlling of the importing process. + * @returns {number} + */ + public get concurrency() { + return 10; + } +} diff --git a/packages/server/src/services/Import/ImportableRegistry.ts b/packages/server/src/services/Import/ImportableRegistry.ts index c7b0120f2..c260c5bd5 100644 --- a/packages/server/src/services/Import/ImportableRegistry.ts +++ b/packages/server/src/services/Import/ImportableRegistry.ts @@ -1,13 +1,18 @@ import { camelCase, upperFirst } from 'lodash'; +import { Importable } from './Importable'; export class ImportableRegistry { private static instance: ImportableRegistry; - private importables: Record; + private importables: Record; private constructor() { this.importables = {}; } + /** + * Gets singleton instance of registry. + * @returns {ImportableRegistry} + */ public static getInstance(): ImportableRegistry { if (!ImportableRegistry.instance) { ImportableRegistry.instance = new ImportableRegistry(); @@ -15,12 +20,22 @@ export class ImportableRegistry { return ImportableRegistry.instance; } - public registerImportable(resource: string, importable: any): void { + /** + * Registers the given importable service. + * @param {string} resource + * @param {Importable} importable + */ + public registerImportable(resource: string, importable: Importable): void { const _resource = this.sanitizeResourceName(resource); this.importables[_resource] = importable; } - public getImportable(name: string): any { + /** + * Retrieves the importable service instance of the given resource name. + * @param {string} name + * @returns {Importable} + */ + public getImportable(name: string): Importable { const _name = this.sanitizeResourceName(name); return this.importables[_name]; } diff --git a/packages/server/src/services/Import/ImportableResources.ts b/packages/server/src/services/Import/ImportableResources.ts index f0da491f6..3f4297075 100644 --- a/packages/server/src/services/Import/ImportableResources.ts +++ b/packages/server/src/services/Import/ImportableResources.ts @@ -1,12 +1,11 @@ import Container, { Service } from 'typedi'; -import { AccountsImportable } from './AccountsImportable'; +import { AccountsImportable } from '../Accounts/AccountsImportable'; import { ImportableRegistry } from './ImportableRegistry'; @Service() export class ImportableResources { private static registry: ImportableRegistry; - constructor() { this.boot(); }