From b1d5390bfce8918ea5f08fb34ebb52e196344f04 Mon Sep 17 00:00:00 2001 From: Ahmed Bouhuolia Date: Sun, 10 Mar 2024 14:53:10 +0200 Subject: [PATCH] WIP --- packages/server/package.json | 3 + .../controllers/Import/ImportController.ts | 149 ++++++++++++++++++ packages/server/src/api/index.ts | 4 + .../20231209230719_create_imports_table.js | 14 ++ packages/server/src/loaders/tenantModels.ts | 4 +- .../Accounts/CreateAccountDTOSchema.ts | 32 ++++ .../src/services/Import/ImportFileUpload.ts | 54 +++++++ .../Import/ImportResourceApplication.ts | 29 ++++ .../Import/ImportResourceInjectable.ts | 120 ++++++++++++++ packages/server/src/services/Import/_utils.ts | 12 ++ pnpm-lock.yaml | 42 +++++ 11 files changed, 462 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/api/controllers/Import/ImportController.ts create mode 100644 packages/server/src/database/migrations/20231209230719_create_imports_table.js create mode 100644 packages/server/src/services/Accounts/CreateAccountDTOSchema.ts create mode 100644 packages/server/src/services/Import/ImportFileUpload.ts create mode 100644 packages/server/src/services/Import/ImportResourceApplication.ts create mode 100644 packages/server/src/services/Import/ImportResourceInjectable.ts create mode 100644 packages/server/src/services/Import/_utils.ts diff --git a/packages/server/package.json b/packages/server/package.json index d359376de..39bc8a217 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -36,6 +36,8 @@ "bcryptjs": "^2.4.3", "bluebird": "^3.7.2", "body-parser": "^1.20.2", + "class-transformer": "^0.5.1", + "class-validator": "^0.14.0", "compression": "^1.7.4", "country-codes-list": "^1.6.8", "cpy": "^8.1.2", @@ -77,6 +79,7 @@ "moment-timezone": "^0.5.43", "mongodb": "^6.1.0", "mongoose": "^5.10.0", + "multer": "1.4.5-lts.1", "mustache": "^3.0.3", "mysql": "^2.17.1", "mysql2": "^1.6.5", diff --git a/packages/server/src/api/controllers/Import/ImportController.ts b/packages/server/src/api/controllers/Import/ImportController.ts new file mode 100644 index 000000000..cfdcb3dc6 --- /dev/null +++ b/packages/server/src/api/controllers/Import/ImportController.ts @@ -0,0 +1,149 @@ +import { Inject, Service } from 'typedi'; +import { Router, Request, Response, NextFunction } from 'express'; +import { query } from 'express-validator'; +import Multer from 'multer'; +import BaseController from '@/api/controllers/BaseController'; +import { ServiceError } from '@/exceptions'; +import { ImportResourceInjectable } from '@/services/Import/ImportResourceInjectable'; +import { ImportResourceApplication } from '@/services/Import/ImportResourceApplication'; + +const upload = Multer({ + dest: './public/imports', + limits: { fileSize: 5 * 1024 * 1024 }, +}); + +@Service() +export class ImportController extends BaseController { + @Inject() + private importResource: ImportResourceInjectable; + + @Inject() + private importResourceApp: ImportResourceApplication; + + /** + * Router constructor method. + */ + router() { + const router = Router(); + + // router.post( + // '/:import_id/import', + // this.asyncMiddleware(this.import.bind(this)), + // this.catchServiceErrors + // ); + router.post( + '/file', + // [...this.importValidationSchema], + upload.single('file'), + this.asyncMiddleware(this.fileUpload.bind(this)) + // this.catchServiceErrors + ); + // router.post( + // '/:import_id/mapping', + // this.asyncMiddleware(this.mapping.bind(this)), + // this.catchServiceErrors + // ); + // router.get( + // '/:import_id/preview', + // this.asyncMiddleware(this.preview.bind(this)), + // this.catchServiceErrors + // ); + return router; + } + + /** + * Import validation schema. + * @returns {ValidationSchema[]} + */ + get importValidationSchema() { + return [query('resource').exists().isString().toString()]; + } + + /** + * Imports xlsx/csv to the given resource type. + * + * - Save the xlsx/csv file and give it a random name. + * - Save the file metadata on the DB storage. + * - + * + * @param {Request} req - + * @param {Response} res - + * @param {NextFunction} next - + */ + async fileUpload(req: Request, res: Response, next: NextFunction) { + const { tenantId } = req; + + try { + const data = await this.importResourceApp.import( + tenantId, + req.body.resource, + req.file.path, + req.file.filename + ); + return res.status(200).send(data); + } catch (error) { + next(error); + } + } + + /** + * + * @param req + * @param res + * @param next + */ + async mapping(req: Request, res: Response, next: NextFunction) { + const { tenantId } = req; + + try { + await this.importResource.mapping(tenantId); + } catch (error) { + next(error); + } + } + + /** + * + * @param req + * @param res + * @param next + */ + async preview(req: Request, res: Response, next: NextFunction) {} + + /** + * + * @param req + * @param res + * @param next + */ + async import(req: Request, res: Response, next: NextFunction) { + const { tenantId } = req; + const { import_id: importId } = req.params; + + try { + await this.importResource.importFile(tenantId, importId); + + return res.status(200).send({}); + } catch (error) { + next(error); + } + } + + /** + * Transforms service errors to response. + * @param {Error} + * @param {Request} req + * @param {Response} res + * @param {ServiceError} error + */ + private catchServiceErrors( + error, + req: Request, + res: Response, + next: NextFunction + ) { + if (error instanceof ServiceError) { + } + next(error); + } +} diff --git a/packages/server/src/api/index.ts b/packages/server/src/api/index.ts index 6a41c8304..1a3ff17c6 100644 --- a/packages/server/src/api/index.ts +++ b/packages/server/src/api/index.ts @@ -56,6 +56,7 @@ import { ProjectsController } from './controllers/Projects/Projects'; import { ProjectTasksController } from './controllers/Projects/Tasks'; import { ProjectTimesController } from './controllers/Projects/Times'; import { TaxRatesController } from './controllers/TaxRates/TaxRates'; +import { ImportController } from './controllers/Import/ImportController'; export default () => { const app = Router(); @@ -131,6 +132,9 @@ export default () => { dashboard.use('/warehouses', Container.get(WarehousesController).router()); dashboard.use('/projects', Container.get(ProjectsController).router()); dashboard.use('/tax-rates', Container.get(TaxRatesController).router()); + + dashboard.use('/import', Container.get(ImportController).router()); + dashboard.use('/', Container.get(ProjectTasksController).router()); dashboard.use('/', Container.get(ProjectTimesController).router()); diff --git a/packages/server/src/database/migrations/20231209230719_create_imports_table.js b/packages/server/src/database/migrations/20231209230719_create_imports_table.js new file mode 100644 index 000000000..69d0a0f7e --- /dev/null +++ b/packages/server/src/database/migrations/20231209230719_create_imports_table.js @@ -0,0 +1,14 @@ +exports.up = function (knex) { + return knex.schema.createTable('imports', (table) => { + table.increments(); + table.string('filename'); + table.string('import_id'); + table.string('resource'); + table.json('mapping'); + table.timestamps(); + }); +}; + +exports.down = function (knex) { + return knex.schema.dropTableIfExists('imports'); +}; diff --git a/packages/server/src/loaders/tenantModels.ts b/packages/server/src/loaders/tenantModels.ts index fcf1936be..8640819d4 100644 --- a/packages/server/src/loaders/tenantModels.ts +++ b/packages/server/src/loaders/tenantModels.ts @@ -61,6 +61,7 @@ import Task from 'models/Task'; import TaxRate from 'models/TaxRate'; import TaxRateTransaction from 'models/TaxRateTransaction'; import Attachment from 'models/Attachment'; +import Import from 'models/Import'; export default (knex) => { const models = { @@ -124,7 +125,8 @@ export default (knex) => { Task, TaxRate, TaxRateTransaction, - Attachment + Attachment, + Import }; return mapValues(models, (model) => model.bindKnex(knex)); }; diff --git a/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts b/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts new file mode 100644 index 000000000..a4f9eab2e --- /dev/null +++ b/packages/server/src/services/Accounts/CreateAccountDTOSchema.ts @@ -0,0 +1,32 @@ +import { DATATYPES_LENGTH } from '@/data/DataTypes'; +import { IsInt, IsOptional, IsString, Length, Min, Max } from 'class-validator'; + +export class AccountDTOSchema { + @IsString() + @Length(3, DATATYPES_LENGTH.STRING) + name: string; + + // @IsString() + // @IsInt() + @IsOptional() + // @Length(3, 6) + code?: string; + + @IsOptional() + currencyCode?: string; + + @IsString() + @Length(3, DATATYPES_LENGTH.STRING) + accountType: string; + + @IsString() + @IsOptional() + @Length(0, DATATYPES_LENGTH.TEXT) + description?: string; + + @IsInt() + @IsOptional() + @Min(0) + @Max(DATATYPES_LENGTH.INT_10) + parentAccountId?: number; +} diff --git a/packages/server/src/services/Import/ImportFileUpload.ts b/packages/server/src/services/Import/ImportFileUpload.ts new file mode 100644 index 000000000..fd537640d --- /dev/null +++ b/packages/server/src/services/Import/ImportFileUpload.ts @@ -0,0 +1,54 @@ +import { first } from 'lodash'; +import { Inject, Service } from 'typedi'; +import { snakeCase, upperFirst } from 'lodash'; +import XLSX from 'xlsx'; +import * as R from 'ramda'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { trimObject } from './_utils'; +const fs = require('fs').promises; + +@Service() +export class ImportFileUploadService { + @Inject() + private tenancy: HasTenancyService; + + /** + * Reads the imported file and stores the import file meta under unqiue id. + * @param {number} tenantId - + * @param {string} filePath - + * @param {string} fileName - + * @returns + */ + public async import( + tenantId: number, + resource: string, + filePath: string, + filename: string + ) { + const { Import } = this.tenancy.models(tenantId); + const buffer = await fs.readFile(filePath); + const workbook = XLSX.read(buffer, { type: 'buffer' }); + + const firstSheetName = workbook.SheetNames[0]; + const worksheet = workbook.Sheets[firstSheetName]; + const jsonData = XLSX.utils.sheet_to_json(worksheet); + + const _resource = upperFirst(snakeCase(resource)); + + const exportFile = await Import.query().insert({ + filename, + importId: filename, + resource: _resource, + }); + const columns = this.getColumns(jsonData); + + return { + ...exportFile, + columns, + }; + } + + private getColumns(json: unknown[]) { + return R.compose(Object.keys, trimObject, first)(json); + } +} diff --git a/packages/server/src/services/Import/ImportResourceApplication.ts b/packages/server/src/services/Import/ImportResourceApplication.ts new file mode 100644 index 000000000..f17d1fa32 --- /dev/null +++ b/packages/server/src/services/Import/ImportResourceApplication.ts @@ -0,0 +1,29 @@ +import { Inject } from 'typedi'; +import { ImportFileUploadService } from './ImportFileUpload'; + +@Inject() +export class ImportResourceApplication { + @Inject() + private importFileService: ImportFileUploadService; + + /** + * Reads the imported file and stores the import file meta under unqiue id. + * @param {number} tenantId - + * @param {string} filePath - + * @param {string} fileName - + * @returns + */ + public async import( + tenantId: number, + resource: string, + filePath: string, + filename: string + ) { + return this.importFileService.import( + tenantId, + resource, + filePath, + filename + ); + } +} diff --git a/packages/server/src/services/Import/ImportResourceInjectable.ts b/packages/server/src/services/Import/ImportResourceInjectable.ts new file mode 100644 index 000000000..0bfc184a5 --- /dev/null +++ b/packages/server/src/services/Import/ImportResourceInjectable.ts @@ -0,0 +1,120 @@ +import XLSX, { readFile } from 'xlsx'; +import * as R from 'ramda'; +import async from 'async'; +import { camelCase, snakeCase, upperFirst } from 'lodash'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { Inject, Service } from 'typedi'; +import { first } from 'lodash'; +import { ServiceError } from '@/exceptions'; +import { validate } from 'class-validator'; +import { AccountDTOSchema } from '../Accounts/CreateAccountDTOSchema'; +import { AccountsApplication } from '../Accounts/AccountsApplication'; +import { plainToClass, plainToInstance } from 'class-transformer'; +const fs = require('fs').promises; + +const ERRORS = { + IMPORT_ID_NOT_FOUND: 'IMPORT_ID_NOT_FOUND', +}; + + +@Service() +export class ImportResourceInjectable { + @Inject() + private tenancy: HasTenancyService; + + @Inject() + private accountsApplication: AccountsApplication; + + public async mapping( + tenantId: number, + importId: number, + maps: { from: string; to: string }[] + ) { + const { Import } = this.tenancy.models(tenantId); + + const importFile = await Import.query().find('filename', importId); + + if (!importFile) { + throw new ServiceError(ERRORS.IMPORT_ID_NOT_FOUND); + } + // + await Import.query() + .findById(importFile.id) + .update({ + maps: JSON.stringify(maps), + }); + // - Validate the to columns. + // - Store the mapping in the import table. + // - + } + + public async preview(tenantId: number, importId: string) {} + + /** + * + * @param tenantId + * @param importId + */ + public async importFile(tenantId: number, importId: string) { + const { Import } = this.tenancy.models(tenantId); + + const importFile = await Import.query().where('importId', importId).first(); + + if (!importFile) { + throw new ServiceError(ERRORS.IMPORT_ID_NOT_FOUND); + } + const buffer = await fs.readFile(`public/imports/${importFile.filename}`); + const workbook = XLSX.read(buffer, { type: 'buffer' }); + + const firstSheetName = workbook.SheetNames[0]; + const worksheet = workbook.Sheets[firstSheetName]; + const jsonData = XLSX.utils.sheet_to_json(worksheet); + + const data = R.compose(R.map(Object.keys), R.map(trimObject))(jsonData); + + const header = first(data); + const body = jsonData; + + const mapping = JSON.parse(importFile.mapping) || []; + const newData = []; + + const findToAttr = (from: string) => { + const found = mapping.find((item) => { + return item.from === from; + }); + return found?.to; + }; + + body.forEach((row) => { + const obj = {}; + + header.forEach((key, index) => { + const toIndex = camelCase(findToAttr(key)); + obj[toIndex] = row[key]; + }); + newData.push(obj); + }); + + const saveJob = async (data) => { + const account = {}; + + Object.keys(data).map((key) => { + account[key] = data[key]; + }); + const accountClass = plainToInstance(AccountDTOSchema, account); + const errors = await validate(accountClass); + + if (errors.length > 0) { + console.log('validation failed. errors: ', errors); + } else { + return this.accountsApplication.createAccount(tenantId, account); + } + }; + const saveDataQueue = async.queue(saveJob, 10); + + newData.forEach((data) => { + saveDataQueue.push(data); + }); + await saveDataQueue.drain(); + } +} diff --git a/packages/server/src/services/Import/_utils.ts b/packages/server/src/services/Import/_utils.ts new file mode 100644 index 000000000..3700a313e --- /dev/null +++ b/packages/server/src/services/Import/_utils.ts @@ -0,0 +1,12 @@ +export function trimObject(obj) { + return Object.entries(obj).reduce((acc, [key, value]) => { + // Trim the key + const trimmedKey = key.trim(); + + // Trim the value if it's a string, otherwise leave it as is + const trimmedValue = typeof value === 'string' ? value.trim() : value; + + // Assign the trimmed key and value to the accumulator object + return { ...acc, [trimmedKey]: trimmedValue }; + }, {}); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9482a23bf..25a16e3de 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -80,6 +80,12 @@ importers: body-parser: specifier: ^1.20.2 version: 1.20.2 + class-transformer: + specifier: ^0.5.1 + version: 0.5.1 + class-validator: + specifier: ^0.14.0 + version: 0.14.0 compression: specifier: ^1.7.4 version: 1.7.4 @@ -203,6 +209,9 @@ importers: mongoose: specifier: ^5.10.0 version: 5.13.20 + multer: + specifier: 1.4.5-lts.1 + version: 1.4.5-lts.1 mustache: specifier: ^3.0.3 version: 3.2.1 @@ -6481,6 +6490,10 @@ packages: resolution: {integrity: sha512-IDaobHimLQhjwsQ/NMwRVfa/yL7L/wriQPMhw1ZJall0KX6E1oxk29XMDeilW5qTIg5aoiqf5Udy8U/51aNoQQ==} dev: false + /@types/validator@13.11.7: + resolution: {integrity: sha512-q0JomTsJ2I5Mv7dhHhQLGjMvX0JJm5dyZ1DXQySIUzU1UlwzB8bt+R6+LODUbz0UDIOvEzGc28tk27gBJw2N8Q==} + dev: false + /@types/webidl-conversions@7.0.0: resolution: {integrity: sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==} dev: false @@ -7458,6 +7471,10 @@ packages: buffer-equal: 1.0.1 dev: false + /append-field@1.0.0: + resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==} + dev: false + /append-transform@1.0.0: resolution: {integrity: sha512-P009oYkeHyU742iSZJzZZywj4QRJdnTWffaKuJQLablCZ1uz6/cW4yaRgcDaoQ+uwOxxnt0gRUcwfsNP2ri0gw==} engines: {node: '>=4'} @@ -9026,6 +9043,10 @@ packages: resolution: {integrity: sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==} dev: false + /class-transformer@0.5.1: + resolution: {integrity: sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==} + dev: false + /class-utils@0.3.6: resolution: {integrity: sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==} engines: {node: '>=0.10.0'} @@ -9036,6 +9057,14 @@ packages: static-extend: 0.1.2 dev: false + /class-validator@0.14.0: + resolution: {integrity: sha512-ct3ltplN8I9fOwUd8GrP8UQixwff129BkEtuWDKL5W45cQuLd19xqmTLu5ge78YDm/fdje6FMt0hGOhl0lii3A==} + dependencies: + '@types/validator': 13.11.7 + libphonenumber-js: 1.10.19 + validator: 13.9.0 + dev: false + /classnames@2.3.2: resolution: {integrity: sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==} dev: false @@ -17307,6 +17336,19 @@ packages: /ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + /multer@1.4.5-lts.1: + resolution: {integrity: sha512-ywPWvcDMeH+z9gQq5qYHCCy+ethsk4goepZ45GLD63fOu0YcNecQxi64nDs3qluZB+murG3/D4dJ7+dGctcCQQ==} + engines: {node: '>= 6.0.0'} + dependencies: + append-field: 1.0.0 + busboy: 1.6.0 + concat-stream: 1.6.2 + mkdirp: 0.5.6 + object-assign: 4.1.1 + type-is: 1.6.18 + xtend: 4.0.2 + dev: false + /multicast-dns@7.2.5: resolution: {integrity: sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==} hasBin: true