From 93bf6d9d3da3b983a262324d9d661cd1963c68d5 Mon Sep 17 00:00:00 2001 From: Ahmed Bouhuolia Date: Thu, 19 Dec 2024 12:48:24 +0200 Subject: [PATCH] refactor: wip migrate ot nestjs --- .../src/libs/logic-evaluation/Lexer.js | 172 +++++++ .../src/libs/logic-evaluation/Parser.js | 159 ++++++ .../src/libs/logic-evaluation/QueryParser.js | 61 +++ packages/server-nest/src/models/Model.ts | 1 + .../modules/Accounts/Account.transformer.ts | 42 +- .../src/modules/Accounts/Accounts.module.ts | 1 - .../Accounts/AccountsApplication.service.ts | 14 +- .../modules/Accounts/GetAccounts.service.ts | 15 +- .../modules/Accounts/models/Account.model.ts | 95 ++-- .../modules/Banking/models/PlaidItem.model.ts | 41 ++ .../DynamicFilter/DynamicFilter.ts | 89 ++++ .../DynamicFilter/DynamicFilter.types.ts | 40 ++ .../DynamicFilter/DynamicFilterAbstractor.ts | 55 +++ .../DynamicFilterAdvancedFilter.ts | 27 ++ .../DynamicFilter/DynamicFilterFilterRoles.ts | 51 ++ .../DynamicFilter/DynamicFilterQueryParser.ts | 72 +++ .../DynamicFilterRoleAbstractor.ts | 388 +++++++++++++++ .../DynamicFilter/DynamicFilterSearch.ts | 48 ++ .../DynamicFilter/DynamicFilterSortBy.ts | 92 ++++ .../DynamicFilter/DynamicFilterViews.ts | 56 +++ .../DynamicListing/DynamicFilter/constants.ts | 43 ++ .../DynamicListing/DynamicFilter/index.ts | 11 + .../DynamicListing/DynamicListAbstract.ts | 1 + .../DynamicListing/DynamicListCustomView.ts | 45 ++ .../DynamicListing/DynamicListFilterRoles.ts | 104 ++++ .../DynamicListing/DynamicListSearch.ts | 15 + .../DynamicListing/DynamicListService.ts | 101 ++++ .../DynamicListing/DynamicListSortBy.ts | 41 ++ .../src/modules/DynamicListing/constants.ts | 6 + .../src/modules/DynamicListing/validators.ts | 0 .../src/modules/Export/ExportAls.ts | 48 ++ .../src/modules/Export/ExportApplication.ts | 22 + .../src/modules/Export/ExportPdf.ts | 47 ++ .../src/modules/Export/ExportRegistery.ts | 50 ++ .../src/modules/Export/ExportResources.ts | 76 +++ .../src/modules/Export/ExportService.ts | 228 +++++++++ .../src/modules/Export/Exportable.ts | 22 + .../server-nest/src/modules/Export/common.ts | 9 + .../src/modules/Export/constants.ts | 2 + .../server-nest/src/modules/Export/utils.ts | 45 ++ .../src/modules/Import/ImportALS.ts | 105 ++++ .../src/modules/Import/ImportFileCommon.ts | 175 +++++++ .../Import/ImportFileDataTransformer.ts | 151 ++++++ .../modules/Import/ImportFileDataValidator.ts | 46 ++ .../src/modules/Import/ImportFileMapping.ts | 156 ++++++ .../src/modules/Import/ImportFileMeta.ts | 33 ++ .../Import/ImportFileMetaTransformer.ts | 19 + .../src/modules/Import/ImportFilePreview.ts | 53 ++ .../src/modules/Import/ImportFileProcess.ts | 104 ++++ .../modules/Import/ImportFileProcessCommit.ts | 66 +++ .../src/modules/Import/ImportFileUpload.ts | 123 +++++ .../Import/ImportRemoveExpiredFiles.ts | 34 ++ .../Import/ImportResourceApplication.ts | 106 ++++ .../src/modules/Import/ImportSample.ts | 46 ++ .../src/modules/Import/Importable.ts | 72 +++ .../src/modules/Import/ImportableRegistry.ts | 46 ++ .../src/modules/Import/ImportableResources.ts | 73 +++ .../src/modules/Import/_constants.ts | 3 + .../server-nest/src/modules/Import/_utils.ts | 459 ++++++++++++++++++ .../src/modules/Import/interfaces.ts | 77 +++ .../jobs/ImportDeleteExpiredFilesJob.ts | 28 ++ .../src/modules/Import/sheet_utils.ts | 56 +++ .../Views/GetResourceColumns.service.ts | 0 .../modules/Views/GetResourceViews.service.ts | 23 + .../src/modules/Views/Views.types.ts | 61 +++ .../src/modules/Views/models/View.model.ts | 72 +++ .../modules/Views/models/ViewColumn.model.ts | 17 + .../modules/Views/models/ViewRole.model.ts | 46 ++ .../utils/assoc-depth-level-to-object-tree.ts | 16 + .../src/utils/flat-to-nested-array.ts | 24 + .../server-nest/src/utils/format-number.ts | 8 +- .../src/utils/nested-array-to-flatten.ts | 33 ++ packages/server-nest/tsconfig.build.json | 13 +- 73 files changed, 4683 insertions(+), 96 deletions(-) create mode 100644 packages/server-nest/src/libs/logic-evaluation/Lexer.js create mode 100644 packages/server-nest/src/libs/logic-evaluation/Parser.js create mode 100644 packages/server-nest/src/libs/logic-evaluation/QueryParser.js create mode 100644 packages/server-nest/src/modules/Banking/models/PlaidItem.model.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilter.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilter.types.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterAbstractor.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterAdvancedFilter.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterFilterRoles.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterQueryParser.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterRoleAbstractor.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterSearch.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterSortBy.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterViews.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicFilter/constants.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicFilter/index.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicListAbstract.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicListCustomView.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicListFilterRoles.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicListSearch.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicListService.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/DynamicListSortBy.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/constants.ts create mode 100644 packages/server-nest/src/modules/DynamicListing/validators.ts create mode 100644 packages/server-nest/src/modules/Export/ExportAls.ts create mode 100644 packages/server-nest/src/modules/Export/ExportApplication.ts create mode 100644 packages/server-nest/src/modules/Export/ExportPdf.ts create mode 100644 packages/server-nest/src/modules/Export/ExportRegistery.ts create mode 100644 packages/server-nest/src/modules/Export/ExportResources.ts create mode 100644 packages/server-nest/src/modules/Export/ExportService.ts create mode 100644 packages/server-nest/src/modules/Export/Exportable.ts create mode 100644 packages/server-nest/src/modules/Export/common.ts create mode 100644 packages/server-nest/src/modules/Export/constants.ts create mode 100644 packages/server-nest/src/modules/Export/utils.ts create mode 100644 packages/server-nest/src/modules/Import/ImportALS.ts create mode 100644 packages/server-nest/src/modules/Import/ImportFileCommon.ts create mode 100644 packages/server-nest/src/modules/Import/ImportFileDataTransformer.ts create mode 100644 packages/server-nest/src/modules/Import/ImportFileDataValidator.ts create mode 100644 packages/server-nest/src/modules/Import/ImportFileMapping.ts create mode 100644 packages/server-nest/src/modules/Import/ImportFileMeta.ts create mode 100644 packages/server-nest/src/modules/Import/ImportFileMetaTransformer.ts create mode 100644 packages/server-nest/src/modules/Import/ImportFilePreview.ts create mode 100644 packages/server-nest/src/modules/Import/ImportFileProcess.ts create mode 100644 packages/server-nest/src/modules/Import/ImportFileProcessCommit.ts create mode 100644 packages/server-nest/src/modules/Import/ImportFileUpload.ts create mode 100644 packages/server-nest/src/modules/Import/ImportRemoveExpiredFiles.ts create mode 100644 packages/server-nest/src/modules/Import/ImportResourceApplication.ts create mode 100644 packages/server-nest/src/modules/Import/ImportSample.ts create mode 100644 packages/server-nest/src/modules/Import/Importable.ts create mode 100644 packages/server-nest/src/modules/Import/ImportableRegistry.ts create mode 100644 packages/server-nest/src/modules/Import/ImportableResources.ts create mode 100644 packages/server-nest/src/modules/Import/_constants.ts create mode 100644 packages/server-nest/src/modules/Import/_utils.ts create mode 100644 packages/server-nest/src/modules/Import/interfaces.ts create mode 100644 packages/server-nest/src/modules/Import/jobs/ImportDeleteExpiredFilesJob.ts create mode 100644 packages/server-nest/src/modules/Import/sheet_utils.ts create mode 100644 packages/server-nest/src/modules/Views/GetResourceColumns.service.ts create mode 100644 packages/server-nest/src/modules/Views/GetResourceViews.service.ts create mode 100644 packages/server-nest/src/modules/Views/Views.types.ts create mode 100644 packages/server-nest/src/modules/Views/models/View.model.ts create mode 100644 packages/server-nest/src/modules/Views/models/ViewColumn.model.ts create mode 100644 packages/server-nest/src/modules/Views/models/ViewRole.model.ts create mode 100644 packages/server-nest/src/utils/assoc-depth-level-to-object-tree.ts create mode 100644 packages/server-nest/src/utils/flat-to-nested-array.ts create mode 100644 packages/server-nest/src/utils/nested-array-to-flatten.ts diff --git a/packages/server-nest/src/libs/logic-evaluation/Lexer.js b/packages/server-nest/src/libs/logic-evaluation/Lexer.js new file mode 100644 index 000000000..3cfc04f41 --- /dev/null +++ b/packages/server-nest/src/libs/logic-evaluation/Lexer.js @@ -0,0 +1,172 @@ + +const OperationType = { + LOGIC: 'LOGIC', + STRING: 'STRING', + COMPARISON: 'COMPARISON', + MATH: 'MATH', +}; + +export class Lexer { + // operation table + static get optable() { + return { + '=': OperationType.LOGIC, + '&': OperationType.LOGIC, + '|': OperationType.LOGIC, + '?': OperationType.LOGIC, + ':': OperationType.LOGIC, + + '\'': OperationType.STRING, + '"': OperationType.STRING, + + '!': OperationType.COMPARISON, + '>': OperationType.COMPARISON, + '<': OperationType.COMPARISON, + + '(': OperationType.MATH, + ')': OperationType.MATH, + '+': OperationType.MATH, + '-': OperationType.MATH, + '*': OperationType.MATH, + '/': OperationType.MATH, + '%': OperationType.MATH, + }; + } + + /** + * Constructor + * @param {*} expression - + */ + constructor(expression) { + this.currentIndex = 0; + this.input = expression; + this.tokenList = []; + } + + getTokens() { + let tok; + do { + // read current token, so step should be -1 + tok = this.pickNext(-1); + const pos = this.currentIndex; + switch (Lexer.optable[tok]) { + case OperationType.LOGIC: + // == && || === + this.readLogicOpt(tok); + break; + + case OperationType.STRING: + this.readString(tok); + break; + + case OperationType.COMPARISON: + this.readCompare(tok); + break; + + case OperationType.MATH: + this.receiveToken(); + break; + + default: + this.readValue(tok); + } + + // if the pos not changed, this loop will go into a infinite loop, every step of while loop, + // we must move the pos forward + // so here we should throw error, for example `1 & 2` + if (pos === this.currentIndex && tok !== undefined) { + const err = new Error(`unkonw token ${tok} from input string ${this.input}`); + err.name = 'UnknowToken'; + throw err; + } + } while (tok !== undefined) + + return this.tokenList; + } + + /** + * read next token, the index param can set next step, default go foward 1 step + * + * @param index next postion + */ + pickNext(index = 0) { + return this.input[index + this.currentIndex + 1]; + } + + /** + * Store token into result tokenList, and move the pos index + * + * @param index + */ + receiveToken(index = 1) { + const tok = this.input.slice(this.currentIndex, this.currentIndex + index).trim(); + // skip empty string + if (tok) { + this.tokenList.push(tok); + } + + this.currentIndex += index; + } + + // ' or " + readString(tok) { + let next; + let index = 0; + do { + next = this.pickNext(index); + index += 1; + } while (next !== tok && next !== undefined); + this.receiveToken(index + 1); + } + + // > or < or >= or <= or !== + // tok in (>, <, !) + readCompare(tok) { + if (this.pickNext() !== '=') { + this.receiveToken(1); + return; + } + // !== + if (tok === '!' && this.pickNext(1) === '=') { + this.receiveToken(3); + return; + } + this.receiveToken(2); + } + + // === or == + // && || + readLogicOpt(tok) { + if (this.pickNext() === tok) { + // === + if (tok === '=' && this.pickNext(1) === tok) { + return this.receiveToken(3); + } + // == && || + return this.receiveToken(2); + } + // handle as && + // a ? b : c is equal to a && b || c + if (tok === '?' || tok === ':') { + return this.receiveToken(1); + } + } + + readValue(tok) { + if (!tok) { + return; + } + + let index = 0; + while (!Lexer.optable[tok] && tok !== undefined) { + tok = this.pickNext(index); + index += 1; + } + this.receiveToken(index); + } +} + +export default function token(expression) { + const lexer = new Lexer(expression); + return lexer.getTokens(); +} diff --git a/packages/server-nest/src/libs/logic-evaluation/Parser.js b/packages/server-nest/src/libs/logic-evaluation/Parser.js new file mode 100644 index 000000000..8e7156592 --- /dev/null +++ b/packages/server-nest/src/libs/logic-evaluation/Parser.js @@ -0,0 +1,159 @@ +export const OPERATION = { + '!': 5, + '*': 4, + '/': 4, + '%': 4, + '+': 3, + '-': 3, + '>': 2, + '<': 2, + '>=': 2, + '<=': 2, + '===': 2, + '!==': 2, + '==': 2, + '!=': 2, + '&&': 1, + '||': 1, + '?': 1, + ':': 1, +}; + +// export interface Node { +// left: Node | string | null; +// right: Node | string | null; +// operation: string; +// grouped?: boolean; +// }; + +export default class Parser { + + constructor(token) { + this.index = -1; + this.blockLevel = 0; + this.token = token; + } + + /** + * + * @return {Node | string} =- + */ + parse() { + let tok; + let root = { + left: null, + right: null, + operation: null, + }; + + do { + tok = this.parseStatement(); + + if (tok === null || tok === undefined) { + break; + } + + if (root.left === null) { + root.left = tok; + root.operation = this.nextToken(); + + if (!root.operation) { + return tok; + } + + root.right = this.parseStatement(); + } else { + if (typeof tok !== 'string') { + throw new Error('operation must be string, but get ' + JSON.stringify(tok)); + } + root = this.addNode(tok, this.parseStatement(), root); + } + } while (tok); + + return root; + } + + nextToken() { + this.index += 1; + return this.token[this.index]; + } + + prevToken() { + return this.token[this.index - 1]; + } + + /** + * + * @param {string} operation + * @param {Node|String|null} right + * @param {Node} root + */ + addNode(operation, right, root) { + let pre = root; + + if (this.compare(pre.operation, operation) < 0 && !pre.grouped) { + + while (pre.right !== null && + typeof pre.right !== 'string' && + this.compare(pre.right.operation, operation) < 0 && !pre.right.grouped) { + pre = pre.right; + } + + pre.right = { + operation, + left: pre.right, + right, + }; + return root; + } + return { + left: pre, + right, + operation, + } + } + + /** + * + * @param {String} a + * @param {String} b + */ + compare(a, b) { + if (!OPERATION.hasOwnProperty(a) || !OPERATION.hasOwnProperty(b)) { + throw new Error(`unknow operation ${a} or ${b}`); + } + return OPERATION[a] - OPERATION[b]; + } + + /** + * @return string | Node | null + */ + parseStatement() { + const token = this.nextToken(); + if (token === '(') { + this.blockLevel += 1; + const node = this.parse(); + this.blockLevel -= 1; + + if (typeof node !== 'string') { + node.grouped = true; + } + return node; + } + + if (token === ')') { + return null; + } + + if (token === '!') { + return { left: null, operation: token, right: this.parseStatement() } + } + + // 3 > -12 or -12 + 10 + if (token === '-' && (OPERATION[this.prevToken()] > 0 || this.prevToken() === undefined)) { + return { left: '0', operation: token, right: this.parseStatement(), grouped: true }; + } + + return token; + } +} diff --git a/packages/server-nest/src/libs/logic-evaluation/QueryParser.js b/packages/server-nest/src/libs/logic-evaluation/QueryParser.js new file mode 100644 index 000000000..cd31c128d --- /dev/null +++ b/packages/server-nest/src/libs/logic-evaluation/QueryParser.js @@ -0,0 +1,61 @@ +import { OPERATION } from './Parser'; + +export default class QueryParser { + + constructor(tree, queries) { + this.tree = tree; + this.queries = queries; + this.query = null; + } + + setQuery(query) { + this.query = query.clone(); + } + + parse() { + return this.parseNode(this.tree); + } + + parseNode(node) { + if (typeof node === 'string') { + const nodeQuery = this.getQuery(node); + return (query) => { nodeQuery(query); }; + } + if (OPERATION[node.operation] === undefined) { + throw new Error(`unknow expression ${node.operation}`); + } + const leftQuery = this.getQuery(node.left); + const rightQuery = this.getQuery(node.right); + + switch (node.operation) { + case '&&': + case 'AND': + default: + return (nodeQuery) => nodeQuery.where((query) => { + query.where((q) => { leftQuery(q); }); + query.andWhere((q) => { rightQuery(q); }); + }); + case '||': + case 'OR': + return (nodeQuery) => nodeQuery.where((query) => { + query.where((q) => { leftQuery(q); }); + query.orWhere((q) => { rightQuery(q); }); + }); + } + } + + getQuery(node) { + if (typeof node !== 'string' && node !== null) { + return this.parseNode(node); + } + const value = parseFloat(node); + + if (!isNaN(value)) { + if (typeof this.queries[node] === 'undefined') { + throw new Error(`unknow query under index ${node}`); + } + return this.queries[node]; + } + return null; + } +} \ No newline at end of file diff --git a/packages/server-nest/src/models/Model.ts b/packages/server-nest/src/models/Model.ts index 5afc6ba77..9551179f6 100644 --- a/packages/server-nest/src/models/Model.ts +++ b/packages/server-nest/src/models/Model.ts @@ -2,4 +2,5 @@ import { Model } from 'objection'; export class BaseModel extends Model { public readonly id: number; + public readonly tableName: string; } \ No newline at end of file diff --git a/packages/server-nest/src/modules/Accounts/Account.transformer.ts b/packages/server-nest/src/modules/Accounts/Account.transformer.ts index 94ac3cee1..560d19f82 100644 --- a/packages/server-nest/src/modules/Accounts/Account.transformer.ts +++ b/packages/server-nest/src/modules/Accounts/Account.transformer.ts @@ -1,11 +1,9 @@ -// import { IAccountsStructureType } from './Accounts.types'; -// import { -// assocDepthLevelToObjectTree, -// flatToNestedArray, -// nestedArrayToFlatten, -// } from 'utils'; import { Transformer } from '../Transformer/Transformer'; import { AccountModel } from './models/Account.model'; +import { flatToNestedArray } from '@/utils/flat-to-nested-array'; +import { assocDepthLevelToObjectTree } from '@/utils/assoc-depth-level-to-object-tree'; +import { nestedArrayToFlatten } from '@/utils/nested-array-to-flatten'; +import { IAccountsStructureType } from './Accounts.types'; export class AccountTransformer extends Transformer { /** @@ -113,20 +111,20 @@ export class AccountTransformer extends Transformer { * @param {IAccount[]} * @returns {IAccount[]} */ - // protected postCollectionTransform = (accounts: AccountModel[]) => { - // // Transfom the flatten to accounts tree. - // const transformed = flatToNestedArray(accounts, { - // id: 'id', - // parentId: 'parentAccountId', - // }); - // // Associate `accountLevel` attr to indicate object depth. - // const transformed2 = assocDepthLevelToObjectTree( - // transformed, - // 1, - // 'accountLevel', - // ); - // return this.options.structure === IAccountsStructureType.Flat - // ? nestedArrayToFlatten(transformed2) - // : transformed2; - // }; + protected postCollectionTransform = (accounts: AccountModel[]) => { + // Transfom the flatten to accounts tree. + const transformed = flatToNestedArray(accounts, { + id: 'id', + parentId: 'parentAccountId', + }); + // Associate `accountLevel` attr to indicate object depth. + const transformed2 = assocDepthLevelToObjectTree( + transformed, + 1, + 'accountLevel', + ); + return this.options.structure === IAccountsStructureType.Flat + ? nestedArrayToFlatten(transformed2) + : transformed2; + }; } diff --git a/packages/server-nest/src/modules/Accounts/Accounts.module.ts b/packages/server-nest/src/modules/Accounts/Accounts.module.ts index e3d22a583..49bc23240 100644 --- a/packages/server-nest/src/modules/Accounts/Accounts.module.ts +++ b/packages/server-nest/src/modules/Accounts/Accounts.module.ts @@ -32,7 +32,6 @@ import { GetAccountTransactionsService } from './GetAccountTransactions.service' ActivateAccount, GetAccountTypesService, GetAccountTransactionsService, - // GetAccountsService, ], }) export class AccountsModule {} diff --git a/packages/server-nest/src/modules/Accounts/AccountsApplication.service.ts b/packages/server-nest/src/modules/Accounts/AccountsApplication.service.ts index e91a9461d..9510196dc 100644 --- a/packages/server-nest/src/modules/Accounts/AccountsApplication.service.ts +++ b/packages/server-nest/src/modules/Accounts/AccountsApplication.service.ts @@ -1,20 +1,8 @@ import { Knex } from 'knex'; import { Injectable } from '@nestjs/common'; -// import { -// IAccount, -// IAccountCreateDTO, -// IAccountEditDTO, -// IAccountResponse, -// IAccountsFilter, -// IAccountsTransactionsFilter, -// IFilterMeta, -// IGetAccountTransactionPOJO, -// } from '@/interfaces'; import { CreateAccountService } from './CreateAccount.service'; import { DeleteAccount } from './DeleteAccount.service'; import { EditAccount } from './EditAccount.service'; -// import { GetAccounts } from './GetAccounts.service'; -// import { GetAccountTransactions } from './GetAccountTransactions.service'; import { CreateAccountDTO } from './CreateAccount.dto'; import { AccountModel } from './models/Account.model'; import { EditAccountDTO } from './EditAccount.dto'; @@ -31,8 +19,8 @@ import { export class AccountsApplication { constructor( private readonly createAccountService: CreateAccountService, - private readonly deleteAccountService: DeleteAccount, private readonly editAccountService: EditAccount, + private readonly deleteAccountService: DeleteAccount, private readonly activateAccountService: ActivateAccount, private readonly getAccountTypesService: GetAccountTypesService, private readonly getAccountService: GetAccount, diff --git a/packages/server-nest/src/modules/Accounts/GetAccounts.service.ts b/packages/server-nest/src/modules/Accounts/GetAccounts.service.ts index 1d3c44840..968e7b3db 100644 --- a/packages/server-nest/src/modules/Accounts/GetAccounts.service.ts +++ b/packages/server-nest/src/modules/Accounts/GetAccounts.service.ts @@ -4,20 +4,19 @@ // IAccountsFilter, // IAccountResponse, // IFilterMeta, -// } from '@/interfaces'; -// import { DynamicListingService } from '@/services/DynamicListing/DynamicListService'; +// } from './Accounts.types'; +// import { DynamicListService } from '../DynamicListing/DynamicListService'; // import { AccountTransformer } from './Account.transformer'; -// import { TransformerService } from '@/lib/Transformer/TransformerService'; -// import { flatToNestedArray } from '@/utils'; -// import { Account } from './Account.model'; +// import { TransformerInjectable } from '../Transformer/TransformerInjectable.service'; +// import { AccountModel } from './models/Account.model'; // import { AccountRepository } from './repositories/Account.repository'; // @Injectable() // export class GetAccountsService { // constructor( -// private readonly dynamicListService: DynamicListingService, -// private readonly transformerService: TransformerService, -// private readonly accountModel: typeof Account, +// private readonly dynamicListService: DynamicListService, +// private readonly transformerService: TransformerInjectable, +// private readonly accountModel: typeof AccountModel, // private readonly accountRepository: AccountRepository, // ) {} diff --git a/packages/server-nest/src/modules/Accounts/models/Account.model.ts b/packages/server-nest/src/modules/Accounts/models/Account.model.ts index 192cb082d..ef0c97bb4 100644 --- a/packages/server-nest/src/modules/Accounts/models/Account.model.ts +++ b/packages/server-nest/src/modules/Accounts/models/Account.model.ts @@ -11,6 +11,7 @@ import { TenantModel } from '@/modules/System/models/TenantModel'; // import { CustomViewBaseModel } from '@/modules/CustomViews/CustomViewBaseModel'; // import { ModelSettings } from '@/modules/Settings/ModelSettings'; import { AccountTypesUtils } from '@/libs/accounts-utils/AccountTypesUtils'; +import { Model } from 'objection'; // import AccountSettings from './Account.Settings'; // import { DEFAULT_VIEWS } from '@/modules/Accounts/constants'; // import { buildFilterQuery, buildSortColumnQuery } from '@/lib/ViewRolesBuilder'; @@ -204,50 +205,50 @@ export class AccountModel extends TenantModel { * Relationship mapping. */ static get relationMappings() { - // const AccountTransaction = require('models/AccountTransaction'); - // const Item = require('models/Item'); + const { AccountTransaction } = require('./AccountTransaction.model'); + const { Item } = require('../../Items/models/Item'); // const InventoryAdjustment = require('models/InventoryAdjustment'); // const ManualJournalEntry = require('models/ManualJournalEntry'); // const Expense = require('models/Expense'); // const ExpenseEntry = require('models/ExpenseCategory'); // const ItemEntry = require('models/ItemEntry'); // const UncategorizedTransaction = require('models/UncategorizedCashflowTransaction'); - // const PlaidItem = require('models/PlaidItem'); + const { PlaidItem } = require('../../Banking/models/PlaidItem.model'); return { - // /** - // * Account model may has many transactions. - // */ - // transactions: { - // relation: Model.HasManyRelation, - // modelClass: AccountTransaction.default, - // join: { - // from: 'accounts.id', - // to: 'accounts_transactions.accountId', - // }, - // }, - // /** - // * - // */ - // itemsCostAccount: { - // relation: Model.HasManyRelation, - // modelClass: Item.default, - // join: { - // from: 'accounts.id', - // to: 'items.costAccountId', - // }, - // }, - // /** - // * - // */ - // itemsSellAccount: { - // relation: Model.HasManyRelation, - // modelClass: Item.default, - // join: { - // from: 'accounts.id', - // to: 'items.sellAccountId', - // }, - // }, + /** + * Account model may has many transactions. + */ + transactions: { + relation: Model.HasManyRelation, + modelClass: AccountTransaction, + join: { + from: 'accounts.id', + to: 'accounts_transactions.accountId', + }, + }, + /** + * Account may has many items as cost account. + */ + itemsCostAccount: { + relation: Model.HasManyRelation, + modelClass: Item, + join: { + from: 'accounts.id', + to: 'items.costAccountId', + }, + }, + /** + * Account may has many items as sell account. + */ + itemsSellAccount: { + relation: Model.HasManyRelation, + modelClass: Item, + join: { + from: 'accounts.id', + to: 'items.sellAccountId', + }, + }, // /** // * // */ @@ -328,17 +329,17 @@ export class AccountModel extends TenantModel { // query.where('categorized', false); // }, // }, - // /** - // * Account model may belongs to a Plaid item. - // */ - // plaidItem: { - // relation: Model.BelongsToOneRelation, - // modelClass: PlaidItem.default, - // join: { - // from: 'accounts.plaidItemId', - // to: 'plaid_items.plaidItemId', - // }, - // }, + /** + * Account model may belongs to a Plaid item. + */ + plaidItem: { + relation: Model.BelongsToOneRelation, + modelClass: PlaidItem, + join: { + from: 'accounts.plaidItemId', + to: 'plaid_items.plaidItemId', + }, + }, }; } diff --git a/packages/server-nest/src/modules/Banking/models/PlaidItem.model.ts b/packages/server-nest/src/modules/Banking/models/PlaidItem.model.ts new file mode 100644 index 000000000..ac3600d7a --- /dev/null +++ b/packages/server-nest/src/modules/Banking/models/PlaidItem.model.ts @@ -0,0 +1,41 @@ +import { BaseModel } from '@/models/Model'; + +export class PlaidItem extends BaseModel { + pausedAt: Date; + + /** + * Table name. + */ + static get tableName() { + return 'plaid_items'; + } + + /** + * Timestamps columns. + */ + get timestamps() { + return []; + } + + /** + * Relationship mapping. + */ + static get relationMappings() { + return {}; + } + + /** + * Virtual attributes. + */ + static get virtualAttributes() { + return ['isPaused']; + } + + /** + * Detarmines whether the Plaid item feeds syncing is paused. + * @return {boolean} + */ + get isPaused() { + return !!this.pausedAt; + } +} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilter.ts b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilter.ts new file mode 100644 index 000000000..ddbb31ce3 --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilter.ts @@ -0,0 +1,89 @@ +import { forEach } from 'lodash'; +import { DynamicFilterAbstractor } from './DynamicFilterAbstractor'; +import { IDynamicFilter, IFilterRole, IModel } from '@/interfaces'; +import { BaseModel } from '@/models/Model'; + +export class DynamicFilter extends DynamicFilterAbstractor { + private model: BaseModel; + private dynamicFilters: IDynamicFilter[]; + + /** + * Constructor. + * @param {String} tableName - + */ + constructor(model: BaseModel) { + super(); + + this.model = model; + this.dynamicFilters = []; + } + + /** + * Registers the given dynamic filter. + * @param {IDynamicFilter} filterRole - Filter role. + */ + public setFilter = (dynamicFilter: IDynamicFilter) => { + dynamicFilter.setModel(this.model); + dynamicFilter.onInitialize(); + + this.dynamicFilters.push(dynamicFilter); + }; + + /** + * Retrieve dynamic filter build queries. + * @returns + */ + private dynamicFiltersBuildQuery = () => { + return this.dynamicFilters.map((filter) => { + return filter.buildQuery(); + }); + }; + + /** + * Retrieve dynamic filter roles. + * @returns {IFilterRole[]} + */ + private dynamicFilterTableColumns = (): IFilterRole[] => { + const localFilterRoles = []; + + this.dynamicFilters.forEach((dynamicFilter) => { + const { filterRoles } = dynamicFilter; + + localFilterRoles.push( + ...(Array.isArray(filterRoles) ? filterRoles : [filterRoles]), + ); + }); + return localFilterRoles; + }; + + /** + * Builds queries of filter roles. + */ + public buildQuery = () => { + const buildersCallbacks = this.dynamicFiltersBuildQuery(); + const tableColumns = this.dynamicFilterTableColumns(); + + return (builder) => { + buildersCallbacks.forEach((builderCallback) => { + builderCallback(builder); + }); + this.buildFilterRolesJoins(builder); + }; + }; + + /** + * Retrieve response metadata from all filters adapters. + */ + public getResponseMeta = () => { + const responseMeta = {}; + + this.dynamicFilters.forEach((filter) => { + const { responseMeta: filterMeta } = filter; + + forEach(filterMeta, (value, key) => { + responseMeta[key] = value; + }); + }); + return responseMeta; + }; +} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilter.types.ts b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilter.types.ts new file mode 100644 index 000000000..2329e2ebf --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilter.types.ts @@ -0,0 +1,40 @@ +import { BaseModel } from '@/models/Model'; +// import { IModel, ISortOrder } from "./Model"; + +export type ISortOrder = 'DESC' | 'ASC'; + +export interface IDynamicFilter { + setModel(model: BaseModel): void; + buildQuery(): void; + getResponseMeta(); +} +export interface IFilterRole { + fieldKey: string; + value: string; + condition?: string; + index?: number; + comparator?: string; +} +export interface IDynamicListFilter { + customViewId?: number; + filterRoles?: IFilterRole[]; + columnSortBy: ISortOrder; + sortOrder: string; + stringifiedFilterRoles: string; + searchKeyword?: string; + viewSlug?: string; +} + +export interface IDynamicListService { + dynamicList( + model: any, + filter: IDynamicListFilter, + ): Promise; + handlerErrorsToResponse(error, req, res, next): void; +} + +// Search role. +export interface ISearchRole { + fieldKey: string; + comparator: string; +} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterAbstractor.ts b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterAbstractor.ts new file mode 100644 index 000000000..7acc8c170 --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterAbstractor.ts @@ -0,0 +1,55 @@ +import { BaseModel } from '@/models/Model'; +import { IDynamicFilter } from './DynamicFilter.types'; + +export class DynamicFilterAbstractor { + model: BaseModel; + dynamicFilters: IDynamicFilter[]; + + /** + * Extract relation table name from relation. + * @param {String} column - + * @return {String} - join relation table. + */ + protected getTableFromRelationColumn = (column: string) => { + const splitedColumn = column.split('.'); + return splitedColumn.length > 0 ? splitedColumn[0] : ''; + }; + + /** + * Builds view roles join queries. + * @param {String} tableName - Table name. + * @param {Array} roles - Roles. + */ + protected buildFilterRolesJoins = (builder) => { + this.dynamicFilters.forEach((dynamicFilter) => { + const relationsFields = dynamicFilter.relationFields; + + this.buildFieldsJoinQueries(builder, relationsFields); + }); + }; + + /** + * Builds join queries of fields. + * @param builder - + * @param {string[]} fieldsRelations - + */ + private buildFieldsJoinQueries = (builder, fieldsRelations: string[]) => { + fieldsRelations.forEach((fieldRelation) => { + const relation = this.model.relationMappings[fieldRelation]; + + if (relation) { + const splitToRelation = relation.join.to.split('.'); + const relationTable = splitToRelation[0] || ''; + + builder.join(relationTable, relation.join.from, '=', relation.join.to); + } + }); + }; + + /** + * Retrieve the dynamic filter mode. + */ + protected getModel() { + return this.model; + } +} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterAdvancedFilter.ts b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterAdvancedFilter.ts new file mode 100644 index 000000000..e597d07b5 --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterAdvancedFilter.ts @@ -0,0 +1,27 @@ +import { IFilterRole } from './DynamicFilter.types'; +import { DynamicFilterFilterRoles } from './DynamicFilterFilterRoles'; + +export class DynamicFilterAdvancedFilter extends DynamicFilterFilterRoles { + private filterRoles: IFilterRole[]; + + /** + * Constructor method. + * @param {Array} filterRoles - + * @param {Array} resourceFields - + */ + constructor(filterRoles: IFilterRole[]) { + super(); + + this.filterRoles = filterRoles; + this.setResponseMeta(); + } + + /** + * Sets response meta. + */ + private setResponseMeta() { + this.responseMeta = { + filterRoles: this.filterRoles, + }; + } +} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterFilterRoles.ts b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterFilterRoles.ts new file mode 100644 index 000000000..00b70b9db --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterFilterRoles.ts @@ -0,0 +1,51 @@ +import { DynamicFilterAbstractor } from './DynamicFilterRoleAbstractor'; +import { IFilterRole } from '@/interfaces'; + +export class DynamicFilterFilterRoles extends DynamicFilterAbstractor { + private filterRoles: IFilterRole[]; + /** + * On initialize filter roles. + */ + public onInitialize() { + super.onInitialize(); + this.setFilterRolesRelations(); + } + + /** + * Builds filter roles logic expression. + * @return {string} + */ + private buildLogicExpression(): string { + let expression = ''; + + this.filterRoles.forEach((role, index) => { + expression += + index === 0 ? `${role.index} ` : `${role.condition} ${role.index} `; + }); + return expression.trim(); + } + + /** + * Builds database query of view roles. + */ + protected buildQuery() { + const logicExpression = this.buildLogicExpression(); + + return (builder) => { + this.buildFilterQuery( + this.model, + this.filterRoles, + logicExpression + )(builder); + }; + } + + /** + * Sets filter roles relations if field was relation type. + */ + private setFilterRolesRelations() { + this.filterRoles.forEach((relationRole) => { + this.setRelationIfRelationField(relationRole.fieldKey); + }); + } +} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterQueryParser.ts b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterQueryParser.ts new file mode 100644 index 000000000..caf863b6c --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterQueryParser.ts @@ -0,0 +1,72 @@ +import { OPERATION } from '@/libs/logic-evaluation/Parser'; + +export default class QueryParser { + constructor(tree, queries) { + this.tree = tree; + this.queries = queries; + this.query = null; + } + + setQuery(query) { + this.query = query.clone(); + } + + parse() { + return this.parseNode(this.tree); + } + + parseNode(node) { + if (typeof node === 'string') { + const nodeQuery = this.getQuery(node); + return (query) => { + nodeQuery(query); + }; + } + if (OPERATION[node.operation] === undefined) { + throw new Error(`unknow expression ${node.operation}`); + } + const leftQuery = this.getQuery(node.left); + const rightQuery = this.getQuery(node.right); + + switch (node.operation) { + case '&&': + case 'AND': + default: + return (nodeQuery) => + nodeQuery.where((query) => { + query.where((q) => { + leftQuery(q); + }); + query.andWhere((q) => { + rightQuery(q); + }); + }); + case '||': + case 'OR': + return (nodeQuery) => + nodeQuery.where((query) => { + query.where((q) => { + leftQuery(q); + }); + query.orWhere((q) => { + rightQuery(q); + }); + }); + } + } + + getQuery(node) { + if (typeof node !== 'string' && node !== null) { + return this.parseNode(node); + } + const value = parseFloat(node); + + if (!isNaN(value)) { + if (typeof this.queries[node] === 'undefined') { + throw new Error(`unknow query under index ${node}`); + } + return this.queries[node]; + } + return null; + } +} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterRoleAbstractor.ts b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterRoleAbstractor.ts new file mode 100644 index 000000000..2eceaff33 --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterRoleAbstractor.ts @@ -0,0 +1,388 @@ +import moment from 'moment'; +import * as R from 'ramda'; +import { IFilterRole, IDynamicFilter, } from './DynamicFilter.types'; +import Parser from '@/libs/logic-evaluation/Parser'; +import { Lexer } from '@/libs/logic-evaluation/Lexer'; +import DynamicFilterQueryParser from './DynamicFilterQueryParser'; +import { COMPARATOR_TYPE, FIELD_TYPE } from './constants'; +import { BaseModel } from '@/models/Model'; + +export abstract class DynamicFilterAbstractor + implements IDynamicFilter +{ + protected filterRoles: IFilterRole[] = []; + protected tableName: string; + protected model: BaseModel; + protected responseMeta: { [key: string]: any } = {}; + public relationFields = []; + + /** + * Sets model the dynamic filter service. + * @param {IModel} model + */ + public setModel(model: BaseModel) { + this.model = model; + this.tableName = model.tableName; + } + + /** + * Transformes filter roles to map by index. + * @param {IModel} model + * @param {IFilterRole[]} roles + * @returns + */ + protected convertRolesMapByIndex = (model, roles) => { + const rolesIndexSet = {}; + + roles.forEach((role) => { + rolesIndexSet[role.index] = this.buildRoleQuery(model, role); + }); + return rolesIndexSet; + }; + + /** + * Builds database query from stored view roles. + * @param {Array} roles - + * @return {Function} + */ + protected buildFilterRolesQuery = ( + model: IModel, + roles: IFilterRole[], + logicExpression: string = '' + ) => { + const rolesIndexSet = this.convertRolesMapByIndex(model, roles); + + // Lexer for logic expression. + const lexer = new Lexer(logicExpression); + const tokens = lexer.getTokens(); + + // Parse the logic expression. + const parser = new Parser(tokens); + const parsedTree = parser.parse(); + + const queryParser = new DynamicFilterQueryParser(parsedTree, rolesIndexSet); + + return queryParser.parse(); + }; + + /** + * Parses the logic expression to base expression. + * @param {string} logicExpression - + * @return {string} + */ + private parseLogicExpression(logicExpression: string): string { + return R.compose( + R.replace(/or|OR/g, '||'), + R.replace(/and|AND/g, '&&'), + )(logicExpression); + } + + /** + * Builds filter query for query builder. + * @param {String} tableName - Table name. + * @param {Array} roles - Filter roles. + * @param {String} logicExpression - Logic expression. + */ + protected buildFilterQuery = ( + model: IModel, + roles: IFilterRole[], + logicExpression: string + ) => { + const basicExpression = this.parseLogicExpression(logicExpression); + + return (builder) => { + this.buildFilterRolesQuery(model, roles, basicExpression)(builder); + }; + }; + + /** + * Retrieve relation column of comparator fieldŲ² + */ + private getFieldComparatorRelationColumn(field) { + const relation = this.model.relationMappings[field.relationKey]; + + if (relation) { + const relationModel = relation.modelClass; + const relationColumn = + field.relationEntityKey === 'id' + ? 'id' + : relationModel.getField(field.relationEntityKey, 'column'); + + return `${relationModel.tableName}.${relationColumn}`; + } + } + + /** + * Retrieve the comparator field column. + * @param {IModel} model - + * @param {} - + */ + private getFieldComparatorColumn = (field) => { + return field.fieldType === FIELD_TYPE.RELATION + ? this.getFieldComparatorRelationColumn(field) + : `${this.tableName}.${field.column}`; + }; + + /** + * Builds roles queries. + * @param {IModel} model - + * @param {Object} role - + */ + protected buildRoleQuery = (model: BaseModel, role: IFilterRole) => { + const field = model.getField(role.fieldKey); + const comparatorColumn = this.getFieldComparatorColumn(field); + + // Field relation custom query. + if (typeof field.filterCustomQuery !== 'undefined') { + return (builder) => { + field.filterCustomQuery(builder, role); + }; + } + switch (field.fieldType) { + case FIELD_TYPE.BOOLEAN: + case FIELD_TYPE.ENUMERATION: + return this.booleanRoleQueryBuilder(role, comparatorColumn); + case FIELD_TYPE.NUMBER: + return this.numberRoleQueryBuilder(role, comparatorColumn); + case FIELD_TYPE.DATE: + return this.dateQueryBuilder(role, comparatorColumn); + case FIELD_TYPE.TEXT: + default: + return this.textRoleQueryBuilder(role, comparatorColumn); + } + }; + + /** + * Boolean column query builder. + * @param {IFilterRole} role + * @param {string} comparatorColumn + * @returns + */ + protected booleanRoleQueryBuilder = ( + role: IFilterRole, + comparatorColumn: string + ) => { + switch (role.comparator) { + case COMPARATOR_TYPE.EQUALS: + case COMPARATOR_TYPE.EQUAL: + case COMPARATOR_TYPE.IS: + default: + return (builder) => { + builder.where(comparatorColumn, '=', role.value); + }; + case COMPARATOR_TYPE.NOT_EQUAL: + case COMPARATOR_TYPE.NOT_EQUALS: + case COMPARATOR_TYPE.IS_NOT: + return (builder) => { + builder.where(comparatorColumn, '<>', role.value); + }; + } + }; + + /** + * Numeric column query builder. + * @param {IFilterRole} role + * @param {string} comparatorColumn + * @returns + */ + protected numberRoleQueryBuilder = ( + role: IFilterRole, + comparatorColumn: string + ) => { + switch (role.comparator) { + case COMPARATOR_TYPE.EQUALS: + case COMPARATOR_TYPE.EQUAL: + default: + return (builder) => { + builder.where(comparatorColumn, '=', role.value); + }; + case COMPARATOR_TYPE.NOT_EQUAL: + case COMPARATOR_TYPE.NOT_EQUALS: + return (builder) => { + builder.whereNot(comparatorColumn, role.value); + }; + case COMPARATOR_TYPE.BIGGER_THAN: + case COMPARATOR_TYPE.BIGGER: + return (builder) => { + builder.where(comparatorColumn, '>', role.value); + }; + case COMPARATOR_TYPE.BIGGER_OR_EQUALS: + return (builder) => { + builder.where(comparatorColumn, '>=', role.value); + }; + case COMPARATOR_TYPE.SMALLER_THAN: + case COMPARATOR_TYPE.SMALLER: + return (builder) => { + builder.where(comparatorColumn, '<', role.value); + }; + case COMPARATOR_TYPE.SMALLER_OR_EQUALS: + return (builder) => { + builder.where(comparatorColumn, '<=', role.value); + }; + } + }; + + /** + * Text column query builder. + * @param {IFilterRole} role + * @param {string} comparatorColumn + * @returns {Function} + */ + protected textRoleQueryBuilder = ( + role: IFilterRole, + comparatorColumn: string + ) => { + switch (role.comparator) { + case COMPARATOR_TYPE.EQUAL: + case COMPARATOR_TYPE.EQUALS: + case COMPARATOR_TYPE.IS: + default: + return (builder) => { + builder.where(comparatorColumn, role.value); + }; + case COMPARATOR_TYPE.NOT_EQUALS: + case COMPARATOR_TYPE.NOT_EQUAL: + case COMPARATOR_TYPE.IS_NOT: + return (builder) => { + builder.whereNot(comparatorColumn, role.value); + }; + case COMPARATOR_TYPE.CONTAIN: + case COMPARATOR_TYPE.CONTAINS: + return (builder) => { + builder.where(comparatorColumn, 'LIKE', `%${role.value}%`); + }; + case COMPARATOR_TYPE.NOT_CONTAIN: + case COMPARATOR_TYPE.NOT_CONTAINS: + return (builder) => { + builder.whereNot(comparatorColumn, 'LIKE', `%${role.value}%`); + }; + case COMPARATOR_TYPE.STARTS_WITH: + case COMPARATOR_TYPE.START_WITH: + return (builder) => { + builder.where(comparatorColumn, 'LIKE', `${role.value}%`); + }; + case COMPARATOR_TYPE.ENDS_WITH: + case COMPARATOR_TYPE.END_WITH: + return (builder) => { + builder.where(comparatorColumn, 'LIKE', `%${role.value}`); + }; + + } + }; + + /** + * Date column query builder. + * @param {IFilterRole} role + * @param {string} comparatorColumn + * @returns {Function} + */ + protected dateQueryBuilder = ( + role: IFilterRole, + comparatorColumn: string + ) => { + switch (role.comparator) { + case COMPARATOR_TYPE.AFTER: + case COMPARATOR_TYPE.BEFORE: + return (builder) => { + this.dateQueryAfterBeforeComparator(role, comparatorColumn, builder); + }; + case COMPARATOR_TYPE.IN: + return (builder) => { + this.dateQueryInComparator(role, comparatorColumn, builder); + }; + } + }; + + /** + * Date query 'IN' comparator type. + * @param {IFilterRole} role + * @param {string} comparatorColumn + * @param builder + */ + protected dateQueryInComparator = ( + role: IFilterRole, + comparatorColumn: string, + builder + ) => { + const hasTimeFormat = moment( + role.value, + 'YYYY-MM-DD HH:MM', + true + ).isValid(); + const dateFormat = 'YYYY-MM-DD HH:MM:SS'; + + if (hasTimeFormat) { + const targetDateTime = moment(role.value).format(dateFormat); + builder.where(comparatorColumn, '=', targetDateTime); + } else { + const startDate = moment(role.value).startOf('day'); + const endDate = moment(role.value).endOf('day'); + + builder.where(comparatorColumn, '>=', startDate.format(dateFormat)); + builder.where(comparatorColumn, '<=', endDate.format(dateFormat)); + } + }; + + /** + * Date query after/before comparator type. + * @param {IFilterRole} role + * @param {string} comparatorColumn - Column. + * @param builder + */ + protected dateQueryAfterBeforeComparator = ( + role: IFilterRole, + comparatorColumn: string, + builder + ) => { + const comparator = role.comparator === COMPARATOR_TYPE.BEFORE ? '<' : '>'; + const hasTimeFormat = moment( + role.value, + 'YYYY-MM-DD HH:MM', + true + ).isValid(); + const targetDate = moment(role.value); + const dateFormat = 'YYYY-MM-DD HH:MM:SS'; + + if (!hasTimeFormat) { + if (role.comparator === COMPARATOR_TYPE.BEFORE) { + targetDate.startOf('day'); + } else { + targetDate.endOf('day'); + } + } + const comparatorValue = targetDate.format(dateFormat); + builder.where(comparatorColumn, comparator, comparatorValue); + }; + + /** + * Registers relation field if the given field was relation type + * and not registered. + * @param {string} fieldKey - Field key. + */ + protected setRelationIfRelationField = (fieldKey: string): void => { + const field = this.model.getField(fieldKey); + const isAlreadyRegistered = this.relationFields.some( + (field) => field === fieldKey + ); + + if ( + !isAlreadyRegistered && + field && + field.fieldType === FIELD_TYPE.RELATION + ) { + this.relationFields.push(field.relationKey); + } + }; + + /** + * Retrieve the model. + */ + getModel() { + return this.model; + } + + /** + * On initialize the registered dynamic filter. + */ + onInitialize() {} +} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterSearch.ts b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterSearch.ts new file mode 100644 index 000000000..7ea70c29d --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterSearch.ts @@ -0,0 +1,48 @@ +import { IFilterRole } from './DynamicFilter.types'; +import { DynamicFilterFilterRoles } from './DynamicFilterFilterRoles'; + +export class DynamicFilterSearch extends DynamicFilterFilterRoles { + private searchKeyword: string; + private filterRoles: IFilterRole[]; + + /** + * Constructor method. + * @param {string} searchKeyword - Search keyword. + */ + constructor(searchKeyword: string) { + super(); + this.searchKeyword = searchKeyword; + } + + /** + * On initialize the dynamic filter. + */ + public onInitialize() { + super.onInitialize(); + this.filterRoles = this.getModelSearchFilterRoles(this.searchKeyword); + } + + /** + * Retrieve the filter roles from model search roles. + * @param {string} searchKeyword + * @returns {IFilterRole[]} + */ + private getModelSearchFilterRoles(searchKeyword: string): IFilterRole[] { + const model = this.getModel(); + + return model.searchRoles.map((searchRole, index) => ({ + ...searchRole, + value: searchKeyword, + index: index + 1, + })); + } + + /** + * + */ + setResponseMeta() { + this.responseMeta = { + searchKeyword: this.searchKeyword, + }; + } +} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterSortBy.ts b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterSortBy.ts new file mode 100644 index 000000000..7fed63a65 --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterSortBy.ts @@ -0,0 +1,92 @@ +import { FIELD_TYPE } from './constants'; +import { DynamicFilterAbstractor } from './DynamicFilterAbstractor'; + +interface ISortRole { + fieldKey: string; + order: string; +} + +export class DynamicFilterSortBy extends DynamicFilterAbstractor { + private sortRole: ISortRole = {}; + + /** + * Constructor method. + * @param {string} sortByFieldKey + * @param {string} sortDirection + */ + constructor(sortByFieldKey: string, sortDirection: string) { + super(); + + this.sortRole = { + fieldKey: sortByFieldKey, + order: sortDirection, + }; + this.setResponseMeta(); + } + + /** + * On initialize the dyanmic sort by. + */ + public onInitialize() { + this.setRelationIfRelationField(this.sortRole.fieldKey); + } + + /** + * Retrieve field comparator relatin column. + * @param field + * @returns {string} + */ + private getFieldComparatorRelationColumn = (field): string => { + const relation = this.model.relationMappings[field.relationKey]; + + if (relation) { + const relationModel = relation.modelClass; + const relationField = relationModel.getField(field.relationEntityLabel); + + return `${relationModel.tableName}.${relationField.column}`; + } + return ''; + }; + + /** + * Retrieve the comparator field column. + * @param {IModel} field + * @returns {string} + */ + private getFieldComparatorColumn = (field): string => { + return field.fieldType === FIELD_TYPE.RELATION + ? this.getFieldComparatorRelationColumn(field) + : `${this.tableName}.${field.column}`; + }; + + /** + * Builds database query of sort by column on the given direction. + */ + public buildQuery = () => { + const field = this.model.getField(this.sortRole.fieldKey); + const comparatorColumn = this.getFieldComparatorColumn(field); + + // Sort custom query. + if (typeof field.sortCustomQuery !== 'undefined') { + return (builder) => { + field.sortCustomQuery(builder, this.sortRole); + }; + } + + return (builder) => { + if (this.sortRole.fieldKey) { + builder.orderBy(`${comparatorColumn}`, this.sortRole.order); + } + }; + }; + + /** + * Sets response meta. + */ + public setResponseMeta() { + this.responseMeta = { + sortOrder: this.sortRole.fieldKey, + sortBy: this.sortRole.order, + }; + } +} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterViews.ts b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterViews.ts new file mode 100644 index 000000000..56c976a31 --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/DynamicFilterViews.ts @@ -0,0 +1,56 @@ +import { omit } from 'lodash'; +import { IView, IViewRole } from '@/interfaces'; +import { DynamicFilterAbstractor } from './DynamicFilterAbstractor'; + +export class DynamicFilterViews extends DynamicFilterAbstractor { + private viewSlug: string; + private logicExpression: string; + private filterRoles: IViewRole[]; + private viewColumns = []; + + /** + * Constructor method. + * @param {IView} view - + */ + constructor(view: IView) { + super(); + + this.viewSlug = view.slug; + this.filterRoles = view.roles; + this.viewColumns = view.columns; + this.logicExpression = view.rolesLogicExpression + .replace('AND', '&&') + .replace('OR', '||'); + + this.setResponseMeta(); + } + + /** + * Builds database query of view roles. + */ + public buildQuery() { + return (builder) => { + this.buildFilterQuery( + this.model, + this.filterRoles, + this.logicExpression + )(builder); + }; + } + + /** + * Sets response meta. + */ + public setResponseMeta() { + this.responseMeta = { + view: { + logicExpression: this.logicExpression, + filterRoles: this.filterRoles.map((filterRole) => ({ + ...omit(filterRole, ['id', 'viewId']), + })), + viewSlug: this.viewSlug, + viewColumns: this.viewColumns, + }, + }; + } +} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicFilter/constants.ts b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/constants.ts new file mode 100644 index 000000000..f845e16c9 --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/constants.ts @@ -0,0 +1,43 @@ +export const COMPARATOR_TYPE = { + EQUAL: 'equal', + EQUALS: 'equals', + + NOT_EQUAL: 'not_equal', + NOT_EQUALS: 'not_equals', + + BIGGER_THAN: 'bigger_than', + BIGGER: 'bigger', + BIGGER_OR_EQUALS: 'bigger_or_equals', + + SMALLER_THAN: 'smaller_than', + SMALLER: 'smaller', + SMALLER_OR_EQUALS: 'smaller_or_equals', + + IS: 'is', + IS_NOT: 'is_not', + + CONTAINS: 'contains', + CONTAIN: 'contain', + NOT_CONTAINS: 'contains', + NOT_CONTAIN: 'contain', + + AFTER: 'after', + BEFORE: 'before', + IN: 'in', + + STARTS_WITH: 'starts_with', + START_WITH: 'start_with', + + ENDS_WITH: 'ends_with', + END_WITH: 'end_with' +}; + +export const FIELD_TYPE = { + TEXT: 'text', + NUMBER: 'number', + ENUMERATION: 'enumeration', + BOOLEAN: 'boolean', + RELATION: 'relation', + DATE: 'date', + COMPUTED: 'computed' +}; diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicFilter/index.ts b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/index.ts new file mode 100644 index 000000000..49850caff --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicFilter/index.ts @@ -0,0 +1,11 @@ +import { DynamicFilter } from './DynamicFilter'; +import { DynamicFilterSortBy } from './DynamicFilterSortBy'; +import { DynamicFilterViews } from './DynamicFilterViews'; +import { DynamicFilterFilterRoles } from './DynamicFilterFilterRoles'; + +export { + DynamicFilter, + DynamicFilterSortBy, + DynamicFilterViews, + DynamicFilterFilterRoles, +}; diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicListAbstract.ts b/packages/server-nest/src/modules/DynamicListing/DynamicListAbstract.ts new file mode 100644 index 000000000..37a98d0b4 --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicListAbstract.ts @@ -0,0 +1 @@ +export class DynamicListAbstract {} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicListCustomView.ts b/packages/server-nest/src/modules/DynamicListing/DynamicListCustomView.ts new file mode 100644 index 000000000..5dcbdddd3 --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicListCustomView.ts @@ -0,0 +1,45 @@ +import { Injectable } from '@nestjs/common'; +import { DynamicListAbstract } from './DynamicListAbstract'; +import { ERRORS } from './constants'; +import { DynamicFilterViews } from './DynamicFilter'; +import { ServiceError } from '../Items/ServiceError'; +import { BaseModel } from '@/models/Model'; + +@Injectable() +export class DynamicListCustomView extends DynamicListAbstract { + /** + * Retreive custom view or throws error not found. + * @param {number} tenantId + * @param {number} viewId + * @return {Promise} + */ + private getCustomViewOrThrowError = async ( + viewSlug: string, + model: BaseModel, + ) => { + // Finds the default view by the given view slug. + const defaultView = model.getDefaultViewBySlug(viewSlug); + + if (!defaultView) { + throw new ServiceError(ERRORS.VIEW_NOT_FOUND); + } + return defaultView; + }; + + /** + * Dynamic list custom view. + * @param {IModel} model + * @param {number} customViewId + * @returns + */ + public dynamicListCustomView = async ( + dynamicFilter: any, + customViewSlug: string, + ) => { + const model = dynamicFilter.getModel(); + + // Retrieve the custom view or throw not found. + const view = await this.getCustomViewOrThrowError(customViewSlug, model); + return new DynamicFilterViews(view); + }; +} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicListFilterRoles.ts b/packages/server-nest/src/modules/DynamicListing/DynamicListFilterRoles.ts new file mode 100644 index 000000000..681179ebe --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicListFilterRoles.ts @@ -0,0 +1,104 @@ +import * as R from 'ramda'; +import { Injectable } from '@nestjs/common'; +import validator from 'is-my-json-valid'; +import { IFilterRole } from './DynamicFilter/DynamicFilter.types'; +import { DynamicListAbstract } from './DynamicListAbstract'; +import { DynamicFilterAdvancedFilter } from './DynamicFilter/DynamicFilterAdvancedFilter'; +import { ERRORS } from './constants'; +import { ServiceError } from '../Items/ServiceError'; +import { BaseModel } from '@/models/Model'; + +@Injectable() +export class DynamicListFilterRoles extends DynamicListAbstract { + /** + * Validates filter roles schema. + * @param {IFilterRole[]} filterRoles - Filter roles. + */ + private validateFilterRolesSchema = (filterRoles: IFilterRole[]) => { + const validate = validator({ + required: ['fieldKey', 'value'], + type: 'object', + properties: { + condition: { type: 'string' }, + fieldKey: { type: 'string' }, + value: { type: 'string' }, + }, + }); + const invalidFields = filterRoles.filter((filterRole) => { + return !validate(filterRole); + }); + if (invalidFields.length > 0) { + throw new ServiceError(ERRORS.STRINGIFIED_FILTER_ROLES_INVALID); + } + }; + + /** + * Retrieve filter roles fields key that not exists on the given model. + * @param {BaseModel} model + * @param {IFilterRole} filterRoles + * @returns {string[]} + */ + private getFilterRolesFieldsNotExist = ( + model: BaseModel, + filterRoles: IFilterRole[], + ): string[] => { + return filterRoles + .filter((filterRole) => !model.getField(filterRole.fieldKey)) + .map((filterRole) => filterRole.fieldKey); + }; + + /** + * Validates existance the fields of filter roles. + * @param {BaseModel} model + * @param {IFilterRole[]} filterRoles + * @throws {ServiceError} + */ + private validateFilterRolesFieldsExistance = ( + model: BaseModel, + filterRoles: IFilterRole[], + ) => { + const invalidFieldsKeys = this.getFilterRolesFieldsNotExist( + model, + filterRoles, + ); + if (invalidFieldsKeys.length > 0) { + throw new ServiceError(ERRORS.FILTER_ROLES_FIELDS_NOT_FOUND); + } + }; + + /** + * Associate index to filter roles. + * @param {IFilterRole[]} filterRoles + * @returns {IFilterRole[]} + */ + private incrementFilterRolesIndex = ( + filterRoles: IFilterRole[], + ): IFilterRole[] => { + return filterRoles.map((filterRole, index) => ({ + ...filterRole, + index: index + 1, + })); + }; + + /** + * Dynamic list filter roles. + * @param {BaseModel} model + * @param {IFilterRole[]} filterRoles + * @returns {DynamicFilterFilterRoles} + */ + public dynamicList = ( + model: BaseModel, + filterRoles: IFilterRole[], + ): DynamicFilterAdvancedFilter => { + const filterRolesParsed = R.compose(this.incrementFilterRolesIndex)( + filterRoles, + ); + // Validate filter roles json schema. + this.validateFilterRolesSchema(filterRolesParsed); + + // Validate the model resource fields. + this.validateFilterRolesFieldsExistance(model, filterRoles); + + return new DynamicFilterAdvancedFilter(filterRolesParsed); + }; +} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicListSearch.ts b/packages/server-nest/src/modules/DynamicListing/DynamicListSearch.ts new file mode 100644 index 000000000..d4d620742 --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicListSearch.ts @@ -0,0 +1,15 @@ +import { Injectable } from '@nestjs/common'; +import { DynamicListAbstract } from './DynamicListAbstract'; +import { DynamicFilterSearch } from './DynamicFilter/DynamicFilterSearch'; + +@Injectable() +export class DynamicListSearch extends DynamicListAbstract { + /** + * Dynamic list filter roles. + * @param {string} searchKeyword - Search keyword. + * @returns {DynamicFilterFilterRoles} + */ + public dynamicSearch = (searchKeyword: string) => { + return new DynamicFilterSearch(searchKeyword); + }; +} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicListService.ts b/packages/server-nest/src/modules/DynamicListing/DynamicListService.ts new file mode 100644 index 000000000..e97f27229 --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicListService.ts @@ -0,0 +1,101 @@ +import { castArray, isEmpty } from 'lodash'; +import { + IDynamicListFilter, + IDynamicListService, +} from './DynamicFilter/DynamicFilter.types'; +import { DynamicListSortBy } from './DynamicListSortBy'; +import { DynamicListSearch } from './DynamicListSearch'; +import { DynamicListCustomView } from './DynamicListCustomView'; +import { Injectable } from '@nestjs/common'; +import { DynamicListFilterRoles } from './DynamicListFilterRoles'; +import { DynamicFilter } from './DynamicFilter'; +import { BaseModel } from '@/models/Model'; + +@Injectable() +export class DynamicListService implements IDynamicListService { + constructor( + private dynamicListFilterRoles: DynamicListFilterRoles, + private dynamicListSearch: DynamicListSearch, + private dynamicListSortBy: DynamicListSortBy, + private dynamicListView: DynamicListCustomView, + ) {} + + /** + * Parses filter DTO. + * @param {IMode} model - + * @param {} filterDTO - + */ + private parseFilterObject = (model, filterDTO) => { + return { + // Merges the default properties with filter object. + ...(model.defaultSort + ? { + sortOrder: model.defaultSort.sortOrder, + columnSortBy: model.defaultSort.sortOrder, + } + : {}), + ...filterDTO, + }; + }; + + /** + * Dynamic listing. + * @param {number} tenantId - Tenant id. + * @param {IModel} model - Model. + * @param {IDynamicListFilter} filter - Dynamic filter DTO. + */ + public dynamicList = async (model: BaseModel, filter: IDynamicListFilter) => { + const dynamicFilter = new DynamicFilter(model); + + // Parses the filter object. + const parsedFilter = this.parseFilterObject(model, filter); + + // Search by keyword. + if (filter.searchKeyword) { + const dynamicListSearch = this.dynamicListSearch.dynamicSearch( + filter.searchKeyword, + ); + dynamicFilter.setFilter(dynamicListSearch); + } + // Custom view filter roles. + if (filter.viewSlug) { + const dynamicListCustomView = + await this.dynamicListView.dynamicListCustomView( + dynamicFilter, + filter.viewSlug, + ); + dynamicFilter.setFilter(dynamicListCustomView); + } + // Sort by the given column. + if (parsedFilter.columnSortBy) { + const dynmaicListSortBy = this.dynamicListSortBy.dynamicSortBy( + model, + parsedFilter.columnSortBy, + parsedFilter.sortOrder, + ); + dynamicFilter.setFilter(dynmaicListSortBy); + } + // Filter roles. + if (!isEmpty(parsedFilter.filterRoles)) { + const dynamicFilterRoles = this.dynamicListFilterRoles.dynamicList( + model, + parsedFilter.filterRoles, + ); + dynamicFilter.setFilter(dynamicFilterRoles); + } + return dynamicFilter; + }; + + /** + * Parses stringified filter roles. + * @param {string} stringifiedFilterRoles - Stringified filter roles. + */ + public parseStringifiedFilter = (filterRoles: IDynamicListFilter) => { + return { + ...filterRoles, + filterRoles: filterRoles.stringifiedFilterRoles + ? castArray(JSON.parse(filterRoles.stringifiedFilterRoles)) + : [], + }; + }; +} diff --git a/packages/server-nest/src/modules/DynamicListing/DynamicListSortBy.ts b/packages/server-nest/src/modules/DynamicListing/DynamicListSortBy.ts new file mode 100644 index 000000000..734238070 --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/DynamicListSortBy.ts @@ -0,0 +1,41 @@ +import { Injectable } from '@nestjs/common'; +import { DynamicListAbstract } from './DynamicListAbstract'; +import { ISortOrder } from './DynamicFilter/DynamicFilter.types'; +import { ERRORS } from './constants'; +import { DynamicFilterSortBy } from './DynamicFilter'; +import { ServiceError } from '../Items/ServiceError'; +import { BaseModel } from '@/models/Model'; + +@Injectable() +export class DynamicListSortBy extends DynamicListAbstract { + /** + * Dynamic list sort by. + * @param {BaseModel} model + * @param {string} columnSortBy + * @param {ISortOrder} sortOrder + * @returns {DynamicFilterSortBy} + */ + public dynamicSortBy( + model: BaseModel, + columnSortBy: string, + sortOrder: ISortOrder, + ) { + this.validateSortColumnExistance(model, columnSortBy); + + return new DynamicFilterSortBy(columnSortBy, sortOrder); + } + + /** + * Validates the sort column whether exists. + * @param {IModel} model - Model. + * @param {string} columnSortBy - Sort column + * @throws {ServiceError} + */ + private validateSortColumnExistance(model: any, columnSortBy: string) { + const field = model.getField(columnSortBy); + + if (!field) { + throw new ServiceError(ERRORS.SORT_COLUMN_NOT_FOUND); + } + } +} diff --git a/packages/server-nest/src/modules/DynamicListing/constants.ts b/packages/server-nest/src/modules/DynamicListing/constants.ts new file mode 100644 index 000000000..414de46ac --- /dev/null +++ b/packages/server-nest/src/modules/DynamicListing/constants.ts @@ -0,0 +1,6 @@ +export const ERRORS = { + STRINGIFIED_FILTER_ROLES_INVALID: 'stringified_filter_roles_invalid', + VIEW_NOT_FOUND: 'view_not_found', + SORT_COLUMN_NOT_FOUND: 'sort_column_not_found', + FILTER_ROLES_FIELDS_NOT_FOUND: 'filter_roles_fields_not_found', +}; diff --git a/packages/server-nest/src/modules/DynamicListing/validators.ts b/packages/server-nest/src/modules/DynamicListing/validators.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/server-nest/src/modules/Export/ExportAls.ts b/packages/server-nest/src/modules/Export/ExportAls.ts new file mode 100644 index 000000000..eb940a3a2 --- /dev/null +++ b/packages/server-nest/src/modules/Export/ExportAls.ts @@ -0,0 +1,48 @@ +import { Injectable } from '@nestjs/common'; +import { AsyncLocalStorage } from 'async_hooks'; + +@Injectable() +export class ExportAls { + private als: AsyncLocalStorage>; + + constructor() { + this.als = new AsyncLocalStorage(); + } + + /** + * Runs a callback function within the context of a new AsyncLocalStorage store. + * @param callback The function to be executed within the AsyncLocalStorage context. + * @returns The result of the callback function. + */ + public run(callback: () => T): T { + return this.als.run(new Map(), () => { + this.markAsExport(); + + return callback(); + }); + } + + /** + * Retrieves the current AsyncLocalStorage store. + * @returns The current store or undefined if not in a valid context. + */ + public getStore(): Map | undefined { + return this.als.getStore(); + } + + /** + * Marks the current context as an export operation. + * @param flag Boolean flag to set or unset the export status. Defaults to true. + */ + public markAsExport(flag: boolean = true): void { + const store = this.getStore(); + store?.set('isExport', flag); + } + /** + * Checks if the current context is an export operation. + * @returns {boolean} True if the context is an export operation, false otherwise. + */ + public get isExport(): boolean { + return !!this.getStore()?.get('isExport'); + } +} diff --git a/packages/server-nest/src/modules/Export/ExportApplication.ts b/packages/server-nest/src/modules/Export/ExportApplication.ts new file mode 100644 index 000000000..b218c722e --- /dev/null +++ b/packages/server-nest/src/modules/Export/ExportApplication.ts @@ -0,0 +1,22 @@ +import { Injectable } from '@nestjs/common'; +import { ExportResourceService } from './ExportService'; +import { ExportFormat } from './common'; + +@Injectable() +export class ExportApplication { + /** + * Constructor method. + */ + constructor( + private readonly exportResource: ExportResourceService, + ) {} + + /** + * Exports the given resource to csv, xlsx or pdf format. + * @param {string} reosurce + * @param {ExportFormat} format + */ + public export(tenantId: number, resource: string, format: ExportFormat) { + return this.exportResource.export(tenantId, resource, format); + } +} diff --git a/packages/server-nest/src/modules/Export/ExportPdf.ts b/packages/server-nest/src/modules/Export/ExportPdf.ts new file mode 100644 index 000000000..74a643c32 --- /dev/null +++ b/packages/server-nest/src/modules/Export/ExportPdf.ts @@ -0,0 +1,47 @@ +// @ts-nocheck +// import { ChromiumlyTenancy } from '../ChromiumlyTenancy/ChromiumlyTenancy'; +// import { TemplateInjectable } from '../TemplateInjectable/TemplateInjectable'; +import { mapPdfRows } from './utils'; +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class ExportPdf { + constructor( + private readonly templateInjectable: TemplateInjectable, + private readonly chromiumlyTenancy: ChromiumlyTenancy, + ) {} + + /** + * Generates the pdf table sheet for the given data and columns. + * @param {number} tenantId + * @param {} columns + * @param {Record} data + * @param {string} sheetTitle + * @param {string} sheetDescription + * @returns + */ + public async pdf( + tenantId: number, + columns: { accessor: string }, + data: Record, + sheetTitle: string = '', + sheetDescription: string = '' + ) { + const rows = mapPdfRows(columns, data); + + const htmlContent = await this.templateInjectable.render( + tenantId, + 'modules/export-resource-table', + { + table: { rows, columns }, + sheetTitle, + sheetDescription, + } + ); + // Convert the HTML content to PDF + return this.chromiumlyTenancy.convertHtmlContent(tenantId, htmlContent, { + margins: { top: 0.2, bottom: 0.2, left: 0.2, right: 0.2 }, + landscape: true, + }); + } +} diff --git a/packages/server-nest/src/modules/Export/ExportRegistery.ts b/packages/server-nest/src/modules/Export/ExportRegistery.ts new file mode 100644 index 000000000..b9f8fa943 --- /dev/null +++ b/packages/server-nest/src/modules/Export/ExportRegistery.ts @@ -0,0 +1,50 @@ +// @ts-nocheck +import { camelCase, upperFirst } from 'lodash'; +import { Exportable } from './Exportable'; + +export class ExportableRegistry { + private static instance: ExportableRegistry; + private exportables: Record; + + /** + * Constructor method. + */ + constructor() { + this.exportables = {}; + } + + /** + * Gets singleton instance of registry. + * @returns {ExportableRegistry} + */ + public static getInstance(): ExportableRegistry { + if (!ExportableRegistry.instance) { + ExportableRegistry.instance = new ExportableRegistry(); + } + return ExportableRegistry.instance; + } + + /** + * Registers the given importable service. + * @param {string} resource + * @param {Exportable} importable + */ + public registerExportable(resource: string, importable: Exportable): void { + const _resource = this.sanitizeResourceName(resource); + this.exportables[_resource] = importable; + } + + /** + * Retrieves the importable service instance of the given resource name. + * @param {string} name + * @returns {Exportable} + */ + public getExportable(name: string): Exportable { + const _name = this.sanitizeResourceName(name); + return this.exportables[_name]; + } + + private sanitizeResourceName(resource: string) { + return upperFirst(camelCase(resource)); + } +} diff --git a/packages/server-nest/src/modules/Export/ExportResources.ts b/packages/server-nest/src/modules/Export/ExportResources.ts new file mode 100644 index 000000000..c590a8682 --- /dev/null +++ b/packages/server-nest/src/modules/Export/ExportResources.ts @@ -0,0 +1,76 @@ +// @ts-nocheck +// import Container, { Service } from 'typedi'; +// import { AccountsExportable } from '../Accounts/AccountsExportable'; +// import { ExportableRegistry } from './ExportRegistery'; +// import { ItemsExportable } from '../Items/ItemsExportable'; +// import { CustomersExportable } from '../Contacts/Customers/CustomersExportable'; +// import { VendorsExportable } from '../Contacts/Vendors/VendorsExportable'; +// import { ExpensesExportable } from '../Expenses/ExpensesExportable'; +// import { SaleInvoicesExportable } from '../Sales/Invoices/SaleInvoicesExportable'; +// import { SaleEstimatesExportable } from '../Sales/Estimates/SaleEstimatesExportable'; +// import { SaleReceiptsExportable } from '../Sales/Receipts/SaleReceiptsExportable'; +// import { BillsExportable } from '../Purchases/Bills/BillsExportable'; +// import { PaymentsReceivedExportable } from '../Sales/PaymentReceived/PaymentsReceivedExportable'; +// import { BillPaymentExportable } from '../Purchases/BillPayments/BillPaymentExportable'; +// import { ManualJournalsExportable } from '../ManualJournals/ManualJournalExportable'; +// import { CreditNotesExportable } from '../CreditNotes/CreditNotesExportable'; +// import { VendorCreditsExportable } from '../Purchases/VendorCredits/VendorCreditsExportable'; +// import { ItemCategoriesExportable } from '../ItemCategories/ItemCategoriesExportable'; +// import { TaxRatesExportable } from '../TaxRates/TaxRatesExportable'; + +import { Injectable } from "@nestjs/common"; +import { ExportableRegistry } from "./ExportRegistery"; + +@Injectable() +export class ExportableResources { + + constructor( + private readonly exportRegistry: ExportableRegistry, + ) { + this.boot(); + } + + /** + * Importable instances. + */ + private importables = [ + // { resource: 'Account', exportable: AccountsExportable }, + // { resource: 'Item', exportable: ItemsExportable }, + // { resource: 'ItemCategory', exportable: ItemCategoriesExportable }, + // { resource: 'Customer', exportable: CustomersExportable }, + // { resource: 'Vendor', exportable: VendorsExportable }, + // { resource: 'Expense', exportable: ExpensesExportable }, + // { resource: 'SaleInvoice', exportable: SaleInvoicesExportable }, + // { resource: 'SaleEstimate', exportable: SaleEstimatesExportable }, + // { resource: 'SaleReceipt', exportable: SaleReceiptsExportable }, + // { resource: 'Bill', exportable: BillsExportable }, + // { resource: 'PaymentReceive', exportable: PaymentsReceivedExportable }, + // { resource: 'BillPayment', exportable: BillPaymentExportable }, + // { resource: 'ManualJournal', exportable: ManualJournalsExportable }, + // { resource: 'CreditNote', exportable: CreditNotesExportable }, + // { resource: 'VendorCredit', exportable: VendorCreditsExportable }, + // { resource: 'TaxRate', exportable: TaxRatesExportable }, + ]; + + /** + * + */ + public get registry() { + return ExportableResources.registry; + } + + /** + * Boots all the registered importables. + */ + public boot() { + if (!ExportableResources.registry) { + const instance = ExportableRegistry.getInstance(); + + this.importables.forEach((importable) => { + const importableInstance = Container.get(importable.exportable); + instance.registerExportable(importable.resource, importableInstance); + }); + ExportableResources.registry = instance; + } + } +} diff --git a/packages/server-nest/src/modules/Export/ExportService.ts b/packages/server-nest/src/modules/Export/ExportService.ts new file mode 100644 index 000000000..a6bf3ef97 --- /dev/null +++ b/packages/server-nest/src/modules/Export/ExportService.ts @@ -0,0 +1,228 @@ +// @ts-nocheck +import xlsx from 'xlsx'; +import * as R from 'ramda'; +import { get } from 'lodash'; +import { sanitizeResourceName } from '../Import/_utils'; +import { ExportableResources } from './ExportResources'; +import { ServiceError } from '@/exceptions'; +import { Errors, ExportFormat } from './common'; +import { IModelMeta, IModelMetaColumn } from '@/interfaces'; +import { flatDataCollections, getDataAccessor } from './utils'; +import { ExportPdf } from './ExportPdf'; +import { ExportAls } from './ExportAls'; +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class ExportResourceService { + constructor( + private readonly exportAls: ExportAls, + private readonly exportPdf: ExportPdf, + private readonly exportableResources: ExportableResources, + private readonly resourceService: ResourceService, + ) {} + + /** + * + * @param {number} tenantId + * @param {string} resourceName + * @param {ExportFormat} format + * @returns + */ + public async export( + tenantId: number, + resourceName: string, + format: ExportFormat = ExportFormat.Csv + ) { + return this.exportAls.run(() => + this.exportAlsRun(tenantId, resourceName, format) + ); + } + + /** + * Exports the given resource data through csv, xlsx or pdf. + * @param {number} tenantId - Tenant id. + * @param {string} resourceName - Resource name. + * @param {ExportFormat} format - File format. + */ + public async exportAlsRun( + tenantId: number, + resourceName: string, + format: ExportFormat = ExportFormat.Csv + ) { + const resource = sanitizeResourceName(resourceName); + const resourceMeta = this.getResourceMeta(tenantId, resource); + const resourceColumns = this.resourceService.getResourceColumns( + tenantId, + resource + ); + this.validateResourceMeta(resourceMeta); + + const data = await this.getExportableData(tenantId, resource); + const transformed = this.transformExportedData(tenantId, resource, data); + + // Returns the csv, xlsx format. + if (format === ExportFormat.Csv || format === ExportFormat.Xlsx) { + const exportableColumns = this.getExportableColumns(resourceColumns); + const workbook = this.createWorkbook(transformed, exportableColumns); + + return this.exportWorkbook(workbook, format); + // Returns the pdf format. + } else if (format === ExportFormat.Pdf) { + const printableColumns = this.getPrintableColumns(resourceMeta); + + return this.exportPdf.pdf( + tenantId, + printableColumns, + transformed, + resourceMeta?.print?.pageTitle + ); + } + } + + /** + * Retrieves metadata for a specific resource. + * @param {number} tenantId - The tenant identifier. + * @param {string} resource - The name of the resource. + * @returns The metadata of the resource. + */ + private getResourceMeta(tenantId: number, resource: string) { + return this.resourceService.getResourceMeta(tenantId, resource); + } + + /** + * Validates if the resource metadata is exportable. + * @param {any} resourceMeta - The metadata of the resource. + * @throws {ServiceError} If the resource is not exportable or lacks columns. + */ + private validateResourceMeta(resourceMeta: any) { + if (!resourceMeta.exportable || !resourceMeta.columns) { + throw new ServiceError(Errors.RESOURCE_NOT_EXPORTABLE); + } + } + + /** + * Transforms the exported data based on the resource metadata. + * If the resource metadata specifies a flattening attribute (`exportFlattenOn`), + * the data will be flattened based on this attribute using the `flatDataCollections` utility function. + * + * @param {number} tenantId - The tenant identifier. + * @param {string} resource - The name of the resource. + * @param {Array>} data - The original data to be transformed. + * @returns {Array>} - The transformed data. + */ + private transformExportedData( + tenantId: number, + resource: string, + data: Array> + ): Array> { + const resourceMeta = this.getResourceMeta(tenantId, resource); + + return R.when>, Array>>( + R.always(Boolean(resourceMeta.exportFlattenOn)), + (data) => flatDataCollections(data, resourceMeta.exportFlattenOn), + data + ); + } + /** + * Fetches exportable data for a given resource. + * @param {number} tenantId - The tenant identifier. + * @param {string} resource - The name of the resource. + * @returns A promise that resolves to the exportable data. + */ + private async getExportableData(tenantId: number, resource: string) { + const exportable = + this.exportableResources.registry.getExportable(resource); + + return exportable.exportable(tenantId, {}); + } + + /** + * Extracts columns that are marked as exportable from the resource metadata. + * @param {IModelMeta} resourceMeta - The metadata of the resource. + * @returns An array of exportable columns. + */ + private getExportableColumns(resourceColumns: any) { + const processColumns = ( + columns: { [key: string]: IModelMetaColumn }, + parent = '' + ) => { + return Object.entries(columns) + .filter(([_, value]) => value.exportable !== false) + .flatMap(([key, value]) => { + if (value.type === 'collection' && value.collectionOf === 'object') { + return processColumns(value.columns, key); + } else { + const group = parent; + return [ + { + name: value.name, + type: value.type || 'text', + accessor: value.accessor || key, + group, + }, + ]; + } + }); + }; + return processColumns(resourceColumns); + } + + private getPrintableColumns(resourceMeta: IModelMeta) { + const processColumns = ( + columns: { [key: string]: IModelMetaColumn }, + parent = '' + ) => { + return Object.entries(columns) + .filter(([_, value]) => value.printable !== false) + .flatMap(([key, value]) => { + if (value.type === 'collection' && value.collectionOf === 'object') { + return processColumns(value.columns, key); + } else { + const group = parent; + return [ + { + name: value.name, + type: value.type || 'text', + accessor: value.accessor || key, + group, + }, + ]; + } + }); + }; + return processColumns(resourceMeta.columns); + } + + /** + * Creates a workbook from the provided data and columns. + * @param {any[]} data - The data to be included in the workbook. + * @param {any[]} exportableColumns - The columns to be included in the workbook. + * @returns The created workbook. + */ + private createWorkbook(data: any[], exportableColumns: any[]) { + const workbook = xlsx.utils.book_new(); + const worksheetData = data.map((item) => + exportableColumns.map((col) => get(item, getDataAccessor(col))) + ); + worksheetData.unshift(exportableColumns.map((col) => col.name)); + + const worksheet = xlsx.utils.aoa_to_sheet(worksheetData); + xlsx.utils.book_append_sheet(workbook, worksheet, 'Exported Data'); + + return workbook; + } + + /** + * Exports the workbook in the specified format. + * @param {any} workbook - The workbook to be exported. + * @param {string} format - The format to export the workbook in. + * @returns The exported workbook data. + */ + private exportWorkbook(workbook: any, format: string) { + if (format.toLowerCase() === 'csv') { + return xlsx.write(workbook, { type: 'buffer', bookType: 'csv' }); + } else if (format.toLowerCase() === 'xlsx') { + return xlsx.write(workbook, { type: 'buffer', bookType: 'xlsx' }); + } + } +} diff --git a/packages/server-nest/src/modules/Export/Exportable.ts b/packages/server-nest/src/modules/Export/Exportable.ts new file mode 100644 index 000000000..0e8801678 --- /dev/null +++ b/packages/server-nest/src/modules/Export/Exportable.ts @@ -0,0 +1,22 @@ +export class Exportable { + /** + * + * @param tenantId + * @returns + */ + public async exportable( + tenantId: number, + query: Record + ): Promise>> { + return []; + } + + /** + * + * @param data + * @returns + */ + public transform(data: Record) { + return data; + } +} diff --git a/packages/server-nest/src/modules/Export/common.ts b/packages/server-nest/src/modules/Export/common.ts new file mode 100644 index 000000000..71f6ef281 --- /dev/null +++ b/packages/server-nest/src/modules/Export/common.ts @@ -0,0 +1,9 @@ +export enum Errors { + RESOURCE_NOT_EXPORTABLE = 'RESOURCE_NOT_EXPORTABLE', +} + +export enum ExportFormat { + Csv = 'csv', + Pdf = 'pdf', + Xlsx = 'xlsx', +} diff --git a/packages/server-nest/src/modules/Export/constants.ts b/packages/server-nest/src/modules/Export/constants.ts new file mode 100644 index 000000000..b7723d38c --- /dev/null +++ b/packages/server-nest/src/modules/Export/constants.ts @@ -0,0 +1,2 @@ +export const EXPORT_SIZE_LIMIT = 9999999; +export const EXPORT_DTE_FORMAT = 'YYYY-MM-DD'; diff --git a/packages/server-nest/src/modules/Export/utils.ts b/packages/server-nest/src/modules/Export/utils.ts new file mode 100644 index 000000000..f21515c46 --- /dev/null +++ b/packages/server-nest/src/modules/Export/utils.ts @@ -0,0 +1,45 @@ +import { flatMap, get } from 'lodash'; +/** + * Flattens the data based on a specified attribute. + * @param data - The data to be flattened. + * @param flattenAttr - The attribute to be flattened. + * @returns - The flattened data. + */ +export const flatDataCollections = ( + data: Record, + flattenAttr: string +): Record[] => { + return flatMap(data, (item) => + item[flattenAttr].map((entry) => ({ + ...item, + [flattenAttr]: entry, + })) + ); +}; + +/** + * Gets the data accessor for a given column. + * @param col - The column to get the data accessor for. + * @returns - The data accessor. + */ +export const getDataAccessor = (col: any) => { + return col.group ? `${col.group}.${col.accessor}` : col.accessor; +}; + +/** + * Maps the data retrieved from the service layer to the pdf document. + * @param {any} columns + * @param {Record} data + * @returns + */ +export const mapPdfRows = (columns: any, data: Record) => { + return data.map((item) => { + const cells = columns.map((column) => { + return { + key: column.accessor, + value: get(item, getDataAccessor(column)), + }; + }); + return { cells, classNames: '' }; + }); +}; diff --git a/packages/server-nest/src/modules/Import/ImportALS.ts b/packages/server-nest/src/modules/Import/ImportALS.ts new file mode 100644 index 000000000..d1298e5bd --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportALS.ts @@ -0,0 +1,105 @@ +import { Service } from 'typedi'; +import { AsyncLocalStorage } from 'async_hooks'; + +@Service() +export class ImportAls { + private als: AsyncLocalStorage>; + + constructor() { + this.als = new AsyncLocalStorage(); + } + + /** + * Runs a callback function within the context of a new AsyncLocalStorage store. + * @param callback The function to be executed within the AsyncLocalStorage context. + * @returns The result of the callback function. + */ + public run(callback: () => T): T { + return this.als.run(new Map(), callback); + } + + /** + * Runs a callback function in preview mode within the AsyncLocalStorage context. + * @param callback The function to be executed in preview mode. + * @returns The result of the callback function. + */ + public runPreview(callback: () => T): T { + return this.run(() => { + this.markAsImport(); + this.markAsImportPreview(); + return callback(); + }); + } + + /** + * Runs a callback function in commit mode within the AsyncLocalStorage context. + * @param {() => T} callback - The function to be executed in commit mode. + * @returns {T} The result of the callback function. + */ + public runCommit(callback: () => T): T { + return this.run(() => { + this.markAsImport(); + this.markAsImportCommit(); + return callback(); + }); + } + + /** + * Retrieves the current AsyncLocalStorage store. + * @returns The current store or undefined if not in a valid context. + */ + public getStore(): Map | undefined { + return this.als.getStore(); + } + + /** + * Marks the current context as an import operation. + * @param flag Boolean flag to set or unset the import status. Defaults to true. + */ + public markAsImport(flag: boolean = true): void { + const store = this.getStore(); + store?.set('isImport', flag); + } + + /** + * Marks the current context as an import commit operation. + * @param flag Boolean flag to set or unset the import commit status. Defaults to true. + */ + public markAsImportCommit(flag: boolean = true): void { + const store = this.getStore(); + store?.set('isImportCommit', flag); + } + + /** + * Marks the current context as an import preview operation. + * @param {boolean} flag - Boolean flag to set or unset the import preview status. Defaults to true. + */ + public markAsImportPreview(flag: boolean = true): void { + const store = this.getStore(); + store?.set('isImportPreview', flag); + } + + /** + * Checks if the current context is an import operation. + * @returns {boolean} True if the context is an import operation, false otherwise. + */ + public get isImport(): boolean { + return !!this.getStore()?.get('isImport'); + } + + /** + * Checks if the current context is an import commit operation. + * @returns {boolean} True if the context is an import commit operation, false otherwise. + */ + public get isImportCommit(): boolean { + return !!this.getStore()?.get('isImportCommit'); + } + + /** + * Checks if the current context is an import preview operation. + * @returns {boolean} True if the context is an import preview operation, false otherwise. + */ + public get isImportPreview(): boolean { + return !!this.getStore()?.get('isImportPreview'); + } +} diff --git a/packages/server-nest/src/modules/Import/ImportFileCommon.ts b/packages/server-nest/src/modules/Import/ImportFileCommon.ts new file mode 100644 index 000000000..e8391882e --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportFileCommon.ts @@ -0,0 +1,175 @@ +import bluebird from 'bluebird'; +import * as R from 'ramda'; +import { Inject, Service } from 'typedi'; +import { first } from 'lodash'; +import { ImportFileDataValidator } from './ImportFileDataValidator'; +import { Knex } from 'knex'; +import { + ImportInsertError, + ImportOperError, + ImportOperSuccess, + ImportableContext, +} from './interfaces'; +import { ServiceError } from '@/exceptions'; +import { getUniqueImportableValue, trimObject } from './_utils'; +import { ImportableResources } from './ImportableResources'; +import ResourceService from '../Resource/ResourceService'; +import { Import } from '@/system/models'; + +@Service() +export class ImportFileCommon { + @Inject() + private importFileValidator: ImportFileDataValidator; + + @Inject() + private importable: ImportableResources; + + @Inject() + private resource: ResourceService; + + /** + * Imports the given parsed data to the resource storage through registered importable service. + * @param {number} tenantId - + * @param {string} resourceName - Resource name. + * @param {Record} parsedData - Parsed data. + * @param {Knex.Transaction} trx - Knex transaction. + * @returns {Promise<[ImportOperSuccess[], ImportOperError[]]>} + */ + public async import( + tenantId: number, + importFile: Import, + parsedData: Record[], + trx?: Knex.Transaction + ): Promise<[ImportOperSuccess[], ImportOperError[]]> { + const resourceFields = this.resource.getResourceFields2( + tenantId, + importFile.resource + ); + const ImportableRegistry = this.importable.registry; + const importable = ImportableRegistry.getImportable(importFile.resource); + + const concurrency = importable.concurrency || 10; + + const success: ImportOperSuccess[] = []; + const failed: ImportOperError[] = []; + + const importAsync = async (objectDTO, index: number): Promise => { + const context: ImportableContext = { + rowIndex: index, + import: importFile, + }; + const transformedDTO = importable.transform(objectDTO, context); + const rowNumber = index + 1; + const uniqueValue = getUniqueImportableValue(resourceFields, objectDTO); + const errorContext = { + rowNumber, + uniqueValue, + }; + try { + // Validate the DTO object before passing it to the service layer. + await this.importFileValidator.validateData( + resourceFields, + transformedDTO + ); + try { + // Run the importable function and listen to the errors. + const data = await importable.importable( + tenantId, + transformedDTO, + trx + ); + success.push({ index, data }); + } catch (err) { + if (err instanceof ServiceError) { + const error: ImportInsertError[] = [ + { + errorCode: 'ServiceError', + errorMessage: err.message || err.errorType, + ...errorContext, + }, + ]; + failed.push({ index, error }); + } else { + const error: ImportInsertError[] = [ + { + errorCode: 'UnknownError', + errorMessage: 'Unknown error occurred', + ...errorContext, + }, + ]; + failed.push({ index, error }); + } + } + } catch (errors) { + const error = errors.map((er) => ({ ...er, ...errorContext })); + failed.push({ index, error }); + } + }; + await bluebird.map(parsedData, importAsync, { concurrency }); + + return [success, failed]; + } + + /** + * + * @param {string} resourceName + * @param {Record} params + */ + public async validateParamsSchema( + resourceName: string, + params: Record + ) { + const ImportableRegistry = this.importable.registry; + const importable = ImportableRegistry.getImportable(resourceName); + + const yupSchema = importable.paramsValidationSchema(); + + try { + await yupSchema.validate(params, { abortEarly: false }); + } catch (validationError) { + const errors = validationError.inner.map((error) => ({ + errorCode: 'ParamsValidationError', + errorMessage: error.errors, + })); + throw errors; + } + } + + /** + * + * @param {string} resourceName + * @param {Record} params + */ + public async validateParams( + tenantId: number, + resourceName: string, + params: Record + ) { + const ImportableRegistry = this.importable.registry; + const importable = ImportableRegistry.getImportable(resourceName); + + await importable.validateParams(tenantId, params); + } + + /** + * + * @param {string} resourceName + * @param {Record} params + * @returns + */ + public transformParams(resourceName: string, params: Record) { + const ImportableRegistry = this.importable.registry; + const importable = ImportableRegistry.getImportable(resourceName); + + return importable.transformParams(params); + } + + /** + * Retrieves the sheet columns from the given sheet data. + * @param {unknown[]} json + * @returns {string[]} + */ + public parseSheetColumns(json: unknown[]): string[] { + return R.compose(Object.keys, trimObject, first)(json); + } +} diff --git a/packages/server-nest/src/modules/Import/ImportFileDataTransformer.ts b/packages/server-nest/src/modules/Import/ImportFileDataTransformer.ts new file mode 100644 index 000000000..5e391fc6e --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportFileDataTransformer.ts @@ -0,0 +1,151 @@ +import { Inject, Service } from 'typedi'; +import bluebird from 'bluebird'; +import { isUndefined, pickBy, set } from 'lodash'; +import { Knex } from 'knex'; +import { ImportMappingAttr, ResourceMetaFieldsMap } from './interfaces'; +import { + valueParser, + parseKey, + getFieldKey, + aggregate, + sanitizeSheetData, + getMapToPath, +} from './_utils'; +import ResourceService from '../Resource/ResourceService'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { CurrencyParsingDTOs } from './_constants'; + +@Service() +export class ImportFileDataTransformer { + @Inject() + private resource: ResourceService; + + @Inject() + private tenancy: HasTenancyService; + + /** + * Parses the given sheet data before passing to the service layer. + * based on the mapped fields and the each field type. + * @param {number} tenantId - + * @param {} + */ + public async parseSheetData( + tenantId: number, + importFile: any, + importableFields: ResourceMetaFieldsMap, + data: Record[], + trx?: Knex.Transaction + ): Promise[]> { + // Sanitize the sheet data. + const sanitizedData = sanitizeSheetData(data); + + // Map the sheet columns key with the given map. + const mappedDTOs = this.mapSheetColumns( + sanitizedData, + importFile.mappingParsed + ); + // Parse the mapped sheet values. + const parsedValues = await this.parseExcelValues( + tenantId, + importableFields, + mappedDTOs, + trx + ); + const aggregateValues = this.aggregateParsedValues( + tenantId, + importFile.resource, + parsedValues + ); + return aggregateValues; + } + + /** + * Aggregates parsed data based on resource metadata configuration. + * @param {number} tenantId + * @param {string} resourceName + * @param {Record} parsedData + * @returns {Record[]} + */ + public aggregateParsedValues = ( + tenantId: number, + resourceName: string, + parsedData: Record[] + ): Record[] => { + let _value = parsedData; + const meta = this.resource.getResourceMeta(tenantId, resourceName); + + if (meta.importAggregator === 'group') { + _value = aggregate( + _value, + meta.importAggregateBy, + meta.importAggregateOn + ); + } + return _value; + }; + + /** + * Maps the columns of the imported data based on the provided mapping attributes. + * @param {Record[]} body - The array of data objects to map. + * @param {ImportMappingAttr[]} map - The mapping attributes. + * @returns {Record[]} - The mapped data objects. + */ + public mapSheetColumns( + body: Record[], + map: ImportMappingAttr[] + ): Record[] { + return body.map((item) => { + const newItem = {}; + map + .filter((mapping) => !isUndefined(item[mapping.from])) + .forEach((mapping) => { + const toPath = getMapToPath(mapping.to, mapping.group); + newItem[toPath] = item[mapping.from]; + }); + return newItem; + }); + } + + /** + * Parses sheet values before passing to the service layer. + * @param {ResourceMetaFieldsMap} fields - + * @param {Record} valueDTOS - + * @returns {Record} + */ + public async parseExcelValues( + tenantId: number, + fields: ResourceMetaFieldsMap, + valueDTOs: Record[], + trx?: Knex.Transaction + ): Promise[]> { + const tenantModels = this.tenancy.models(tenantId); + const _valueParser = valueParser(fields, tenantModels, trx); + const _keyParser = parseKey(fields); + + const parseAsync = async (valueDTO) => { + // Clean up the undefined keys that not exist in resource fields. + const _valueDTO = pickBy( + valueDTO, + (value, key) => !isUndefined(fields[getFieldKey(key)]) + ); + // Keys of mapped values. key structure: `group.key` or `key`. + const keys = Object.keys(_valueDTO); + + // Map the object values. + return bluebird.reduce( + keys, + async (acc, key) => { + const parsedValue = await _valueParser(_valueDTO[key], key); + const parsedKey = await _keyParser(key); + + set(acc, parsedKey, parsedValue); + return acc; + }, + {} + ); + }; + return bluebird.map(valueDTOs, parseAsync, { + concurrency: CurrencyParsingDTOs, + }); + } +} diff --git a/packages/server-nest/src/modules/Import/ImportFileDataValidator.ts b/packages/server-nest/src/modules/Import/ImportFileDataValidator.ts new file mode 100644 index 000000000..143533c66 --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportFileDataValidator.ts @@ -0,0 +1,46 @@ +import { Service } from 'typedi'; +import { ImportInsertError, ResourceMetaFieldsMap } from './interfaces'; +import { ERRORS, convertFieldsToYupValidation } from './_utils'; +import { IModelMeta } from '@/interfaces'; +import { ServiceError } from '@/exceptions'; + +@Service() +export class ImportFileDataValidator { + /** + * Validates the given resource is importable. + * @param {IModelMeta} resourceMeta + */ + public validateResourceImportable(resourceMeta: IModelMeta) { + // Throw service error if the resource does not support importing. + if (!resourceMeta.importable) { + throw new ServiceError(ERRORS.RESOURCE_NOT_IMPORTABLE); + } + } + + /** + * Validates the given mapped DTOs and returns errors with their index. + * @param {Record} mappedDTOs + * @returns {Promise} + */ + public async validateData( + importableFields: ResourceMetaFieldsMap, + data: Record + ): Promise { + const YupSchema = convertFieldsToYupValidation(importableFields); + const _data = { ...data }; + + try { + await YupSchema.validate(_data, { abortEarly: false }); + } catch (validationError) { + const errors = validationError.inner.reduce((errors, error) => { + const newErrors = error.errors.map((errMsg) => ({ + errorCode: 'ValidationError', + errorMessage: errMsg, + })); + return [...errors, ...newErrors]; + }, []); + + throw errors; + } + } +} diff --git a/packages/server-nest/src/modules/Import/ImportFileMapping.ts b/packages/server-nest/src/modules/Import/ImportFileMapping.ts new file mode 100644 index 000000000..3ef321239 --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportFileMapping.ts @@ -0,0 +1,156 @@ +import { fromPairs, isUndefined } from 'lodash'; +import { Inject, Service } from 'typedi'; +import { + ImportDateFormats, + ImportFileMapPOJO, + ImportMappingAttr, +} from './interfaces'; +import ResourceService from '../Resource/ResourceService'; +import { ServiceError } from '@/exceptions'; +import { ERRORS } from './_utils'; +import { Import } from '@/system/models'; + +@Service() +export class ImportFileMapping { + @Inject() + private resource: ResourceService; + + /** + * Mapping the excel sheet columns with resource columns. + * @param {number} tenantId + * @param {number} importId + * @param {ImportMappingAttr} maps + */ + public async mapping( + tenantId: number, + importId: string, + maps: ImportMappingAttr[] + ): Promise { + const importFile = await Import.query() + .findOne('filename', importId) + .throwIfNotFound(); + + // Invalidate the from/to map attributes. + this.validateMapsAttrs(tenantId, importFile, maps); + + // @todo validate the required fields. + + // Validate the diplicated relations of map attrs. + this.validateDuplicatedMapAttrs(maps); + + // Validate the date format mapping. + this.validateDateFormatMapping(tenantId, importFile.resource, maps); + + const mappingStringified = JSON.stringify(maps); + + await Import.query().findById(importFile.id).patch({ + mapping: mappingStringified, + }); + return { + import: { + importId: importFile.importId, + resource: importFile.resource, + }, + }; + } + + /** + * Validate the mapping attributes. + * @param {number} tenantId - + * @param {} importFile - + * @param {ImportMappingAttr[]} maps + * @throws {ServiceError(ERRORS.INVALID_MAP_ATTRS)} + */ + private validateMapsAttrs( + tenantId: number, + importFile: any, + maps: ImportMappingAttr[] + ) { + const fields = this.resource.getResourceFields2( + tenantId, + importFile.resource + ); + const columnsMap = fromPairs( + importFile.columnsParsed.map((field) => [field, '']) + ); + const invalid = []; + + // is not empty, is not undefined or map.group + maps.forEach((map) => { + let _invalid = true; + + if (!map.group && fields[map.to]) { + _invalid = false; + } + if (map.group && fields[map.group] && fields[map.group]?.fields[map.to]) { + _invalid = false; + } + if (columnsMap[map.from]) { + _invalid = false; + } + if (_invalid) { + invalid.push(map); + } + }); + if (invalid.length > 0) { + throw new ServiceError(ERRORS.INVALID_MAP_ATTRS); + } + } + + /** + * Validate the map attrs relation should be one-to-one relation only. + * @param {ImportMappingAttr[]} maps + */ + private validateDuplicatedMapAttrs(maps: ImportMappingAttr[]) { + const fromMap = {}; + const toMap = {}; + + maps.forEach((map) => { + if (fromMap[map.from]) { + throw new ServiceError(ERRORS.DUPLICATED_FROM_MAP_ATTR); + } else { + fromMap[map.from] = true; + } + const toPath = !isUndefined(map?.group) + ? `${map.group}.${map.to}` + : map.to; + + if (toMap[toPath]) { + throw new ServiceError(ERRORS.DUPLICATED_TO_MAP_ATTR); + } else { + toMap[toPath] = true; + } + }); + } + + /** + * Validates the date format mapping. + * @param {number} tenantId + * @param {string} resource + * @param {ImportMappingAttr[]} maps + */ + private validateDateFormatMapping( + tenantId: number, + resource: string, + maps: ImportMappingAttr[] + ) { + const fields = this.resource.getResourceImportableFields( + tenantId, + resource + ); + // @todo Validate date type of the nested fields. + maps.forEach((map) => { + if ( + typeof fields[map.to] !== 'undefined' && + fields[map.to].fieldType === 'date' + ) { + if ( + typeof map.dateFormat !== 'undefined' && + ImportDateFormats.indexOf(map.dateFormat) === -1 + ) { + throw new ServiceError(ERRORS.INVALID_MAP_DATE_FORMAT); + } + } + }); + } +} diff --git a/packages/server-nest/src/modules/Import/ImportFileMeta.ts b/packages/server-nest/src/modules/Import/ImportFileMeta.ts new file mode 100644 index 000000000..c20d49120 --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportFileMeta.ts @@ -0,0 +1,33 @@ +import { Inject, Service } from 'typedi'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { TransformerInjectable } from '@/lib/Transformer/TransformerInjectable'; +import { ImportFileMetaTransformer } from './ImportFileMetaTransformer'; +import { Import } from '@/system/models'; + +@Service() +export class ImportFileMeta { + @Inject() + private tenancy: HasTenancyService; + + @Inject() + private transformer: TransformerInjectable; + + /** + * Retrieves the import meta of the given import model id. + * @param {number} tenantId + * @param {number} importId + * @returns {} + */ + async getImportMeta(tenantId: number, importId: string) { + const importFile = await Import.query() + .where('tenantId', tenantId) + .findOne('importId', importId); + + // Retrieves the transformed accounts collection. + return this.transformer.transform( + tenantId, + importFile, + new ImportFileMetaTransformer() + ); + } +} diff --git a/packages/server-nest/src/modules/Import/ImportFileMetaTransformer.ts b/packages/server-nest/src/modules/Import/ImportFileMetaTransformer.ts new file mode 100644 index 000000000..6c50c1e37 --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportFileMetaTransformer.ts @@ -0,0 +1,19 @@ +import { Transformer } from '@/lib/Transformer/Transformer'; + +export class ImportFileMetaTransformer extends Transformer { + /** + * Include these attributes to sale invoice object. + * @returns {Array} + */ + public includeAttributes = (): string[] => { + return ['map']; + }; + + public excludeAttributes = (): string[] => { + return ['id', 'filename', 'columns', 'mappingParsed', 'mapping']; + } + + map(importFile) { + return importFile.mappingParsed; + } +} diff --git a/packages/server-nest/src/modules/Import/ImportFilePreview.ts b/packages/server-nest/src/modules/Import/ImportFilePreview.ts new file mode 100644 index 000000000..e8b566bbe --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportFilePreview.ts @@ -0,0 +1,53 @@ +import { Inject, Service } from 'typedi'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { ImportFilePreviewPOJO } from './interfaces'; +import { ImportFileProcess } from './ImportFileProcess'; +import { ImportAls } from './ImportALS'; + +@Service() +export class ImportFilePreview { + @Inject() + private tenancy: HasTenancyService; + + @Inject() + private importFile: ImportFileProcess; + + @Inject() + private importAls: ImportAls; + + /** + * Preview the imported file results before commiting the transactions. + * @param {number} tenantId - + * @param {string} importId - + * @returns {Promise} + */ + public async preview( + tenantId: number, + importId: string + ): Promise { + return this.importAls.runPreview>(() => + this.previewAlsRun(tenantId, importId) + ); + } + + /** + * Preview the imported file results before commiting the transactions. + * @param {number} tenantId + * @param {number} importId + * @returns {Promise} + */ + public async previewAlsRun( + tenantId: number, + importId: string + ): Promise { + const knex = this.tenancy.knex(tenantId); + const trx = await knex.transaction({ isolationLevel: 'read uncommitted' }); + + const meta = await this.importFile.import(tenantId, importId, trx); + + // Rollback the successed transaction. + await trx.rollback(); + + return meta; + } +} diff --git a/packages/server-nest/src/modules/Import/ImportFileProcess.ts b/packages/server-nest/src/modules/Import/ImportFileProcess.ts new file mode 100644 index 000000000..405b0da43 --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportFileProcess.ts @@ -0,0 +1,104 @@ +import { Inject, Service } from 'typedi'; +import { chain } from 'lodash'; +import { Knex } from 'knex'; +import { ServiceError } from '@/exceptions'; +import { ERRORS, getUnmappedSheetColumns, readImportFile } from './_utils'; +import { ImportFileCommon } from './ImportFileCommon'; +import { ImportFileDataTransformer } from './ImportFileDataTransformer'; +import ResourceService from '../Resource/ResourceService'; +import UnitOfWork from '../UnitOfWork'; +import { ImportFilePreviewPOJO } from './interfaces'; +import { Import } from '@/system/models'; +import { parseSheetData } from './sheet_utils'; + +@Service() +export class ImportFileProcess { + @Inject() + private resource: ResourceService; + + @Inject() + private importCommon: ImportFileCommon; + + @Inject() + private importParser: ImportFileDataTransformer; + + @Inject() + private uow: UnitOfWork; + + /** + * Preview the imported file results before commiting the transactions. + * @param {number} tenantId + * @param {number} importId + * @returns {Promise} + */ + public async import( + tenantId: number, + importId: string, + trx?: Knex.Transaction + ): Promise { + const importFile = await Import.query() + .findOne('importId', importId) + .where('tenantId', tenantId) + .throwIfNotFound(); + + // Throw error if the import file is not mapped yet. + if (!importFile.isMapped) { + throw new ServiceError(ERRORS.IMPORT_FILE_NOT_MAPPED); + } + // Read the imported file and parse the given buffer to get columns + // and sheet data in json format. + const buffer = await readImportFile(importFile.filename); + const [sheetData, sheetColumns] = parseSheetData(buffer); + + const resource = importFile.resource; + const resourceFields = this.resource.getResourceFields2(tenantId, resource); + + // Runs the importing operation with ability to return errors that will happen. + const [successedImport, failedImport, allData] = + await this.uow.withTransaction( + tenantId, + async (trx: Knex.Transaction) => { + // Prases the sheet json data. + const parsedData = await this.importParser.parseSheetData( + tenantId, + importFile, + resourceFields, + sheetData, + trx + ); + const [successedImport, failedImport] = + await this.importCommon.import( + tenantId, + importFile, + parsedData, + trx + ); + return [successedImport, failedImport, parsedData]; + }, + trx + ); + const mapping = importFile.mappingParsed; + const errors = chain(failedImport) + .map((oper) => oper.error) + .flatten() + .value(); + + const unmappedColumns = getUnmappedSheetColumns(sheetColumns, mapping); + const totalCount = allData.length; + + const createdCount = successedImport.length; + const errorsCount = failedImport.length; + const skippedCount = errorsCount; + + return { + resource, + createdCount, + skippedCount, + totalCount, + errorsCount, + errors, + unmappedColumns: unmappedColumns, + unmappedColumnsCount: unmappedColumns.length, + }; + } +} diff --git a/packages/server-nest/src/modules/Import/ImportFileProcessCommit.ts b/packages/server-nest/src/modules/Import/ImportFileProcessCommit.ts new file mode 100644 index 000000000..f9cb640fc --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportFileProcessCommit.ts @@ -0,0 +1,66 @@ +import { Inject, Service } from 'typedi'; +import HasTenancyService from '../Tenancy/TenancyService'; +import { ImportFilePreviewPOJO } from './interfaces'; +import { ImportFileProcess } from './ImportFileProcess'; +import { EventPublisher } from '@/lib/EventPublisher/EventPublisher'; +import events from '@/subscribers/events'; +import { IImportFileCommitedEventPayload } from '@/interfaces/Import'; +import { ImportAls } from './ImportALS'; + +@Service() +export class ImportFileProcessCommit { + @Inject() + private tenancy: HasTenancyService; + + @Inject() + private importFile: ImportFileProcess; + + @Inject() + private importAls: ImportAls; + + @Inject() + private eventPublisher: EventPublisher; + + /** + * Commits the imported file under ALS. + * @param {number} tenantId + * @param {string} importId + * @returns {Promise} + */ + public commit( + tenantId: number, + importId: string + ): Promise { + return this.importAls.runCommit>(() => + this.commitAlsRun(tenantId, importId) + ); + } + + /** + * Commits the imported file. + * @param {number} tenantId + * @param {number} importId + * @returns {Promise} + */ + public async commitAlsRun( + tenantId: number, + importId: string + ): Promise { + const knex = this.tenancy.knex(tenantId); + const trx = await knex.transaction({ isolationLevel: 'read uncommitted' }); + + const meta = await this.importFile.import(tenantId, importId, trx); + + // Commit the successed transaction. + await trx.commit(); + + // Triggers `onImportFileCommitted` event. + await this.eventPublisher.emitAsync(events.import.onImportCommitted, { + meta, + importId, + tenantId, + } as IImportFileCommitedEventPayload); + + return meta; + } +} diff --git a/packages/server-nest/src/modules/Import/ImportFileUpload.ts b/packages/server-nest/src/modules/Import/ImportFileUpload.ts new file mode 100644 index 000000000..9f954de9d --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportFileUpload.ts @@ -0,0 +1,123 @@ +import { Inject, Service } from 'typedi'; +import { + deleteImportFile, + getResourceColumns, + readImportFile, + sanitizeResourceName, + validateSheetEmpty, +} from './_utils'; +import ResourceService from '../Resource/ResourceService'; +import { ImportFileCommon } from './ImportFileCommon'; +import { ImportFileDataValidator } from './ImportFileDataValidator'; +import { ImportFileUploadPOJO } from './interfaces'; +import { Import } from '@/system/models'; +import { parseSheetData } from './sheet_utils'; + +@Service() +export class ImportFileUploadService { + @Inject() + private resourceService: ResourceService; + + @Inject() + private importFileCommon: ImportFileCommon; + + @Inject() + private importValidator: ImportFileDataValidator; + + /** + * Imports the specified file for the given resource. + * Deletes the file if an error occurs during the import process. + * @param {number} tenantId + * @param {string} resourceName + * @param {string} filename + * @param {Record} params + * @returns {Promise} + */ + public async import( + tenantId: number, + resourceName: string, + filename: string, + params: Record + ): Promise { + try { + return await this.importUnhandled( + tenantId, + resourceName, + filename, + params + ); + } catch (err) { + deleteImportFile(filename); + throw err; + } + } + + /** + * Reads the imported file and stores the import file meta under unqiue id. + * @param {number} tenantId - Tenant id. + * @param {string} resource - Resource name. + * @param {string} filePath - File path. + * @param {string} fileName - File name. + * @returns {Promise} + */ + public async importUnhandled( + tenantId: number, + resourceName: string, + filename: string, + params: Record + ): Promise { + const resource = sanitizeResourceName(resourceName); + const resourceMeta = this.resourceService.getResourceMeta( + tenantId, + resource + ); + // Throw service error if the resource does not support importing. + this.importValidator.validateResourceImportable(resourceMeta); + + // Reads the imported file into buffer. + const buffer = await readImportFile(filename); + + // Parse the buffer file to array data. + const [sheetData, sheetColumns] = parseSheetData(buffer); + const coumnsStringified = JSON.stringify(sheetColumns); + + // Throws service error if the sheet data is empty. + validateSheetEmpty(sheetData); + + try { + // Validates the params Yup schema. + await this.importFileCommon.validateParamsSchema(resource, params); + + // Validates importable params asyncly. + await this.importFileCommon.validateParams(tenantId, resource, params); + } catch (error) { + throw error; + } + const _params = this.importFileCommon.transformParams(resource, params); + const paramsStringified = JSON.stringify(_params); + + // Store the import model with related metadata. + const importFile = await Import.query().insert({ + filename, + resource, + tenantId, + importId: filename, + columns: coumnsStringified, + params: paramsStringified, + }); + const resourceColumnsMap = this.resourceService.getResourceFields2( + tenantId, + resource + ); + const resourceColumns = getResourceColumns(resourceColumnsMap); + + return { + import: { + importId: importFile.importId, + resource: importFile.resource, + }, + sheetColumns, + resourceColumns, + }; + } +} diff --git a/packages/server-nest/src/modules/Import/ImportRemoveExpiredFiles.ts b/packages/server-nest/src/modules/Import/ImportRemoveExpiredFiles.ts new file mode 100644 index 000000000..c20f486f5 --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportRemoveExpiredFiles.ts @@ -0,0 +1,34 @@ +import moment from 'moment'; +import bluebird from 'bluebird'; +import { Import } from '@/system/models'; +import { deleteImportFile } from './_utils'; +import { Service } from 'typedi'; + +@Service() +export class ImportDeleteExpiredFiles { + /** + * Delete expired files. + */ + async deleteExpiredFiles() { + const yesterday = moment().subtract(1, 'hour').format('YYYY-MM-DD HH:mm'); + + const expiredImports = await Import.query().where( + 'createdAt', + '<', + yesterday + ); + await bluebird.map( + expiredImports, + async (expiredImport) => { + await deleteImportFile(expiredImport.filename); + }, + { concurrency: 10 } + ); + const expiredImportsIds = expiredImports.map( + (expiredImport) => expiredImport.id + ); + if (expiredImportsIds.length > 0) { + await Import.query().whereIn('id', expiredImportsIds).delete(); + } + } +} diff --git a/packages/server-nest/src/modules/Import/ImportResourceApplication.ts b/packages/server-nest/src/modules/Import/ImportResourceApplication.ts new file mode 100644 index 000000000..6e000804c --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportResourceApplication.ts @@ -0,0 +1,106 @@ +import { Inject } from 'typedi'; +import { ImportFileUploadService } from './ImportFileUpload'; +import { ImportFileMapping } from './ImportFileMapping'; +import { ImportMappingAttr } from './interfaces'; +import { ImportFileProcess } from './ImportFileProcess'; +import { ImportFilePreview } from './ImportFilePreview'; +import { ImportSampleService } from './ImportSample'; +import { ImportFileMeta } from './ImportFileMeta'; +import { ImportFileProcessCommit } from './ImportFileProcessCommit'; + +@Inject() +export class ImportResourceApplication { + @Inject() + private importFileService: ImportFileUploadService; + + @Inject() + private importMappingService: ImportFileMapping; + + @Inject() + private importProcessService: ImportFileProcess; + + @Inject() + private ImportFilePreviewService: ImportFilePreview; + + @Inject() + private importSampleService: ImportSampleService; + + @Inject() + private importMetaService: ImportFileMeta; + + @Inject() + private importProcessCommit: ImportFileProcessCommit; + + /** + * Reads the imported file and stores the import file meta under unqiue id. + * @param {number} tenantId - + * @param {string} resource - Resource name. + * @param {string} fileName - File name. + * @returns {Promise} + */ + public async import( + tenantId: number, + resource: string, + filename: string, + params: Record + ) { + return this.importFileService.import(tenantId, resource, filename, params); + } + + /** + * Mapping the excel sheet columns with resource columns. + * @param {number} tenantId + * @param {number} importId - Import id. + * @param {ImportMappingAttr} maps + */ + public async mapping( + tenantId: number, + importId: string, + maps: ImportMappingAttr[] + ) { + return this.importMappingService.mapping(tenantId, importId, maps); + } + + /** + * Preview the mapped results before process importing. + * @param {number} tenantId + * @param {number} importId - Import id. + * @returns {Promise} + */ + public async preview(tenantId: number, importId: string) { + return this.ImportFilePreviewService.preview(tenantId, importId); + } + + /** + * Process the import file sheet through service for creating entities. + * @param {number} tenantId + * @param {number} importId + * @returns {Promise} + */ + public async process(tenantId: number, importId: string) { + return this.importProcessCommit.commit(tenantId, importId); + } + + /** + * Retrieves the import meta of the given import id. + * @param {number} tenantId - + * @param {string} importId - Import id. + * @returns {} + */ + public importMeta(tenantId: number, importId: string) { + return this.importMetaService.getImportMeta(tenantId, importId); + } + + /** + * Retrieves the csv/xlsx sample sheet of the given + * @param {number} tenantId + * @param {number} resource - Resource name. + */ + public sample( + tenantId: number, + resource: string, + format: 'csv' | 'xlsx' = 'csv' + ) { + return this.importSampleService.sample(tenantId, resource, format); + } +} diff --git a/packages/server-nest/src/modules/Import/ImportSample.ts b/packages/server-nest/src/modules/Import/ImportSample.ts new file mode 100644 index 000000000..db2fe0543 --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportSample.ts @@ -0,0 +1,46 @@ +import XLSX from 'xlsx'; +import { Inject, Service } from 'typedi'; +import { ImportableResources } from './ImportableResources'; +import { sanitizeResourceName } from './_utils'; + +@Service() +export class ImportSampleService { + @Inject() + private importable: ImportableResources; + + /** + * Retrieves the sample sheet of the given resource. + * @param {number} tenantId + * @param {string} resource + * @param {string} format + * @returns {Buffer | string} + */ + public sample( + tenantId: number, + resource: string, + format: 'csv' | 'xlsx' + ): Buffer | string { + const _resource = sanitizeResourceName(resource); + + const ImportableRegistry = this.importable.registry; + const importable = ImportableRegistry.getImportable(_resource); + + const data = importable.sampleData(); + + const workbook = XLSX.utils.book_new(); + const worksheet = XLSX.utils.json_to_sheet(data); + XLSX.utils.book_append_sheet(workbook, worksheet, 'Sheet1'); + + // Determine the output format + if (format === 'csv') { + const csvOutput = XLSX.utils.sheet_to_csv(worksheet); + return csvOutput; + } else { + const xlsxOutput = XLSX.write(workbook, { + bookType: 'xlsx', + type: 'buffer', + }); + return xlsxOutput; + } + } +} diff --git a/packages/server-nest/src/modules/Import/Importable.ts b/packages/server-nest/src/modules/Import/Importable.ts new file mode 100644 index 000000000..9cb82b56c --- /dev/null +++ b/packages/server-nest/src/modules/Import/Importable.ts @@ -0,0 +1,72 @@ +import { Knex } from 'knex'; +import * as Yup from 'yup'; +import { ImportableContext } from './interfaces'; + +export abstract class Importable { + /** + * + * @param {number} tenantId + * @param {any} createDTO + * @param {Knex.Transaction} trx + */ + public importable(tenantId: number, createDTO: any, trx?: Knex.Transaction) { + throw new Error( + 'The `importable` function is not defined in service importable.' + ); + } + + /** + * Transformes the DTO before passing it to importable and validation. + * @param {Record} createDTO + * @param {ImportableContext} context + * @returns {Record} + */ + public transform(createDTO: Record, context: ImportableContext) { + return createDTO; + } + + /** + * Concurrency controlling of the importing process. + * @returns {number} + */ + public get concurrency() { + return 10; + } + + /** + * Retrieves the sample data of importable. + * @returns {Array} + */ + public sampleData(): Array { + return []; + } + + // ------------------ + // # Params + // ------------------ + /** + * Params Yup validation schema. + * @returns {Yup.ObjectSchema} + */ + public paramsValidationSchema(): Yup.ObjectSchema { + return Yup.object().nullable(); + } + + /** + * Validates the params of the importable service. + * @param {Record} + * @returns {Promise} - True means passed and false failed. + */ + public async validateParams( + tenantId: number, + params: Record + ): Promise {} + + /** + * Transformes the import params before storing them. + * @param {Record} parmas + */ + public transformParams(parmas: Record) { + return parmas; + } +} \ No newline at end of file diff --git a/packages/server-nest/src/modules/Import/ImportableRegistry.ts b/packages/server-nest/src/modules/Import/ImportableRegistry.ts new file mode 100644 index 000000000..9fe25ec43 --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportableRegistry.ts @@ -0,0 +1,46 @@ +import { camelCase, upperFirst } from 'lodash'; +import { Importable } from './Importable'; + +export class ImportableRegistry { + private static instance: ImportableRegistry; + private importables: Record; + + constructor() { + this.importables = {}; + } + + /** + * Gets singleton instance of registry. + * @returns {ImportableRegistry} + */ + public static getInstance(): ImportableRegistry { + if (!ImportableRegistry.instance) { + ImportableRegistry.instance = new ImportableRegistry(); + } + return ImportableRegistry.instance; + } + + /** + * Registers the given importable service. + * @param {string} resource + * @param {Importable} importable + */ + public registerImportable(resource: string, importable: Importable): void { + const _resource = this.sanitizeResourceName(resource); + this.importables[_resource] = importable; + } + + /** + * Retrieves the importable service instance of the given resource name. + * @param {string} name + * @returns {Importable} + */ + public getImportable(name: string): Importable { + const _name = this.sanitizeResourceName(name); + return this.importables[_name]; + } + + private sanitizeResourceName(resource: string) { + return upperFirst(camelCase(resource)); + } +} diff --git a/packages/server-nest/src/modules/Import/ImportableResources.ts b/packages/server-nest/src/modules/Import/ImportableResources.ts new file mode 100644 index 000000000..f79ceb36f --- /dev/null +++ b/packages/server-nest/src/modules/Import/ImportableResources.ts @@ -0,0 +1,73 @@ +import Container, { Service } from 'typedi'; +import { AccountsImportable } from '../Accounts/AccountsImportable'; +import { ImportableRegistry } from './ImportableRegistry'; +import { UncategorizedTransactionsImportable } from '../Cashflow/UncategorizedTransactionsImportable'; +import { CustomersImportable } from '../Contacts/Customers/CustomersImportable'; +import { VendorsImportable } from '../Contacts/Vendors/VendorsImportable'; +import { ItemsImportable } from '../Items/ItemsImportable'; +import { ItemCategoriesImportable } from '../ItemCategories/ItemCategoriesImportable'; +import { ManualJournalImportable } from '../ManualJournals/ManualJournalsImport'; +import { BillsImportable } from '../Purchases/Bills/BillsImportable'; +import { ExpensesImportable } from '../Expenses/ExpensesImportable'; +import { SaleInvoicesImportable } from '../Sales/Invoices/SaleInvoicesImportable'; +import { SaleEstimatesImportable } from '../Sales/Estimates/SaleEstimatesImportable'; +import { BillPaymentsImportable } from '../Purchases/BillPayments/BillPaymentsImportable'; +import { VendorCreditsImportable } from '../Purchases/VendorCredits/VendorCreditsImportable'; +import { PaymentsReceivedImportable } from '../Sales/PaymentReceived/PaymentsReceivedImportable'; +import { CreditNotesImportable } from '../CreditNotes/CreditNotesImportable'; +import { SaleReceiptsImportable } from '../Sales/Receipts/SaleReceiptsImportable'; +import { TaxRatesImportable } from '../TaxRates/TaxRatesImportable'; + +@Service() +export class ImportableResources { + private static registry: ImportableRegistry; + + constructor() { + this.boot(); + } + + /** + * Importable instances. + */ + private importables = [ + { resource: 'Account', importable: AccountsImportable }, + { + resource: 'UncategorizedCashflowTransaction', + importable: UncategorizedTransactionsImportable, + }, + { resource: 'Customer', importable: CustomersImportable }, + { resource: 'Vendor', importable: VendorsImportable }, + { resource: 'Item', importable: ItemsImportable }, + { resource: 'ItemCategory', importable: ItemCategoriesImportable }, + { resource: 'ManualJournal', importable: ManualJournalImportable }, + { resource: 'Bill', importable: BillsImportable }, + { resource: 'Expense', importable: ExpensesImportable }, + { resource: 'SaleInvoice', importable: SaleInvoicesImportable }, + { resource: 'SaleEstimate', importable: SaleEstimatesImportable }, + { resource: 'BillPayment', importable: BillPaymentsImportable }, + { resource: 'PaymentReceive', importable: PaymentsReceivedImportable }, + { resource: 'VendorCredit', importable: VendorCreditsImportable }, + { resource: 'CreditNote', importable: CreditNotesImportable }, + { resource: 'SaleReceipt', importable: SaleReceiptsImportable }, + { resource: 'TaxRate', importable: TaxRatesImportable }, + ]; + + public get registry() { + return ImportableResources.registry; + } + + /** + * Boots all the registered importables. + */ + public boot() { + if (!ImportableResources.registry) { + const instance = ImportableRegistry.getInstance(); + + this.importables.forEach((importable) => { + const importableInstance = Container.get(importable.importable); + instance.registerImportable(importable.resource, importableInstance); + }); + ImportableResources.registry = instance; + } + } +} diff --git a/packages/server-nest/src/modules/Import/_constants.ts b/packages/server-nest/src/modules/Import/_constants.ts new file mode 100644 index 000000000..1e9d37cbf --- /dev/null +++ b/packages/server-nest/src/modules/Import/_constants.ts @@ -0,0 +1,3 @@ + + +export const CurrencyParsingDTOs = 10; \ No newline at end of file diff --git a/packages/server-nest/src/modules/Import/_utils.ts b/packages/server-nest/src/modules/Import/_utils.ts new file mode 100644 index 000000000..c3127f110 --- /dev/null +++ b/packages/server-nest/src/modules/Import/_utils.ts @@ -0,0 +1,459 @@ +import * as Yup from 'yup'; +import moment from 'moment'; +import * as R from 'ramda'; +import { Knex } from 'knex'; +import fs from 'fs/promises'; +import path from 'path'; +import { + defaultTo, + upperFirst, + camelCase, + first, + isUndefined, + pickBy, + isEmpty, + castArray, + get, + head, + split, + last, +} from 'lodash'; +import pluralize from 'pluralize'; +import { ResourceMetaFieldsMap } from './interfaces'; +import { IModelMetaField, IModelMetaField2 } from '@/interfaces'; +import { ServiceError } from '@/exceptions'; +import { multiNumberParse } from '@/utils/multi-number-parse'; + +export const ERRORS = { + RESOURCE_NOT_IMPORTABLE: 'RESOURCE_NOT_IMPORTABLE', + INVALID_MAP_ATTRS: 'INVALID_MAP_ATTRS', + DUPLICATED_FROM_MAP_ATTR: 'DUPLICATED_FROM_MAP_ATTR', + DUPLICATED_TO_MAP_ATTR: 'DUPLICATED_TO_MAP_ATTR', + IMPORT_FILE_NOT_MAPPED: 'IMPORT_FILE_NOT_MAPPED', + INVALID_MAP_DATE_FORMAT: 'INVALID_MAP_DATE_FORMAT', + MAP_DATE_FORMAT_NOT_DEFINED: 'MAP_DATE_FORMAT_NOT_DEFINED', + IMPORTED_SHEET_EMPTY: 'IMPORTED_SHEET_EMPTY', +}; + +/** + * Trimms the imported object string values before parsing. + * @param {Record} obj + * @returns {} + */ +export function trimObject(obj: Record) { + return Object.entries(obj).reduce((acc, [key, value]) => { + // Trim the key + const trimmedKey = key.trim(); + + // Trim the value if it's a string, otherwise leave it as is + const trimmedValue = typeof value === 'string' ? value.trim() : value; + + // Assign the trimmed key and value to the accumulator object + return { ...acc, [trimmedKey]: trimmedValue }; + }, {}); +} + +/** + * Generates the Yup validation schema based on the given resource fields. + * @param {ResourceMetaFieldsMap} fields + * @returns {Yup} + */ +export const convertFieldsToYupValidation = (fields: ResourceMetaFieldsMap) => { + const yupSchema = {}; + + Object.keys(fields).forEach((fieldName: string) => { + const field = fields[fieldName] as IModelMetaField; + let fieldSchema; + fieldSchema = Yup.string().label(field.name); + + if (field.fieldType === 'text') { + if (!isUndefined(field.minLength)) { + fieldSchema = fieldSchema.min( + field.minLength, + `Minimum length is ${field.minLength} characters` + ); + } + if (!isUndefined(field.maxLength)) { + fieldSchema = fieldSchema.max( + field.maxLength, + `Maximum length is ${field.maxLength} characters` + ); + } + } else if (field.fieldType === 'number') { + fieldSchema = Yup.number().label(field.name); + + if (!isUndefined(field.max)) { + fieldSchema = fieldSchema.max(field.max); + } + if (!isUndefined(field.min)) { + fieldSchema = fieldSchema.min(field.min); + } + } else if (field.fieldType === 'boolean') { + fieldSchema = Yup.boolean().label(field.name); + } else if (field.fieldType === 'enumeration') { + const options = field.options.reduce((acc, option) => { + acc[option.key] = option.label; + return acc; + }, {}); + fieldSchema = Yup.string().oneOf(Object.keys(options)).label(field.name); + // Validate date field type. + } else if (field.fieldType === 'date') { + fieldSchema = fieldSchema.test( + 'date validation', + 'Invalid date or format. The string should be a valid YYYY-MM-DD format.', + (val) => { + if (!val) { + return true; + } + return moment(val, 'YYYY-MM-DD', true).isValid(); + } + ); + } else if (field.fieldType === 'url') { + fieldSchema = fieldSchema.url(); + } else if (field.fieldType === 'collection') { + const nestedFieldShema = convertFieldsToYupValidation(field.fields); + fieldSchema = Yup.array().label(field.name); + + if (!isUndefined(field.collectionMaxLength)) { + fieldSchema = fieldSchema.max(field.collectionMaxLength); + } + if (!isUndefined(field.collectionMinLength)) { + fieldSchema = fieldSchema.min(field.collectionMinLength); + } + fieldSchema = fieldSchema.of(nestedFieldShema); + } + if (field.required) { + fieldSchema = fieldSchema.required(); + } + const _fieldName = parseFieldName(fieldName, field); + + yupSchema[_fieldName] = fieldSchema; + }); + return Yup.object().shape(yupSchema); +}; + +const parseFieldName = (fieldName: string, field: IModelMetaField) => { + let _key = fieldName; + + if (field.dataTransferObjectKey) { + _key = field.dataTransferObjectKey; + } + return _key; +}; + +/** + * Retrieves the unmapped sheet columns. + * @param columns + * @param mapping + * @returns + */ +export const getUnmappedSheetColumns = (columns, mapping) => { + return columns.filter( + (column) => !mapping.some((map) => map.from === column) + ); +}; + +export const sanitizeResourceName = (resourceName: string) => { + return upperFirst(camelCase(pluralize.singular(resourceName))); +}; + +export const getSheetColumns = (sheetData: unknown[]) => { + return Object.keys(first(sheetData)); +}; + +/** + * Retrieves the unique value from the given imported object DTO based on the + * configured unique resource field. + * @param {{ [key: string]: IModelMetaField }} importableFields - + * @param {} + * @returns {string} + */ +export const getUniqueImportableValue = ( + importableFields: { [key: string]: IModelMetaField2 }, + objectDTO: Record +) => { + const uniqueImportableValue = pickBy( + importableFields, + (field) => field.unique + ); + const uniqueImportableKeys = Object.keys(uniqueImportableValue); + const uniqueImportableKey = first(uniqueImportableKeys); + + return defaultTo(objectDTO[uniqueImportableKey], ''); +}; + +/** + * Throws service error the given sheet is empty. + * @param {Array} sheetData + */ +export const validateSheetEmpty = (sheetData: Array) => { + if (isEmpty(sheetData)) { + throw new ServiceError(ERRORS.IMPORTED_SHEET_EMPTY); + } +}; + +const booleanValuesRepresentingTrue: string[] = ['true', 'yes', 'y', 't', '1']; +const booleanValuesRepresentingFalse: string[] = ['false', 'no', 'n', 'f', '0']; + +/** + * Parses the given string value to boolean. + * @param {string} value + * @returns {string|null} + */ +export const parseBoolean = (value: string): boolean | null => { + const normalizeValue = (value: string): string => + value.toString().trim().toLowerCase(); + + const normalizedValue = normalizeValue(value); + const valuesRepresentingTrue = + booleanValuesRepresentingTrue.map(normalizeValue); + const valueRepresentingFalse = + booleanValuesRepresentingFalse.map(normalizeValue); + + if (valuesRepresentingTrue.includes(normalizedValue)) { + return true; + } else if (valueRepresentingFalse.includes(normalizedValue)) { + return false; + } + return null; +}; + +export const transformInputToGroupedFields = (input) => { + const output = []; + + // Group for non-nested fields + const mainGroup = { + groupLabel: '', + groupKey: '', + fields: [], + }; + input.forEach((item) => { + if (!item.fields) { + // If the item does not have nested fields, add it to the main group + mainGroup.fields.push(item); + } else { + // If the item has nested fields, create a new group for these fields + output.push({ + groupLabel: item.name, + groupKey: item.key, + fields: item.fields, + }); + } + }); + // Add the main group to the output if it contains any fields + if (mainGroup.fields.length > 0) { + output.unshift(mainGroup); // Add the main group at the beginning + } + return output; +}; + +export const getResourceColumns = (resourceColumns: { + [key: string]: IModelMetaField2; +}) => { + const mapColumn = + (group: string) => + ([fieldKey, { name, importHint, required, order, ...field }]: [ + string, + IModelMetaField2 + ]) => { + const extra: Record = {}; + const key = fieldKey; + + if (group) { + extra.group = group; + } + if (field.fieldType === 'collection') { + extra.fields = mapColumns(field.fields, key); + } + return { + key, + name, + required, + hint: importHint, + order, + ...extra, + }; + }; + const sortColumn = (a, b) => + a.order && b.order ? a.order - b.order : a.order ? -1 : b.order ? 1 : 0; + + const mapColumns = (columns, parentKey = '') => + Object.entries(columns).map(mapColumn(parentKey)).sort(sortColumn); + + return R.compose(transformInputToGroupedFields, mapColumns)(resourceColumns); +}; + +// Prases the given object value based on the field key type. +export const valueParser = + (fields: ResourceMetaFieldsMap, tenantModels: any, trx?: Knex.Transaction) => + async (value: any, key: string, group = '') => { + let _value = value; + + const fieldKey = key.includes('.') ? key.split('.')[0] : key; + const field = group ? fields[group]?.fields[fieldKey] : fields[fieldKey]; + + // Parses the boolean value. + if (field.fieldType === 'boolean') { + _value = parseBoolean(value); + + // Parses the enumeration value. + } else if (field.fieldType === 'enumeration') { + const option = get(field, 'options', []).find( + (option) => option.label?.toLowerCase() === value?.toLowerCase() + ); + _value = get(option, 'key'); + // Parses the numeric value. + } else if (field.fieldType === 'number') { + _value = multiNumberParse(value); + // Parses the relation value. + } else if (field.fieldType === 'relation') { + const RelationModel = tenantModels[field.relationModel]; + + if (!RelationModel) { + throw new Error(`The relation model of ${key} field is not exist.`); + } + const relationQuery = RelationModel.query(trx); + const relationKeys = castArray(field?.relationImportMatch); + + relationQuery.where(function () { + relationKeys.forEach((relationKey: string) => { + this.orWhereRaw('LOWER(??) = LOWER(?)', [relationKey, value]); + }); + }); + const result = await relationQuery.first(); + _value = get(result, 'id'); + } else if (field.fieldType === 'collection') { + const ObjectFieldKey = key.includes('.') ? key.split('.')[1] : key; + const _valueParser = valueParser(fields, tenantModels); + _value = await _valueParser(value, ObjectFieldKey, fieldKey); + } + return _value; + }; + +/** + * Parses the field key and detarmines the key path. + * @param {{ [key: string]: IModelMetaField2 }} fields + * @param {string} key - Mapped key path. formats: `group.key` or `key`. + * @returns {string} + */ +export const parseKey = R.curry( + (fields: { [key: string]: IModelMetaField2 }, key: string) => { + const fieldKey = getFieldKey(key); + const field = fields[fieldKey]; + let _key = key; + + if (field.fieldType === 'collection') { + if (field.collectionOf === 'object') { + const nestedFieldKey = last(key.split('.')); + _key = `${fieldKey}[0].${nestedFieldKey}`; + } else if ( + field.collectionOf === 'string' || + field.collectionOf || + 'numberic' + ) { + _key = `${fieldKey}`; + } + } + return _key; + } +); + +/** + * Retrieves the field root key, for instance: I -> entries.itemId O -> entries. + * @param {string} input + * @returns {string} + */ +export const getFieldKey = (input: string) => { + const keys = split(input, '.'); + const firstKey = head(keys).split('[')[0]; // Split by "[" in case of array notation + return firstKey; +}; + +/** +{ * Aggregates the input array of objects based on a comparator attribute and groups the entries. + * This function is useful for combining multiple entries into a single entry based on a specific attribute, + * while aggregating other attributes into an array.} + * + * @param {Array} input - The array of objects to be aggregated. + * @param {string} comparatorAttr - The attribute of the objects used for comparison to aggregate. + * @param {string} groupOn - The attribute of the objects where the grouped entries will be pushed. + * @returns {Array} - The aggregated array of objects. + * + * @example + * // Example input: + * const input = [ + * { id: 1, name: 'John', entries: ['entry1'] }, + * { id: 2, name: 'Jane', entries: ['entry2'] }, + * { id: 1, name: 'John', entries: ['entry3'] }, + * ]; + * const comparatorAttr = 'id'; + * const groupOn = 'entries'; + * + * // Example output: + * const output = [ + * { id: 1, name: 'John', entries: ['entry1', 'entry3'] }, + * { id: 2, name: 'Jane', entries: ['entry2'] }, + * ]; + */ +export function aggregate( + input: Array, + comparatorAttr: string, + groupOn: string +): Array> { + return input.reduce((acc, curr) => { + const existingEntry = acc.find( + (entry) => entry[comparatorAttr] === curr[comparatorAttr] + ); + + if (existingEntry) { + existingEntry[groupOn].push(...curr.entries); + } else { + acc.push({ ...curr }); + } + return acc; + }, []); +} + +/** + * Sanitizes the data in the imported sheet by trimming object keys. + * @param json - The JSON data representing the imported sheet. + * @returns {string[][]} - The sanitized data with trimmed object keys. + */ +export const sanitizeSheetData = (json) => { + return R.compose(R.map(trimObject))(json); +}; + +/** + * Returns the path to map a value to based on the 'to' and 'group' parameters. + * @param {string} to - The target key to map the value to. + * @param {string} group - The group key to nest the target key under. + * @returns {string} - The path to map the value to. + */ +export const getMapToPath = (to: string, group = '') => + group ? `${group}.${to}` : to; + +export const getImportsStoragePath = () => { + return path.join(global.__storage_dir, `/imports`); +}; + +/** + * Deletes the imported file from the storage and database. + * @param {string} filename + */ +export const deleteImportFile = async (filename: string) => { + const filePath = getImportsStoragePath(); + + // Deletes the imported file. + await fs.unlink(`${filePath}/${filename}`); +}; + +/** + * Reads the import file. + * @param {string} filename + * @returns {Promise} + */ +export const readImportFile = (filename: string) => { + const filePath = getImportsStoragePath(); + + return fs.readFile(`${filePath}/${filename}`); +}; diff --git a/packages/server-nest/src/modules/Import/interfaces.ts b/packages/server-nest/src/modules/Import/interfaces.ts new file mode 100644 index 000000000..65d53ff59 --- /dev/null +++ b/packages/server-nest/src/modules/Import/interfaces.ts @@ -0,0 +1,77 @@ +import { IModelMetaField, IModelMetaField2 } from '@/interfaces'; +import Import from '@/models/Import'; + +export interface ImportMappingAttr { + from: string; + to: string; + group?: string; + dateFormat?: string; +} + +export interface ImportValidationError { + index: number; + property: string; + constraints: Record; +} + +export type ResourceMetaFieldsMap = { [key: string]: IModelMetaField2 }; + +export interface ImportInsertError { + rowNumber: number; + errorCode: string; + errorMessage: string; +} + +export interface ImportFileUploadPOJO { + import: { + importId: string; + resource: string; + }; + sheetColumns: string[]; + resourceColumns: { + key: string; + name: string; + required?: boolean; + hint?: string; + }[]; +} + +export interface ImportFileMapPOJO { + import: { + importId: string; + resource: string; + }; +} + +export interface ImportFilePreviewPOJO { + resource: string; + createdCount: number; + skippedCount: number; + totalCount: number; + errorsCount: number; + errors: ImportInsertError[]; + unmappedColumns: string[]; + unmappedColumnsCount: number; +} + +export interface ImportOperSuccess { + data: unknown; + index: number; +} + +export interface ImportOperError { + error: ImportInsertError[]; + index: number; +} + +export interface ImportableContext { + import: Import; + rowIndex: number; +} + +export const ImportDateFormats = [ + 'yyyy-MM-dd', + 'dd.MM.yy', + 'MM/dd/yy', + 'dd/MMM/yyyy', +]; diff --git a/packages/server-nest/src/modules/Import/jobs/ImportDeleteExpiredFilesJob.ts b/packages/server-nest/src/modules/Import/jobs/ImportDeleteExpiredFilesJob.ts new file mode 100644 index 000000000..74ce6a7c8 --- /dev/null +++ b/packages/server-nest/src/modules/Import/jobs/ImportDeleteExpiredFilesJob.ts @@ -0,0 +1,28 @@ +import Container, { Service } from 'typedi'; +import { ImportDeleteExpiredFiles } from '../ImportRemoveExpiredFiles'; + +@Service() +export class ImportDeleteExpiredFilesJobs { + /** + * Constructor method. + */ + constructor(agenda) { + agenda.define('delete-expired-imported-files', this.handler); + } + + /** + * Triggers sending invoice mail. + */ + private handler = async (job, done: Function) => { + const importDeleteExpiredFiles = Container.get(ImportDeleteExpiredFiles); + + try { + console.log('Delete expired import files has started.'); + await importDeleteExpiredFiles.deleteExpiredFiles(); + done(); + } catch (error) { + console.log(error); + done(error); + } + }; +} diff --git a/packages/server-nest/src/modules/Import/sheet_utils.ts b/packages/server-nest/src/modules/Import/sheet_utils.ts new file mode 100644 index 000000000..b21f07320 --- /dev/null +++ b/packages/server-nest/src/modules/Import/sheet_utils.ts @@ -0,0 +1,56 @@ +import XLSX from 'xlsx'; +import { first } from 'lodash'; + +/** + * Parses the given sheet buffer to worksheet. + * @param {Buffer} buffer + * @returns {XLSX.WorkSheet} + */ +export function parseFirstSheet(buffer: Buffer): XLSX.WorkSheet { + const workbook = XLSX.read(buffer, { type: 'buffer', raw: true }); + + const firstSheetName = workbook.SheetNames[0]; + const worksheet = workbook.Sheets[firstSheetName]; + + return worksheet; +} + +/** + * Extracts the given worksheet to columns. + * @param {XLSX.WorkSheet} worksheet + * @returns {Array} + */ +export function extractSheetColumns(worksheet: XLSX.WorkSheet): Array { + // By default, sheet_to_json scans the first row and uses the values as headers. + // With the header: 1 option, the function exports an array of arrays of values. + const sheetCells = XLSX.utils.sheet_to_json(worksheet, { header: 1 }); + const sheetCols = first(sheetCells) as Array; + + return sheetCols.filter((col) => col); +} + +/** + * Parses the given worksheet to json values. the keys are columns labels. + * @param {XLSX.WorkSheet} worksheet + * @returns {Array>} + */ +export function parseSheetToJson( + worksheet: XLSX.WorkSheet +): Array> { + return XLSX.utils.sheet_to_json(worksheet, {}); +} + +/** + * Parses the given sheet buffer then retrieves the sheet data and columns. + * @param {Buffer} buffer + */ +export function parseSheetData( + buffer: Buffer +): [Array>, string[]] { + const worksheet = parseFirstSheet(buffer); + + const columns = extractSheetColumns(worksheet); + const data = parseSheetToJson(worksheet); + + return [data, columns]; +} diff --git a/packages/server-nest/src/modules/Views/GetResourceColumns.service.ts b/packages/server-nest/src/modules/Views/GetResourceColumns.service.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/server-nest/src/modules/Views/GetResourceViews.service.ts b/packages/server-nest/src/modules/Views/GetResourceViews.service.ts new file mode 100644 index 000000000..d3827fb40 --- /dev/null +++ b/packages/server-nest/src/modules/Views/GetResourceViews.service.ts @@ -0,0 +1,23 @@ +import { Injectable } from '@nestjs/common'; +import ResourceService from '@/services/Resource/ResourceService'; +import { BaseModel } from '@/models/Model'; +import { View } from './models/View.model'; + +@Injectable() +export class GetResourceViewsService { + constructor(private readonly resourceService: ResourceService) {} + /** + * Listing resource views. + * @param {number} tenantId - + * @param {string} resourceModel - + */ + public async getResourceViews(resourceName: string): Promise { + // Validate the resource model name is valid. + const resourceModel = this.resourceService.getResourceModel(resourceName); + + // Default views. + const defaultViews = resourceModel.getDefaultViews(); + + return defaultViews; + } +} diff --git a/packages/server-nest/src/modules/Views/Views.types.ts b/packages/server-nest/src/modules/Views/Views.types.ts new file mode 100644 index 000000000..857f45808 --- /dev/null +++ b/packages/server-nest/src/modules/Views/Views.types.ts @@ -0,0 +1,61 @@ + +export interface IView { + id: number, + name: string, + slug: string; + predefined: boolean, + resourceModel: string, + favourite: boolean, + rolesLogicExpression: string, + + roles: IViewRole[], + columns: IViewHasColumn[], +}; + +export interface IViewRole { + id: number, + fieldKey: string, + index: number, + comparator: string, + value: string, + viewId: number, +}; + +export interface IViewHasColumn { + id :number, + viewId: number, + fieldId: number, + index: number, +} + +export interface IViewRoleDTO { + index: number, + fieldKey: string, + comparator: string, + value: string, + viewId: number, +} + +export interface IViewColumnDTO { + id: number, + index: number, + viewId: number, + fieldKey: string, +}; + +export interface IViewDTO { + name: string, + logicExpression: string, + resourceModel: string, + + roles: IViewRoleDTO[], + columns: IViewColumnDTO[], +}; + +export interface IViewEditDTO { + name: string, + logicExpression: string, + + roles: IViewRoleDTO[], + columns: IViewColumnDTO[], +}; diff --git a/packages/server-nest/src/modules/Views/models/View.model.ts b/packages/server-nest/src/modules/Views/models/View.model.ts new file mode 100644 index 000000000..9749b1b2c --- /dev/null +++ b/packages/server-nest/src/modules/Views/models/View.model.ts @@ -0,0 +1,72 @@ +import { Model } from 'objection'; +import { BaseModel } from '@/models/Model'; + +export class View extends BaseModel { + /** + * Table name. + */ + static get tableName() { + return 'views'; + } + + /** + * Model timestamps. + */ + get timestamps() { + return ['createdAt', 'updatedAt']; + } + + static get modifiers() { + const TABLE_NAME = View.tableName; + + return { + allMetadata(query) { + query.withGraphFetched('roles.field'); + query.withGraphFetched('columns'); + }, + + specificOrFavourite(query, viewId) { + if (viewId) { + query.where('id', viewId); + } else { + query.where('favourite', true); + } + return query; + }, + }; + } + + /** + * Relationship mapping. + */ + static get relationMappings() { + const { ViewColumn } = require('./ViewColumn.model'); + const { ViewRole } = require('./ViewRole.model'); + + return { + /** + * View model may has many columns. + */ + columns: { + relation: Model.HasManyRelation, + modelClass: ViewColumn.default, + join: { + from: 'views.id', + to: 'view_has_columns.viewId', + }, + }, + + /** + * View model may has many view roles. + */ + roles: { + relation: Model.HasManyRelation, + modelClass: ViewRole.default, + join: { + from: 'views.id', + to: 'view_roles.viewId', + }, + }, + }; + } +} diff --git a/packages/server-nest/src/modules/Views/models/ViewColumn.model.ts b/packages/server-nest/src/modules/Views/models/ViewColumn.model.ts new file mode 100644 index 000000000..7a7b5c1a0 --- /dev/null +++ b/packages/server-nest/src/modules/Views/models/ViewColumn.model.ts @@ -0,0 +1,17 @@ +import { BaseModel } from '@/models/Model'; + +export class ViewColumn extends BaseModel { + /** + * Table name. + */ + static get tableName() { + return 'view_has_columns'; + } + + /** + * Relationship mapping. + */ + static get relationMappings() { + return {}; + } +} diff --git a/packages/server-nest/src/modules/Views/models/ViewRole.model.ts b/packages/server-nest/src/modules/Views/models/ViewRole.model.ts new file mode 100644 index 000000000..f9531d0e8 --- /dev/null +++ b/packages/server-nest/src/modules/Views/models/ViewRole.model.ts @@ -0,0 +1,46 @@ +import { BaseModel } from '@/models/Model'; +import { Model } from 'objection'; + +export class ViewRole extends BaseModel { + + /** + * Virtual attributes. + */ + static get virtualAttributes() { + return ['comparators']; + } + + static get comparators() { + return [ + 'equals', 'not_equal', 'contains', 'not_contain', + ]; + } + + /** + * Table name. + */ + static get tableName() { + return 'view_roles'; + } + + /** + * Relationship mapping. + */ + static get relationMappings() { + const View = require('./View.model'); + + return { + /** + * View role model may belongs to view model. + */ + view: { + relation: Model.BelongsToOneRelation, + modelClass: View.default, + join: { + from: 'view_roles.viewId', + to: 'views.id', + }, + }, + }; + } +} diff --git a/packages/server-nest/src/utils/assoc-depth-level-to-object-tree.ts b/packages/server-nest/src/utils/assoc-depth-level-to-object-tree.ts new file mode 100644 index 000000000..88a6919ca --- /dev/null +++ b/packages/server-nest/src/utils/assoc-depth-level-to-object-tree.ts @@ -0,0 +1,16 @@ + +export const assocDepthLevelToObjectTree = ( + objects, + level = 1, + propertyName = 'level' +) => { + for (let i = 0; i < objects.length; i++) { + const object = objects[i]; + object[propertyName] = level; + + if (object.children) { + assocDepthLevelToObjectTree(object.children, level + 1, propertyName); + } + } + return objects; +}; \ No newline at end of file diff --git a/packages/server-nest/src/utils/flat-to-nested-array.ts b/packages/server-nest/src/utils/flat-to-nested-array.ts new file mode 100644 index 000000000..5cdaf3d7f --- /dev/null +++ b/packages/server-nest/src/utils/flat-to-nested-array.ts @@ -0,0 +1,24 @@ +export const flatToNestedArray = ( + data, + config = { id: 'id', parentId: 'parent_id' }, +) => { + const map = {}; + const nestedArray = []; + + data.forEach((item) => { + map[item[config.id]] = item; + map[item[config.id]].children = []; + }); + + data.forEach((item) => { + const parentItemId = item[config.parentId]; + + if (!item[config.parentId]) { + nestedArray.push(item); + } + if (parentItemId) { + map[parentItemId].children.push(item); + } + }); + return nestedArray; +}; diff --git a/packages/server-nest/src/utils/format-number.ts b/packages/server-nest/src/utils/format-number.ts index 539da998e..00767a827 100644 --- a/packages/server-nest/src/utils/format-number.ts +++ b/packages/server-nest/src/utils/format-number.ts @@ -1,6 +1,6 @@ -import _ from 'lodash'; -import accounting from 'accounting'; -import Currencies from 'js-money/lib/currency'; +import { get } from 'lodash'; +import * as accounting from 'accounting'; +import * as Currencies from 'js-money/lib/currency'; const getNegativeFormat = (formatName) => { switch (formatName) { @@ -12,7 +12,7 @@ const getNegativeFormat = (formatName) => { }; const getCurrencySign = (currencyCode) => { - return _.get(Currencies, `${currencyCode}.symbol`); + return get(Currencies, `${currencyCode}.symbol`); }; export const formatNumber = ( diff --git a/packages/server-nest/src/utils/nested-array-to-flatten.ts b/packages/server-nest/src/utils/nested-array-to-flatten.ts new file mode 100644 index 000000000..d1b3cfb8d --- /dev/null +++ b/packages/server-nest/src/utils/nested-array-to-flatten.ts @@ -0,0 +1,33 @@ +import { omit, concat } from 'lodash'; + +export const nestedArrayToFlatten = ( + collection, + property = 'children', + parseItem = (a, level) => a, + level = 1, +) => { + const parseObject = (obj) => + parseItem( + { + ...omit(obj, [property]), + }, + level, + ); + + return collection.reduce((items, currentValue, index) => { + let localItems = [...items]; + const parsedItem = parseObject(currentValue); + localItems.push(parsedItem); + + if (Array.isArray(currentValue[property])) { + const flattenArray = nestedArrayToFlatten( + currentValue[property], + property, + parseItem, + level + 1, + ); + localItems = concat(localItems, flattenArray); + } + return localItems; + }, []); +}; diff --git a/packages/server-nest/tsconfig.build.json b/packages/server-nest/tsconfig.build.json index 64f86c6bd..760558e7a 100644 --- a/packages/server-nest/tsconfig.build.json +++ b/packages/server-nest/tsconfig.build.json @@ -1,4 +1,15 @@ { "extends": "./tsconfig.json", - "exclude": ["node_modules", "test", "dist", "**/*spec.ts"] + "exclude": [ + "node_modules", + "test", + "dist", + "**/*spec.ts", + // "./src/modules/DynamicListing/**/*.ts", + "./src/modules/Export", + "./src/modules/Import", + "./src/modules/DynamicListing", + // "./src/modules/DynamicListing", + "./src/modules/Views" + ] }