From 5bfff51093eb931f9e21464f4f42ebce19b9bd7d Mon Sep 17 00:00:00 2001 From: Ahmed Bouhuolia Date: Sat, 12 Apr 2025 08:38:29 +0200 Subject: [PATCH 1/2] refactor(nestjs): import module --- .../src/modules/Import/Import.controller.ts | 126 ++++++++++++++++++ .../src/modules/Import/Import.module.ts | 6 +- .../src/modules/Import/ImportFileCommon.ts | 33 +++-- .../Import/ImportFileDataTransformer.ts | 4 +- .../modules/Import/ImportFileDataValidator.ts | 3 +- .../src/modules/Import/ImportFileMeta.ts | 2 +- .../src/modules/Import/ImportMulter.utils.ts | 39 ++++++ .../Import/ImportRemoveExpiredFiles.ts | 2 +- .../server/src/modules/Import/ImportSample.ts | 19 +-- .../src/modules/Import/ImportableRegistry.ts | 45 ++----- .../src/modules/Import/ImportableResources.ts | 73 ---------- .../Import/decorators/Import.decorator.ts | 37 +++++ .../ItemCategoriesImportable.ts | 3 + .../models/ItemCategory.model.ts | 2 + .../server/src/modules/Items/Items.module.ts | 4 +- .../modules/Items/ItemsImportable.service.ts | 3 + .../server/src/modules/Items/models/Item.ts | 2 + packages/server/src/utils/parse-json.ts | 7 + 18 files changed, 271 insertions(+), 139 deletions(-) create mode 100644 packages/server/src/modules/Import/Import.controller.ts create mode 100644 packages/server/src/modules/Import/ImportMulter.utils.ts delete mode 100644 packages/server/src/modules/Import/ImportableResources.ts create mode 100644 packages/server/src/modules/Import/decorators/Import.decorator.ts create mode 100644 packages/server/src/utils/parse-json.ts diff --git a/packages/server/src/modules/Import/Import.controller.ts b/packages/server/src/modules/Import/Import.controller.ts new file mode 100644 index 000000000..d1d4a74c7 --- /dev/null +++ b/packages/server/src/modules/Import/Import.controller.ts @@ -0,0 +1,126 @@ +import { Response, NextFunction } from 'express'; +import { FileInterceptor } from '@nestjs/platform-express'; +import { defaultTo } from 'lodash'; +import { + Controller, + Post, + Get, + Body, + Param, + Query, + Res, + Next, + UseInterceptors, + UploadedFile, +} from '@nestjs/common'; +import { ApiTags, ApiOperation, ApiResponse } from '@nestjs/swagger'; +import { ImportResourceApplication } from './ImportResourceApplication'; +import { uploadImportFileMulterOptions } from './ImportMulter.utils'; +import { parseJsonSafe } from '@/utils/parse-json'; + +@Controller('import') +@ApiTags('import') +export class ImportController { + constructor(private readonly importResourceApp: ImportResourceApplication) {} + + /** + * Imports xlsx/csv to the given resource type. + */ + @Post('/file') + @ApiOperation({ summary: 'Upload import file' }) + @ApiResponse({ status: 200, description: 'File uploaded successfully' }) + @UseInterceptors( + FileInterceptor('file', { storage: uploadImportFileMulterOptions }), + ) + async fileUpload( + @Res() res: Response, + @Next() next: NextFunction, + @UploadedFile() file: Express.Multer.File, + @Body('resource') resource: string, + @Body('params') rawParams?: string, + ) { + const params = defaultTo(parseJsonSafe(rawParams), {}); + + try { + const data = await this.importResourceApp.import( + resource, + file.filename, + params, + ); + return res.status(200).send(data); + } catch (error) { + next(error); + } + } + + /** + * Maps the columns of the imported file. + */ + @Post('/:import_id/mapping') + @ApiOperation({ summary: 'Map import columns' }) + @ApiResponse({ status: 200, description: 'Mapping successful' }) + async mapping( + @Res() res: Response, + @Param('import_id') importId: string, + @Body('mapping') + mapping: Array<{ group?: string; from: string; to: string }>, + ) { + const result = await this.importResourceApp.mapping(importId, mapping); + + return res.status(200).send(result); + } + + /** + * Preview the imported file before actual importing. + */ + @Get('/:import_id/preview') + @ApiOperation({ summary: 'Preview import data' }) + @ApiResponse({ status: 200, description: 'Preview data' }) + async preview(@Res() res: Response, @Param('import_id') importId: string) { + const preview = await this.importResourceApp.preview(importId); + + return res.status(200).send(preview); + } + + /** + * Importing the imported file to the application storage. + */ + @Post('/:import_id/import') + @ApiOperation({ summary: 'Process import' }) + @ApiResponse({ status: 200, description: 'Import processed successfully' }) + async import(@Res() res: Response, @Param('import_id') importId: string) { + const result = await this.importResourceApp.process(importId); + + return res.status(200).send(result); + } + + /** + * Retrieves the csv/xlsx sample sheet of the given resource name. + */ + @Get('/sample') + @ApiOperation({ summary: 'Get import sample' }) + @ApiResponse({ status: 200, description: 'Sample data' }) + async downloadImportSample( + @Res() res: Response, + @Query('resource') resource: string, + @Query('format') format?: 'csv' | 'xlsx', + ) { + const result = await this.importResourceApp.sample(resource, format); + + return res.status(200).send(result); + } + + /** + * Retrieves the import file meta. + */ + @Get('/:import_id') + @ApiOperation({ summary: 'Get import metadata' }) + @ApiResponse({ status: 200, description: 'Import metadata' }) + async getImportFileMeta( + @Res() res: Response, + @Param('import_id') importId: string, + ) { + const result = await this.importResourceApp.importMeta(importId); + return res.status(200).send(result); + } +} diff --git a/packages/server/src/modules/Import/Import.module.ts b/packages/server/src/modules/Import/Import.module.ts index 032e34508..4b37a9974 100644 --- a/packages/server/src/modules/Import/Import.module.ts +++ b/packages/server/src/modules/Import/Import.module.ts @@ -12,17 +12,17 @@ import { ImportFileMapping } from './ImportFileMapping'; import { ImportFileDataValidator } from './ImportFileDataValidator'; import { ImportFileDataTransformer } from './ImportFileDataTransformer'; import { ImportFileCommon } from './ImportFileCommon'; -import { ImportableResources } from './ImportableResources'; import { ResourceModule } from '../Resource/Resource.module'; import { TenancyModule } from '../Tenancy/Tenancy.module'; import { AccountsModule } from '../Accounts/Accounts.module'; +import { ImportController } from './Import.controller'; +import { ImportableRegistry } from './ImportableRegistry'; @Module({ imports: [ResourceModule, TenancyModule, AccountsModule], providers: [ ImportAls, ImportSampleService, - ImportableResources, ImportResourceApplication, ImportDeleteExpiredFiles, ImportFileUploadService, @@ -34,7 +34,9 @@ import { AccountsModule } from '../Accounts/Accounts.module'; ImportFileDataValidator, ImportFileDataTransformer, ImportFileCommon, + ImportableRegistry ], + controllers: [ImportController], exports: [ImportAls], }) export class ImportModule {} diff --git a/packages/server/src/modules/Import/ImportFileCommon.ts b/packages/server/src/modules/Import/ImportFileCommon.ts index 7423ab771..f1c968302 100644 --- a/packages/server/src/modules/Import/ImportFileCommon.ts +++ b/packages/server/src/modules/Import/ImportFileCommon.ts @@ -10,18 +10,18 @@ import { ImportableContext, } from './interfaces'; import { getUniqueImportableValue, trimObject } from './_utils'; -import { ImportableResources } from './ImportableResources'; import { ResourceService } from '../Resource/ResourceService'; import { Injectable } from '@nestjs/common'; import { ServiceError } from '../Items/ServiceError'; import { ImportModelShape } from './models/Import'; +import { ImportableRegistry } from './ImportableRegistry'; @Injectable() export class ImportFileCommon { constructor( private readonly importFileValidator: ImportFileDataValidator, - private readonly importable: ImportableResources, private readonly resource: ResourceService, + private readonly importableRegistry: ImportableRegistry, ) {} /** @@ -39,9 +39,9 @@ export class ImportFileCommon { const resourceFields = this.resource.getResourceFields2( importFile.resource, ); - const ImportableRegistry = this.importable.registry; - const importable = ImportableRegistry.getImportable(importFile.resource); - + const importable = await this.importableRegistry.getImportable( + importFile.resource, + ); const concurrency = importable.concurrency || 10; const success: ImportOperSuccess[] = []; @@ -67,10 +67,7 @@ export class ImportFileCommon { ); try { // Run the importable function and listen to the errors. - const data = await importable.importable( - transformedDTO, - trx, - ); + const data = await importable.importable(transformedDTO, trx); success.push({ index, data }); } catch (err) { if (err instanceof ServiceError) { @@ -112,9 +109,8 @@ export class ImportFileCommon { resourceName: string, params: Record, ) { - const ImportableRegistry = this.importable.registry; - const importable = ImportableRegistry.getImportable(resourceName); - + const importable = + await this.importableRegistry.getImportable(resourceName); const yupSchema = importable.paramsValidationSchema(); try { @@ -137,8 +133,8 @@ export class ImportFileCommon { resourceName: string, params: Record, ) { - const ImportableRegistry = this.importable.registry; - const importable = ImportableRegistry.getImportable(resourceName); + const importable = + await this.importableRegistry.getImportable(resourceName); await importable.validateParams(params); } @@ -149,9 +145,12 @@ export class ImportFileCommon { * @param {Record} params * @returns */ - public transformParams(resourceName: string, params: Record) { - const ImportableRegistry = this.importable.registry; - const importable = ImportableRegistry.getImportable(resourceName); + public async transformParams( + resourceName: string, + params: Record, + ) { + const importable = + await this.importableRegistry.getImportable(resourceName); return importable.transformParams(params); } diff --git a/packages/server/src/modules/Import/ImportFileDataTransformer.ts b/packages/server/src/modules/Import/ImportFileDataTransformer.ts index fe8505c22..005698ee6 100644 --- a/packages/server/src/modules/Import/ImportFileDataTransformer.ts +++ b/packages/server/src/modules/Import/ImportFileDataTransformer.ts @@ -1,6 +1,7 @@ -import bluebird from 'bluebird'; +import * as bluebird from 'bluebird'; import { isUndefined, pickBy, set } from 'lodash'; import { Knex } from 'knex'; +import { Injectable } from '@nestjs/common'; import { ImportMappingAttr, ResourceMetaFieldsMap } from './interfaces'; import { valueParser, @@ -12,7 +13,6 @@ import { } from './_utils'; import { ResourceService } from '../Resource/ResourceService'; import { CurrencyParsingDTOs } from './_constants'; -import { Injectable } from '@nestjs/common'; @Injectable() export class ImportFileDataTransformer { diff --git a/packages/server/src/modules/Import/ImportFileDataValidator.ts b/packages/server/src/modules/Import/ImportFileDataValidator.ts index 172a9076b..2df188acc 100644 --- a/packages/server/src/modules/Import/ImportFileDataValidator.ts +++ b/packages/server/src/modules/Import/ImportFileDataValidator.ts @@ -1,7 +1,6 @@ - +import { Injectable } from '@nestjs/common'; import { ImportInsertError, ResourceMetaFieldsMap } from './interfaces'; import { ERRORS, convertFieldsToYupValidation } from './_utils'; -import { Injectable } from '@nestjs/common'; import { IModelMeta } from '@/interfaces/Model'; import { ServiceError } from '../Items/ServiceError'; diff --git a/packages/server/src/modules/Import/ImportFileMeta.ts b/packages/server/src/modules/Import/ImportFileMeta.ts index e92cac775..2a9f10903 100644 --- a/packages/server/src/modules/Import/ImportFileMeta.ts +++ b/packages/server/src/modules/Import/ImportFileMeta.ts @@ -1,6 +1,6 @@ +import { Inject, Injectable } from '@nestjs/common'; import { ImportModel } from './models/Import'; import { ImportFileMetaTransformer } from './ImportFileMetaTransformer'; -import { Inject, Injectable } from '@nestjs/common'; import { TransformerInjectable } from '../Transformer/TransformerInjectable.service'; import { TenancyContext } from '../Tenancy/TenancyContext.service'; diff --git a/packages/server/src/modules/Import/ImportMulter.utils.ts b/packages/server/src/modules/Import/ImportMulter.utils.ts new file mode 100644 index 000000000..4c07658a4 --- /dev/null +++ b/packages/server/src/modules/Import/ImportMulter.utils.ts @@ -0,0 +1,39 @@ +import * as Multer from 'multer'; +import * as path from 'path'; +import { ServiceError } from '../Items/ServiceError'; + +export const getImportsStoragePath = () => { + return path.join(global.__static_dirname, `/imports`); +}; + + +export function allowSheetExtensions(req, file, cb) { + if ( + file.mimetype !== 'text/csv' && + file.mimetype !== 'application/vnd.ms-excel' && + file.mimetype !== + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + ) { + cb(new ServiceError('IMPORTED_FILE_EXTENSION_INVALID')); + return; + } + cb(null, true); +} + +const storage = Multer.diskStorage({ + destination: function (req, file, cb) { + const path = getImportsStoragePath(); + cb(null, path); + }, + filename: function (req, file, cb) { + // Add the creation timestamp to clean up temp files later. + const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1e9); + cb(null, uniqueSuffix); + }, +}); + +export const uploadImportFileMulterOptions = { + storage, + limits: { fileSize: 5 * 1024 * 1024 }, + fileFilter: allowSheetExtensions, +}; \ No newline at end of file diff --git a/packages/server/src/modules/Import/ImportRemoveExpiredFiles.ts b/packages/server/src/modules/Import/ImportRemoveExpiredFiles.ts index bce8d926e..1c479745f 100644 --- a/packages/server/src/modules/Import/ImportRemoveExpiredFiles.ts +++ b/packages/server/src/modules/Import/ImportRemoveExpiredFiles.ts @@ -1,5 +1,5 @@ import * as moment from 'moment'; -import bluebird from 'bluebird'; +import * as bluebird from 'bluebird'; import { deleteImportFile } from './_utils'; import { Inject, Injectable } from '@nestjs/common'; import { ImportModel } from './models/Import'; diff --git a/packages/server/src/modules/Import/ImportSample.ts b/packages/server/src/modules/Import/ImportSample.ts index 7e0ca9e8e..13e8c8280 100644 --- a/packages/server/src/modules/Import/ImportSample.ts +++ b/packages/server/src/modules/Import/ImportSample.ts @@ -1,28 +1,28 @@ -import XLSX from 'xlsx'; -import { ImportableResources } from './ImportableResources'; +import * as XLSX from 'xlsx'; import { sanitizeResourceName } from './_utils'; import { Injectable } from '@nestjs/common'; +import { getImportableService } from './decorators/Import.decorator'; +import { ImportableRegistry } from './ImportableRegistry'; @Injectable() export class ImportSampleService { constructor( - private readonly importable: ImportableResources, - ) {} + private readonly importableRegistry: ImportableRegistry, + ) { + } /** * Retrieves the sample sheet of the given resource. * @param {string} resource * @param {string} format * @returns {Buffer | string} */ - public sample( + public async sample( resource: string, format: 'csv' | 'xlsx' - ): Buffer | string { + ): Promise { const _resource = sanitizeResourceName(resource); - - const ImportableRegistry = this.importable.registry; - const importable = ImportableRegistry.getImportable(_resource); + const importable = await this.importableRegistry.getImportable(_resource); const data = importable.sampleData(); @@ -30,6 +30,7 @@ export class ImportSampleService { const worksheet = XLSX.utils.json_to_sheet(data); XLSX.utils.book_append_sheet(workbook, worksheet, 'Sheet1'); + // Determine the output format if (format === 'csv') { const csvOutput = XLSX.utils.sheet_to_csv(worksheet); diff --git a/packages/server/src/modules/Import/ImportableRegistry.ts b/packages/server/src/modules/Import/ImportableRegistry.ts index 9fe25ec43..5d4f6563b 100644 --- a/packages/server/src/modules/Import/ImportableRegistry.ts +++ b/packages/server/src/modules/Import/ImportableRegistry.ts @@ -1,43 +1,26 @@ import { camelCase, upperFirst } from 'lodash'; import { Importable } from './Importable'; +import { getImportableService } from './decorators/Import.decorator'; +import { Injectable } from '@nestjs/common'; +import { ContextIdFactory, ModuleRef } from '@nestjs/core'; +@Injectable() export class ImportableRegistry { - private static instance: ImportableRegistry; - private importables: Record; - - constructor() { - this.importables = {}; - } - - /** - * Gets singleton instance of registry. - * @returns {ImportableRegistry} - */ - public static getInstance(): ImportableRegistry { - if (!ImportableRegistry.instance) { - ImportableRegistry.instance = new ImportableRegistry(); - } - return ImportableRegistry.instance; - } - - /** - * Registers the given importable service. - * @param {string} resource - * @param {Importable} importable - */ - public registerImportable(resource: string, importable: Importable): void { - const _resource = this.sanitizeResourceName(resource); - this.importables[_resource] = importable; - } - + constructor(private readonly moduleRef: ModuleRef) {} /** * Retrieves the importable service instance of the given resource name. - * @param {string} name + * @param {string} name * @returns {Importable} */ - public getImportable(name: string): Importable { + public async getImportable(name: string) { const _name = this.sanitizeResourceName(name); - return this.importables[_name]; + const importable = getImportableService(_name); + const contextId = ContextIdFactory.create(); + + const importableInstance = await this.moduleRef.resolve(importable, contextId, { + strict: false, + }); + return importableInstance; } private sanitizeResourceName(resource: string) { diff --git a/packages/server/src/modules/Import/ImportableResources.ts b/packages/server/src/modules/Import/ImportableResources.ts deleted file mode 100644 index a16c4306a..000000000 --- a/packages/server/src/modules/Import/ImportableResources.ts +++ /dev/null @@ -1,73 +0,0 @@ -// import { AccountsImportable } from '../Accounts/AccountsImportable'; -import { Injectable } from '@nestjs/common'; -import { ImportableRegistry } from './ImportableRegistry'; -// import { UncategorizedTransactionsImportable } from '../BankingCategorize/commands/UncategorizedTransactionsImportable'; -// import { CustomersImportable } from '../Contacts/Customers/CustomersImportable'; -// import { VendorsImportable } from '../Contacts/Vendors/VendorsImportable'; -// import { ItemsImportable } from '../Items/ItemsImportable'; -// import { ItemCategoriesImportable } from '../ItemCategories/ItemCategoriesImportable'; -// import { ManualJournalImportable } from '../ManualJournals/commands/ManualJournalsImport'; -// import { BillsImportable } from '../Purchases/Bills/BillsImportable'; -// import { ExpensesImportable } from '../Expenses/ExpensesImportable'; -// import { SaleInvoicesImportable } from '../Sales/Invoices/SaleInvoicesImportable'; -// import { SaleEstimatesImportable } from '../Sales/Estimates/SaleEstimatesImportable'; -// import { BillPaymentsImportable } from '../Purchases/BillPayments/BillPaymentsImportable'; -// import { VendorCreditsImportable } from '../Purchases/VendorCredits/VendorCreditsImportable'; -// import { PaymentsReceivedImportable } from '../Sales/PaymentReceived/PaymentsReceivedImportable'; -// import { CreditNotesImportable } from '../CreditNotes/commands/CreditNotesImportable'; -// import { SaleReceiptsImportable } from '../Sales/Receipts/SaleReceiptsImportable'; -// import { TaxRatesImportable } from '../TaxRates/TaxRatesImportable'; - -@Injectable() -export class ImportableResources { - private static registry: ImportableRegistry; - - constructor() { - this.boot(); - } - - /** - * Importable instances. - */ - private importables = [ - // { resource: 'Account', importable: AccountsImportable }, - // { - // resource: 'UncategorizedCashflowTransaction', - // importable: UncategorizedTransactionsImportable, - // }, - // { resource: 'Customer', importable: CustomersImportable }, - // { resource: 'Vendor', importable: VendorsImportable }, - // { resource: 'Item', importable: ItemsImportable }, - // { resource: 'ItemCategory', importable: ItemCategoriesImportable }, - // { resource: 'ManualJournal', importable: ManualJournalImportable }, - // { resource: 'Bill', importable: BillsImportable }, - // { resource: 'Expense', importable: ExpensesImportable }, - // { resource: 'SaleInvoice', importable: SaleInvoicesImportable }, - // { resource: 'SaleEstimate', importable: SaleEstimatesImportable }, - // { resource: 'BillPayment', importable: BillPaymentsImportable }, - // { resource: 'PaymentReceive', importable: PaymentsReceivedImportable }, - // { resource: 'VendorCredit', importable: VendorCreditsImportable }, - // { resource: 'CreditNote', importable: CreditNotesImportable }, - // { resource: 'SaleReceipt', importable: SaleReceiptsImportable }, - // { resource: 'TaxRate', importable: TaxRatesImportable }, - ]; - - public get registry() { - return ImportableResources.registry; - } - - /** - * Boots all the registered importables. - */ - public boot() { - if (!ImportableResources.registry) { - const instance = ImportableRegistry.getInstance(); - - this.importables.forEach((importable) => { - // const importableInstance = Container.get(importable.importable); - // instance.registerImportable(importable.resource, importableInstance); - }); - ImportableResources.registry = instance; - } - } -} diff --git a/packages/server/src/modules/Import/decorators/Import.decorator.ts b/packages/server/src/modules/Import/decorators/Import.decorator.ts new file mode 100644 index 000000000..6134041b1 --- /dev/null +++ b/packages/server/src/modules/Import/decorators/Import.decorator.ts @@ -0,0 +1,37 @@ +import { Global } from "@nestjs/common"; + +const importableModels = new Map(); +const importableService = new Map() + +/** + * Decorator that marks a model as exportable and registers its metadata. + * @param metadata Model metadata configuration for export + */ +export function ImportableModel() { + return function (target: any) { + const modelName = target.name; + importableModels.set(modelName, true); + }; +} + +export function ImportableService({ name }: { name: string }) { + return function (target: any) { + importableService.set(name, target); + + // Apply the @Global() decorator to make the service globally available + Global()(target); + }; +} + +/** + * Gets the registered exportable model metadata + * @param modelName Name of the model class + */ +export function getImportableModelMeta(modelName: string): boolean | undefined { + return importableModels.get(modelName); +} + + +export function getImportableService(modelName: string) { + return importableService.get(modelName); +} \ No newline at end of file diff --git a/packages/server/src/modules/ItemCategories/ItemCategoriesImportable.ts b/packages/server/src/modules/ItemCategories/ItemCategoriesImportable.ts index d65c55e70..2266c157a 100644 --- a/packages/server/src/modules/ItemCategories/ItemCategoriesImportable.ts +++ b/packages/server/src/modules/ItemCategories/ItemCategoriesImportable.ts @@ -4,8 +4,11 @@ import { ItemCategoriesSampleData } from './constants'; import { Injectable } from '@nestjs/common'; import { CreateItemCategoryDto } from './dtos/ItemCategory.dto'; import { ItemCategoryApplication } from './ItemCategory.application'; +import { ImportableService } from '../Import/decorators/Import.decorator'; +import { ItemCategory } from './models/ItemCategory.model'; @Injectable() +@ImportableService({ name: ItemCategory.name }) export class ItemCategoriesImportable extends Importable { constructor(private readonly itemCategoriesApp: ItemCategoryApplication) { super(); diff --git a/packages/server/src/modules/ItemCategories/models/ItemCategory.model.ts b/packages/server/src/modules/ItemCategories/models/ItemCategory.model.ts index a75803403..bb6e3a7b3 100644 --- a/packages/server/src/modules/ItemCategories/models/ItemCategory.model.ts +++ b/packages/server/src/modules/ItemCategories/models/ItemCategory.model.ts @@ -3,8 +3,10 @@ import { ExportableModel } from '@/modules/Export/decorators/ExportableModel.dec import { TenantBaseModel } from '@/modules/System/models/TenantBaseModel'; import { InjectModelMeta } from '@/modules/Tenancy/TenancyModels/decorators/InjectModelMeta.decorator'; import { ItemCategoryMeta } from './ItemCategory.meta'; +import { ImportableModel } from '@/modules/Import/decorators/Import.decorator'; @ExportableModel() +@ImportableModel() @InjectModelMeta(ItemCategoryMeta) export class ItemCategory extends TenantBaseModel { name!: string; diff --git a/packages/server/src/modules/Items/Items.module.ts b/packages/server/src/modules/Items/Items.module.ts index cc0c4fb1f..8cc26d759 100644 --- a/packages/server/src/modules/Items/Items.module.ts +++ b/packages/server/src/modules/Items/Items.module.ts @@ -17,6 +17,7 @@ import { GetItemsService } from './GetItems.service'; import { DynamicListModule } from '../DynamicListing/DynamicList.module'; import { InventoryAdjustmentsModule } from '../InventoryAdjutments/InventoryAdjustments.module'; import { ItemsExportable } from './ItemsExportable.service'; +import { ItemsImportable } from './ItemsImportable.service'; @Module({ imports: [ @@ -40,7 +41,8 @@ import { ItemsExportable } from './ItemsExportable.service'; TransformerInjectable, ItemsEntriesService, ItemsExportable, + ItemsImportable ], - exports: [ItemsEntriesService, ItemsExportable], + exports: [ItemsEntriesService, ItemsExportable, ItemsImportable], }) export class ItemsModule {} diff --git a/packages/server/src/modules/Items/ItemsImportable.service.ts b/packages/server/src/modules/Items/ItemsImportable.service.ts index 091473c87..b17ee1f76 100644 --- a/packages/server/src/modules/Items/ItemsImportable.service.ts +++ b/packages/server/src/modules/Items/ItemsImportable.service.ts @@ -4,8 +4,11 @@ import { Importable } from '../Import/Importable'; import { CreateItemService } from './CreateItem.service'; import { CreateItemDto } from './dtos/Item.dto'; import { ItemsSampleData } from './Items.constants'; +import { ImportableService } from '../Import/decorators/Import.decorator'; +import { Item } from './models/Item'; @Injectable() +@ImportableService({ name: Item.name }) export class ItemsImportable extends Importable { constructor( private readonly createItemService: CreateItemService, diff --git a/packages/server/src/modules/Items/models/Item.ts b/packages/server/src/modules/Items/models/Item.ts index c9f561c81..c6c9be47e 100644 --- a/packages/server/src/modules/Items/models/Item.ts +++ b/packages/server/src/modules/Items/models/Item.ts @@ -4,8 +4,10 @@ import { Model } from 'objection'; import { ExportableModel } from '@/modules/Export/decorators/ExportableModel.decorator'; import { InjectModelMeta } from '@/modules/Tenancy/TenancyModels/decorators/InjectModelMeta.decorator'; import { ItemMeta } from './Item.meta'; +import { ImportableModel } from '@/modules/Import/decorators/Import.decorator'; @ExportableModel() +@ImportableModel() @InjectModelMeta(ItemMeta) export class Item extends TenantBaseModel { public readonly quantityOnHand: number; diff --git a/packages/server/src/utils/parse-json.ts b/packages/server/src/utils/parse-json.ts new file mode 100644 index 000000000..393558ba8 --- /dev/null +++ b/packages/server/src/utils/parse-json.ts @@ -0,0 +1,7 @@ +export const parseJsonSafe = (value: string) => { + try { + return JSON.parse(value); + } catch { + return null; + } +}; \ No newline at end of file From b9755ff01c40b906e0e53e316fc79cda23c0458a Mon Sep 17 00:00:00 2001 From: Ahmed Bouhuolia Date: Sat, 12 Apr 2025 13:39:17 +0200 Subject: [PATCH 2/2] refactor(nestjs): import module --- packages/server/package.json | 7 +-- packages/server/src/modules/App/App.module.ts | 2 + .../src/modules/Import/Import.controller.ts | 2 +- .../src/modules/Import/Import.module.ts | 11 ++++- .../src/modules/Import/ImportFileCommon.ts | 2 +- .../src/modules/Import/ImportFileMapping.ts | 6 +-- .../src/modules/Import/ImportFileProcess.ts | 1 - .../src/modules/Import/ImportFileUpload.ts | 7 ++- .../src/modules/Import/ImportMulter.utils.ts | 3 +- .../Import/ImportRemoveExpiredFiles.ts | 1 + packages/server/src/modules/Import/_utils.ts | 2 +- .../jobs/ImportDeleteExpiredFilesJob.ts | 47 +++++++++---------- .../src/modules/Import/models/Import.ts | 4 +- .../server/src/modules/Import/sheet_utils.ts | 2 +- .../modules/Items/ItemsImportable.service.ts | 3 +- pnpm-lock.yaml | 35 +++++++++++++- 16 files changed, 84 insertions(+), 51 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 9eb37819a..9d09782e2 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -28,7 +28,6 @@ "@casl/ability": "^5.4.3", "@lemonsqueezy/lemonsqueezy.js": "^2.2.0", "@liaoliaots/nestjs-redis": "^10.0.0", - "@types/multer": "^1.4.11", "@nestjs/bull": "^10.2.1", "@nestjs/bullmq": "^10.2.2", "@nestjs/cache-manager": "^2.2.2", @@ -39,9 +38,11 @@ "@nestjs/jwt": "^10.2.0", "@nestjs/passport": "^11.0.5", "@nestjs/platform-express": "^10.0.0", + "@nestjs/schedule": "^4.1.2", "@nestjs/swagger": "^7.4.2", "@nestjs/throttler": "^6.2.1", "@supercharge/promise-pool": "^3.2.0", + "@types/multer": "^1.4.11", "@types/nodemailer": "^6.4.17", "@types/passport-local": "^1.0.38", "@types/ramda": "^0.30.2", @@ -74,10 +75,10 @@ "moment": "^2.30.1", "moment-range": "^4.0.2", "moment-timezone": "^0.5.43", - "mysql": "^2.18.1", - "mysql2": "^3.11.3", "multer": "1.4.5-lts.1", "multer-s3": "^3.0.1", + "mysql": "^2.18.1", + "mysql2": "^3.11.3", "nestjs-cls": "^5.2.0", "nestjs-i18n": "^10.4.9", "nestjs-redis": "^1.3.3", diff --git a/packages/server/src/modules/App/App.module.ts b/packages/server/src/modules/App/App.module.ts index 28791abb3..5ffc2ed98 100644 --- a/packages/server/src/modules/App/App.module.ts +++ b/packages/server/src/modules/App/App.module.ts @@ -12,6 +12,7 @@ import { QueryResolver, } from 'nestjs-i18n'; import { BullModule } from '@nestjs/bullmq'; +import { ScheduleModule } from '@nestjs/schedule'; import { PassportModule } from '@nestjs/passport'; import { ClsModule, ClsService } from 'nestjs-cls'; import { AppController } from './App.controller'; @@ -139,6 +140,7 @@ import { ImportModule } from '../Import/Import.module'; }), inject: [ConfigService], }), + ScheduleModule.forRoot(), TenancyDatabaseModule, TenancyModelsModule, TenancyModule, diff --git a/packages/server/src/modules/Import/Import.controller.ts b/packages/server/src/modules/Import/Import.controller.ts index d1d4a74c7..dcc1e29e4 100644 --- a/packages/server/src/modules/Import/Import.controller.ts +++ b/packages/server/src/modules/Import/Import.controller.ts @@ -30,7 +30,7 @@ export class ImportController { @ApiOperation({ summary: 'Upload import file' }) @ApiResponse({ status: 200, description: 'File uploaded successfully' }) @UseInterceptors( - FileInterceptor('file', { storage: uploadImportFileMulterOptions }), + FileInterceptor('file', uploadImportFileMulterOptions), ) async fileUpload( @Res() res: Response, diff --git a/packages/server/src/modules/Import/Import.module.ts b/packages/server/src/modules/Import/Import.module.ts index 4b37a9974..bc01e4c7c 100644 --- a/packages/server/src/modules/Import/Import.module.ts +++ b/packages/server/src/modules/Import/Import.module.ts @@ -17,10 +17,16 @@ import { TenancyModule } from '../Tenancy/Tenancy.module'; import { AccountsModule } from '../Accounts/Accounts.module'; import { ImportController } from './Import.controller'; import { ImportableRegistry } from './ImportableRegistry'; +import { InjectSystemModel } from '../System/SystemModels/SystemModels.module'; +import { ImportModel } from './models/Import'; +import { ImportDeleteExpiredFilesJobs } from './jobs/ImportDeleteExpiredFilesJob'; + +const models = [InjectSystemModel(ImportModel)]; @Module({ imports: [ResourceModule, TenancyModule, AccountsModule], providers: [ + ...models, ImportAls, ImportSampleService, ImportResourceApplication, @@ -34,9 +40,10 @@ import { ImportableRegistry } from './ImportableRegistry'; ImportFileDataValidator, ImportFileDataTransformer, ImportFileCommon, - ImportableRegistry + ImportableRegistry, + ImportDeleteExpiredFilesJobs ], controllers: [ImportController], - exports: [ImportAls], + exports: [ImportAls, ...models], }) export class ImportModule {} diff --git a/packages/server/src/modules/Import/ImportFileCommon.ts b/packages/server/src/modules/Import/ImportFileCommon.ts index f1c968302..345bfbb85 100644 --- a/packages/server/src/modules/Import/ImportFileCommon.ts +++ b/packages/server/src/modules/Import/ImportFileCommon.ts @@ -1,4 +1,4 @@ -import bluebird from 'bluebird'; +import * as bluebird from 'bluebird'; import * as R from 'ramda'; import { first } from 'lodash'; import { ImportFileDataValidator } from './ImportFileDataValidator'; diff --git a/packages/server/src/modules/Import/ImportFileMapping.ts b/packages/server/src/modules/Import/ImportFileMapping.ts index da97bdcdb..01a822d85 100644 --- a/packages/server/src/modules/Import/ImportFileMapping.ts +++ b/packages/server/src/modules/Import/ImportFileMapping.ts @@ -16,7 +16,7 @@ export class ImportFileMapping { private readonly resource: ResourceService, @Inject(ImportModel.name) - private readonly importModel: () => typeof ImportModel, + private readonly importModel: typeof ImportModel, ) {} /** @@ -28,7 +28,7 @@ export class ImportFileMapping { importId: string, maps: ImportMappingAttr[], ): Promise { - const importFile = await this.importModel() + const importFile = await this.importModel .query() .findOne('filename', importId) .throwIfNotFound(); @@ -46,7 +46,7 @@ export class ImportFileMapping { const mappingStringified = JSON.stringify(maps); - await this.importModel().query().findById(importFile.id).patch({ + await this.importModel.query().findById(importFile.id).patch({ mapping: mappingStringified, }); return { diff --git a/packages/server/src/modules/Import/ImportFileProcess.ts b/packages/server/src/modules/Import/ImportFileProcess.ts index 736247405..b9cdf049d 100644 --- a/packages/server/src/modules/Import/ImportFileProcess.ts +++ b/packages/server/src/modules/Import/ImportFileProcess.ts @@ -27,7 +27,6 @@ export class ImportFileProcess { /** * Preview the imported file results before commiting the transactions. - * @param {number} tenantId * @param {number} importId * @returns {Promise} */ diff --git a/packages/server/src/modules/Import/ImportFileUpload.ts b/packages/server/src/modules/Import/ImportFileUpload.ts index 5d4926f20..2ef5b6afb 100644 --- a/packages/server/src/modules/Import/ImportFileUpload.ts +++ b/packages/server/src/modules/Import/ImportFileUpload.ts @@ -1,3 +1,4 @@ +import { Inject, Injectable } from '@nestjs/common'; import { deleteImportFile, getResourceColumns, @@ -10,7 +11,6 @@ import { ImportFileCommon } from './ImportFileCommon'; import { ImportFileDataValidator } from './ImportFileDataValidator'; import { ImportFileUploadPOJO } from './interfaces'; import { parseSheetData } from './sheet_utils'; -import { Inject, Injectable } from '@nestjs/common'; import { ImportModel } from './models/Import'; import { TenancyContext } from '../Tenancy/TenancyContext.service'; @@ -29,9 +29,8 @@ export class ImportFileUploadService { /** * Imports the specified file for the given resource. * Deletes the file if an error occurs during the import process. - * @param {number} tenantId - * @param {string} resourceName - * @param {string} filename + * @param {string} resourceName - Resource name. + * @param {string} filename - File name. * @param {Record} params * @returns {Promise} */ diff --git a/packages/server/src/modules/Import/ImportMulter.utils.ts b/packages/server/src/modules/Import/ImportMulter.utils.ts index 4c07658a4..162c8c182 100644 --- a/packages/server/src/modules/Import/ImportMulter.utils.ts +++ b/packages/server/src/modules/Import/ImportMulter.utils.ts @@ -6,7 +6,6 @@ export const getImportsStoragePath = () => { return path.join(global.__static_dirname, `/imports`); }; - export function allowSheetExtensions(req, file, cb) { if ( file.mimetype !== 'text/csv' && @@ -35,5 +34,5 @@ const storage = Multer.diskStorage({ export const uploadImportFileMulterOptions = { storage, limits: { fileSize: 5 * 1024 * 1024 }, - fileFilter: allowSheetExtensions, + // fileFilter: allowSheetExtensions, }; \ No newline at end of file diff --git a/packages/server/src/modules/Import/ImportRemoveExpiredFiles.ts b/packages/server/src/modules/Import/ImportRemoveExpiredFiles.ts index 1c479745f..653be77ea 100644 --- a/packages/server/src/modules/Import/ImportRemoveExpiredFiles.ts +++ b/packages/server/src/modules/Import/ImportRemoveExpiredFiles.ts @@ -19,6 +19,7 @@ export class ImportDeleteExpiredFiles { const expiredImports = await this.importModel .query() .where('createdAt', '<', yesterday); + await bluebird.map( expiredImports, async (expiredImport) => { diff --git a/packages/server/src/modules/Import/_utils.ts b/packages/server/src/modules/Import/_utils.ts index 63d58e25e..588252da0 100644 --- a/packages/server/src/modules/Import/_utils.ts +++ b/packages/server/src/modules/Import/_utils.ts @@ -434,7 +434,7 @@ export const getMapToPath = (to: string, group = '') => group ? `${group}.${to}` : to; export const getImportsStoragePath = () => { - return path.join(global.__storage_dir, `/imports`); + return path.join(global.__static_dirname, `/imports`); }; /** diff --git a/packages/server/src/modules/Import/jobs/ImportDeleteExpiredFilesJob.ts b/packages/server/src/modules/Import/jobs/ImportDeleteExpiredFilesJob.ts index 849703546..3a76037fc 100644 --- a/packages/server/src/modules/Import/jobs/ImportDeleteExpiredFilesJob.ts +++ b/packages/server/src/modules/Import/jobs/ImportDeleteExpiredFilesJob.ts @@ -1,28 +1,23 @@ -// import Container, { Service } from 'typedi'; -// import { ImportDeleteExpiredFiles } from '../ImportRemoveExpiredFiles'; +import { Cron } from '@nestjs/schedule'; +import { Injectable } from '@nestjs/common'; +import { ImportDeleteExpiredFiles } from '../ImportRemoveExpiredFiles'; -// @Service() -// export class ImportDeleteExpiredFilesJobs { -// /** -// * Constructor method. -// */ -// constructor(agenda) { -// agenda.define('delete-expired-imported-files', this.handler); -// } +@Injectable() +export class ImportDeleteExpiredFilesJobs { + constructor( + private readonly importDeleteExpiredFiles: ImportDeleteExpiredFiles, + ) {} -// /** -// * Triggers sending invoice mail. -// */ -// private handler = async (job, done: Function) => { -// const importDeleteExpiredFiles = Container.get(ImportDeleteExpiredFiles); - -// try { -// console.log('Delete expired import files has started.'); -// await importDeleteExpiredFiles.deleteExpiredFiles(); -// done(); -// } catch (error) { -// console.log(error); -// done(error); -// } -// }; -// } + /** + * Triggers sending invoice mail. + */ + @Cron('* * * * *') + async importDeleteExpiredJob() { + try { + console.log('Delete expired import files has started.'); + await this.importDeleteExpiredFiles.deleteExpiredFiles(); + } catch (error) { + console.log(error); + } + } +} diff --git a/packages/server/src/modules/Import/models/Import.ts b/packages/server/src/modules/Import/models/Import.ts index 680ad8584..6a61a1597 100644 --- a/packages/server/src/modules/Import/models/Import.ts +++ b/packages/server/src/modules/Import/models/Import.ts @@ -67,7 +67,7 @@ export class ImportModel extends BaseModel { * Relationship mapping. */ static get relationMappings() { - const Tenant = require('system/models/Tenant'); + const { TenantModel } = require('../../System/models/TenantModel'); return { /** @@ -75,7 +75,7 @@ export class ImportModel extends BaseModel { */ tenant: { relation: Model.BelongsToOneRelation, - modelClass: Tenant.default, + modelClass: TenantModel, join: { from: 'imports.tenantId', to: 'tenants.id', diff --git a/packages/server/src/modules/Import/sheet_utils.ts b/packages/server/src/modules/Import/sheet_utils.ts index b21f07320..6fecb5425 100644 --- a/packages/server/src/modules/Import/sheet_utils.ts +++ b/packages/server/src/modules/Import/sheet_utils.ts @@ -1,4 +1,4 @@ -import XLSX from 'xlsx'; +import * as XLSX from 'xlsx'; import { first } from 'lodash'; /** diff --git a/packages/server/src/modules/Items/ItemsImportable.service.ts b/packages/server/src/modules/Items/ItemsImportable.service.ts index b17ee1f76..d5bc5aecc 100644 --- a/packages/server/src/modules/Items/ItemsImportable.service.ts +++ b/packages/server/src/modules/Items/ItemsImportable.service.ts @@ -18,8 +18,7 @@ export class ItemsImportable extends Importable { /** * Mapps the imported data to create a new item service. - * @param {number} tenantId - * @param {ICustomerNewDTO} createDTO + * @param {CreateItemDto} createDTO * @param {Knex.Transaction} trx * @returns {Promise} */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 61518ffee..9728c3275 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -93,6 +93,9 @@ importers: '@nestjs/platform-express': specifier: ^10.0.0 version: 10.4.7(@nestjs/common@10.4.7)(@nestjs/core@10.4.7) + '@nestjs/schedule': + specifier: ^4.1.2 + version: 4.1.2(@nestjs/common@10.4.7)(@nestjs/core@10.4.7) '@nestjs/swagger': specifier: ^7.4.2 version: 7.4.2(@nestjs/common@10.4.7)(@nestjs/core@10.4.7)(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2) @@ -7084,6 +7087,18 @@ packages: transitivePeerDependencies: - supports-color + /@nestjs/schedule@4.1.2(@nestjs/common@10.4.7)(@nestjs/core@10.4.7): + resolution: {integrity: sha512-hCTQ1lNjIA5EHxeu8VvQu2Ed2DBLS1GSC6uKPYlBiQe6LL9a7zfE9iVSK+zuK8E2odsApteEBmfAQchc8Hx0Gg==} + peerDependencies: + '@nestjs/common': ^8.0.0 || ^9.0.0 || ^10.0.0 + '@nestjs/core': ^8.0.0 || ^9.0.0 || ^10.0.0 + dependencies: + '@nestjs/common': 10.4.7(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.1) + '@nestjs/core': 10.4.7(@nestjs/common@10.4.7)(@nestjs/platform-express@10.4.7)(reflect-metadata@0.2.2)(rxjs@7.8.1) + cron: 3.2.1 + uuid: 11.0.3 + dev: false + /@nestjs/schematics@10.2.3(chokidar@3.6.0)(typescript@5.6.3): resolution: {integrity: sha512-4e8gxaCk7DhBxVUly2PjYL4xC2ifDFexCqq1/u4TtivLGXotVk0wHdYuPYe1tHTHuR1lsOkRbfOCpkdTnigLVg==} peerDependencies: @@ -11637,6 +11652,10 @@ packages: /@types/lodash@4.17.4: resolution: {integrity: sha512-wYCP26ZLxaT3R39kiN2+HcJ4kTd3U1waI/cY7ivWYqFP6pW3ZNpvi6Wd6PHZx7T/t8z0vlkXMg3QYLa7DZ/IJQ==} + /@types/luxon@3.4.2: + resolution: {integrity: sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==} + dev: false + /@types/mathjs@6.0.12: resolution: {integrity: sha512-bpKs8CDJ0aOiiJguywryE/U6Wre/uftJ89xhp4aCgF4oRb3Yug2VyZ87958gmSeq4WMsvWPMs2Q5TtFv+dJtaA==} dependencies: @@ -15212,6 +15231,13 @@ packages: luxon: 3.5.0 dev: false + /cron@3.2.1: + resolution: {integrity: sha512-w2n5l49GMmmkBFEsH9FIDhjZ1n1QgTMOCMGuQtOXs5veNiosZmso6bQGuqOJSYAXXrG84WQFVneNk+Yt0Ua9iw==} + dependencies: + '@types/luxon': 3.4.2 + luxon: 3.5.0 + dev: false + /cross-env@7.0.3: resolution: {integrity: sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==} engines: {node: '>=10.14', npm: '>=6', yarn: '>=1'} @@ -28560,9 +28586,9 @@ packages: webpack: ^5.0.0 dependencies: chalk: 4.1.2 - enhanced-resolve: 5.16.1 + enhanced-resolve: 5.17.1 micromatch: 4.0.7 - semver: 7.6.2 + semver: 7.6.3 source-map: 0.7.4 typescript: 5.6.3 webpack: 5.91.0(esbuild@0.18.20)(webpack-cli@5.1.4) @@ -29242,6 +29268,11 @@ packages: hasBin: true dev: false + /uuid@11.0.3: + resolution: {integrity: sha512-d0z310fCWv5dJwnX1Y/MncBAqGMKEzlBb1AOf7z9K8ALnd0utBX/msg/fA0+sbyN1ihbMsLhrBlnl1ak7Wa0rg==} + hasBin: true + dev: false + /uuid@8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} hasBin: true