diff --git a/packages/server/public/.DS_Store b/packages/server/public/.DS_Store new file mode 100644 index 000000000..e02f2f664 Binary files /dev/null and b/packages/server/public/.DS_Store differ diff --git a/packages/server/public/imports/.DS_Store b/packages/server/public/imports/.DS_Store new file mode 100644 index 000000000..5008ddfcf Binary files /dev/null and b/packages/server/public/imports/.DS_Store differ diff --git a/packages/server/src/api/controllers/Import/ImportController.ts b/packages/server/src/api/controllers/Import/ImportController.ts index 7db23fd1d..5567aca78 100644 --- a/packages/server/src/api/controllers/Import/ImportController.ts +++ b/packages/server/src/api/controllers/Import/ImportController.ts @@ -48,6 +48,7 @@ export class ImportController extends BaseController { router.get( '/sample', [query('resource').exists(), query('format').optional()], + this.validationResult, this.downloadImportSample.bind(this), this.catchServiceErrors ); diff --git a/packages/server/src/api/controllers/Import/_utils.ts b/packages/server/src/api/controllers/Import/_utils.ts index 0621ba999..3ad867941 100644 --- a/packages/server/src/api/controllers/Import/_utils.ts +++ b/packages/server/src/api/controllers/Import/_utils.ts @@ -4,18 +4,29 @@ import { ServiceError } from '@/exceptions'; export function allowSheetExtensions(req, file, cb) { if ( file.mimetype !== 'text/csv' && - file.mimetype !== 'application/vnd.ms-excel' && - file.mimetype !== 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + file.mimetype !== 'application/vnd.ms-excel' && + file.mimetype !== + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ) { cb(new ServiceError('IMPORTED_FILE_EXTENSION_INVALID')); - return; } cb(null, true); } +const storage = Multer.diskStorage({ + destination: function (req, file, cb) { + cb(null, './public/imports'); + }, + filename: function (req, file, cb) { + // Add the creation timestamp to clean up temp files later. + const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1e9); + cb(null, uniqueSuffix); + }, +}); + export const uploadImportFile = Multer({ - dest: './public/imports', + storage, limits: { fileSize: 5 * 1024 * 1024 }, fileFilter: allowSheetExtensions, }); diff --git a/packages/server/src/loaders/jobs.ts b/packages/server/src/loaders/jobs.ts index 3215c9558..58da23291 100644 --- a/packages/server/src/loaders/jobs.ts +++ b/packages/server/src/loaders/jobs.ts @@ -11,6 +11,7 @@ import { SendSaleEstimateMailJob } from '@/services/Sales/Estimates/SendSaleEsti import { SaleReceiptMailNotificationJob } from '@/services/Sales/Receipts/SaleReceiptMailNotificationJob'; import { PaymentReceiveMailNotificationJob } from '@/services/Sales/PaymentReceives/PaymentReceiveMailNotificationJob'; import { PlaidFetchTransactionsJob } from '@/services/Banking/Plaid/PlaidFetchTransactionsJob'; +import { ImportDeleteExpiredFilesJobs } from '@/services/Import/jobs/ImportDeleteExpiredFilesJob'; export default ({ agenda }: { agenda: Agenda }) => { new ResetPasswordMailJob(agenda); @@ -25,6 +26,9 @@ export default ({ agenda }: { agenda: Agenda }) => { new SaleReceiptMailNotificationJob(agenda); new PaymentReceiveMailNotificationJob(agenda); new PlaidFetchTransactionsJob(agenda); + new ImportDeleteExpiredFilesJobs(agenda); - agenda.start(); + agenda.start().then(() => { + agenda.every('1 hours', 'delete-expired-imported-files', {}); + }); }; diff --git a/packages/server/src/loaders/tenantModels.ts b/packages/server/src/loaders/tenantModels.ts index 0608d5e94..c3d08ab6f 100644 --- a/packages/server/src/loaders/tenantModels.ts +++ b/packages/server/src/loaders/tenantModels.ts @@ -61,7 +61,6 @@ import Task from 'models/Task'; import TaxRate from 'models/TaxRate'; import TaxRateTransaction from 'models/TaxRateTransaction'; import Attachment from 'models/Attachment'; -import Import from 'models/Import'; import PlaidItem from 'models/PlaidItem'; import UncategorizedCashflowTransaction from 'models/UncategorizedCashflowTransaction'; @@ -128,7 +127,6 @@ export default (knex) => { TaxRate, TaxRateTransaction, Attachment, - Import, PlaidItem, UncategorizedCashflowTransaction }; diff --git a/packages/server/src/services/Import/ImportFileCommon.ts b/packages/server/src/services/Import/ImportFileCommon.ts index f41ec2b50..739927e6c 100644 --- a/packages/server/src/services/Import/ImportFileCommon.ts +++ b/packages/server/src/services/Import/ImportFileCommon.ts @@ -1,4 +1,3 @@ -import fs from 'fs/promises'; import XLSX from 'xlsx'; import bluebird from 'bluebird'; import * as R from 'ramda'; @@ -17,7 +16,7 @@ import { getUniqueImportableValue, trimObject } from './_utils'; import { ImportableResources } from './ImportableResources'; import ResourceService from '../Resource/ResourceService'; import HasTenancyService from '../Tenancy/TenancyService'; -import Import from '@/models/Import'; +import { Import } from '@/system/models'; @Service() export class ImportFileCommon { @@ -48,14 +47,6 @@ export class ImportFileCommon { return XLSX.utils.sheet_to_json(worksheet, {}); } - /** - * Reads the import file. - * @param {string} filename - * @returns {Promise} - */ - public readImportFile(filename: string) { - return fs.readFile(`public/imports/${filename}`); - } /** * Imports the given parsed data to the resource storage through registered importable service. @@ -202,19 +193,4 @@ export class ImportFileCommon { public parseSheetColumns(json: unknown[]): string[] { return R.compose(Object.keys, trimObject, first)(json); } - - /** - * Deletes the imported file from the storage and database. - * @param {number} tenantId - * @param {} importFile - */ - public async deleteImportFile(tenantId: number, importFile: any) { - const { Import } = this.tenancy.models(tenantId); - - // Deletes the import row. - await Import.query().findById(importFile.id).delete(); - - // Deletes the imported file. - await fs.unlink(`public/imports/${importFile.filename}`); - } } diff --git a/packages/server/src/services/Import/ImportFileDataTransformer.ts b/packages/server/src/services/Import/ImportFileDataTransformer.ts index 8fd47b584..5e391fc6e 100644 --- a/packages/server/src/services/Import/ImportFileDataTransformer.ts +++ b/packages/server/src/services/Import/ImportFileDataTransformer.ts @@ -9,14 +9,11 @@ import { getFieldKey, aggregate, sanitizeSheetData, + getMapToPath, } from './_utils'; import ResourceService from '../Resource/ResourceService'; import HasTenancyService from '../Tenancy/TenancyService'; - -const CurrencyParsingDTOs = 10; - -const getMapToPath = (to: string, group = '') => - group ? `${group}.${to}` : to; +import { CurrencyParsingDTOs } from './_constants'; @Service() export class ImportFileDataTransformer { diff --git a/packages/server/src/services/Import/ImportFileMapping.ts b/packages/server/src/services/Import/ImportFileMapping.ts index 8d8f062ac..036fde757 100644 --- a/packages/server/src/services/Import/ImportFileMapping.ts +++ b/packages/server/src/services/Import/ImportFileMapping.ts @@ -1,6 +1,5 @@ import { fromPairs, isUndefined } from 'lodash'; import { Inject, Service } from 'typedi'; -import HasTenancyService from '../Tenancy/TenancyService'; import { ImportDateFormats, ImportFileMapPOJO, @@ -9,12 +8,10 @@ import { import ResourceService from '../Resource/ResourceService'; import { ServiceError } from '@/exceptions'; import { ERRORS } from './_utils'; +import { Import } from '@/system/models'; @Service() export class ImportFileMapping { - @Inject() - private tenancy: HasTenancyService; - @Inject() private resource: ResourceService; @@ -29,8 +26,6 @@ export class ImportFileMapping { importId: number, maps: ImportMappingAttr[] ): Promise { - const { Import } = this.tenancy.models(tenantId); - const importFile = await Import.query() .findOne('filename', importId) .throwIfNotFound(); diff --git a/packages/server/src/services/Import/ImportFileMeta.ts b/packages/server/src/services/Import/ImportFileMeta.ts index 758f94fe4..c20d49120 100644 --- a/packages/server/src/services/Import/ImportFileMeta.ts +++ b/packages/server/src/services/Import/ImportFileMeta.ts @@ -2,6 +2,7 @@ import { Inject, Service } from 'typedi'; import HasTenancyService from '../Tenancy/TenancyService'; import { TransformerInjectable } from '@/lib/Transformer/TransformerInjectable'; import { ImportFileMetaTransformer } from './ImportFileMetaTransformer'; +import { Import } from '@/system/models'; @Service() export class ImportFileMeta { @@ -12,15 +13,15 @@ export class ImportFileMeta { private transformer: TransformerInjectable; /** - * + * Retrieves the import meta of the given import model id. * @param {number} tenantId * @param {number} importId * @returns {} */ async getImportMeta(tenantId: number, importId: string) { - const { Import } = this.tenancy.models(tenantId); - - const importFile = await Import.query().findOne('importId', importId); + const importFile = await Import.query() + .where('tenantId', tenantId) + .findOne('importId', importId); // Retrieves the transformed accounts collection. return this.transformer.transform( diff --git a/packages/server/src/services/Import/ImportFileProcess.ts b/packages/server/src/services/Import/ImportFileProcess.ts index 3c57f2b69..8eebccede 100644 --- a/packages/server/src/services/Import/ImportFileProcess.ts +++ b/packages/server/src/services/Import/ImportFileProcess.ts @@ -2,19 +2,16 @@ import { Inject, Service } from 'typedi'; import { chain } from 'lodash'; import { Knex } from 'knex'; import { ServiceError } from '@/exceptions'; -import { ERRORS, getSheetColumns, getUnmappedSheetColumns } from './_utils'; -import HasTenancyService from '../Tenancy/TenancyService'; +import { ERRORS, getSheetColumns, getUnmappedSheetColumns, readImportFile } from './_utils'; import { ImportFileCommon } from './ImportFileCommon'; import { ImportFileDataTransformer } from './ImportFileDataTransformer'; import ResourceService from '../Resource/ResourceService'; import UnitOfWork from '../UnitOfWork'; import { ImportFilePreviewPOJO } from './interfaces'; +import { Import } from '@/system/models'; @Service() export class ImportFileProcess { - @Inject() - private tenancy: HasTenancyService; - @Inject() private resource: ResourceService; @@ -38,10 +35,9 @@ export class ImportFileProcess { importId: number, trx?: Knex.Transaction ): Promise { - const { Import } = this.tenancy.models(tenantId); - const importFile = await Import.query() .findOne('importId', importId) + .where('tenantId', tenantId) .throwIfNotFound(); // Throw error if the import file is not mapped yet. @@ -49,7 +45,7 @@ export class ImportFileProcess { throw new ServiceError(ERRORS.IMPORT_FILE_NOT_MAPPED); } // Read the imported file. - const buffer = await this.importCommon.readImportFile(importFile.filename); + const buffer = await readImportFile(importFile.filename); const sheetData = this.importCommon.parseXlsxSheet(buffer); const header = getSheetColumns(sheetData); diff --git a/packages/server/src/services/Import/ImportFileUpload.ts b/packages/server/src/services/Import/ImportFileUpload.ts index 11dfc52c0..89c9d73d8 100644 --- a/packages/server/src/services/Import/ImportFileUpload.ts +++ b/packages/server/src/services/Import/ImportFileUpload.ts @@ -1,7 +1,8 @@ import { Inject, Service } from 'typedi'; -import HasTenancyService from '../Tenancy/TenancyService'; import { + deleteImportFile, getResourceColumns, + readImportFile, sanitizeResourceName, validateSheetEmpty, } from './_utils'; @@ -9,12 +10,10 @@ import ResourceService from '../Resource/ResourceService'; import { ImportFileCommon } from './ImportFileCommon'; import { ImportFileDataValidator } from './ImportFileDataValidator'; import { ImportFileUploadPOJO } from './interfaces'; +import { Import } from '@/system/models'; @Service() export class ImportFileUploadService { - @Inject() - private tenancy: HasTenancyService; - @Inject() private resourceService: ResourceService; @@ -25,11 +24,12 @@ export class ImportFileUploadService { private importValidator: ImportFileDataValidator; /** - * Reads the imported file and stores the import file meta under unqiue id. - * @param {number} tenantId - Tenant id. - * @param {string} resource - Resource name. - * @param {string} filePath - File path. - * @param {string} fileName - File name. + * Imports the specified file for the given resource. + * Deletes the file if an error occurs during the import process. + * @param {number} tenantId + * @param {string} resourceName + * @param {string} filename + * @param {Record} params * @returns {Promise} */ public async import( @@ -38,8 +38,35 @@ export class ImportFileUploadService { filename: string, params: Record ): Promise { - const { Import } = this.tenancy.models(tenantId); + console.log(filename, 'filename'); + try { + return await this.importUnhandled( + tenantId, + resourceName, + filename, + params + ); + } catch (err) { + // deleteImportFile(filename); + throw err; + } + } + + /** + * Reads the imported file and stores the import file meta under unqiue id. + * @param {number} tenantId - Tenant id. + * @param {string} resource - Resource name. + * @param {string} filePath - File path. + * @param {string} fileName - File name. + * @returns {Promise} + */ + public async importUnhandled( + tenantId: number, + resourceName: string, + filename: string, + params: Record + ): Promise { const resource = sanitizeResourceName(resourceName); const resourceMeta = this.resourceService.getResourceMeta( tenantId, @@ -49,7 +76,7 @@ export class ImportFileUploadService { this.importValidator.validateResourceImportable(resourceMeta); // Reads the imported file into buffer. - const buffer = await this.importFileCommon.readImportFile(filename); + const buffer = await readImportFile(filename); // Parse the buffer file to array data. const sheetData = this.importFileCommon.parseXlsxSheet(buffer); @@ -76,6 +103,7 @@ export class ImportFileUploadService { const importFile = await Import.query().insert({ filename, resource, + tenantId, importId: filename, columns: coumnsStringified, params: paramsStringified, diff --git a/packages/server/src/services/Import/ImportRemoveExpiredFiles.ts b/packages/server/src/services/Import/ImportRemoveExpiredFiles.ts new file mode 100644 index 000000000..c20f486f5 --- /dev/null +++ b/packages/server/src/services/Import/ImportRemoveExpiredFiles.ts @@ -0,0 +1,34 @@ +import moment from 'moment'; +import bluebird from 'bluebird'; +import { Import } from '@/system/models'; +import { deleteImportFile } from './_utils'; +import { Service } from 'typedi'; + +@Service() +export class ImportDeleteExpiredFiles { + /** + * Delete expired files. + */ + async deleteExpiredFiles() { + const yesterday = moment().subtract(1, 'hour').format('YYYY-MM-DD HH:mm'); + + const expiredImports = await Import.query().where( + 'createdAt', + '<', + yesterday + ); + await bluebird.map( + expiredImports, + async (expiredImport) => { + await deleteImportFile(expiredImport.filename); + }, + { concurrency: 10 } + ); + const expiredImportsIds = expiredImports.map( + (expiredImport) => expiredImport.id + ); + if (expiredImportsIds.length > 0) { + await Import.query().whereIn('id', expiredImportsIds).delete(); + } + } +} diff --git a/packages/server/src/services/Import/ImportSample.ts b/packages/server/src/services/Import/ImportSample.ts index aed610603..db2fe0543 100644 --- a/packages/server/src/services/Import/ImportSample.ts +++ b/packages/server/src/services/Import/ImportSample.ts @@ -10,9 +10,9 @@ export class ImportSampleService { /** * Retrieves the sample sheet of the given resource. - * @param {number} tenantId - * @param {string} resource - * @param {string} format + * @param {number} tenantId + * @param {string} resource + * @param {string} format * @returns {Buffer | string} */ public sample( diff --git a/packages/server/src/services/Import/Importable.ts b/packages/server/src/services/Import/Importable.ts index 5be4f1fac..9cb82b56c 100644 --- a/packages/server/src/services/Import/Importable.ts +++ b/packages/server/src/services/Import/Importable.ts @@ -69,4 +69,4 @@ export abstract class Importable { public transformParams(parmas: Record) { return parmas; } -} +} \ No newline at end of file diff --git a/packages/server/src/services/Import/_constants.ts b/packages/server/src/services/Import/_constants.ts new file mode 100644 index 000000000..1e9d37cbf --- /dev/null +++ b/packages/server/src/services/Import/_constants.ts @@ -0,0 +1,3 @@ + + +export const CurrencyParsingDTOs = 10; \ No newline at end of file diff --git a/packages/server/src/services/Import/_utils.ts b/packages/server/src/services/Import/_utils.ts index 80afe17f2..8e1b5fe59 100644 --- a/packages/server/src/services/Import/_utils.ts +++ b/packages/server/src/services/Import/_utils.ts @@ -2,6 +2,7 @@ import * as Yup from 'yup'; import moment from 'moment'; import * as R from 'ramda'; import { Knex } from 'knex'; +import fs from 'fs/promises'; import { defaultTo, upperFirst, @@ -141,9 +142,9 @@ const parseFieldName = (fieldName: string, field: IModelMetaField) => { /** * Retrieves the unmapped sheet columns. - * @param columns - * @param mapping - * @returns + * @param columns + * @param mapping + * @returns */ export const getUnmappedSheetColumns = (columns, mapping) => { return columns.filter( @@ -421,3 +422,30 @@ export function aggregate( export const sanitizeSheetData = (json) => { return R.compose(R.map(trimObject))(json); }; + +/** + * Returns the path to map a value to based on the 'to' and 'group' parameters. + * @param {string} to - The target key to map the value to. + * @param {string} group - The group key to nest the target key under. + * @returns {string} - The path to map the value to. + */ +export const getMapToPath = (to: string, group = '') => + group ? `${group}.${to}` : to; + +/** + * Deletes the imported file from the storage and database. + * @param {string} filename + */ +export const deleteImportFile = async (filename: string) => { + // Deletes the imported file. + await fs.unlink(`public/imports/${filename}`); +}; + +/** + * Reads the import file. + * @param {string} filename + * @returns {Promise} + */ +export const readImportFile = (filename: string) => { + return fs.readFile(`public/imports/${filename}`); +}; diff --git a/packages/server/src/services/Import/jobs/ImportDeleteExpiredFilesJob.ts b/packages/server/src/services/Import/jobs/ImportDeleteExpiredFilesJob.ts index 89a6f5b21..74ce6a7c8 100644 --- a/packages/server/src/services/Import/jobs/ImportDeleteExpiredFilesJob.ts +++ b/packages/server/src/services/Import/jobs/ImportDeleteExpiredFilesJob.ts @@ -1 +1,28 @@ -export class ImportDeleteExpiredFilesJobs {} +import Container, { Service } from 'typedi'; +import { ImportDeleteExpiredFiles } from '../ImportRemoveExpiredFiles'; + +@Service() +export class ImportDeleteExpiredFilesJobs { + /** + * Constructor method. + */ + constructor(agenda) { + agenda.define('delete-expired-imported-files', this.handler); + } + + /** + * Triggers sending invoice mail. + */ + private handler = async (job, done: Function) => { + const importDeleteExpiredFiles = Container.get(ImportDeleteExpiredFiles); + + try { + console.log('Delete expired import files has started.'); + await importDeleteExpiredFiles.deleteExpiredFiles(); + done(); + } catch (error) { + console.log(error); + done(error); + } + }; +} diff --git a/packages/server/src/system/migrations/20231209230719_create_imports_table.js b/packages/server/src/system/migrations/20231209230719_create_imports_table.js new file mode 100644 index 000000000..ef3f73cd4 --- /dev/null +++ b/packages/server/src/system/migrations/20231209230719_create_imports_table.js @@ -0,0 +1,22 @@ +exports.up = function (knex) { + return knex.schema.createTable('imports', (table) => { + table.increments(); + table.string('filename'); + table.string('import_id'); + table.string('resource'); + table.json('columns'); + table.json('mapping'); + table.json('params'); + table + .bigInteger('tenant_id') + .unsigned() + .index() + .references('id') + .inTable('tenants'); + table.timestamps(); + }); +}; + +exports.down = function (knex) { + return knex.schema.dropTableIfExists('imports'); +}; diff --git a/packages/server/src/models/Import.ts b/packages/server/src/system/models/Import.ts similarity index 58% rename from packages/server/src/models/Import.ts rename to packages/server/src/system/models/Import.ts index 05b1c858c..c7dfd2b90 100644 --- a/packages/server/src/models/Import.ts +++ b/packages/server/src/system/models/Import.ts @@ -1,11 +1,13 @@ -import TenantModel from 'models/TenantModel'; +import { Model, ModelObject } from 'objection'; +import SystemModel from './SystemModel'; -export default class Import extends TenantModel { - resource!: string; +export class Import extends SystemModel { + resource: string; + tenantId: number; mapping!: string; columns!: string; - params!: Record; - + params!: string; + /** * Table name. */ @@ -24,14 +26,7 @@ export default class Import extends TenantModel { * Timestamps columns. */ get timestamps() { - return []; - } - - /** - * Relationship mapping. - */ - static get relationMappings() { - return {}; + return ['createdAt', 'updatedAt']; } /** @@ -50,7 +45,6 @@ export default class Import extends TenantModel { } } - public get paramsParsed() { try { return JSON.parse(this.params); @@ -66,4 +60,27 @@ export default class Import extends TenantModel { return []; } } + + /** + * Relationship mapping. + */ + static get relationMappings() { + const Tenant = require('system/models/Tenant'); + + return { + /** + * System user may belongs to tenant model. + */ + tenant: { + relation: Model.BelongsToOneRelation, + modelClass: Tenant.default, + join: { + from: 'imports.tenantId', + to: 'tenants.id', + }, + }, + }; + } } + +export type ImportShape = ModelObject; diff --git a/packages/server/src/system/models/Tenant.ts b/packages/server/src/system/models/Tenant.ts index 53436cd17..b3376a3a3 100644 --- a/packages/server/src/system/models/Tenant.ts +++ b/packages/server/src/system/models/Tenant.ts @@ -5,6 +5,11 @@ import BaseModel from 'models/Model'; import TenantMetadata from './TenantMetadata'; export default class Tenant extends BaseModel { + upgradeJobId: string; + buildJobId: string; + initializedAt!: Date | null; + seededAt!: Date | null; + /** * Table name. */ @@ -14,6 +19,7 @@ export default class Tenant extends BaseModel { /** * Timestamps columns. + * @returns {string[]} */ get timestamps() { return ['createdAt', 'updatedAt']; @@ -21,6 +27,7 @@ export default class Tenant extends BaseModel { /** * Virtual attributes. + * @returns {string[]} */ static get virtualAttributes() { return ['isReady', 'isBuildRunning', 'isUpgradeRunning']; @@ -28,6 +35,7 @@ export default class Tenant extends BaseModel { /** * Tenant is ready. + * @returns {boolean} */ get isReady() { return !!(this.initializedAt && this.seededAt); @@ -35,6 +43,7 @@ export default class Tenant extends BaseModel { /** * Detarimes the tenant whether is build currently running. + * @returns {boolean} */ get isBuildRunning() { return !!this.buildJobId; @@ -42,6 +51,7 @@ export default class Tenant extends BaseModel { /** * Detarmines the tenant whether is upgrade currently running. + * @returns {boolean} */ get isUpgradeRunning() { return !!this.upgradeJobId; @@ -64,6 +74,7 @@ export default class Tenant extends BaseModel { }, }; } + /** * Creates a new tenant with random organization id. */ diff --git a/packages/server/src/system/models/index.ts b/packages/server/src/system/models/index.ts index 61fa5b708..b6e3c1f45 100644 --- a/packages/server/src/system/models/index.ts +++ b/packages/server/src/system/models/index.ts @@ -4,6 +4,7 @@ import SystemUser from './SystemUser'; import PasswordReset from './PasswordReset'; import Invite from './Invite'; import SystemPlaidItem from './SystemPlaidItem'; +import { Import } from './Import'; export { Tenant, @@ -12,4 +13,5 @@ export { PasswordReset, Invite, SystemPlaidItem, + Import, };