feat: wip import resources

packages/server/package.json

@@ -37,8 +37,6 @@
     "bcryptjs": "^2.4.3",
     "bluebird": "^3.7.2",
     "body-parser": "^1.20.2",
-    "class-transformer": "^0.5.1",
-    "class-validator": "^0.14.0",
     "compression": "^1.7.4",
     "country-codes-list": "^1.6.8",
     "cpy": "^8.1.2",
@@ -56,7 +54,6 @@
     "express": "^4.17.1",
     "express-basic-auth": "^1.2.0",
     "express-boom": "^3.0.0",
-    "express-fileupload": "^1.1.7-alpha.3",
     "express-oauth-server": "^2.0.0",
     "express-validator": "^6.12.2",
     "form-data": "^4.0.0",

packages/server/src/api/controllers/Import/ImportController.ts

@@ -1,15 +1,10 @@
 import { Inject, Service } from 'typedi';
 import { Router, Request, Response, NextFunction } from 'express';
 import { body, param } from 'express-validator';
-import Multer from 'multer';
 import BaseController from '@/api/controllers/BaseController';
 import { ServiceError } from '@/exceptions';
 import { ImportResourceApplication } from '@/services/Import/ImportResourceApplication';
-
+import { uploadImportFile } from './_utils';
-const upload = Multer({
-  dest: './public/imports',
-  limits: { fileSize: 5 * 1024 * 1024 },
-});
 
 @Service()
 export class ImportController extends BaseController {
@@ -24,7 +19,7 @@ export class ImportController extends BaseController {
 
     router.post(
       '/file',
-      upload.single('file'),
+      uploadImportFile.single('file'),
       this.importValidationSchema,
       this.validationResult,
       this.asyncMiddleware(this.fileUpload.bind(this)),
@@ -60,27 +55,11 @@ export class ImportController extends BaseController {
    * @returns {ValidationSchema[]}
    */
   private get importValidationSchema() {
-    return [
+    return [body('resource').exists()];
-      body('resource').exists(),
-      //   body('file').custom((value, { req }) => {
-      //     if (!value) {
-      //       throw new Error('File is required');
-      //     }
-      //     if (!['xlsx', 'csv'].includes(value.split('.').pop())) {
-      //       throw new Error('File must be in xlsx or csv format');
-      //     }
-      //     return true;
-      //   }),
-    ];
   }
 
   /**
    * Imports xlsx/csv to the given resource type.
-   *
-   * - Save the xlsx/csv file and give it a random name.
-   * - Save the file metadata on the DB storage.
-   * -
-   *
    * @param {Request} req -
    * @param {Response} res -
    * @param {NextFunction} next -
@@ -117,7 +96,6 @@ export class ImportController extends BaseController {
         importId,
         body?.mapping
       );
-
       return res.status(200).send(mapping);
     } catch (error) {
       next(error);
@@ -144,22 +122,19 @@ export class ImportController extends BaseController {
   }
 
   /**
-   *
+   * Importing the imported file to the application storage.
-   * @param req
+   * @param {Request} req
-   * @param res
+   * @param {Response} res
-   * @param next
+   * @param {NextFunction} next
    */
   private async import(req: Request, res: Response, next: NextFunction) {
     const { tenantId } = req;
     const { import_id: importId } = req.params;
 
     try {
-      await this.importResourceApp.process(tenantId, importId);
+      const result = await this.importResourceApp.process(tenantId, importId);
 
-      return res.status(200).send({
+      return res.status(200).send(result);
-        id: importId,
-        message: 'Importing the uploaded file is importing.',
-      });
     } catch (error) {
       next(error);
     }
@@ -194,6 +169,11 @@ export class ImportController extends BaseController {
           errors: [{ type: 'DUPLICATED_TO_MAP_ATTR' }],
         });
       }
+      if (error.errorType === 'IMPORTED_FILE_EXTENSION_INVALID') {
+        return res.status(400).send({
+          errors: [{ type: 'IMPORTED_FILE_EXTENSION_INVALID' }],
+        });
+      }
     }
     next(error);
   }

packages/server/src/api/controllers/Import/_utils.ts

@@ -0,0 +1,20 @@
+import Multer from 'multer';
+import { ServiceError } from '@/exceptions';
+
+export function allowSheetExtensions(req, file, cb) {
+  if (
+    file.mimetype !== 'text/csv' &&
+    file.mimetype !== 'application/vnd.ms-excel'
+  ) {
+    cb(new ServiceError('IMPORTED_FILE_EXTENSION_INVALID'));
+
+    return;
+  }
+  cb(null, true);
+}
+
+export const uploadImportFile = Multer({
+  dest: './public/imports',
+  limits: { fileSize: 5 * 1024 * 1024 },
+  fileFilter: allowSheetExtensions,
+});

packages/server/src/loaders/express.ts

@@ -4,7 +4,6 @@ import helmet from 'helmet';
 import boom from 'express-boom';
 import errorHandler from 'errorhandler';
 import bodyParser from 'body-parser';
-import fileUpload from 'express-fileupload';
 import { Server } from 'socket.io';
 import Container from 'typedi';
 import routes from 'api';
@@ -47,13 +46,6 @@ export default ({ app }) => {
 
   app.use('/public', express.static(path.join(global.__storage_dir)));
 
-  // Handle multi-media requests.
-  // app.use(
-  //   fileUpload({
-  //     createParentPath: true,
-  //   })
-  // );
-
   // Logger middleware.
   app.use(LoggerMiddleware);
 

packages/server/src/services/Accounts/AccountsImportable.ts

@@ -1,15 +1,16 @@
 import { Inject, Service } from 'typedi';
 import { Knex } from 'knex';
 import { IAccountCreateDTO } from '@/interfaces';
-import { CreateAccount } from '../Accounts/CreateAccount';
+import { CreateAccount } from './CreateAccount';
+import { Importable } from '../Import/Importable';
 
 @Service()
-export class AccountsImportable {
+export class AccountsImportable extends Importable {
   @Inject()
   private createAccountService: CreateAccount;
 
   /**
-   *
+   * Importing to account service.
    * @param {number} tenantId
    * @param {IAccountCreateDTO} createAccountDTO
    * @returns
@@ -27,20 +28,10 @@ export class AccountsImportable {
   }
 
   /**
-   *
+   * Concurrrency controlling of the importing process.
-   * @param data
+   * @returns {number}
-   * @returns
    */
-  public transform(data) {
+  public get concurrency() {
-    return { ...data };
+    return 1;
-  }
-
-  /**
-   *
-   * @param data
-   * @returns
-   */
-  public preTransform(data) {
-    return { ...data };
   }
 }

packages/server/src/services/Accounts/CreateAccountDTOSchema.ts

@@ -1,33 +0,0 @@
-import { DATATYPES_LENGTH } from '@/data/DataTypes';
-import { IsInt, IsOptional, IsString, Length, Min, Max, IsNotEmpty } from 'class-validator';
-
-export class AccountDTOSchema {
-  @IsString()
-  @Length(3, DATATYPES_LENGTH.STRING)
-  @IsNotEmpty()
-  name: string;
-
-  @IsString()
-  @IsOptional()
-  @Length(3, 6)
-  code?: string;
-
-  @IsOptional()
-  currencyCode?: string;
-
-  @IsString()
-  @Length(3, DATATYPES_LENGTH.STRING)
-  @IsNotEmpty()
-  accountType: string;
-
-  @IsString()
-  @IsOptional()
-  @Length(0, DATATYPES_LENGTH.TEXT)
-  description?: string;
-
-  @IsInt()
-  @IsOptional()
-  @Min(0)
-  @Max(DATATYPES_LENGTH.INT_10)
-  parentAccountId?: number;
-}

packages/server/src/services/Import/ImportFileCommon.ts

@@ -11,7 +11,7 @@ import {
   ImportOperError,
   ImportOperSuccess,
 } from './interfaces';
-import { AccountsImportable } from './AccountsImportable';
+import { AccountsImportable } from '../Accounts/AccountsImportable';
 import { ServiceError } from '@/exceptions';
 import { trimObject } from './_utils';
 import { ImportableResources } from './ImportableResources';
@@ -57,12 +57,12 @@ export class ImportFileCommon {
   }
 
   /**
-   *
+   * Imports the given parsed data to the resource storage through registered importable service. 
    * @param {number} tenantId -
    * @param {string} resourceName - Resource name.
-   * @param {Record<string, any>} parsedData -
+   * @param {Record<string, any>} parsedData - Parsed data.
-   * @param {Knex.Transaction} trx
+   * @param {Knex.Transaction} trx - Knex transaction.
-   * @returns
+   * @returns {Promise<[ImportOperSuccess[], ImportOperError[]]>}
    */
   public async import(
     tenantId: number,
@@ -77,6 +77,8 @@ export class ImportFileCommon {
     const ImportableRegistry = this.importable.registry;
     const importable = ImportableRegistry.getImportable(resourceName);
 
+    const concurrency = importable.concurrency || 10;
+
     const success: ImportOperSuccess[] = [];
     const failed: ImportOperError[] = [];
 
@@ -108,7 +110,7 @@ export class ImportFileCommon {
         failed.push({ index, error });
       }
     };
-    await bluebird.map(parsedData, importAsync, { concurrency: 2 });
+    await bluebird.map(parsedData, importAsync, { concurrency });
 
     return [success, failed];
   }
@@ -127,7 +129,7 @@ export class ImportFileCommon {
    * @param {number} tenantId
    * @param {} importFile
    */
-  private async deleteImportFile(tenantId: number, importFile: any) {
+  public async deleteImportFile(tenantId: number, importFile: any) {
     const { Import } = this.tenancy.models(tenantId);
 
     // Deletes the import row.

packages/server/src/services/Import/ImportFileDataTransformer.ts

@@ -1,10 +1,9 @@
-import { Inject, Service } from 'typedi';
+import { Service } from 'typedi';
 import * as R from 'ramda';
-import { isUndefined, mapValues, get, pickBy, chain } from 'lodash';
+import { isUndefined, get, chain } from 'lodash';
 import { ImportMappingAttr, ResourceMetaFieldsMap } from './interfaces';
 import { parseBoolean } from '@/utils';
 import { trimObject } from './_utils';
-import ResourceService from '../Resource/ResourceService';
 
 @Service()
 export class ImportFileDataTransformer {

packages/server/src/services/Import/ImportFileUpload.ts

@@ -27,7 +27,7 @@ export class ImportFileUploadService {
    * @param {string} resource - Resource name.
    * @param {string} filePath - File path.
    * @param {string} fileName - File name.
-   * @returns
+   * @returns {Promise<ImportFileUploadPOJO>}
    */
   public async import(
     tenantId: number,

packages/server/src/services/Import/ImportResourceApplication.ts

@@ -52,7 +52,7 @@ export class ImportResourceApplication {
    * Preview the mapped results before process importing.
    * @param {number} tenantId
    * @param {number} importId
-   * @returns {}
+   * @returns {Promise<ImportFilePreviewPOJO>}
    */
   public async preview(tenantId: number, importId: number) {
     return this.ImportFilePreviewService.preview(tenantId, importId);
@@ -62,7 +62,7 @@ export class ImportResourceApplication {
    * Process the import file sheet through service for creating entities.
    * @param {number} tenantId
    * @param {number} importId
-   * @returns {Promise<void>}
+   * @returns {Promise<ImportFilePreviewPOJO>}
    */
   public async process(tenantId: number, importId: number) {
     return this.importProcessService.import(tenantId, importId);

packages/server/src/services/Import/ImportResourceRegistry.ts

@@ -1,7 +0,0 @@
-import { Service } from "typedi";
-
-
-@Service()
-export class ImportResourceRegistry {
-
-}

packages/server/src/services/Import/Importable.ts

@@ -1,7 +1,23 @@
+import { Knex } from 'knex';
 
+export abstract class Importable {
+  /**
+   *
+   * @param {number} tenantId
+   * @param {any} createDTO
+   * @param {Knex.Transaction} trx
+   */
+  public importable(tenantId: number, createDTO: any, trx?: Knex.Transaction) {
+    throw new Error(
+      'The `importable` function is not defined in service importable.'
+    );
+  }
 
-abstract class importable {
+  /**
-
+   * Concurrency controlling of the importing process.
-
+   * @returns {number}
-  
+   */
+  public get concurrency() {
+    return 10;
+  }
 }

packages/server/src/services/Import/ImportableRegistry.ts

@@ -1,13 +1,18 @@
 import { camelCase, upperFirst } from 'lodash';
+import { Importable } from './Importable';
 
 export class ImportableRegistry {
   private static instance: ImportableRegistry;
-  private importables: Record<string, any>;
+  private importables: Record<string, Importable>;
 
   private constructor() {
     this.importables = {};
   }
 
+  /**
+   * Gets singleton instance of registry.
+   * @returns {ImportableRegistry}
+   */
   public static getInstance(): ImportableRegistry {
     if (!ImportableRegistry.instance) {
       ImportableRegistry.instance = new ImportableRegistry();
@@ -15,12 +20,22 @@ export class ImportableRegistry {
     return ImportableRegistry.instance;
   }
 
-  public registerImportable(resource: string, importable: any): void {
+  /**
+   * Registers the given importable service.
+   * @param {string} resource 
+   * @param {Importable} importable 
+   */
+  public registerImportable(resource: string, importable: Importable): void {
     const _resource = this.sanitizeResourceName(resource);
     this.importables[_resource] = importable;
   }
 
-  public getImportable(name: string): any {
+  /**
+   * Retrieves the importable service instance of the given resource name.
+   * @param {string} name 
+   * @returns {Importable}
+   */
+  public getImportable(name: string): Importable {
     const _name = this.sanitizeResourceName(name);
     return this.importables[_name];
   }

packages/server/src/services/Import/ImportableResources.ts

@@ -1,12 +1,11 @@
 import Container, { Service } from 'typedi';
-import { AccountsImportable } from './AccountsImportable';
+import { AccountsImportable } from '../Accounts/AccountsImportable';
 import { ImportableRegistry } from './ImportableRegistry';
 
 @Service()
 export class ImportableResources {
   private static registry: ImportableRegistry;
 
-  
   constructor() {
     this.boot();
   }