WIP

packages/server/package.json

@@ -36,6 +36,8 @@
     "bcryptjs": "^2.4.3",
     "bluebird": "^3.7.2",
     "body-parser": "^1.20.2",
+    "class-transformer": "^0.5.1",
+    "class-validator": "^0.14.0",
     "compression": "^1.7.4",
     "country-codes-list": "^1.6.8",
     "cpy": "^8.1.2",
@@ -77,6 +79,7 @@
     "moment-timezone": "^0.5.43",
     "mongodb": "^6.1.0",
     "mongoose": "^5.10.0",
+    "multer": "1.4.5-lts.1",
     "mustache": "^3.0.3",
     "mysql": "^2.17.1",
     "mysql2": "^1.6.5",

packages/server/src/api/controllers/Import/ImportController.ts

@@ -0,0 +1,149 @@
+import { Inject, Service } from 'typedi';
+import { Router, Request, Response, NextFunction } from 'express';
+import { query } from 'express-validator';
+import Multer from 'multer';
+import BaseController from '@/api/controllers/BaseController';
+import { ServiceError } from '@/exceptions';
+import { ImportResourceInjectable } from '@/services/Import/ImportResourceInjectable';
+import { ImportResourceApplication } from '@/services/Import/ImportResourceApplication';
+
+const upload = Multer({
+  dest: './public/imports',
+  limits: { fileSize: 5 * 1024 * 1024 },
+});
+
+@Service()
+export class ImportController extends BaseController {
+  @Inject()
+  private importResource: ImportResourceInjectable;
+
+  @Inject()
+  private importResourceApp: ImportResourceApplication;
+
+  /**
+   * Router constructor method.
+   */
+  router() {
+    const router = Router();
+
+    // router.post(
+    //   '/:import_id/import',
+    //   this.asyncMiddleware(this.import.bind(this)),
+    //   this.catchServiceErrors
+    // );
+    router.post(
+      '/file',
+      // [...this.importValidationSchema],
+      upload.single('file'),
+      this.asyncMiddleware(this.fileUpload.bind(this))
+      // this.catchServiceErrors
+    );
+    // router.post(
+    //   '/:import_id/mapping',
+    //   this.asyncMiddleware(this.mapping.bind(this)),
+    //   this.catchServiceErrors
+    // );
+    // router.get(
+    //   '/:import_id/preview',
+    //   this.asyncMiddleware(this.preview.bind(this)),
+    //   this.catchServiceErrors
+    // );
+    return router;
+  }
+
+  /**
+   * Import validation schema.
+   * @returns {ValidationSchema[]}
+   */
+  get importValidationSchema() {
+    return [query('resource').exists().isString().toString()];
+  }
+
+  /**
+   * Imports xlsx/csv to the given resource type.
+   *
+   * - Save the xlsx/csv file and give it a random name.
+   * - Save the file metadata on the DB storage.
+   * -
+   *
+   * @param {Request} req -
+   * @param {Response} res -
+   * @param {NextFunction} next -
+   */
+  async fileUpload(req: Request, res: Response, next: NextFunction) {
+    const { tenantId } = req;
+
+    try {
+      const data = await this.importResourceApp.import(
+        tenantId,
+        req.body.resource,
+        req.file.path,
+        req.file.filename
+      );
+      return res.status(200).send(data);
+    } catch (error) {
+      next(error);
+    }
+  }
+
+  /**
+   *
+   * @param req
+   * @param res
+   * @param next
+   */
+  async mapping(req: Request, res: Response, next: NextFunction) {
+    const { tenantId } = req;
+
+    try {
+      await this.importResource.mapping(tenantId);
+    } catch (error) {
+      next(error);
+    }
+  }
+
+  /**
+   *
+   * @param req
+   * @param res
+   * @param next
+   */
+  async preview(req: Request, res: Response, next: NextFunction) {}
+
+  /**
+   *
+   * @param req
+   * @param res
+   * @param next
+   */
+  async import(req: Request, res: Response, next: NextFunction) {
+    const { tenantId } = req;
+    const { import_id: importId } = req.params;
+
+    try {
+      await this.importResource.importFile(tenantId, importId);
+
+      return res.status(200).send({});
+    } catch (error) {
+      next(error);
+    }
+  }
+
+  /**
+   * Transforms service errors to response.
+   * @param {Error}
+   * @param {Request} req
+   * @param {Response} res
+   * @param {ServiceError} error
+   */
+  private catchServiceErrors(
+    error,
+    req: Request,
+    res: Response,
+    next: NextFunction
+  ) {
+    if (error instanceof ServiceError) {
+    }
+    next(error);
+  }
+}

packages/server/src/api/index.ts

@@ -56,6 +56,7 @@ import { ProjectsController } from './controllers/Projects/Projects';
 import { ProjectTasksController } from './controllers/Projects/Tasks';
 import { ProjectTimesController } from './controllers/Projects/Times';
 import { TaxRatesController } from './controllers/TaxRates/TaxRates';
+import { ImportController } from './controllers/Import/ImportController';
 
 export default () => {
   const app = Router();
@@ -131,6 +132,9 @@ export default () => {
   dashboard.use('/warehouses', Container.get(WarehousesController).router());
   dashboard.use('/projects', Container.get(ProjectsController).router());
   dashboard.use('/tax-rates', Container.get(TaxRatesController).router());
+
+  dashboard.use('/import', Container.get(ImportController).router());
+
   dashboard.use('/', Container.get(ProjectTasksController).router());
   dashboard.use('/', Container.get(ProjectTimesController).router());
 

packages/server/src/database/migrations/20231209230719_create_imports_table.js

@@ -0,0 +1,14 @@
+exports.up = function (knex) {
+  return knex.schema.createTable('imports', (table) => {
+    table.increments();
+    table.string('filename');
+    table.string('import_id');
+    table.string('resource');
+    table.json('mapping');
+    table.timestamps();
+  });
+};
+
+exports.down = function (knex) {
+  return knex.schema.dropTableIfExists('imports');
+};

packages/server/src/loaders/tenantModels.ts

@@ -61,6 +61,7 @@ import Task from 'models/Task';
 import TaxRate from 'models/TaxRate';
 import TaxRateTransaction from 'models/TaxRateTransaction';
 import Attachment from 'models/Attachment';
+import Import from 'models/Import';
 
 export default (knex) => {
   const models = {
@@ -124,7 +125,8 @@ export default (knex) => {
     Task,
     TaxRate,
     TaxRateTransaction,
-    Attachment
+    Attachment,
+    Import
   };
   return mapValues(models, (model) => model.bindKnex(knex));
 };

packages/server/src/services/Accounts/CreateAccountDTOSchema.ts

@@ -0,0 +1,32 @@
+import { DATATYPES_LENGTH } from '@/data/DataTypes';
+import { IsInt, IsOptional, IsString, Length, Min, Max } from 'class-validator';
+
+export class AccountDTOSchema {
+  @IsString()
+  @Length(3, DATATYPES_LENGTH.STRING)
+  name: string;
+
+  // @IsString()
+  // @IsInt()
+  @IsOptional()
+  // @Length(3, 6)
+  code?: string;
+
+  @IsOptional()
+  currencyCode?: string;
+
+  @IsString()
+  @Length(3, DATATYPES_LENGTH.STRING)
+  accountType: string;
+
+  @IsString()
+  @IsOptional()
+  @Length(0, DATATYPES_LENGTH.TEXT)
+  description?: string;
+
+  @IsInt()
+  @IsOptional()
+  @Min(0)
+  @Max(DATATYPES_LENGTH.INT_10)
+  parentAccountId?: number;
+}

packages/server/src/services/Import/ImportFileUpload.ts

@@ -0,0 +1,54 @@
+import { first } from 'lodash';
+import { Inject, Service } from 'typedi';
+import { snakeCase, upperFirst } from 'lodash';
+import XLSX from 'xlsx';
+import * as R from 'ramda';
+import HasTenancyService from '../Tenancy/TenancyService';
+import { trimObject } from './_utils';
+const fs = require('fs').promises;
+
+@Service()
+export class ImportFileUploadService {
+  @Inject()
+  private tenancy: HasTenancyService;
+
+  /**
+   * Reads the imported file and stores the import file meta under unqiue id.
+   * @param {number} tenantId -
+   * @param {string} filePath -
+   * @param {string} fileName -
+   * @returns
+   */
+  public async import(
+    tenantId: number,
+    resource: string,
+    filePath: string,
+    filename: string
+  ) {
+    const { Import } = this.tenancy.models(tenantId);
+    const buffer = await fs.readFile(filePath);
+    const workbook = XLSX.read(buffer, { type: 'buffer' });
+
+    const firstSheetName = workbook.SheetNames[0];
+    const worksheet = workbook.Sheets[firstSheetName];
+    const jsonData = XLSX.utils.sheet_to_json(worksheet);
+
+    const _resource = upperFirst(snakeCase(resource));
+
+    const exportFile = await Import.query().insert({
+      filename,
+      importId: filename,
+      resource: _resource,
+    });
+    const columns = this.getColumns(jsonData);
+
+    return {
+      ...exportFile,
+      columns,
+    };
+  }
+
+  private getColumns(json: unknown[]) {
+    return R.compose(Object.keys, trimObject, first)(json);
+  }
+}

packages/server/src/services/Import/ImportResourceApplication.ts

@@ -0,0 +1,29 @@
+import { Inject } from 'typedi';
+import { ImportFileUploadService } from './ImportFileUpload';
+
+@Inject()
+export class ImportResourceApplication {
+  @Inject()
+  private importFileService: ImportFileUploadService;
+
+  /**
+   * Reads the imported file and stores the import file meta under unqiue id.
+   * @param {number} tenantId -
+   * @param {string} filePath -
+   * @param {string} fileName -
+   * @returns
+   */
+  public async import(
+    tenantId: number,
+    resource: string,
+    filePath: string,
+    filename: string
+  ) {
+    return this.importFileService.import(
+      tenantId,
+      resource,
+      filePath,
+      filename
+    );
+  }
+}

packages/server/src/services/Import/ImportResourceInjectable.ts

@@ -0,0 +1,120 @@
+import XLSX, { readFile } from 'xlsx';
+import * as R from 'ramda';
+import async from 'async';
+import { camelCase, snakeCase, upperFirst } from 'lodash';
+import HasTenancyService from '../Tenancy/TenancyService';
+import { Inject, Service } from 'typedi';
+import { first } from 'lodash';
+import { ServiceError } from '@/exceptions';
+import { validate } from 'class-validator';
+import { AccountDTOSchema } from '../Accounts/CreateAccountDTOSchema';
+import { AccountsApplication } from '../Accounts/AccountsApplication';
+import { plainToClass, plainToInstance } from 'class-transformer';
+const fs = require('fs').promises;
+
+const ERRORS = {
+  IMPORT_ID_NOT_FOUND: 'IMPORT_ID_NOT_FOUND',
+};
+
+
+@Service()
+export class ImportResourceInjectable {
+  @Inject()
+  private tenancy: HasTenancyService;
+
+  @Inject()
+  private accountsApplication: AccountsApplication;
+
+  public async mapping(
+    tenantId: number,
+    importId: number,
+    maps: { from: string; to: string }[]
+  ) {
+    const { Import } = this.tenancy.models(tenantId);
+
+    const importFile = await Import.query().find('filename', importId);
+
+    if (!importFile) {
+      throw new ServiceError(ERRORS.IMPORT_ID_NOT_FOUND);
+    }
+    //
+    await Import.query()
+      .findById(importFile.id)
+      .update({
+        maps: JSON.stringify(maps),
+      });
+    // - Validate the to columns.
+    // - Store the mapping in the import table.
+    // -
+  }
+
+  public async preview(tenantId: number, importId: string) {}
+
+  /**
+   *
+   * @param tenantId
+   * @param importId
+   */
+  public async importFile(tenantId: number, importId: string) {
+    const { Import } = this.tenancy.models(tenantId);
+
+    const importFile = await Import.query().where('importId', importId).first();
+
+    if (!importFile) {
+      throw new ServiceError(ERRORS.IMPORT_ID_NOT_FOUND);
+    }
+    const buffer = await fs.readFile(`public/imports/${importFile.filename}`);
+    const workbook = XLSX.read(buffer, { type: 'buffer' });
+
+    const firstSheetName = workbook.SheetNames[0];
+    const worksheet = workbook.Sheets[firstSheetName];
+    const jsonData = XLSX.utils.sheet_to_json(worksheet);
+
+    const data = R.compose(R.map(Object.keys), R.map(trimObject))(jsonData);
+
+    const header = first(data);
+    const body = jsonData;
+
+    const mapping = JSON.parse(importFile.mapping) || [];
+    const newData = [];
+
+    const findToAttr = (from: string) => {
+      const found = mapping.find((item) => {
+        return item.from === from;
+      });
+      return found?.to;
+    };
+
+    body.forEach((row) => {
+      const obj = {};
+
+      header.forEach((key, index) => {
+        const toIndex = camelCase(findToAttr(key));
+        obj[toIndex] = row[key];
+      });
+      newData.push(obj);
+    });
+
+    const saveJob = async (data) => {
+      const account = {};
+
+      Object.keys(data).map((key) => {
+        account[key] = data[key];
+      });
+      const accountClass = plainToInstance(AccountDTOSchema, account);
+      const errors = await validate(accountClass);
+
+      if (errors.length > 0) {
+        console.log('validation failed. errors: ', errors);
+      } else {
+        return this.accountsApplication.createAccount(tenantId, account);
+      }
+    };
+    const saveDataQueue = async.queue(saveJob, 10);
+
+    newData.forEach((data) => {
+      saveDataQueue.push(data);
+    });
+    await saveDataQueue.drain();
+  }
+}

packages/server/src/services/Import/_utils.ts

@@ -0,0 +1,12 @@
+export function trimObject(obj) {
+  return Object.entries(obj).reduce((acc, [key, value]) => {
+    // Trim the key
+    const trimmedKey = key.trim();
+
+    // Trim the value if it's a string, otherwise leave it as is
+    const trimmedValue = typeof value === 'string' ? value.trim() : value;
+
+    // Assign the trimmed key and value to the accumulator object
+    return { ...acc, [trimmedKey]: trimmedValue };
+  }, {});
+}

pnpm-lock.yaml

@@ -80,6 +80,12 @@ importers:
       body-parser:
         specifier: ^1.20.2
         version: 1.20.2
+      class-transformer:
+        specifier: ^0.5.1
+        version: 0.5.1
+      class-validator:
+        specifier: ^0.14.0
+        version: 0.14.0
       compression:
         specifier: ^1.7.4
         version: 1.7.4
@@ -203,6 +209,9 @@ importers:
       mongoose:
         specifier: ^5.10.0
         version: 5.13.20
+      multer:
+        specifier: 1.4.5-lts.1
+        version: 1.4.5-lts.1
       mustache:
         specifier: ^3.0.3
         version: 3.2.1
@@ -6481,6 +6490,10 @@ packages:
     resolution: {integrity: sha512-IDaobHimLQhjwsQ/NMwRVfa/yL7L/wriQPMhw1ZJall0KX6E1oxk29XMDeilW5qTIg5aoiqf5Udy8U/51aNoQQ==}
     dev: false
 
+  /@types/validator@13.11.7:
+    resolution: {integrity: sha512-q0JomTsJ2I5Mv7dhHhQLGjMvX0JJm5dyZ1DXQySIUzU1UlwzB8bt+R6+LODUbz0UDIOvEzGc28tk27gBJw2N8Q==}
+    dev: false
+
   /@types/webidl-conversions@7.0.0:
     resolution: {integrity: sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==}
     dev: false
@@ -7458,6 +7471,10 @@ packages:
       buffer-equal: 1.0.1
     dev: false
 
+  /append-field@1.0.0:
+    resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==}
+    dev: false
+
   /append-transform@1.0.0:
     resolution: {integrity: sha512-P009oYkeHyU742iSZJzZZywj4QRJdnTWffaKuJQLablCZ1uz6/cW4yaRgcDaoQ+uwOxxnt0gRUcwfsNP2ri0gw==}
     engines: {node: '>=4'}
@@ -9026,6 +9043,10 @@ packages:
     resolution: {integrity: sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==}
     dev: false
 
+  /class-transformer@0.5.1:
+    resolution: {integrity: sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==}
+    dev: false
+
   /class-utils@0.3.6:
     resolution: {integrity: sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==}
     engines: {node: '>=0.10.0'}
@@ -9036,6 +9057,14 @@ packages:
       static-extend: 0.1.2
     dev: false
 
+  /class-validator@0.14.0:
+    resolution: {integrity: sha512-ct3ltplN8I9fOwUd8GrP8UQixwff129BkEtuWDKL5W45cQuLd19xqmTLu5ge78YDm/fdje6FMt0hGOhl0lii3A==}
+    dependencies:
+      '@types/validator': 13.11.7
+      libphonenumber-js: 1.10.19
+      validator: 13.9.0
+    dev: false
+
   /classnames@2.3.2:
     resolution: {integrity: sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==}
     dev: false
@@ -17307,6 +17336,19 @@ packages:
   /ms@2.1.3:
     resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
 
+  /multer@1.4.5-lts.1:
+    resolution: {integrity: sha512-ywPWvcDMeH+z9gQq5qYHCCy+ethsk4goepZ45GLD63fOu0YcNecQxi64nDs3qluZB+murG3/D4dJ7+dGctcCQQ==}
+    engines: {node: '>= 6.0.0'}
+    dependencies:
+      append-field: 1.0.0
+      busboy: 1.6.0
+      concat-stream: 1.6.2
+      mkdirp: 0.5.6
+      object-assign: 4.1.1
+      type-is: 1.6.18
+      xtend: 4.0.2
+    dev: false
+
   /multicast-dns@7.2.5:
     resolution: {integrity: sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==}
     hasBin: true