fix: import resource imporements

packages/server/src/services/Import/ImportFileCommon.ts

@@ -7,16 +7,16 @@ import { first } from 'lodash';
 import { ImportFileDataValidator } from './ImportFileDataValidator';
 import { Knex } from 'knex';
 import {
-  ImportInsertError,
   ImportOperError,
   ImportOperSuccess,
+  ImportableContext,
 } from './interfaces';
-import { AccountsImportable } from '../Accounts/AccountsImportable';
 import { ServiceError } from '@/exceptions';
 import { trimObject } from './_utils';
 import { ImportableResources } from './ImportableResources';
 import ResourceService from '../Resource/ResourceService';
 import HasTenancyService from '../Tenancy/TenancyService';
+import Import from '@/models/Import';
 
 @Service()
 export class ImportFileCommon {
@@ -39,12 +39,12 @@ export class ImportFileCommon {
    * @returns {Record<string, any>[]} - The mapped data objects.
    */
   public parseXlsxSheet(buffer: Buffer): Record<string, unknown>[] {
-    const workbook = XLSX.read(buffer, { type: 'buffer' });
+    const workbook = XLSX.read(buffer, { type: 'buffer', raw: true });
 
     const firstSheetName = workbook.SheetNames[0];
     const worksheet = workbook.Sheets[firstSheetName];
 
-    return XLSX.utils.sheet_to_json(worksheet);
+    return XLSX.utils.sheet_to_json(worksheet, {});
   }
 
   /**
@@ -57,7 +57,7 @@ export class ImportFileCommon {
   }
 
   /**
-   * Imports the given parsed data to the resource storage through registered importable service. 
+   * Imports the given parsed data to the resource storage through registered importable service.
    * @param {number} tenantId -
    * @param {string} resourceName - Resource name.
    * @param {Record<string, any>} parsedData - Parsed data.
@@ -66,16 +66,16 @@ export class ImportFileCommon {
    */
   public async import(
     tenantId: number,
-    resourceName: string,
+    importFile: Import,
     parsedData: Record<string, any>[],
     trx?: Knex.Transaction
   ): Promise<[ImportOperSuccess[], ImportOperError[]]> {
     const importableFields = this.resource.getResourceImportableFields(
       tenantId,
-      resourceName
+      importFile.resource
     );
     const ImportableRegistry = this.importable.registry;
-    const importable = ImportableRegistry.getImportable(resourceName);
+    const importable = ImportableRegistry.getImportable(importFile.resource);
 
     const concurrency = importable.concurrency || 10;
 
@@ -83,15 +83,25 @@ export class ImportFileCommon {
     const failed: ImportOperError[] = [];
 
     const importAsync = async (objectDTO, index: number): Promise<void> => {
+      const context: ImportableContext = {
+        rowIndex: index,
+        import: importFile,
+      };
+      const transformedDTO = importable.transform(objectDTO, context);
+
       try {
         // Validate the DTO object before passing it to the service layer.
         await this.importFileValidator.validateData(
           importableFields,
-          objectDTO
+          transformedDTO
         );
         try {
           // Run the importable function and listen to the errors.
-          const data = await importable.importable(tenantId, objectDTO, trx);
+          const data = await importable.importable(
+            tenantId,
+            transformedDTO,
+            trx
+          );
           success.push({ index, data });
         } catch (err) {
           if (err instanceof ServiceError) {
@@ -115,6 +125,60 @@ export class ImportFileCommon {
     return [success, failed];
   }
 
+  /**
+   *
+   * @param {string} resourceName
+   * @param {Record<string, any>} params
+   */
+  public async validateParamsSchema(
+    resourceName: string,
+    params: Record<string, any>
+  ) {
+    const ImportableRegistry = this.importable.registry;
+    const importable = ImportableRegistry.getImportable(resourceName);
+
+    const yupSchema = importable.paramsValidationSchema();
+
+    try {
+      await yupSchema.validate(params, { abortEarly: false });
+    } catch (validationError) {
+      const errors = validationError.inner.map((error) => ({
+        errorCode: 'ParamsValidationError',
+        errorMessage: error.errors,
+      }));
+      throw errors;
+    }
+  }
+
+  /**
+   *
+   * @param {string} resourceName
+   * @param {Record<string, any>} params
+   */
+  public async validateParams(
+    tenantId: number,
+    resourceName: string,
+    params: Record<string, any>
+  ) {
+    const ImportableRegistry = this.importable.registry;
+    const importable = ImportableRegistry.getImportable(resourceName);
+
+    await importable.validateParams(tenantId, params);
+  }
+
+  /**
+   *
+   * @param {string} resourceName
+   * @param {Record<string, any>} params
+   * @returns
+   */
+  public transformParams(resourceName: string, params: Record<string, any>) {
+    const ImportableRegistry = this.importable.registry;
+    const importable = ImportableRegistry.getImportable(resourceName);
+
+    return importable.transformParams(params);
+  }
+
   /**
    * Retrieves the sheet columns from the given sheet data.
    * @param {unknown[]} json

packages/server/src/services/Import/ImportFileMapping.ts

@@ -1,7 +1,11 @@
 import { fromPairs } from 'lodash';
 import { Inject, Service } from 'typedi';
 import HasTenancyService from '../Tenancy/TenancyService';
-import { ImportFileMapPOJO, ImportMappingAttr } from './interfaces';
+import {
+  ImportDateFormats,
+  ImportFileMapPOJO,
+  ImportMappingAttr,
+} from './interfaces';
 import ResourceService from '../Resource/ResourceService';
 import { ServiceError } from '@/exceptions';
 import { ERRORS } from './_utils';
@@ -37,12 +41,14 @@ export class ImportFileMapping {
     // Validate the diplicated relations of map attrs.
     this.validateDuplicatedMapAttrs(maps);
 
+    // Validate the date format mapping.
+    this.validateDateFormatMapping(tenantId, importFile.resource, maps);
+
     const mappingStringified = JSON.stringify(maps);
 
     await Import.query().findById(importFile.id).patch({
       mapping: mappingStringified,
     });
-
     return {
       import: {
         importId: importFile.importId,
@@ -106,4 +112,34 @@ export class ImportFileMapping {
       }
     });
   }
+
+  /**
+   * Validates the date format mapping.
+   * @param {number} tenantId
+   * @param {string} resource
+   * @param {ImportMappingAttr[]} maps
+   */
+  private validateDateFormatMapping(
+    tenantId: number,
+    resource: string,
+    maps: ImportMappingAttr[]
+  ) {
+    const fields = this.resource.getResourceImportableFields(
+      tenantId,
+      resource
+    );
+    maps.forEach((map) => {
+      if (
+        typeof fields[map.to] !== 'undefined' &&
+        fields[map.to].fieldType === 'date'
+      ) {
+        if (
+          typeof map.dateFormat !== 'undefined' &&
+          ImportDateFormats.indexOf(map.dateFormat) === -1
+        ) {
+          throw new ServiceError(ERRORS.INVALID_MAP_DATE_FORMAT);
+        }
+      }
+    });
+  }
 }

packages/server/src/services/Import/ImportFileMeta.ts

@@ -0,0 +1,32 @@
+import { Inject, Service } from 'typedi';
+import HasTenancyService from '../Tenancy/TenancyService';
+import { TransformerInjectable } from '@/lib/Transformer/TransformerInjectable';
+import { ImportFileMetaTransformer } from './ImportFileMetaTransformer';
+
+@Service()
+export class ImportFileMeta {
+  @Inject()
+  private tenancy: HasTenancyService;
+
+  @Inject()
+  private transformer: TransformerInjectable;
+
+  /**
+   *
+   * @param {number} tenantId
+   * @param {number} importId
+   * @returns {}
+   */
+  async getImportMeta(tenantId: number, importId: string) {
+    const { Import } = this.tenancy.models(tenantId);
+
+    const importFile = await Import.query().findOne('importId', importId);
+
+    // Retrieves the transformed accounts collection.
+    return this.transformer.transform(
+      tenantId,
+      importFile,
+      new ImportFileMetaTransformer()
+    );
+  }
+}

packages/server/src/services/Import/ImportFileMetaTransformer.ts

@@ -0,0 +1,19 @@
+import { Transformer } from '@/lib/Transformer/Transformer';
+
+export class ImportFileMetaTransformer extends Transformer {
+  /**
+   * Include these attributes to sale invoice object.
+   * @returns {Array}
+   */
+  public includeAttributes = (): string[] => {
+    return ['map'];
+  };
+
+  public excludeAttributes = (): string[] => {
+    return ['id', 'filename', 'columns', 'mappingParsed', 'mapping'];
+  }
+
+  map(importFile) {
+    return  importFile.mappingParsed;
+  }
+}

packages/server/src/services/Import/ImportFileProcess.ts

@@ -67,12 +67,7 @@ export class ImportFileProcess {
     const [successedImport, failedImport] = await this.uow.withTransaction(
       tenantId,
       (trx: Knex.Transaction) =>
-        this.importCommon.import(
-          tenantId,
-          importFile.resource,
-          parsedData,
-          trx
-        ),
+        this.importCommon.import(tenantId, importFile, parsedData, trx),
       trx
     );
     const mapping = importFile.mappingParsed;

packages/server/src/services/Import/ImportFileUpload.ts

@@ -6,6 +6,7 @@ import { IModelMetaField } from '@/interfaces';
 import { ImportFileCommon } from './ImportFileCommon';
 import { ImportFileDataValidator } from './ImportFileDataValidator';
 import { ImportFileUploadPOJO } from './interfaces';
+import { ServiceError } from '@/exceptions';
 
 @Service()
 export class ImportFileUploadService {
@@ -32,13 +33,15 @@ export class ImportFileUploadService {
   public async import(
     tenantId: number,
     resourceName: string,
-    filename: string
+    filename: string,
+    params: Record<string, number | string>
   ): Promise<ImportFileUploadPOJO> {
     const { Import } = this.tenancy.models(tenantId);
 
+    const resource = sanitizeResourceName(resourceName);
     const resourceMeta = this.resourceService.getResourceMeta(
       tenantId,
-      resourceName
+      resource
     );
     // Throw service error if the resource does not support importing.
     this.importValidator.validateResourceImportable(resourceMeta);
@@ -48,22 +51,32 @@ export class ImportFileUploadService {
 
     // Parse the buffer file to array data.
     const sheetData = this.importFileCommon.parseXlsxSheet(buffer);
-
     const sheetColumns = this.importFileCommon.parseSheetColumns(sheetData);
     const coumnsStringified = JSON.stringify(sheetColumns);
 
-    const _resourceName = sanitizeResourceName(resourceName);
+    try {
+      // Validates the params Yup schema.
+      await this.importFileCommon.validateParamsSchema(resource, params);
+
+      // Validates importable params asyncly.
+      await this.importFileCommon.validateParams(tenantId, resource, params);
+    } catch (error) {
+      throw error;
+    }
+    const _params = this.importFileCommon.transformParams(resource, params);
+    const paramsStringified = JSON.stringify(_params);
 
     // Store the import model with related metadata.
     const importFile = await Import.query().insert({
       filename,
+      resource,
       importId: filename,
-      resource: _resourceName,
       columns: coumnsStringified,
+      params: paramsStringified,
     });
     const resourceColumnsMap = this.resourceService.getResourceImportableFields(
       tenantId,
-      _resourceName
+      resource
     );
     const resourceColumns = this.getResourceColumns(resourceColumnsMap);
 

packages/server/src/services/Import/ImportResourceApplication.ts

@@ -4,6 +4,8 @@ import { ImportFileMapping } from './ImportFileMapping';
 import { ImportMappingAttr } from './interfaces';
 import { ImportFileProcess } from './ImportFileProcess';
 import { ImportFilePreview } from './ImportFilePreview';
+import { ImportSampleService } from './ImportSample';
+import { ImportFileMeta } from './ImportFileMeta';
 
 @Inject()
 export class ImportResourceApplication {
@@ -19,25 +21,32 @@ export class ImportResourceApplication {
   @Inject()
   private ImportFilePreviewService: ImportFilePreview;
 
+  @Inject()
+  private importSampleService: ImportSampleService;
+
+  @Inject()
+  private importMetaService: ImportFileMeta;
+
   /**
    * Reads the imported file and stores the import file meta under unqiue id.
    * @param {number} tenantId -
-   * @param {string} resource - 
-   * @param {string} fileName -
+   * @param {string} resource - Resource name.
+   * @param {string} fileName - File name.
    * @returns {Promise<ImportFileUploadPOJO>}
    */
   public async import(
     tenantId: number,
     resource: string,
-    filename: string
+    filename: string,
+    params: Record<string, any>
   ) {
-    return this.importFileService.import(tenantId, resource, filename);
+    return this.importFileService.import(tenantId, resource, filename, params);
   }
 
   /**
    * Mapping the excel sheet columns with resource columns.
    * @param {number} tenantId
-   * @param {number} importId
+   * @param {number} importId - Import id.
    * @param {ImportMappingAttr} maps
    */
   public async mapping(
@@ -51,7 +60,7 @@ export class ImportResourceApplication {
   /**
    * Preview the mapped results before process importing.
    * @param {number} tenantId
-   * @param {number} importId
+   * @param {number} importId - Import id.
    * @returns {Promise<ImportFilePreviewPOJO>}
    */
   public async preview(tenantId: number, importId: number) {
@@ -67,4 +76,27 @@ export class ImportResourceApplication {
   public async process(tenantId: number, importId: number) {
     return this.importProcessService.import(tenantId, importId);
   }
+
+  /**
+   * Retrieves the import meta of the given import id.
+   * @param {number} tenantId - 
+   * @param {string} importId - Import id.
+   * @returns {}
+   */
+  public importMeta(tenantId: number, importId: string) {
+    return this.importMetaService.getImportMeta(tenantId, importId);
+  }
+
+  /**
+   * Retrieves the csv/xlsx sample sheet of the given
+   * @param {number} tenantId
+   * @param {number} resource - Resource name.
+   */
+  public sample(
+    tenantId: number,
+    resource: string,
+    format: 'csv' | 'xlsx' = 'csv'
+  ) {
+    return this.importSampleService.sample(tenantId, resource, format);
+  }
 }

packages/server/src/services/Import/ImportSample.ts

@@ -0,0 +1,46 @@
+import XLSX from 'xlsx';
+import { Inject, Service } from 'typedi';
+import { ImportableResources } from './ImportableResources';
+import { sanitizeResourceName } from './_utils';
+
+@Service()
+export class ImportSampleService {
+  @Inject()
+  private importable: ImportableResources;
+
+  /**
+   * Retrieves the sample sheet of the given resource.
+   * @param {number} tenantId 
+   * @param {string} resource 
+   * @param {string} format 
+   * @returns {Buffer | string}
+   */
+  public sample(
+    tenantId: number,
+    resource: string,
+    format: 'csv' | 'xlsx'
+  ): Buffer | string {
+    const _resource = sanitizeResourceName(resource);
+
+    const ImportableRegistry = this.importable.registry;
+    const importable = ImportableRegistry.getImportable(_resource);
+
+    const data = importable.sampleData();
+
+    const workbook = XLSX.utils.book_new();
+    const worksheet = XLSX.utils.json_to_sheet(data);
+    XLSX.utils.book_append_sheet(workbook, worksheet, 'Sheet1');
+
+    // Determine the output format
+    if (format === 'csv') {
+      const csvOutput = XLSX.utils.sheet_to_csv(worksheet);
+      return csvOutput;
+    } else {
+      const xlsxOutput = XLSX.write(workbook, {
+        bookType: 'xlsx',
+        type: 'buffer',
+      });
+      return xlsxOutput;
+    }
+  }
+}

packages/server/src/services/Import/Importable.ts

@@ -1,4 +1,6 @@
 import { Knex } from 'knex';
+import * as Yup from 'yup';
+import { ImportableContext } from './interfaces';
 
 export abstract class Importable {
   /**
@@ -13,6 +15,16 @@ export abstract class Importable {
     );
   }
 
+  /**
+   * Transformes the DTO before passing it to importable and validation.
+   * @param {Record<string, any>} createDTO
+   * @param {ImportableContext} context
+   * @returns {Record<string, any>}
+   */
+  public transform(createDTO: Record<string, any>, context: ImportableContext) {
+    return createDTO;
+  }
+
   /**
    * Concurrency controlling of the importing process.
    * @returns {number}
@@ -20,4 +32,41 @@ export abstract class Importable {
   public get concurrency() {
     return 10;
   }
+
+  /**
+   * Retrieves the sample data of importable.
+   * @returns {Array<any>}
+   */
+  public sampleData(): Array<any> {
+    return [];
+  }
+
+  // ------------------
+  // # Params
+  // ------------------
+  /**
+   * Params Yup validation schema.
+   * @returns {Yup.ObjectSchema<object, object>}
+   */
+  public paramsValidationSchema(): Yup.ObjectSchema<object, object> {
+    return Yup.object().nullable();
+  }
+
+  /**
+   * Validates the params of the importable service.
+   * @param {Record<string, any>}
+   * @returns {Promise<boolean>} - True means passed and false failed.
+   */
+  public async validateParams(
+    tenantId: number,
+    params: Record<string, any>
+  ): Promise<void> {}
+
+  /**
+   * Transformes the import params before storing them.
+   * @param {Record<string, any>} parmas
+   */
+  public transformParams(parmas: Record<string, any>) {
+    return parmas;
+  }
 }

packages/server/src/services/Import/ImportableResources.ts

@@ -1,6 +1,9 @@
 import Container, { Service } from 'typedi';
 import { AccountsImportable } from '../Accounts/AccountsImportable';
 import { ImportableRegistry } from './ImportableRegistry';
+import { UncategorizedTransactionsImportable } from '../Cashflow/UncategorizedTransactionsImportable';
+import { CustomersImportable } from '../Contacts/Customers/CustomersImportable';
+import { VendorsImportable } from '../Contacts/Vendors/VendorsImportable';
 
 @Service()
 export class ImportableResources {
@@ -15,6 +18,12 @@ export class ImportableResources {
    */
   private importables = [
     { resource: 'Account', importable: AccountsImportable },
+    {
+      resource: 'UncategorizedCashflowTransaction',
+      importable: UncategorizedTransactionsImportable,
+    },
+    { resource: 'Customer', importable: CustomersImportable },
+    { resource: 'Vendor', importable: VendorsImportable },
   ];
 
   public get registry() {

packages/server/src/services/Import/_utils.ts

@@ -3,6 +3,17 @@ import { upperFirst, camelCase, first } from 'lodash';
 import pluralize from 'pluralize';
 import { ResourceMetaFieldsMap } from './interfaces';
 import { IModelMetaField } from '@/interfaces';
+import moment from 'moment';
+
+export const ERRORS = {
+  RESOURCE_NOT_IMPORTABLE: 'RESOURCE_NOT_IMPORTABLE',
+  INVALID_MAP_ATTRS: 'INVALID_MAP_ATTRS',
+  DUPLICATED_FROM_MAP_ATTR: 'DUPLICATED_FROM_MAP_ATTR',
+  DUPLICATED_TO_MAP_ATTR: 'DUPLICATED_TO_MAP_ATTR',
+  IMPORT_FILE_NOT_MAPPED: 'IMPORT_FILE_NOT_MAPPED',
+  INVALID_MAP_DATE_FORMAT: 'INVALID_MAP_DATE_FORMAT',
+  MAP_DATE_FORMAT_NOT_DEFINED: 'MAP_DATE_FORMAT_NOT_DEFINED',
+};
 
 export function trimObject(obj) {
   return Object.entries(obj).reduce((acc, [key, value]) => {
@@ -47,6 +58,18 @@ export const convertFieldsToYupValidation = (fields: ResourceMetaFieldsMap) => {
         return acc;
       }, {});
       fieldSchema = Yup.string().oneOf(Object.keys(options)).label(field.name);
+      // Validate date field type.
+    } else if (field.fieldType === 'date') {
+      fieldSchema = fieldSchema.test(
+        'date validation',
+        'Invalid date or format. The string should be a valid YYYY-MM-DD format.',
+        (val) => {
+          if (!val) {
+            return true;
+          }
+          return moment(val, 'YYYY-MM-DD', true).isValid();
+        }
+      );
     }
     if (field.required) {
       fieldSchema = fieldSchema.required();
@@ -56,14 +79,6 @@ export const convertFieldsToYupValidation = (fields: ResourceMetaFieldsMap) => {
   return Yup.object().shape(yupSchema);
 };
 
-export const ERRORS = {
-  RESOURCE_NOT_IMPORTABLE: 'RESOURCE_NOT_IMPORTABLE',
-  INVALID_MAP_ATTRS: 'INVALID_MAP_ATTRS',
-  DUPLICATED_FROM_MAP_ATTR: 'DUPLICATED_FROM_MAP_ATTR',
-  DUPLICATED_TO_MAP_ATTR: 'DUPLICATED_TO_MAP_ATTR',
-  IMPORT_FILE_NOT_MAPPED: 'IMPORT_FILE_NOT_MAPPED',
-};
-
 export const getUnmappedSheetColumns = (columns, mapping) => {
   return columns.filter(
     (column) => !mapping.some((map) => map.from === column)

packages/server/src/services/Import/interfaces.ts

@@ -1,8 +1,10 @@
 import { IModelMetaField } from '@/interfaces';
+import Import from '@/models/Import';
 
 export interface ImportMappingAttr {
   from: string;
   to: string;
+  dateFormat?: string;
 }
 
 export interface ImportValidationError {
@@ -59,3 +61,16 @@ export interface ImportOperError {
   error: ImportInsertError;
   index: number;
 }
+ 
+export interface ImportableContext {
+  import: Import,
+  rowIndex: number;
+}
+
+
+export const ImportDateFormats = [
+  'yyyy-MM-dd',
+  'dd.MM.yy',
+  'MM/dd/yy',
+  'dd/MMM/yyyy'
+]