Merge branch 'develop' into import-relations-mapping

packages/server/src/services/Import/ImportFileDataTransformer.ts

@@ -2,12 +2,12 @@ import { Inject, Service } from 'typedi';
 import * as R from 'ramda';
 import bluebird from 'bluebird';
 import { isUndefined, get, chain, toArray, pickBy, castArray } from 'lodash';
+import { Knex } from 'knex';
 import { ImportMappingAttr, ResourceMetaFieldsMap } from './interfaces';
-import { parseBoolean } from '@/utils';
+import { trimObject, parseBoolean } from './_utils';
-import { trimObject } from './_utils';
 import { Account, Item } from '@/models';
 import ResourceService from '../Resource/ResourceService';
-import { Knex } from 'knex';
+import { multiNumberParse } from '@/utils/multi-number-parse';
 
 const CurrencyParsingDTOs = 10;
 
@@ -17,7 +17,8 @@ export class ImportFileDataTransformer {
   private resource: ResourceService;
 
   /**
-   *
+   * Parses the given sheet data before passing to the service layer.
+   * based on the mapped fields and the each field type .
    * @param {number} tenantId -
    * @param {}
    */
@@ -98,7 +99,7 @@ export class ImportFileDataTransformer {
 
       // Parses the boolean value.
       if (fields[key].fieldType === 'boolean') {
-        _value = parseBoolean(value, false);
+        _value = parseBoolean(value);
 
         // Parses the enumeration value.
       } else if (field.fieldType === 'enumeration') {
@@ -109,7 +110,7 @@ export class ImportFileDataTransformer {
         _value = get(option, 'key');
         // Parses the numeric value.
       } else if (fields[key].fieldType === 'number') {
-        _value = parseFloat(value);
+        _value = multiNumberParse(value);
         // Parses the relation value.
       } else if (field.fieldType === 'relation') {
         const relationModel = resourceModel.relationMappings[field.relationKey];

packages/server/src/services/Import/ImportFileUpload.ts

@@ -1,12 +1,11 @@
 import { Inject, Service } from 'typedi';
 import HasTenancyService from '../Tenancy/TenancyService';
-import { sanitizeResourceName } from './_utils';
+import { sanitizeResourceName, validateSheetEmpty } from './_utils';
 import ResourceService from '../Resource/ResourceService';
 import { IModelMetaField } from '@/interfaces';
 import { ImportFileCommon } from './ImportFileCommon';
 import { ImportFileDataValidator } from './ImportFileDataValidator';
 import { ImportFileUploadPOJO } from './interfaces';
-import { ServiceError } from '@/exceptions';
 
 @Service()
 export class ImportFileUploadService {
@@ -51,6 +50,10 @@ export class ImportFileUploadService {
 
     // Parse the buffer file to array data.
     const sheetData = this.importFileCommon.parseXlsxSheet(buffer);
+
+    // Throws service error if the sheet data is empty.
+    validateSheetEmpty(sheetData);
+
     const sheetColumns = this.importFileCommon.parseSheetColumns(sheetData);
     const coumnsStringified = JSON.stringify(sheetColumns);
 

packages/server/src/services/Import/_utils.ts

@@ -6,11 +6,13 @@ import {
   first,
   isUndefined,
   pickBy,
+  isEmpty,
 } from 'lodash';
 import pluralize from 'pluralize';
 import { ResourceMetaFieldsMap } from './interfaces';
 import { IModelMetaField } from '@/interfaces';
 import moment from 'moment';
+import { ServiceError } from '@/exceptions';
 
 export const ERRORS = {
   RESOURCE_NOT_IMPORTABLE: 'RESOURCE_NOT_IMPORTABLE',
@@ -20,6 +22,7 @@ export const ERRORS = {
   IMPORT_FILE_NOT_MAPPED: 'IMPORT_FILE_NOT_MAPPED',
   INVALID_MAP_DATE_FORMAT: 'INVALID_MAP_DATE_FORMAT',
   MAP_DATE_FORMAT_NOT_DEFINED: 'MAP_DATE_FORMAT_NOT_DEFINED',
+  IMPORTED_SHEET_EMPTY: 'IMPORTED_SHEET_EMPTY',
 };
 
 export function trimObject(obj) {
@@ -143,3 +146,38 @@ export const getUniqueImportableValue = (
 
   return defaultTo(objectDTO[uniqueImportableKey], '');
 };
+
+/**
+ * Throws service error the given sheet is empty.
+ * @param {Array<any>} sheetData
+ */
+export const validateSheetEmpty = (sheetData: Array<any>) => {
+  if (isEmpty(sheetData)) {
+    throw new ServiceError(ERRORS.IMPORTED_SHEET_EMPTY);
+  }
+
+const booleanValuesRepresentingTrue: string[] = ['true', 'yes', 'y', 't', '1'];
+const booleanValuesRepresentingFalse: string[] = ['false', 'no', 'n', 'f', '0'];
+
+/**
+ * Parses the given string value to boolean.
+ * @param {string} value 
+ * @returns {string|null} 
+ */
+export const parseBoolean = (value: string): boolean | null => {
+  const normalizeValue = (value: string): string =>
+    value.toString().trim().toLowerCase();
+
+  const normalizedValue = normalizeValue(value);
+  const valuesRepresentingTrue =
+    booleanValuesRepresentingTrue.map(normalizeValue);
+  const valueRepresentingFalse =
+    booleanValuesRepresentingFalse.map(normalizeValue);
+
+  if (valuesRepresentingTrue.includes(normalizedValue)) {
+    return true;
+  } else if (valueRepresentingFalse.includes(normalizedValue)) {
+    return false;
+  }
+  return null;
+};

packages/server/src/utils/multi-number-parse.test.ts

@@ -0,0 +1,51 @@
+import { assert } from 'chai';
+import { multiNumberParse } from './multi-number-parse';
+
+const correctNumbers = [
+  { actual: '10.5', expected: 10.5 },
+  { actual: '10,5', expected: 10.5 },
+  { actual: '1.235,76', expected: 1235.76 },
+  { actual: '2,543.56', expected: 2543.56 },
+  { actual: '10 654.1234', expected: 10654.1234 },
+  { actual: '2.654$10', expected: 2654.1 },
+  { actual: '5.435.123,645', expected: 5435123.645 },
+  { actual: '2,566,765.234', expected: 2566765.234 },
+  { actual: '2,432,123$23', expected: 2432123.23 },
+  { actual: '2,45EUR', expected: 2.45 },
+  { actual: '4.78€', expected: 4.78 },
+  { actual: '28', expected: 28 },
+  { actual: '-48', expected: -48 },
+  { actual: '39USD', expected: 39 },
+
+  // Some negative numbers
+  { actual: '-2,543.56', expected: -2543.56 },
+  { actual: '-10 654.1234', expected: -10654.1234 },
+  { actual: '-2.654$10', expected: -2654.1 },
+];
+
+const incorrectNumbers = [
+  '10 345,234.21', // too many different separators
+  '1.123.234,534,234', // impossible to detect where's the decimal separator
+  '10.4,2', // malformed digit groups
+  '1.123.2', // also malformed digit groups
+];
+
+describe('Test numbers', () => {
+  correctNumbers.forEach((item) => {
+    it(`"${item.actual}" should return ${item.expected}`, (done) => {
+      const parsed = multiNumberParse(item.actual);
+      assert.isNotNaN(parsed);
+      assert.equal(parsed, item.expected);
+
+      done();
+    });
+  });
+
+  incorrectNumbers.forEach((item) => {
+    it(`"${item}" should return NaN`, (done) => {
+      assert.isNaN(numberParse(item));
+
+      done();
+    });
+  });
+});

packages/server/src/utils/multi-number-parse.ts

@@ -0,0 +1,130 @@
+const validGrouping = (integerPart, sep) =>
+  integerPart.split(sep).reduce((acc, group, idx) => {
+    if (idx > 0) {
+      return acc && group.length === 3;
+    }
+
+    return acc && group.length;
+  }, true);
+
+export const multiNumberParse = (number: number | string, standardDecSep = '.') => {
+  // if it's a number already, this is going to be easy...
+  if (typeof number === 'number') {
+    return number;
+  }
+
+  // check validity of parameters
+  if (!number || typeof number !== 'string') {
+    throw new TypeError('number must be a string');
+  }
+
+  if (typeof standardDecSep !== 'string' || standardDecSep.length !== 1) {
+    throw new TypeError('standardDecSep must be a single character string');
+  }
+
+  // check if negative
+  const negative = number[0] === '-';
+
+  // strip unnecessary chars
+  const stripped = number
+    // get rid of trailing non-numbers
+    .replace(/[^\d]+$/, '')
+    // get rid of the signal
+    .slice(negative ? 1 : 0);
+
+  // analyze separators
+  const separators = (stripped.match(/[^\d]/g) || []).reduce(
+    (acc, sep, idx) => {
+      const sepChr = `str_${sep.codePointAt(0)}`;
+      const cnt = ((acc[sepChr] || {}).cnt || 0) + 1;
+
+      return {
+        ...acc,
+        [sepChr]: {
+          sep,
+          cnt,
+          lastIdx: idx,
+        },
+      };
+    },
+    {}
+  );
+
+  // check correctness of separators
+  const sepKeys = Object.keys(separators);
+
+  if (!sepKeys.length) {
+    // no separator, that's easy-peasy
+    return parseInt(stripped, 10) * (negative ? -1 : 1);
+  }
+
+  if (sepKeys.length > 2) {
+    // there's more than 2 separators, that's wrong
+    return Number.NaN;
+  }
+
+  if (sepKeys.length > 1) {
+    // there's two separators, that's ok by now
+    let sep1 = separators[sepKeys[0]];
+    let sep2 = separators[sepKeys[1]];
+
+    if (sep1.lastIdx > sep2.lastIdx) {
+      // swap
+      [sep1, sep2] = [sep2, sep1];
+    }
+
+    // if more than one separator appears more than once, that's wrong
+    if (sep1.cnt > 1 && sep2.cnt > 1) {
+      return Number.NaN;
+    }
+
+    // check if the last separator is the single one
+    if (sep2.cnt > 1) {
+      return Number.NaN;
+    }
+
+    // check the groupings
+    const [integerPart] = stripped.split(sep2.sep);
+
+    if (!validGrouping(integerPart, sep1.sep)) {
+      return Number.NaN;
+    }
+
+    // ok, we got here! let's handle it
+    return (
+      parseFloat(stripped.split(sep1.sep).join('').replace(sep2.sep, '.')) *
+      (negative ? -1 : 1)
+    );
+  }
+
+  // ok, only one separator, which is nice
+  const sep = separators[sepKeys[0]];
+
+  if (sep.cnt > 1) {
+    // there's more than one separator, which means it's integer
+    // let's check the groupings
+    if (!validGrouping(stripped, sep.sep)) {
+      return Number.NaN;
+    }
+
+    // it's valid, let's return an integer
+    return parseInt(stripped.split(sep.sep).join(''), 10) * (negative ? -1 : 1);
+  }
+
+  // just one separator, let's check last group
+  const groups = stripped.split(sep.sep);
+
+  if (groups[groups.length - 1].length === 3) {
+    // ok, we're in ambiguous territory here
+
+    if (sep.sep !== standardDecSep) {
+      // it's an integer
+      return (
+        parseInt(stripped.split(sep.sep).join(''), 10) * (negative ? -1 : 1)
+      );
+    }
+  }
+
+  // well, it looks like it's a simple float
+  return parseFloat(stripped.replace(sep.sep, '.')) * (negative ? -1 : 1);
+};

packages/webapp/src/containers/Import/AlertsManager.tsx

@@ -0,0 +1,53 @@
+import type { ReactNode } from 'react';
+import { useState, createContext, useContext } from 'react';
+
+interface AlertsManagerContextValue {
+  alerts: (string | number)[];
+  showAlert: (alert: string | number) => void;
+  hideAlert: (alert: string | number) => void;
+
+  hideAlerts: () => void;
+  isAlertActive: (alert: string | number) => boolean;
+  findAlert: (alert: string | number) => string | number | undefined;
+}
+
+const AlertsManagerContext = createContext<AlertsManagerContextValue>(
+  {} as AlertsManagerContextValue,
+);
+
+export function AlertsManager({ children }: { children: ReactNode }) {
+  const [alerts, setAlerts] = useState<(string | number)[]>([]);
+
+  const showAlert = (type: string | number): void => {
+    setAlerts([...alerts, type]);
+  };
+  const hideAlert = (type: string | number): void => {
+    alerts.filter((t) => t !== type);
+  };
+  const hideAlerts = (): void => {
+    setAlerts([]);
+  };
+  const isAlertActive = (type: string | number): boolean => {
+    return alerts.some((t) => t === type);
+  };
+  const findAlert = (type: string | number): number | string | undefined => {
+    return alerts.find((t) => t === type);
+  };
+
+  return (
+    <AlertsManagerContext.Provider
+      value={{
+        alerts,
+        showAlert,
+        hideAlert,
+        hideAlerts,
+        isAlertActive,
+        findAlert,
+      }}
+    >
+      {children}
+    </AlertsManagerContext.Provider>
+  );
+}
+
+export const useAlertsManager = () => useContext(AlertsManagerContext);

packages/webapp/src/containers/Import/ImportDropzone.tsx

@@ -3,8 +3,11 @@ import { Field } from 'formik';
 import { Box, Group, Stack } from '@/components';
 import styles from './ImportDropzone.module.css';
 import { ImportDropzoneField } from './ImportDropzoneFile';
+import { useAlertsManager } from './AlertsManager';
 
 export function ImportDropzone() {
+  const { hideAlerts } = useAlertsManager();
+
   return (
     <Stack spacing={0}>
       <Field id={'file'} name={'file'} type="file">
@@ -12,6 +15,7 @@ export function ImportDropzone() {
           <ImportDropzoneField
             value={form.file}
             onChange={(file) => {
+              hideAlerts();
               form.setFieldValue('file', file);
             }}
           />

packages/webapp/src/containers/Import/ImportFileUploadForm.tsx

@@ -5,7 +5,8 @@ import { Intent } from '@blueprintjs/core';
 import { Formik, Form, FormikHelpers } from 'formik';
 import * as Yup from 'yup';
 import { useImportFileContext } from './ImportFileProvider';
-import { ImportStepperStep } from './_types';
+import { ImportAlert, ImportStepperStep } from './_types';
+import { useAlertsManager } from './AlertsManager';
 
 const initialValues = {
   file: null,
@@ -28,6 +29,7 @@ export function ImportFileUploadForm({
   formikProps,
   formProps,
 }: ImportFileUploadFormProps) {
+  const { showAlert, hideAlerts } = useAlertsManager();
   const { mutateAsync: uploadImportFile } = useImportFileUpload();
   const {
     resource,
@@ -42,6 +44,7 @@ export function ImportFileUploadForm({
     values: ImportFileUploadValues,
     { setSubmitting }: FormikHelpers<ImportFileUploadValues>,
   ) => {
+    hideAlerts();
     if (!values.file) return;
 
     setSubmitting(true);
@@ -69,6 +72,9 @@ export function ImportFileUploadForm({
             message: 'The extenstion of uploaded file is not supported.',
           });
         }
+        if (data.errors.find((er) => er.type === 'IMPORTED_SHEET_EMPTY')) {
+          showAlert(ImportAlert.IMPORTED_SHEET_EMPTY);
+        }
         setSubmitting(false);
       });
   };

packages/webapp/src/containers/Import/ImportFileUploadStep.tsx

@@ -1,5 +1,5 @@
 // @ts-nocheck
-import { Classes } from '@blueprintjs/core';
+import { Callout, Classes, Intent } from '@blueprintjs/core';
 import { Stack } from '@/components';
 import { ImportDropzone } from './ImportDropzone';
 import { ImportSampleDownload } from './ImportSampleDownload';
@@ -7,29 +7,50 @@ import { ImportFileUploadForm } from './ImportFileUploadForm';
 import { ImportFileUploadFooterActions } from './ImportFileFooterActions';
 import { ImportFileContainer } from './ImportFileContainer';
 import { useImportFileContext } from './ImportFileProvider';
+import { AlertsManager, useAlertsManager } from './AlertsManager';
+import { ImportAlert } from './_types';
+
+function ImportFileUploadCallouts() {
+  const { isAlertActive } = useAlertsManager();
+  return (
+    <>
+      {isAlertActive(ImportAlert.IMPORTED_SHEET_EMPTY) && (
+        <Callout intent={Intent.DANGER} icon={null}>
+          The imported sheet is empty.
+        </Callout>
+      )}
+    </>
+  );
+}
 
 export function ImportFileUploadStep() {
   const { exampleDownload } = useImportFileContext();
 
   return (
+    <AlertsManager>
       <ImportFileUploadForm>
         <ImportFileContainer>
           <p
             className={Classes.TEXT_MUTED}
             style={{ marginBottom: 18, lineHeight: 1.6 }}
           >
-          Download a sample file and compare it with your import file to ensure
+            Download a sample file and compare it with your import file to
-          it is properly formatted. It's not necessary for the columns to be in
+            ensure it is properly formatted. It's not necessary for the columns
-          the same order, you can map them later.
+            to be in the same order, you can map them later.
           </p>
 
+          <Stack>
+            <ImportFileUploadCallouts />
+
             <Stack spacing={40}>
               <ImportDropzone />
               {exampleDownload && <ImportSampleDownload />}
             </Stack>
+          </Stack>
         </ImportFileContainer>
 
         <ImportFileUploadFooterActions />
       </ImportFileUploadForm>
+    </AlertsManager>
   );
 }

packages/webapp/src/containers/Import/_types.ts

@@ -3,3 +3,7 @@ export enum ImportStepperStep {
   Mapping = 1,
   Preview = 2,
 }
+
+export enum ImportAlert {
+  IMPORTED_SHEET_EMPTY = 'IMPORTED_SHEET_EMPTY'
+}