diff --git a/packages/server/src/services/Import/ImportFileDataTransformer.ts b/packages/server/src/services/Import/ImportFileDataTransformer.ts index 239fda383..4fb8cf300 100644 --- a/packages/server/src/services/Import/ImportFileDataTransformer.ts +++ b/packages/server/src/services/Import/ImportFileDataTransformer.ts @@ -2,12 +2,12 @@ import { Inject, Service } from 'typedi'; import * as R from 'ramda'; import bluebird from 'bluebird'; import { isUndefined, get, chain, toArray, pickBy, castArray } from 'lodash'; +import { Knex } from 'knex'; import { ImportMappingAttr, ResourceMetaFieldsMap } from './interfaces'; -import { parseBoolean } from '@/utils'; -import { trimObject } from './_utils'; +import { trimObject, parseBoolean } from './_utils'; import { Account, Item } from '@/models'; import ResourceService from '../Resource/ResourceService'; -import { Knex } from 'knex'; +import { multiNumberParse } from '@/utils/multi-number-parse'; const CurrencyParsingDTOs = 10; @@ -17,7 +17,8 @@ export class ImportFileDataTransformer { private resource: ResourceService; /** - * + * Parses the given sheet data before passing to the service layer. + * based on the mapped fields and the each field type . * @param {number} tenantId - * @param {} */ @@ -98,7 +99,7 @@ export class ImportFileDataTransformer { // Parses the boolean value. if (fields[key].fieldType === 'boolean') { - _value = parseBoolean(value, false); + _value = parseBoolean(value); // Parses the enumeration value. } else if (field.fieldType === 'enumeration') { @@ -109,7 +110,7 @@ export class ImportFileDataTransformer { _value = get(option, 'key'); // Parses the numeric value. } else if (fields[key].fieldType === 'number') { - _value = parseFloat(value); + _value = multiNumberParse(value); // Parses the relation value. } else if (field.fieldType === 'relation') { const relationModel = resourceModel.relationMappings[field.relationKey]; diff --git a/packages/server/src/services/Import/ImportFileUpload.ts b/packages/server/src/services/Import/ImportFileUpload.ts index 8dad9a53c..048739e03 100644 --- a/packages/server/src/services/Import/ImportFileUpload.ts +++ b/packages/server/src/services/Import/ImportFileUpload.ts @@ -1,12 +1,11 @@ import { Inject, Service } from 'typedi'; import HasTenancyService from '../Tenancy/TenancyService'; -import { sanitizeResourceName } from './_utils'; +import { sanitizeResourceName, validateSheetEmpty } from './_utils'; import ResourceService from '../Resource/ResourceService'; import { IModelMetaField } from '@/interfaces'; import { ImportFileCommon } from './ImportFileCommon'; import { ImportFileDataValidator } from './ImportFileDataValidator'; import { ImportFileUploadPOJO } from './interfaces'; -import { ServiceError } from '@/exceptions'; @Service() export class ImportFileUploadService { @@ -51,6 +50,10 @@ export class ImportFileUploadService { // Parse the buffer file to array data. const sheetData = this.importFileCommon.parseXlsxSheet(buffer); + + // Throws service error if the sheet data is empty. + validateSheetEmpty(sheetData); + const sheetColumns = this.importFileCommon.parseSheetColumns(sheetData); const coumnsStringified = JSON.stringify(sheetColumns); diff --git a/packages/server/src/services/Import/_utils.ts b/packages/server/src/services/Import/_utils.ts index f1b55ea14..063c7b5b8 100644 --- a/packages/server/src/services/Import/_utils.ts +++ b/packages/server/src/services/Import/_utils.ts @@ -6,11 +6,13 @@ import { first, isUndefined, pickBy, + isEmpty, } from 'lodash'; import pluralize from 'pluralize'; import { ResourceMetaFieldsMap } from './interfaces'; import { IModelMetaField } from '@/interfaces'; import moment from 'moment'; +import { ServiceError } from '@/exceptions'; export const ERRORS = { RESOURCE_NOT_IMPORTABLE: 'RESOURCE_NOT_IMPORTABLE', @@ -20,6 +22,7 @@ export const ERRORS = { IMPORT_FILE_NOT_MAPPED: 'IMPORT_FILE_NOT_MAPPED', INVALID_MAP_DATE_FORMAT: 'INVALID_MAP_DATE_FORMAT', MAP_DATE_FORMAT_NOT_DEFINED: 'MAP_DATE_FORMAT_NOT_DEFINED', + IMPORTED_SHEET_EMPTY: 'IMPORTED_SHEET_EMPTY', }; export function trimObject(obj) { @@ -143,3 +146,38 @@ export const getUniqueImportableValue = ( return defaultTo(objectDTO[uniqueImportableKey], ''); }; + +/** + * Throws service error the given sheet is empty. + * @param {Array} sheetData + */ +export const validateSheetEmpty = (sheetData: Array) => { + if (isEmpty(sheetData)) { + throw new ServiceError(ERRORS.IMPORTED_SHEET_EMPTY); + } + +const booleanValuesRepresentingTrue: string[] = ['true', 'yes', 'y', 't', '1']; +const booleanValuesRepresentingFalse: string[] = ['false', 'no', 'n', 'f', '0']; + +/** + * Parses the given string value to boolean. + * @param {string} value + * @returns {string|null} + */ +export const parseBoolean = (value: string): boolean | null => { + const normalizeValue = (value: string): string => + value.toString().trim().toLowerCase(); + + const normalizedValue = normalizeValue(value); + const valuesRepresentingTrue = + booleanValuesRepresentingTrue.map(normalizeValue); + const valueRepresentingFalse = + booleanValuesRepresentingFalse.map(normalizeValue); + + if (valuesRepresentingTrue.includes(normalizedValue)) { + return true; + } else if (valueRepresentingFalse.includes(normalizedValue)) { + return false; + } + return null; +}; diff --git a/packages/server/src/utils/multi-number-parse.test.ts b/packages/server/src/utils/multi-number-parse.test.ts new file mode 100644 index 000000000..a45ef384c --- /dev/null +++ b/packages/server/src/utils/multi-number-parse.test.ts @@ -0,0 +1,51 @@ +import { assert } from 'chai'; +import { multiNumberParse } from './multi-number-parse'; + +const correctNumbers = [ + { actual: '10.5', expected: 10.5 }, + { actual: '10,5', expected: 10.5 }, + { actual: '1.235,76', expected: 1235.76 }, + { actual: '2,543.56', expected: 2543.56 }, + { actual: '10 654.1234', expected: 10654.1234 }, + { actual: '2.654$10', expected: 2654.1 }, + { actual: '5.435.123,645', expected: 5435123.645 }, + { actual: '2,566,765.234', expected: 2566765.234 }, + { actual: '2,432,123$23', expected: 2432123.23 }, + { actual: '2,45EUR', expected: 2.45 }, + { actual: '4.78€', expected: 4.78 }, + { actual: '28', expected: 28 }, + { actual: '-48', expected: -48 }, + { actual: '39USD', expected: 39 }, + + // Some negative numbers + { actual: '-2,543.56', expected: -2543.56 }, + { actual: '-10 654.1234', expected: -10654.1234 }, + { actual: '-2.654$10', expected: -2654.1 }, +]; + +const incorrectNumbers = [ + '10 345,234.21', // too many different separators + '1.123.234,534,234', // impossible to detect where's the decimal separator + '10.4,2', // malformed digit groups + '1.123.2', // also malformed digit groups +]; + +describe('Test numbers', () => { + correctNumbers.forEach((item) => { + it(`"${item.actual}" should return ${item.expected}`, (done) => { + const parsed = multiNumberParse(item.actual); + assert.isNotNaN(parsed); + assert.equal(parsed, item.expected); + + done(); + }); + }); + + incorrectNumbers.forEach((item) => { + it(`"${item}" should return NaN`, (done) => { + assert.isNaN(numberParse(item)); + + done(); + }); + }); +}); diff --git a/packages/server/src/utils/multi-number-parse.ts b/packages/server/src/utils/multi-number-parse.ts new file mode 100644 index 000000000..261e66335 --- /dev/null +++ b/packages/server/src/utils/multi-number-parse.ts @@ -0,0 +1,130 @@ +const validGrouping = (integerPart, sep) => + integerPart.split(sep).reduce((acc, group, idx) => { + if (idx > 0) { + return acc && group.length === 3; + } + + return acc && group.length; + }, true); + +export const multiNumberParse = (number: number | string, standardDecSep = '.') => { + // if it's a number already, this is going to be easy... + if (typeof number === 'number') { + return number; + } + + // check validity of parameters + if (!number || typeof number !== 'string') { + throw new TypeError('number must be a string'); + } + + if (typeof standardDecSep !== 'string' || standardDecSep.length !== 1) { + throw new TypeError('standardDecSep must be a single character string'); + } + + // check if negative + const negative = number[0] === '-'; + + // strip unnecessary chars + const stripped = number + // get rid of trailing non-numbers + .replace(/[^\d]+$/, '') + // get rid of the signal + .slice(negative ? 1 : 0); + + // analyze separators + const separators = (stripped.match(/[^\d]/g) || []).reduce( + (acc, sep, idx) => { + const sepChr = `str_${sep.codePointAt(0)}`; + const cnt = ((acc[sepChr] || {}).cnt || 0) + 1; + + return { + ...acc, + [sepChr]: { + sep, + cnt, + lastIdx: idx, + }, + }; + }, + {} + ); + + // check correctness of separators + const sepKeys = Object.keys(separators); + + if (!sepKeys.length) { + // no separator, that's easy-peasy + return parseInt(stripped, 10) * (negative ? -1 : 1); + } + + if (sepKeys.length > 2) { + // there's more than 2 separators, that's wrong + return Number.NaN; + } + + if (sepKeys.length > 1) { + // there's two separators, that's ok by now + let sep1 = separators[sepKeys[0]]; + let sep2 = separators[sepKeys[1]]; + + if (sep1.lastIdx > sep2.lastIdx) { + // swap + [sep1, sep2] = [sep2, sep1]; + } + + // if more than one separator appears more than once, that's wrong + if (sep1.cnt > 1 && sep2.cnt > 1) { + return Number.NaN; + } + + // check if the last separator is the single one + if (sep2.cnt > 1) { + return Number.NaN; + } + + // check the groupings + const [integerPart] = stripped.split(sep2.sep); + + if (!validGrouping(integerPart, sep1.sep)) { + return Number.NaN; + } + + // ok, we got here! let's handle it + return ( + parseFloat(stripped.split(sep1.sep).join('').replace(sep2.sep, '.')) * + (negative ? -1 : 1) + ); + } + + // ok, only one separator, which is nice + const sep = separators[sepKeys[0]]; + + if (sep.cnt > 1) { + // there's more than one separator, which means it's integer + // let's check the groupings + if (!validGrouping(stripped, sep.sep)) { + return Number.NaN; + } + + // it's valid, let's return an integer + return parseInt(stripped.split(sep.sep).join(''), 10) * (negative ? -1 : 1); + } + + // just one separator, let's check last group + const groups = stripped.split(sep.sep); + + if (groups[groups.length - 1].length === 3) { + // ok, we're in ambiguous territory here + + if (sep.sep !== standardDecSep) { + // it's an integer + return ( + parseInt(stripped.split(sep.sep).join(''), 10) * (negative ? -1 : 1) + ); + } + } + + // well, it looks like it's a simple float + return parseFloat(stripped.replace(sep.sep, '.')) * (negative ? -1 : 1); +}; diff --git a/packages/webapp/src/containers/Import/AlertsManager.tsx b/packages/webapp/src/containers/Import/AlertsManager.tsx new file mode 100644 index 000000000..ad170f2c3 --- /dev/null +++ b/packages/webapp/src/containers/Import/AlertsManager.tsx @@ -0,0 +1,53 @@ +import type { ReactNode } from 'react'; +import { useState, createContext, useContext } from 'react'; + +interface AlertsManagerContextValue { + alerts: (string | number)[]; + showAlert: (alert: string | number) => void; + hideAlert: (alert: string | number) => void; + + hideAlerts: () => void; + isAlertActive: (alert: string | number) => boolean; + findAlert: (alert: string | number) => string | number | undefined; +} + +const AlertsManagerContext = createContext( + {} as AlertsManagerContextValue, +); + +export function AlertsManager({ children }: { children: ReactNode }) { + const [alerts, setAlerts] = useState<(string | number)[]>([]); + + const showAlert = (type: string | number): void => { + setAlerts([...alerts, type]); + }; + const hideAlert = (type: string | number): void => { + alerts.filter((t) => t !== type); + }; + const hideAlerts = (): void => { + setAlerts([]); + }; + const isAlertActive = (type: string | number): boolean => { + return alerts.some((t) => t === type); + }; + const findAlert = (type: string | number): number | string | undefined => { + return alerts.find((t) => t === type); + }; + + return ( + + {children} + + ); +} + +export const useAlertsManager = () => useContext(AlertsManagerContext); diff --git a/packages/webapp/src/containers/Import/ImportDropzone.tsx b/packages/webapp/src/containers/Import/ImportDropzone.tsx index cae4f6d65..5dae46d06 100644 --- a/packages/webapp/src/containers/Import/ImportDropzone.tsx +++ b/packages/webapp/src/containers/Import/ImportDropzone.tsx @@ -3,8 +3,11 @@ import { Field } from 'formik'; import { Box, Group, Stack } from '@/components'; import styles from './ImportDropzone.module.css'; import { ImportDropzoneField } from './ImportDropzoneFile'; +import { useAlertsManager } from './AlertsManager'; export function ImportDropzone() { + const { hideAlerts } = useAlertsManager(); + return ( @@ -12,6 +15,7 @@ export function ImportDropzone() { { + hideAlerts(); form.setFieldValue('file', file); }} /> diff --git a/packages/webapp/src/containers/Import/ImportFileUploadForm.tsx b/packages/webapp/src/containers/Import/ImportFileUploadForm.tsx index bc0a7e1b5..73db99efc 100644 --- a/packages/webapp/src/containers/Import/ImportFileUploadForm.tsx +++ b/packages/webapp/src/containers/Import/ImportFileUploadForm.tsx @@ -5,7 +5,8 @@ import { Intent } from '@blueprintjs/core'; import { Formik, Form, FormikHelpers } from 'formik'; import * as Yup from 'yup'; import { useImportFileContext } from './ImportFileProvider'; -import { ImportStepperStep } from './_types'; +import { ImportAlert, ImportStepperStep } from './_types'; +import { useAlertsManager } from './AlertsManager'; const initialValues = { file: null, @@ -28,6 +29,7 @@ export function ImportFileUploadForm({ formikProps, formProps, }: ImportFileUploadFormProps) { + const { showAlert, hideAlerts } = useAlertsManager(); const { mutateAsync: uploadImportFile } = useImportFileUpload(); const { resource, @@ -42,6 +44,7 @@ export function ImportFileUploadForm({ values: ImportFileUploadValues, { setSubmitting }: FormikHelpers, ) => { + hideAlerts(); if (!values.file) return; setSubmitting(true); @@ -69,6 +72,9 @@ export function ImportFileUploadForm({ message: 'The extenstion of uploaded file is not supported.', }); } + if (data.errors.find((er) => er.type === 'IMPORTED_SHEET_EMPTY')) { + showAlert(ImportAlert.IMPORTED_SHEET_EMPTY); + } setSubmitting(false); }); }; diff --git a/packages/webapp/src/containers/Import/ImportFileUploadStep.tsx b/packages/webapp/src/containers/Import/ImportFileUploadStep.tsx index b1c6f9e01..c991de56b 100644 --- a/packages/webapp/src/containers/Import/ImportFileUploadStep.tsx +++ b/packages/webapp/src/containers/Import/ImportFileUploadStep.tsx @@ -1,5 +1,5 @@ // @ts-nocheck -import { Classes } from '@blueprintjs/core'; +import { Callout, Classes, Intent } from '@blueprintjs/core'; import { Stack } from '@/components'; import { ImportDropzone } from './ImportDropzone'; import { ImportSampleDownload } from './ImportSampleDownload'; @@ -7,29 +7,50 @@ import { ImportFileUploadForm } from './ImportFileUploadForm'; import { ImportFileUploadFooterActions } from './ImportFileFooterActions'; import { ImportFileContainer } from './ImportFileContainer'; import { useImportFileContext } from './ImportFileProvider'; +import { AlertsManager, useAlertsManager } from './AlertsManager'; +import { ImportAlert } from './_types'; + +function ImportFileUploadCallouts() { + const { isAlertActive } = useAlertsManager(); + return ( + <> + {isAlertActive(ImportAlert.IMPORTED_SHEET_EMPTY) && ( + + The imported sheet is empty. + + )} + + ); +} export function ImportFileUploadStep() { const { exampleDownload } = useImportFileContext(); return ( - - -

- Download a sample file and compare it with your import file to ensure - it is properly formatted. It's not necessary for the columns to be in - the same order, you can map them later. -

+ + + +

+ Download a sample file and compare it with your import file to + ensure it is properly formatted. It's not necessary for the columns + to be in the same order, you can map them later. +

- - - {exampleDownload && } - -
+ + - -
+ + + {exampleDownload && } + +
+ + + + + ); } diff --git a/packages/webapp/src/containers/Import/_types.ts b/packages/webapp/src/containers/Import/_types.ts index 0d7856819..08df767d9 100644 --- a/packages/webapp/src/containers/Import/_types.ts +++ b/packages/webapp/src/containers/Import/_types.ts @@ -3,3 +3,7 @@ export enum ImportStepperStep { Mapping = 1, Preview = 2, } + +export enum ImportAlert { + IMPORTED_SHEET_EMPTY = 'IMPORTED_SHEET_EMPTY' +} \ No newline at end of file