QT/bigcapital
1
0
Fork 0
mirror of https://github.com/bigcapitalhq/bigcapital.git synced 2026-02-18 13:50:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: aggregate rows on import feature

Browse Source
This commit is contained in:
Ahmed Bouhuolia
2024-04-04 05:01:09 +02:00
parent b9651f30d5
commit 3851d34ba4
32 changed files with 1115 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/server/src/services/Import/ImportFileCommon.ts
View File

@@ -71,7 +71,7 @@ export class ImportFileCommon {
parsedData: Record<string, any>[],
trx?: Knex.Transaction
): Promise<[ImportOperSuccess[], ImportOperError[]]> {
const importableFields = this.resource.getResourceImportableFields(
const resourceFields = this.resource.getResourceFields2(
tenantId,
importFile.resource
);
@@ -90,7 +90,7 @@ export class ImportFileCommon {
};
const transformedDTO = importable.transform(objectDTO, context);
const rowNumber = index + 1;
const uniqueValue = getUniqueImportableValue(importableFields, objectDTO);
const uniqueValue = getUniqueImportableValue(resourceFields, objectDTO);
const errorContext = {
rowNumber,
uniqueValue,
@@ -98,7 +98,7 @@ export class ImportFileCommon {
try {
// Validate the DTO object before passing it to the service layer.
await this.importFileValidator.validateData(
importableFields,
resourceFields,
transformedDTO
);
try {

143
packages/server/src/services/Import/ImportFileDataTransformer.ts
View File

@@ -1,63 +1,91 @@
import { Inject, Service } from 'typedi';
import * as R from 'ramda';
import bluebird from 'bluebird';
import { isUndefined, get, chain, toArray, pickBy, castArray } from 'lodash';
import { isUndefined, pickBy, set } from 'lodash';
import { Knex } from 'knex';
import { ImportMappingAttr, ResourceMetaFieldsMap } from './interfaces';
import { trimObject, parseBoolean } from './_utils';
import { Account, Item } from '@/models';
import {
valueParser,
parseKey,
getFieldKey,
aggregate,
sanitizeSheetData,
} from './_utils';
import ResourceService from '../Resource/ResourceService';
import { multiNumberParse } from '@/utils/multi-number-parse';
import HasTenancyService from '../Tenancy/TenancyService';
const CurrencyParsingDTOs = 10;
const getMapToPath = (to: string, group = '') =>
group ? `${group}.${to}` : to;
@Service()
export class ImportFileDataTransformer {
@Inject()
private resource: ResourceService;
@Inject()
private tenancy: HasTenancyService;
/**
* Parses the given sheet data before passing to the service layer.
* based on the mapped fields and the each field type .
* based on the mapped fields and the each field type.
* @param {number} tenantId -
* @param {}
*/
public async parseSheetData(
tenantId: number,
importFile: any,
importableFields: any,
importableFields: ResourceMetaFieldsMap,
data: Record<string, unknown>[],
trx?: Knex.Transaction
) {
): Promise<Record<string, any>[]> {
// Sanitize the sheet data.
const sanitizedData = this.sanitizeSheetData(data);
const sanitizedData = sanitizeSheetData(data);
// Map the sheet columns key with the given map.
const mappedDTOs = this.mapSheetColumns(
sanitizedData,
importFile.mappingParsed
);
const resourceModel = this.resource.getResourceModel(
tenantId,
importFile.resource
);
// Parse the mapped sheet values.
return this.parseExcelValues(
const parsedValues = await this.parseExcelValues(
tenantId,
importableFields,
mappedDTOs,
resourceModel,
trx
);
const aggregateValues = this.aggregateParsedValues(
tenantId,
importFile.resource,
parsedValues
);
return aggregateValues;
}
/**
* Sanitizes the data in the imported sheet by trimming object keys.
* @param json - The JSON data representing the imported sheet.
* @returns {string[][]} - The sanitized data with trimmed object keys.
* Aggregates parsed data based on resource metadata configuration.
* @param {number} tenantId
* @param {string} resourceName
* @param {Record<string, any>} parsedData
* @returns {Record<string, any>[]}
*/
public sanitizeSheetData(json) {
return R.compose(R.map(trimObject))(json);
}
public aggregateParsedValues = (
tenantId: number,
resourceName: string,
parsedData: Record<string, any>[]
): Record<string, any>[] => {
let _value = parsedData;
const meta = this.resource.getResourceMeta(tenantId, resourceName);
if (meta.importAggregator === 'group') {
_value = aggregate(
_value,
meta.importAggregateBy,
meta.importAggregateOn
);
}
return _value;
};
/**
* Maps the columns of the imported data based on the provided mapping attributes.
@@ -74,7 +102,8 @@ export class ImportFileDataTransformer {
map
.filter((mapping) => !isUndefined(item[mapping.from]))
.forEach((mapping) => {
newItem[mapping.to] = item[mapping.from];
const toPath = getMapToPath(mapping.to, mapping.group);
newItem[toPath] = item[mapping.from];
});
return newItem;
});
@@ -87,78 +116,32 @@ export class ImportFileDataTransformer {
* @returns {Record<string, any>}
*/
public async parseExcelValues(
tenantId: number,
fields: ResourceMetaFieldsMap,
valueDTOs: Record<string, any>[],
resourceModel: any,
trx?: Knex.Transaction
): Promise<Record<string, any>> {
// Prases the given object value based on the field key type.
const parser = async (value, key) => {
let _value = value;
const field = fields[key];
): Promise<Record<string, any>[]> {
const tenantModels = this.tenancy.models(tenantId);
const _valueParser = valueParser(fields, tenantModels, trx);
const _keyParser = parseKey(fields);
// Parses the boolean value.
if (fields[key].fieldType === 'boolean') {
_value = parseBoolean(value);
// Parses the enumeration value.
} else if (field.fieldType === 'enumeration') {
const field = fields[key];
const option = get(field, 'options', []).find(
(option) => option.label === value
);
_value = get(option, 'key');
// Parses the numeric value.
} else if (fields[key].fieldType === 'number') {
_value = multiNumberParse(value);
// Parses the relation value.
} else if (field.fieldType === 'relation') {
const relationModel = resourceModel.relationMappings[field.relationKey];
const RelationModel = relationModel?.modelClass;
if (!relationModel || !RelationModel) {
throw new Error(`The relation model of ${key} field is not exist.`);
}
const relationQuery = RelationModel.query(trx);
const relationKeys = field?.importableRelationLabel
? castArray(field?.importableRelationLabel)
: castArray(field?.relationEntityLabel);
relationQuery.where(function () {
relationKeys.forEach((relationKey: string) => {
this.orWhereRaw('LOWER(??) = LOWER(?)', [relationKey, value]);
});
});
const result = await relationQuery.first();
_value = get(result, 'id');
}
return _value;
};
const parseKey = (key: string) => {
const field = fields[key];
let _objectTransferObjectKey = key;
if (field.fieldType === 'relation') {
_objectTransferObjectKey = `${key}Id`;
}
return _objectTransferObjectKey;
};
const parseAsync = async (valueDTO) => {
// Remove the undefined fields.
// Clean up the undefined keys that not exist in resource fields.
const _valueDTO = pickBy(
valueDTO,
(value, key) => !isUndefined(fields[key])
(value, key) => !isUndefined(fields[getFieldKey(key)])
);
// Keys of mapped values. key structure: `group.key` or `key`.
const keys = Object.keys(_valueDTO);
// Map the object values.
return bluebird.reduce(
keys,
async (acc, key) => {
const parsedValue = await parser(_valueDTO[key], key);
const parsedKey = await parseKey(key);
acc[parsedKey] = parsedValue;
const parsedValue = await _valueParser(_valueDTO[key], key);
const parsedKey = await _keyParser(key);
set(acc, parsedKey, parsedValue);
return acc;
},
{}

30
packages/server/src/services/Import/ImportFileMapping.ts
View File

@@ -1,4 +1,4 @@
import { fromPairs } from 'lodash';
import { fromPairs, isUndefined } from 'lodash';
import { Inject, Service } from 'typedi';
import HasTenancyService from '../Tenancy/TenancyService';
import {
@@ -69,7 +69,7 @@ export class ImportFileMapping {
importFile: any,
maps: ImportMappingAttr[]
) {
const fields = this.resource.getResourceImportableFields(
const fields = this.resource.getResourceFields2(
tenantId,
importFile.resource
);
@@ -78,11 +78,20 @@ export class ImportFileMapping {
);
const invalid = [];
// is not empty, is not undefined or map.group
maps.forEach((map) => {
if (
'undefined' === typeof fields[map.to] ||
'undefined' === typeof columnsMap[map.from]
) {
let _invalid = true;
if (!map.group && fields[map.to]) {
_invalid = false;
}
if (map.group && fields[map.group] && fields[map.group]?.fields[map.to]) {
_invalid = false;
}
if (columnsMap[map.from]) {
_invalid = false;
}
if (_invalid) {
invalid.push(map);
}
});
@@ -105,10 +114,14 @@ export class ImportFileMapping {
} else {
fromMap[map.from] = true;
}
if (toMap[map.to]) {
const toPath = !isUndefined(map?.group)
? `${map.group}.${map.to}`
: map.to;
if (toMap[toPath]) {
throw new ServiceError(ERRORS.DUPLICATED_TO_MAP_ATTR);
} else {
toMap[map.to] = true;
toMap[toPath] = true;
}
});
}
@@ -128,6 +141,7 @@ export class ImportFileMapping {
tenantId,
resource
);
// @todo Validate date type of the nested fields.
maps.forEach((map) => {
if (
typeof fields[map.to] !== 'undefined' &&

5
packages/server/src/services/Import/ImportFileProcess.ts
View File

@@ -53,11 +53,10 @@ export class ImportFileProcess {
const sheetData = this.importCommon.parseXlsxSheet(buffer);
const header = getSheetColumns(sheetData);
const importableFields = this.resource.getResourceImportableFields(
const resourceFields = this.resource.getResourceFields2(
tenantId,
importFile.resource
);
// Runs the importing operation with ability to return errors that will happen.
const [successedImport, failedImport, allData] =
await this.uow.withTransaction(
@@ -67,7 +66,7 @@ export class ImportFileProcess {
const parsedData = await this.importParser.parseSheetData(
tenantId,
importFile,
importableFields,
resourceFields,
sheetData,
trx
);

30
packages/server/src/services/Import/ImportFileUpload.ts
View File

@@ -1,8 +1,11 @@
import { Inject, Service } from 'typedi';
import HasTenancyService from '../Tenancy/TenancyService';
import { sanitizeResourceName, validateSheetEmpty } from './_utils';
import {
getResourceColumns,
sanitizeResourceName,
validateSheetEmpty,
} from './_utils';
import ResourceService from '../Resource/ResourceService';
import { IModelMetaField } from '@/interfaces';
import { ImportFileCommon } from './ImportFileCommon';
import { ImportFileDataValidator } from './ImportFileDataValidator';
import { ImportFileUploadPOJO } from './interfaces';
@@ -77,11 +80,11 @@ export class ImportFileUploadService {
columns: coumnsStringified,
params: paramsStringified,
});
const resourceColumnsMap = this.resourceService.getResourceImportableFields(
const resourceColumnsMap = this.resourceService.getResourceFields2(
tenantId,
resource
);
const resourceColumns = this.getResourceColumns(resourceColumnsMap);
const resourceColumns = getResourceColumns(resourceColumnsMap);
return {
import: {
@@ -92,23 +95,4 @@ export class ImportFileUploadService {
resourceColumns,
};
}
getResourceColumns(resourceColumns: { [key: string]: IModelMetaField }) {
return Object.entries(resourceColumns)
.map(
([key, { name, importHint, required, order }]: [
string,
IModelMetaField
]) => ({
key,
name,
required,
hint: importHint,
order,
})
)
.sort((a, b) =>
a.order && b.order ? a.order - b.order : a.order ? -1 : b.order ? 1 : 0
);
}
}

2
packages/server/src/services/Import/ImportableRegistry.ts
View File

@@ -5,7 +5,7 @@ export class ImportableRegistry {
private static instance: ImportableRegistry;
private importables: Record<string, Importable>;
private constructor() {
constructor() {
this.importables = {};
}

4
packages/server/src/services/Import/ImportableResources.ts
View File

@@ -6,6 +6,8 @@ import { CustomersImportable } from '../Contacts/Customers/CustomersImportable';
import { VendorsImportable } from '../Contacts/Vendors/VendorsImportable';
import { ItemsImportable } from '../Items/ItemsImportable';
import { ItemCategoriesImportable } from '../ItemCategories/ItemCategoriesImportable';
import { ManualJournalImportable } from '../ManualJournals/ManualJournalsImport';
import { BillsImportable } from '../Purchases/Bills/BillsImportable';
@Service()
export class ImportableResources {
@@ -28,6 +30,8 @@ export class ImportableResources {
{ resource: 'Vendor', importable: VendorsImportable },
{ resource: 'Item', importable: ItemsImportable },
{ resource: 'ItemCategory', importable: ItemCategoriesImportable },
{ resource: 'ManualJournal', importable: ManualJournalImportable },
{ resource: 'Bill', importable: BillsImportable },
];
public get registry() {

237
packages/server/src/services/Import/_utils.ts
View File

@@ -1,5 +1,7 @@
import * as Yup from 'yup';
import moment from 'moment';
import * as R from 'ramda';
import { Knex } from 'knex';
import {
defaultTo,
upperFirst,
@@ -8,11 +10,17 @@ import {
isUndefined,
pickBy,
isEmpty,
castArray,
get,
head,
split,
last,
} from 'lodash';
import pluralize from 'pluralize';
import { ResourceMetaFieldsMap } from './interfaces';
import { IModelMetaField } from '@/interfaces';
import { IModelMetaField, IModelMetaField2 } from '@/interfaces';
import { ServiceError } from '@/exceptions';
import { multiNumberParse } from '@/utils/multi-number-parse';
export const ERRORS = {
RESOURCE_NOT_IMPORTABLE: 'RESOURCE_NOT_IMPORTABLE',
@@ -40,6 +48,7 @@ export function trimObject(obj) {
export const convertFieldsToYupValidation = (fields: ResourceMetaFieldsMap) => {
const yupSchema = {};
Object.keys(fields).forEach((fieldName: string) => {
const field = fields[fieldName] as IModelMetaField;
let fieldSchema;
@@ -89,6 +98,17 @@ export const convertFieldsToYupValidation = (fields: ResourceMetaFieldsMap) => {
);
} else if (field.fieldType === 'url') {
fieldSchema = fieldSchema.url();
} else if (field.fieldType === 'collection') {
const nestedFieldShema = convertFieldsToYupValidation(field.fields);
fieldSchema = Yup.array().label(field.name);
if (!isUndefined(field.collectionMaxLength)) {
fieldSchema = fieldSchema.max(field.collectionMaxLength);
}
if (!isUndefined(field.collectionMinLength)) {
fieldSchema = fieldSchema.min(field.collectionMinLength);
}
fieldSchema = fieldSchema.of(nestedFieldShema);
}
if (field.required) {
fieldSchema = fieldSchema.required();
@@ -103,9 +123,9 @@ export const convertFieldsToYupValidation = (fields: ResourceMetaFieldsMap) => {
const parseFieldName = (fieldName: string, field: IModelMetaField) => {
let _key = fieldName;
if (field.fieldType === 'relation') {
_key = `${fieldName}Id`;
}
// if (field.fieldType === 'relation') {
// _key = `${fieldName}Id`;
// }
if (field.dataTransferObjectKey) {
_key = field.dataTransferObjectKey;
}
@@ -134,7 +154,7 @@ export const getSheetColumns = (sheetData: unknown[]) => {
* @returns {string}
*/
export const getUniqueImportableValue = (
importableFields: { [key: string]: IModelMetaField },
importableFields: { [key: string]: IModelMetaField2 },
objectDTO: Record<string, any>
) => {
const uniqueImportableValue = pickBy(
@@ -155,15 +175,15 @@ export const validateSheetEmpty = (sheetData: Array<any>) => {
if (isEmpty(sheetData)) {
throw new ServiceError(ERRORS.IMPORTED_SHEET_EMPTY);
}
}
};
const booleanValuesRepresentingTrue: string[] = ['true', 'yes', 'y', 't', '1'];
const booleanValuesRepresentingFalse: string[] = ['false', 'no', 'n', 'f', '0'];
/**
* Parses the given string value to boolean.
* @param {string} value
* @returns {string|null}
* @param {string} value
* @returns {string|null}
*/
export const parseBoolean = (value: string): boolean | null => {
const normalizeValue = (value: string): string =>
@@ -182,3 +202,204 @@ export const parseBoolean = (value: string): boolean | null => {
}
return null;
};
export const transformInputToGroupedFields = (input) => {
const output = [];
// Group for non-nested fields
const mainGroup = {
groupLabel: '',
groupKey: '',
fields: [],
};
input.forEach((item) => {
if (!item.fields) {
// If the item does not have nested fields, add it to the main group
mainGroup.fields.push(item);
} else {
// If the item has nested fields, create a new group for these fields
output.push({
groupLabel: item.name,
groupKey: item.key,
fields: item.fields,
});
}
});
// Add the main group to the output if it contains any fields
if (mainGroup.fields.length > 0) {
output.unshift(mainGroup); // Add the main group at the beginning
}
return output;
};
export const getResourceColumns = (resourceColumns: {
[key: string]: IModelMetaField2;
}) => {
const mapColumn =
(group: string) =>
([fieldKey, { name, importHint, required, order, ...field }]: [
string,
IModelMetaField2
]) => {
const extra: Record<string, any> = {};
const key = fieldKey;
if (group) {
extra.group = group;
}
if (field.fieldType === 'collection') {
extra.fields = mapColumns(field.fields, key);
}
return {
key,
name,
required,
hint: importHint,
order,
...extra,
};
};
const sortColumn = (a, b) =>
a.order && b.order ? a.order - b.order : a.order ? -1 : b.order ? 1 : 0;
const mapColumns = (columns, parentKey = '') =>
Object.entries(columns).map(mapColumn(parentKey)).sort(sortColumn);
return R.compose(transformInputToGroupedFields, mapColumns)(resourceColumns);
};
// Prases the given object value based on the field key type.
export const valueParser =
(fields: ResourceMetaFieldsMap, tenantModels: any, trx?: Knex.Transaction) =>
async (value: any, key: string, group = '') => {
let _value = value;
const fieldKey = key.includes('.') ? key.split('.')[0] : key;
const field = group ? fields[group]?.fields[fieldKey] : fields[fieldKey];
// Parses the boolean value.
if (field.fieldType === 'boolean') {
_value = parseBoolean(value);
// Parses the enumeration value.
} else if (field.fieldType === 'enumeration') {
const option = get(field, 'options', []).find(
(option) => option.label === value
);
_value = get(option, 'key');
// Parses the numeric value.
} else if (field.fieldType === 'number') {
_value = multiNumberParse(value);
// Parses the relation value.
} else if (field.fieldType === 'relation') {
const RelationModel = tenantModels[field.relationModel];
if (!RelationModel) {
throw new Error(`The relation model of ${key} field is not exist.`);
}
const relationQuery = RelationModel.query(trx);
const relationKeys = castArray(field?.relationImportMatch);
relationQuery.where(function () {
relationKeys.forEach((relationKey: string) => {
this.orWhereRaw('LOWER(??) = LOWER(?)', [relationKey, value]);
});
});
const result = await relationQuery.first();
_value = get(result, 'id');
} else if (field.fieldType === 'collection') {
const ObjectFieldKey = key.includes('.') ? key.split('.')[1] : key;
const _valueParser = valueParser(fields, tenantModels);
_value = await _valueParser(value, ObjectFieldKey, fieldKey);
}
return _value;
};
export const parseKey = R.curry(
/**
*
* @param {{ [key: string]: IModelMetaField2 }} fields
* @param {string} key - Mapped key path. formats: `group.key` or `key`.
* @returns {string}
*/
(fields: { [key: string]: IModelMetaField2 }, key: string) => {
const fieldKey = getFieldKey(key);
const field = fields[fieldKey];
let _key = key;
if (field.fieldType === 'collection') {
if (field.collectionOf === 'object') {
const nestedFieldKey = last(key.split('.'));
_key = `${fieldKey}[0].${nestedFieldKey}`;
} else if (
field.collectionOf === 'string' ||
field.collectionOf ||
'numberic'
) {
_key = `${fieldKey}`;
}
}
console.log(_key);
return _key;
}
);
export const getFieldKey = (input: string) => {
const keys = split(input, '.');
const firstKey = head(keys).split('[')[0]; // Split by "[" in case of array notation
return firstKey;
};
/**
{ * Aggregates the input array of objects based on a comparator attribute and groups the entries.
* This function is useful for combining multiple entries into a single entry based on a specific attribute,
* while aggregating other attributes into an array.}
*
* @param {Array} input - The array of objects to be aggregated.
* @param {string} comparatorAttr - The attribute of the objects used for comparison to aggregate.
* @param {string} groupOn - The attribute of the objects where the grouped entries will be pushed.
* @returns {Array} - The aggregated array of objects.
*
* @example
* // Example input:
* const input = [
* { id: 1, name: 'John', entries: ['entry1'] },
* { id: 2, name: 'Jane', entries: ['entry2'] },
* { id: 1, name: 'John', entries: ['entry3'] },
* ];
* const comparatorAttr = 'id';
* const groupOn = 'entries';
*
* // Example output:
* const output = [
* { id: 1, name: 'John', entries: ['entry1', 'entry3'] },
* { id: 2, name: 'Jane', entries: ['entry2'] },
* ];
*/
export function aggregate(
input: Array<any>,
comparatorAttr: string,
groupOn: string
): Array<Record<string, any>> {
return input.reduce((acc, curr) => {
const existingEntry = acc.find(
(entry) => entry[comparatorAttr] === curr[comparatorAttr]
);
if (existingEntry) {
existingEntry[groupOn].push(...curr.entries);
} else {
acc.push({ ...curr });
}
return acc;
}, []);
}
/**
* Sanitizes the data in the imported sheet by trimming object keys.
* @param json - The JSON data representing the imported sheet.
* @returns {string[][]} - The sanitized data with trimmed object keys.
*/
export const sanitizeSheetData = (json) => {
return R.compose(R.map(trimObject))(json);
};

14
packages/server/src/services/Import/interfaces.ts
View File

@@ -1,9 +1,10 @@
import { IModelMetaField } from '@/interfaces';
import { IModelMetaField, IModelMetaField2 } from '@/interfaces';
import Import from '@/models/Import';
export interface ImportMappingAttr {
from: string;
to: string;
group?: string;
dateFormat?: string;
}
@@ -13,7 +14,7 @@ export interface ImportValidationError {
constraints: Record<string, string>;
}
export type ResourceMetaFieldsMap = { [key: string]: IModelMetaField };
export type ResourceMetaFieldsMap = { [key: string]: IModelMetaField2 };
export interface ImportInsertError {
rowNumber: number;
@@ -61,16 +62,15 @@ export interface ImportOperError {
error: ImportInsertError[];
index: number;
}
export interface ImportableContext {
import: Import,
import: Import;
rowIndex: number;
}
export const ImportDateFormats = [
'yyyy-MM-dd',
'dd.MM.yy',
'MM/dd/yy',
'dd/MMM/yyyy'
]
'dd/MMM/yyyy',
];

1
packages/server/src/services/Import/jobs/ImportDeleteExpiredFilesJob.ts Normal file
View File

@@ -0,0 +1 @@
export class ImportDeleteExpiredFilesJobs {}