Merge pull request #393 from bigcapitalhq/import-relations-mapping

feat: linking relation with id in importing
This commit is contained in:
Ahmed Bouhuolia
2024-04-01 03:02:04 +02:00
committed by GitHub
21 changed files with 386 additions and 83 deletions

View File

@@ -245,25 +245,26 @@
"account.field.currency": "Currency",
"account.field.balance": "Balance",
"account.field.created_at": "Created at",
"item.field.type": "Item type",
"item.field.type": "Item Type",
"item.field.type.inventory": "Inventory",
"item.field.type.service": "Service",
"item.field.type.non-inventory": "Non inventory",
"item.field.name": "Name",
"item.field.code": "Code",
"item.field.type.non-inventory": "Non Inventory",
"item.field.name": "Item Name",
"item.field.code": "Item Code",
"item.field.sellable": "Sellable",
"item.field.purchasable": "Purchasable",
"item.field.cost_price": "Cost price",
"item.field.cost_account": "Cost account",
"item.field.sell_account": "Sell account",
"item.field.sell_description": "Sell description",
"item.field.inventory_account": "Inventory account",
"item.field.purchase_description": "Purchase description",
"item.field.quantity_on_hand": "Quantity on hand",
"item.field.cost_price": "Cost Price",
"item.field.sell_price": "Sell Price",
"item.field.cost_account": "Cost Account",
"item.field.sell_account": "Sell Account",
"item.field.sell_description": "Sell Description",
"item.field.inventory_account": "Inventory Account",
"item.field.purchase_description": "Purchase Description",
"item.field.quantity_on_hand": "Quantity on Hand",
"item.field.note": "Note",
"item.field.category": "Category",
"item.field.active": "Active",
"item.field.created_at": "Created at",
"item.field.created_at": "Created At",
"item_category.field.name": "Name",
"item_category.field.description": "Description",
"item_category.field.count": "Count",

View File

@@ -344,7 +344,7 @@ export default class ItemsController extends BaseController {
const filter = {
sortOrder: 'DESC',
columnSortBy: 'created_at',
columnSortBy: 'createdAt',
page: 1,
pageSize: 12,
inactiveMode: false,

View File

@@ -6,7 +6,7 @@ import ItemTransactionsController from './ItemsTransactions';
@Service()
export default class ItemsBaseController {
router() {
public router() {
const router = Router();
router.use('/', Container.get(ItemsController).router());

View File

@@ -32,12 +32,13 @@ export interface IModelMetaFieldCommon {
name: string;
column: string;
columnable?: boolean;
fieldType: IModelColumnType;
customQuery?: Function;
required?: boolean;
importHint?: string;
importableRelationLabel?: string;
order?: number;
unique?: number;
dataTransferObjectKey?: string;
}
export interface IModelMetaFieldText {

View File

@@ -16,45 +16,53 @@ export default {
{ key: 'non-inventory', label: 'item.field.type.non-inventory' },
],
importable: true,
required: true,
},
name: {
name: 'item.field.name',
column: 'name',
fieldType: 'text',
importable: true,
required: true,
unique: true,
},
code: {
name: 'item.field.code',
column: 'code',
fieldType: 'text',
importable: true,
},
sellable: {
name: 'item.field.sellable',
column: 'sellable',
fieldType: 'boolean',
importable: true,
required: true,
},
purchasable: {
name: 'item.field.purchasable',
column: 'purchasable',
fieldType: 'boolean',
importable: true,
required: true,
},
sellPrice: {
name: 'item.field.cost_price',
name: 'item.field.sell_price',
column: 'sell_price',
fieldType: 'number',
importable: true,
required: true,
},
costPrice: {
name: 'item.field.cost_account',
name: 'item.field.cost_price',
column: 'cost_price',
fieldType: 'number',
importable: true,
required: true,
},
costAccount: {
name: 'item.field.sell_account',
name: 'item.field.cost_account',
column: 'cost_account_id',
fieldType: 'relation',
@@ -64,10 +72,13 @@ export default {
relationEntityLabel: 'name',
relationEntityKey: 'slug',
dataTransferObjectKey: 'costAccountId',
importableRelationLabel: ['name', 'code'],
importable: true,
required: true,
},
sellAccount: {
name: 'item.field.sell_description',
name: 'item.field.sell_account',
column: 'sell_account_id',
fieldType: 'relation',
@@ -77,11 +88,15 @@ export default {
relationEntityLabel: 'name',
relationEntityKey: 'slug',
importableRelationLabel: ['name', 'code'],
importable: true,
required: true,
},
inventoryAccount: {
name: 'item.field.inventory_account',
column: 'inventory_account_id',
fieldType: 'relation',
relationType: 'enumeration',
relationKey: 'inventoryAccount',
@@ -89,7 +104,10 @@ export default {
relationEntityLabel: 'name',
relationEntityKey: 'slug',
importableRelationLabel: ['name', 'code'],
importable: true,
required: true,
},
sellDescription: {
name: 'Sell description',
@@ -107,7 +125,6 @@ export default {
name: 'item.field.quantity_on_hand',
column: 'quantity_on_hand',
fieldType: 'number',
importable: true,
},
note: {
name: 'item.field.note',
@@ -118,12 +135,15 @@ export default {
category: {
name: 'item.field.category',
column: 'category_id',
fieldType: 'relation',
relationType: 'enumeration',
relationKey: 'category',
relationEntityLabel: 'name',
relationEntityKey: 'id',
importableRelationLabel: 'name',
importable: true,
},
active: {

View File

@@ -4,16 +4,19 @@ export default {
sortField: 'name',
sortOrder: 'DESC',
},
importable: true,
fields: {
name: {
name: 'item_category.field.name',
column: 'name',
fieldType: 'text',
importable: true,
},
description: {
name: 'item_category.field.description',
column: 'description',
fieldType: 'text',
importable: true,
},
count: {
name: 'item_category.field.count',

View File

@@ -1,22 +1,33 @@
import { Service } from 'typedi';
import { Inject, Service } from 'typedi';
import * as R from 'ramda';
import { isUndefined, get, chain } from 'lodash';
import bluebird from 'bluebird';
import { isUndefined, get, chain, toArray, pickBy, castArray } from 'lodash';
import { Knex } from 'knex';
import { ImportMappingAttr, ResourceMetaFieldsMap } from './interfaces';
import { trimObject, parseBoolean } from './_utils';
import { Account, Item } from '@/models';
import ResourceService from '../Resource/ResourceService';
import { multiNumberParse } from '@/utils/multi-number-parse';
const CurrencyParsingDTOs = 10;
@Service()
export class ImportFileDataTransformer {
@Inject()
private resource: ResourceService;
/**
* Parses the given sheet data before passing to the service layer.
* based on the mapped fields and the each field type .
* @param {number} tenantId -
* @param {}
*/
public parseSheetData(
public async parseSheetData(
tenantId: number,
importFile: any,
importableFields: any,
data: Record<string, unknown>[]
data: Record<string, unknown>[],
trx?: Knex.Transaction
) {
// Sanitize the sheet data.
const sanitizedData = this.sanitizeSheetData(data);
@@ -26,10 +37,17 @@ export class ImportFileDataTransformer {
sanitizedData,
importFile.mappingParsed
);
const resourceModel = this.resource.getResourceModel(
tenantId,
importFile.resource
);
// Parse the mapped sheet values.
const parsedValues = this.parseExcelValues(importableFields, mappedDTOs);
return parsedValues;
return this.parseExcelValues(
importableFields,
mappedDTOs,
resourceModel,
trx
);
}
/**
@@ -68,35 +86,86 @@ export class ImportFileDataTransformer {
* @param {Record<string, any>} valueDTOS -
* @returns {Record<string, any>}
*/
public parseExcelValues(
public async parseExcelValues(
fields: ResourceMetaFieldsMap,
valueDTOs: Record<string, any>[]
): Record<string, any> {
const parser = (value, key) => {
valueDTOs: Record<string, any>[],
resourceModel: any,
trx?: Knex.Transaction
): Promise<Record<string, any>> {
// Prases the given object value based on the field key type.
const parser = async (value, key) => {
let _value = value;
const field = fields[key];
// Parses the boolean value.
if (fields[key].fieldType === 'boolean') {
_value = parseBoolean(value);
// Parses the enumeration value.
} else if (fields[key].fieldType === 'enumeration') {
} else if (field.fieldType === 'enumeration') {
const field = fields[key];
const option = get(field, 'options', []).find(
(option) => option.label === value
);
_value = get(option, 'key');
// Prases the numeric value.
// Parses the numeric value.
} else if (fields[key].fieldType === 'number') {
_value = multiNumberParse(value);
// Parses the relation value.
} else if (field.fieldType === 'relation') {
const relationModel = resourceModel.relationMappings[field.relationKey];
const RelationModel = relationModel?.modelClass;
if (!relationModel || !RelationModel) {
throw new Error(`The relation model of ${key} field is not exist.`);
}
const relationQuery = RelationModel.query(trx);
const relationKeys = field?.importableRelationLabel
? castArray(field?.importableRelationLabel)
: castArray(field?.relationEntityLabel);
relationQuery.where(function () {
relationKeys.forEach((relationKey: string) => {
this.orWhereRaw('LOWER(??) = LOWER(?)', [relationKey, value]);
});
});
const result = await relationQuery.first();
_value = get(result, 'id');
}
return _value;
};
return valueDTOs.map((DTO) => {
return chain(DTO)
.pickBy((value, key) => !isUndefined(fields[key]))
.mapValues(parser)
.value();
const parseKey = (key: string) => {
const field = fields[key];
let _objectTransferObjectKey = key;
if (field.fieldType === 'relation') {
_objectTransferObjectKey = `${key}Id`;
}
return _objectTransferObjectKey;
};
const parseAsync = async (valueDTO) => {
// Remove the undefined fields.
const _valueDTO = pickBy(
valueDTO,
(value, key) => !isUndefined(fields[key])
);
const keys = Object.keys(_valueDTO);
// Map the object values.
return bluebird.reduce(
keys,
async (acc, key) => {
const parsedValue = await parser(_valueDTO[key], key);
const parsedKey = await parseKey(key);
acc[parsedKey] = parsedValue;
return acc;
},
{}
);
};
return bluebird.map(valueDTOs, parseAsync, {
concurrency: CurrencyParsingDTOs,
});
}
}

View File

@@ -57,19 +57,31 @@ export class ImportFileProcess {
tenantId,
importFile.resource
);
// Prases the sheet json data.
const parsedData = this.importParser.parseSheetData(
importFile,
importableFields,
sheetData
);
// Runs the importing operation with ability to return errors that will happen.
const [successedImport, failedImport] = await this.uow.withTransaction(
tenantId,
(trx: Knex.Transaction) =>
this.importCommon.import(tenantId, importFile, parsedData, trx),
trx
);
const [successedImport, failedImport, allData] =
await this.uow.withTransaction(
tenantId,
async (trx: Knex.Transaction) => {
// Prases the sheet json data.
const parsedData = await this.importParser.parseSheetData(
tenantId,
importFile,
importableFields,
sheetData,
trx
);
const [successedImport, failedImport] =
await this.importCommon.import(
tenantId,
importFile,
parsedData,
trx
);
return [successedImport, failedImport, parsedData];
},
trx
);
const mapping = importFile.mappingParsed;
const errors = chain(failedImport)
.map((oper) => oper.error)
@@ -77,7 +89,7 @@ export class ImportFileProcess {
.value();
const unmappedColumns = getUnmappedSheetColumns(header, mapping);
const totalCount = parsedData.length;
const totalCount = allData.length;
const createdCount = successedImport.length;
const errorsCount = failedImport.length;

View File

@@ -4,6 +4,8 @@ import { ImportableRegistry } from './ImportableRegistry';
import { UncategorizedTransactionsImportable } from '../Cashflow/UncategorizedTransactionsImportable';
import { CustomersImportable } from '../Contacts/Customers/CustomersImportable';
import { VendorsImportable } from '../Contacts/Vendors/VendorsImportable';
import { ItemsImportable } from '../Items/ItemsImportable';
import { ItemCategoriesImportable } from '../ItemCategories/ItemCategoriesImportable';
@Service()
export class ImportableResources {
@@ -24,6 +26,8 @@ export class ImportableResources {
},
{ resource: 'Customer', importable: CustomersImportable },
{ resource: 'Vendor', importable: VendorsImportable },
{ resource: 'Item', importable: ItemsImportable },
{ resource: 'ItemCategory', importable: ItemCategoriesImportable },
];
public get registry() {

View File

@@ -93,11 +93,25 @@ export const convertFieldsToYupValidation = (fields: ResourceMetaFieldsMap) => {
if (field.required) {
fieldSchema = fieldSchema.required();
}
yupSchema[fieldName] = fieldSchema;
const _fieldName = parseFieldName(fieldName, field);
yupSchema[_fieldName] = fieldSchema;
});
return Yup.object().shape(yupSchema);
};
const parseFieldName = (fieldName: string, field: IModelMetaField) => {
let _key = fieldName;
if (field.fieldType === 'relation') {
_key = `${fieldName}Id`;
}
if (field.dataTransferObjectKey) {
_key = field.dataTransferObjectKey;
}
return _key;
};
export const getUnmappedSheetColumns = (columns, mapping) => {
return columns.filter(
(column) => !mapping.some((map) => map.from === column)

View File

@@ -0,0 +1,38 @@
import { Inject, Service } from 'typedi';
import ItemCategoriesService from './ItemCategoriesService';
import { Importable } from '../Import/Importable';
import { Knex } from 'knex';
import { IItemCategoryOTD } from '@/interfaces';
import { ItemCategoriesSampleData } from './constants';
@Service()
export class ItemCategoriesImportable extends Importable {
@Inject()
private itemCategoriesService: ItemCategoriesService;
/**
* Importing to create new item category service.
* @param {number} tenantId
* @param {any} createDTO
* @param {Knex.Transaction} trx
*/
public async importable(
tenantId: number,
createDTO: IItemCategoryOTD,
trx?: Knex.Transaction
) {
await this.itemCategoriesService.newItemCategory(
tenantId,
createDTO,
{},
trx
);
}
/**
* Item categories sample data used to download sample sheet file.
*/
public sampleData(): any[] {
return ItemCategoriesSampleData;
}
}

View File

@@ -1,6 +1,6 @@
import { Inject } from 'typedi';
import * as R from 'ramda';
import Knex from 'knex';
import { Knex } from 'knex';
import { ServiceError } from '@/exceptions';
import {
IItemCategory,
@@ -115,7 +115,8 @@ export default class ItemCategoriesService implements IItemCategoriesService {
public async newItemCategory(
tenantId: number,
itemCategoryOTD: IItemCategoryOTD,
authorizedUser: ISystemUser
authorizedUser: ISystemUser,
trx?: Knex.Transaction
): Promise<IItemCategory> {
const { ItemCategory } = this.tenancy.models(tenantId);
@@ -139,20 +140,24 @@ export default class ItemCategoriesService implements IItemCategoriesService {
authorizedUser
);
// Creates item category under unit-of-work evnirement.
return this.uow.withTransaction(tenantId, async (trx: Knex.Transaction) => {
// Inserts the item category.
const itemCategory = await ItemCategory.query(trx).insert({
...itemCategoryObj,
});
// Triggers `onItemCategoryCreated` event.
await this.eventPublisher.emitAsync(events.itemCategory.onCreated, {
itemCategory,
tenantId,
trx,
} as IItemCategoryCreatedPayload);
return this.uow.withTransaction(
tenantId,
async (trx: Knex.Transaction) => {
// Inserts the item category.
const itemCategory = await ItemCategory.query(trx).insert({
...itemCategoryObj,
});
// Triggers `onItemCategoryCreated` event.
await this.eventPublisher.emitAsync(events.itemCategory.onCreated, {
itemCategory,
tenantId,
trx,
} as IItemCategoryCreatedPayload);
return itemCategory;
});
return itemCategory;
},
trx
);
}
/**
@@ -308,7 +313,7 @@ export default class ItemCategoriesService implements IItemCategoriesService {
} as IItemCategoryDeletedPayload);
});
}
/**
* Parses items categories filter DTO.
* @param {} filterDTO

View File

@@ -11,3 +11,25 @@ export const ERRORS = {
INVENTORY_ACCOUNT_NOT_INVENTORY: 'INVENTORY_ACCOUNT_NOT_INVENTORY',
CATEGORY_HAVE_ITEMS: 'CATEGORY_HAVE_ITEMS',
};
export const ItemCategoriesSampleData = [
{
Name: 'Kassulke Group',
Description: 'Optio itaque eaque qui adipisci illo sed.',
},
{
Name: 'Crist, Mraz and Lueilwitz',
Description:
'Dolores veniam deserunt sed commodi error quia veritatis non.',
},
{
Name: 'Gutmann and Sons',
Description:
'Ratione aperiam voluptas rem adipisci assumenda eos neque veritatis tempora.',
},
{
Name: 'Reichel - Raynor',
Description:
'Necessitatibus repellendus placeat possimus dolores excepturi ut.',
},
];

View File

@@ -88,7 +88,11 @@ export class CreateItem {
* @param {IItemDTO} item
* @return {Promise<IItem>}
*/
public async createItem(tenantId: number, itemDTO: IItemDTO): Promise<IItem> {
public async createItem(
tenantId: number,
itemDTO: IItemDTO,
trx?: Knex.Transaction
): Promise<IItem> {
const { Item } = this.tenancy.models(tenantId);
// Authorize the item before creating.
@@ -111,7 +115,8 @@ export class CreateItem {
} as IItemEventCreatedPayload);
return item;
}
},
trx
);
return item;
}

View File

@@ -35,7 +35,10 @@ export class ItemsValidators {
}
});
if (foundItems.length > 0) {
throw new ServiceError(ERRORS.ITEM_NAME_EXISTS);
throw new ServiceError(
ERRORS.ITEM_NAME_EXISTS,
'The item name is already exist.'
);
}
}

View File

@@ -0,0 +1,34 @@
import { Inject, Service } from 'typedi';
import { Knex } from 'knex';
import { Importable } from '@/services/Import/Importable';
import { IItemCreateDTO } from '@/interfaces';
import { CreateItem } from './CreateItem';
@Service()
export class ItemsImportable extends Importable {
@Inject()
private createItemService: CreateItem;
/**
* Mapps the imported data to create a new item service.
* @param {number} tenantId
* @param {ICustomerNewDTO} createDTO
* @param {Knex.Transaction} trx
* @returns {Promise<void>}
*/
public async importable(
tenantId: number,
createDTO: IItemCreateDTO,
trx?: Knex.Transaction<any, any[]>
): Promise<void> {
console.log(createDTO, tenantId, 'XX');
await this.createItemService.createItem(tenantId, createDTO, trx);
}
/**
* Retrieves the sample data of customers used to download sample sheet.
*/
public sampleData(): any[] {
return [];
}
}

View File

@@ -94,6 +94,10 @@ function ItemsActionsBar({
const handleTableRowSizeChange = (size) => {
addSetting('items', 'tableSize', size);
};
// Handles the import button click.
const handleImportBtnClick = () => {
history.push('/items/import');
};
return (
<DashboardActionsBar>
@@ -143,6 +147,7 @@ function ItemsActionsBar({
<Button
className={Classes.MINIMAL}
icon={<Icon icon="file-import-16" iconSize={16} />}
onClick={handleImportBtnClick}
text={<T id={'import'} />}
/>
<Button

View File

@@ -0,0 +1,25 @@
// @ts-nocheck
import { DashboardInsider } from '@/components';
import { ImportView } from '../Import/ImportView';
import { useHistory } from 'react-router-dom';
export default function ItemsImportpage() {
const history = useHistory();
const handleImportSuccess = () => {
history.push('/items');
};
const handleCancelBtnClick = () => {
history.push('/items');
};
return (
<DashboardInsider name={'import-items'}>
<ImportView
resource={'items'}
onImportSuccess={handleImportSuccess}
onCancelClick={handleCancelBtnClick}
exampleTitle="Items Example"
/>
</DashboardInsider>
);
}

View File

@@ -0,0 +1,25 @@
// @ts-nocheck
import { DashboardInsider } from '@/components';
import { ImportView } from '../Import/ImportView';
import { useHistory } from 'react-router-dom';
export default function ItemCategoriesImport() {
const history = useHistory();
const handleImportSuccess = () => {
history.push('/items/categories');
};
const handleCancelBtnClick = () => {
history.push('/items/categories');
};
return (
<DashboardInsider name={'import-item-categories'}>
<ImportView
resource={'item_category'}
onImportSuccess={handleImportSuccess}
onCancelClick={handleCancelBtnClick}
exampleTitle="Item Categories Example"
/>
</DashboardInsider>
);
}

View File

@@ -12,7 +12,7 @@ import {
FormattedMessage as T,
AdvancedFilterPopover,
DashboardFilterButton,
DashboardActionsBar
DashboardActionsBar,
} from '@/components';
import withItemCategories from './withItemCategories';
@@ -22,6 +22,7 @@ import withAlertActions from '@/containers/Alert/withAlertActions';
import { compose } from '@/utils';
import { useItemsCategoriesContext } from './ItemsCategoriesProvider';
import { useHistory } from 'react-router-dom';
/**
* Items categories actions bar.
@@ -41,11 +42,16 @@ function ItemsCategoryActionsBar({
openAlert,
}) {
const { fields } = useItemsCategoriesContext();
const history = useHistory();
const onClickNewCategory = () => {
openDialog('item-category-form', {});
};
const handleImportBtnClick = () => {
history.push('/item/categories/import');
};
// Handle the items categories bulk delete.
const handelBulkDelete = () => {
openAlert('item-categories-bulk-delete', {
@@ -93,6 +99,7 @@ function ItemsCategoryActionsBar({
className={Classes.MINIMAL}
icon={<Icon icon="file-import-16" iconSize={16} />}
text={<T id={'import'} />}
onClick={handleImportBtnClick}
/>
<Button
className={Classes.MINIMAL}

View File

@@ -63,6 +63,16 @@ export const getDashboardRoutes = () => [
defaultSearchResource: RESOURCES_TYPES.MANUAL_JOURNAL,
subscriptionActive: [SUBSCRIPTION_TYPE.MAIN],
},
{
path: `/item/categories/import`,
component: lazy(
() => import('@/containers/ItemsCategories/ItemCategoriesImport'),
),
backLink: true,
pageTitle: 'Item Categories Import',
subscriptionActive: [SUBSCRIPTION_TYPE.MAIN],
defaultSearchResource: RESOURCES_TYPES.ITEM,
},
{
path: `/items/categories`,
component: lazy(
@@ -73,9 +83,16 @@ export const getDashboardRoutes = () => [
defaultSearchResource: RESOURCES_TYPES.ITEM,
subscriptionActive: [SUBSCRIPTION_TYPE.MAIN],
},
// Items.
{
path: `/items/import`,
component: lazy(() => import('@/containers/Items/ItemsImportPage')),
backLink: true,
pageTitle: 'Items Import',
subscriptionActive: [SUBSCRIPTION_TYPE.MAIN],
defaultSearchResource: RESOURCES_TYPES.CUSTOMER,
},
{
path: `/items/:id/edit`,
component: lazy(() => import('@/containers/Items/ItemFormPage')),
@@ -514,12 +531,7 @@ export const getDashboardRoutes = () => [
// Customers
{
path: `/customers/import`,
component: lazy(
() =>
import(
'@/containers/Customers/CustomersImport'
),
),
component: lazy(() => import('@/containers/Customers/CustomersImport')),
backLink: true,
pageTitle: 'Customers Import',
subscriptionActive: [SUBSCRIPTION_TYPE.MAIN],
@@ -577,9 +589,7 @@ export const getDashboardRoutes = () => [
// Vendors
{
path: `/vendors/import`,
component: lazy(
() => import('@/containers/Vendors/VendorsImport'),
),
component: lazy(() => import('@/containers/Vendors/VendorsImport')),
backLink: true,
pageTitle: 'Vendors Import',
subscriptionActive: [SUBSCRIPTION_TYPE.MAIN],