From c3ffb2b13f2310a3aef2a0b9c54ed61f45c90041 Mon Sep 17 00:00:00 2001 From: Firman Ramdhani <33869609+firmanramdhani@users.noreply.github.com> Date: Wed, 3 Jul 2024 14:47:20 +0700 Subject: [PATCH] feat: init query builder report --- .../reports/report/report.controller.ts | 5 + src/modules/reports/report/report.service.ts | 6 +- .../shared/configs/general-report/index.ts | 1 + src/modules/reports/shared/configs/index.ts | 4 + .../shared/configs/tenant-report/index.ts | 1 + .../shared/entities/report-config.entity.ts | 59 ++ src/modules/reports/shared/helpers/index.ts | 1 + .../reports/shared/helpers/query-builder.ts | 506 ++++++++++++++++++ 8 files changed, 582 insertions(+), 1 deletion(-) create mode 100644 src/modules/reports/shared/configs/general-report/index.ts create mode 100644 src/modules/reports/shared/configs/tenant-report/index.ts create mode 100644 src/modules/reports/shared/entities/report-config.entity.ts create mode 100644 src/modules/reports/shared/helpers/index.ts create mode 100644 src/modules/reports/shared/helpers/query-builder.ts diff --git a/src/modules/reports/report/report.controller.ts b/src/modules/reports/report/report.controller.ts index 5ccbb55..315474b 100644 --- a/src/modules/reports/report/report.controller.ts +++ b/src/modules/reports/report/report.controller.ts @@ -21,4 +21,9 @@ export class ReportController { async getReportData(@Body() body: GetReportDataDto) { return await this.service.getReportData(body); } + + @Post('meta') + async getReportMeta(@Body() body: GetReportDataDto) { + return await this.service.getReportMeta(body); + } } diff --git a/src/modules/reports/report/report.service.ts b/src/modules/reports/report/report.service.ts index 390737d..0474f56 100644 --- a/src/modules/reports/report/report.service.ts +++ b/src/modules/reports/report/report.service.ts @@ -2,14 +2,18 @@ import { Injectable } from '@nestjs/common'; import { BaseReportService } from '../shared/services/base-report.service'; import { GetReportConfigDto } from '../shared/dto/report-config.get.dto'; import { GetReportDataDto } from '../shared/dto/report-data.get.dto'; +import { ReportConfigs } from '../shared/configs'; @Injectable() export class ReportService extends BaseReportService { async getReportConfig(query: GetReportConfigDto) { - return 'you hit API for get report config'; + return ReportConfigs; } async getReportData(body: GetReportDataDto) { return 'you hit API for get report data'; } + async getReportMeta(body: GetReportDataDto) { + return 'you hit API for get report meta'; + } } diff --git a/src/modules/reports/shared/configs/general-report/index.ts b/src/modules/reports/shared/configs/general-report/index.ts new file mode 100644 index 0000000..9d6a412 --- /dev/null +++ b/src/modules/reports/shared/configs/general-report/index.ts @@ -0,0 +1 @@ +export const GeneralReportConfig = []; diff --git a/src/modules/reports/shared/configs/index.ts b/src/modules/reports/shared/configs/index.ts index e69de29..1ec9aab 100644 --- a/src/modules/reports/shared/configs/index.ts +++ b/src/modules/reports/shared/configs/index.ts @@ -0,0 +1,4 @@ +import { GeneralReportConfig } from './general-report'; +import { TenantReportConfig } from './tenant-report'; + +export const ReportConfigs = [...GeneralReportConfig, ...TenantReportConfig]; diff --git a/src/modules/reports/shared/configs/tenant-report/index.ts b/src/modules/reports/shared/configs/tenant-report/index.ts new file mode 100644 index 0000000..3893e35 --- /dev/null +++ b/src/modules/reports/shared/configs/tenant-report/index.ts @@ -0,0 +1 @@ +export const TenantReportConfig = []; diff --git a/src/modules/reports/shared/entities/report-config.entity.ts b/src/modules/reports/shared/entities/report-config.entity.ts new file mode 100644 index 0000000..90dc77f --- /dev/null +++ b/src/modules/reports/shared/entities/report-config.entity.ts @@ -0,0 +1,59 @@ +import { DATA_FORMAT, DATA_TYPE, FILTER_TYPE } from '../constant'; + +export interface ReportColumnConfigEntity { + column: string; + query: string; + label: string; + type: DATA_TYPE; + format: DATA_FORMAT; + date_format?: string; +} + +export interface FilterConfigEntity { + filter_column: string; + filter_type: FILTER_TYPE; + + filed_label: string; + field_type: string; + hide_field?: boolean; + + select_data_source_url?: string; + select_custom_options?: string[]; + select_value_key?: string; + select_label_key?: string; +} + +export interface FilterPeriodConfigEntity { + key: string; + type: FILTER_TYPE; + note?: string; + hidden?: boolean; +} + +export interface ReportConfigEntity { + group_name: string; + unique_name: string; + label: string; + + table_schema: string; + main_table_alias?: string; + customVirtualTableSchema?( + filterModel: any, + findQueryConfig: (column: string) => string, + createFilterSql: (key: string, item: any) => string, + ): string; + whereCondition?(filterModel: any): string[]; + whereDefaultConditions?: { + column: string; + filter_type: FILTER_TYPE; + values: string[]; + }[]; + defaultOrderBy?: string[]; + lowLevelOrderBy?: string[]; + + column_configs: ReportColumnConfigEntity[]; + filter_configs?: FilterConfigEntity[]; + filter_period_config?: FilterPeriodConfigEntity; + ignore_filter_keys?: string[]; + customQueryFilter?(column: string): string; +} diff --git a/src/modules/reports/shared/helpers/index.ts b/src/modules/reports/shared/helpers/index.ts new file mode 100644 index 0000000..0fe5354 --- /dev/null +++ b/src/modules/reports/shared/helpers/index.ts @@ -0,0 +1 @@ +export * from './query-builder'; diff --git a/src/modules/reports/shared/helpers/query-builder.ts b/src/modules/reports/shared/helpers/query-builder.ts new file mode 100644 index 0000000..458bfe1 --- /dev/null +++ b/src/modules/reports/shared/helpers/query-builder.ts @@ -0,0 +1,506 @@ +import { FILTER_TYPE } from '../constant'; +import { QueryModelEntity } from '../entities/query-model.entity'; +import { + ReportColumnConfigEntity, + ReportConfigEntity, +} from '../entities/report-config.entity'; + +export class ReportQueryBuilder { + public reportConfig: ReportConfigEntity; + public queryModel: QueryModelEntity; + constructor(reportConfig: ReportConfigEntity, queryModel: QueryModelEntity) { + this.reportConfig = reportConfig; + this.queryModel = queryModel; + } + + getBaseConfig() { + const tableSchema = this.reportConfig.table_schema; + const mainTableAlias = this.reportConfig.main_table_alias ?? 'main'; + const queryModel = this.queryModel; + return { + tableSchema, + mainTableAlias, + queryModel, + }; + } + + // OLD VERSION + buildSql(reportConfig: ReportConfigEntity, queryModel: any) { + const tableSchema = reportConfig.table_schema; + const mainTableAlias = reportConfig.main_table_alias ?? 'main'; + + const selectSql = this.createSelectSql(queryModel, reportConfig); + const selectSqlExport = this.createSelectSqlExport( + queryModel, + reportConfig, + ); + + const whereSql = this.createWhereSql(queryModel, reportConfig); + const limitSql = this.createLimitSql(queryModel); + + const orderBySql = this.createOrderBySql(queryModel, reportConfig); + const orderBySqlExport = this.createOrderBySqlExport( + queryModel, + reportConfig, + ); + + const { groupByQuery, groupByColumn } = this.createGroupBySql( + queryModel, + reportConfig, + ); + const { groupByQueryExport, groupByColumnExport } = + this.createGroupBySqlExport(queryModel, reportConfig); + + const SQL: string = + `SELECT ${selectSql} FROM ${tableSchema} ${whereSql} ${groupByQuery} ${orderBySql} ${limitSql}` as string; + const SQL_COUNT: string = `SELECT COUNT(${ + groupByColumn ? `DISTINCT ${groupByColumn}` : `${mainTableAlias}.id` + }) ${ + groupByColumn + ? `+ COUNT(DISTINCT CASE WHEN ${groupByColumn} IS NULL THEN 1 END)` + : '' + } AS count FROM ${tableSchema} ${whereSql}` as string; + const SQL_EXPORT: string = + `SELECT ${selectSqlExport} FROM ${tableSchema} ${whereSql} ${groupByQueryExport} ${orderBySqlExport}` as string; + const SQL_EXPORT_COUNT: string = `SELECT COUNT(${ + groupByColumnExport + ? `DISTINCT ${groupByColumnExport}` + : `${mainTableAlias}.id` + }) ${ + groupByColumnExport + ? `+ COUNT(DISTINCT CASE WHEN ${groupByColumnExport} IS NULL THEN 1 END)` + : '' + } AS count FROM ${tableSchema} ${whereSql}` as string; + + return { SQL, SQL_COUNT, SQL_EXPORT, SQL_EXPORT_COUNT }; + } + + createSelectSql(queryModel: any, reportConfig: ReportConfigEntity): string { + const configColumns = reportConfig.column_configs; + const mainTableAlias = reportConfig.main_table_alias ?? 'main'; + + const rowGroupCols = queryModel.rowGroupCols; + const valueCols = queryModel.valueCols; + const groupKeys = queryModel.groupKeys; + + if (this.isDoingGrouping(rowGroupCols, groupKeys)) { + const colsToSelect = []; + const rowGroupCol = rowGroupCols[groupKeys.length]; + const rowGroupColChildCount = rowGroupCols[groupKeys.length + 1]; + + colsToSelect.push( + this.findQueryConfig(reportConfig, rowGroupCol.field) + + ` AS ${rowGroupCol.field}`, + ); + + if (rowGroupColChildCount) { + colsToSelect.push( + `COUNT(DISTINCT ${this.findQueryConfig( + reportConfig, + rowGroupColChildCount.field, + )}) AS count_child_group`, + ); + } else { + colsToSelect.push(`COUNT(${mainTableAlias}.id) AS count_child_group`); + } + + // eslint-disable-next-line @typescript-eslint/no-this-alias + const thisSelf = this; + valueCols.forEach(function (valueCol) { + colsToSelect.push( + `${valueCol.aggFunc} (${thisSelf.findQueryConfig( + reportConfig, + valueCol.field, + )}) AS ${valueCol.field}`, + ); + }); + return colsToSelect.join(', '); + } + + const columns = configColumns.map( + (item) => `${item.query} AS ${item.column}`, + ); + + return columns.join(', '); + } + + createSelectSqlExport( + queryModel: any, + reportConfig: ReportConfigEntity, + ): string { + const configColumns = reportConfig.column_configs; + + const rowGroupCols = queryModel.rowGroupCols; + const valueCols = queryModel.valueCols; + + // eslint-disable-next-line @typescript-eslint/no-this-alias + const thisSelf = this; + + if (rowGroupCols.length > 0) { + const colsToSelect = []; + rowGroupCols.forEach(function (rowGroupCol) { + colsToSelect.push( + thisSelf.findQueryConfig(reportConfig, rowGroupCol.field) + + ` AS ${rowGroupCol.field}`, + ); + }); + + valueCols.forEach(function (valueCol) { + colsToSelect.push( + `${valueCol.aggFunc} (${thisSelf.findQueryConfig( + reportConfig, + valueCol.field, + )}) AS ${valueCol.field}`, + ); + }); + + return colsToSelect.join(', '); + } + + const columns = configColumns.map((item) => `${item.query} ${item.column}`); + + return columns.join(', '); + } + + createFilterSql(column: string, item: { type: FILTER_TYPE; filter: any }) { + switch (item.type) { + // TEXT + case FILTER_TYPE.TEXT_EQUAL: + return `${column} = '${item.filter}'`; + + case FILTER_TYPE.TEXT_NOT_EQUAL: + return `${column} != '${item.filter}'`; + + case FILTER_TYPE.TEXT_CONTAINS: + return `${column} ILIKE '%${item.filter}%'`; + + case FILTER_TYPE.TEXT_NOT_CONTAINS: + return `${column} NOT ILIKE '%${item.filter}%'`; + + case FILTER_TYPE.TEXT_START_WITH: + return `${column} ILIKE '${item.filter}%'`; + + case FILTER_TYPE.TEXT_END_WITH: + return `${column} ILIKE '%${item.filter}'`; + + case FILTER_TYPE.TEXT_IN_MEMBER: + return item.filter?.length > 0 + ? `${column} IN(${item.filter.map((i) => `'${i}'`).join(', ')})` + : null; + + case FILTER_TYPE.TEXT_MULTIPLE_CONTAINS: + return item.filter?.length > 0 + ? `${column} ILIKE ANY(ARRAY[${item.filter + .map((i) => `'%${i}%'`) + .join(', ')}])` + : null; + + case FILTER_TYPE.TEXT_MULTIPLE_REGEXP_CONTAINS: + return item.filter?.length > 0 + ? `${column} REGEXP '${item.filter.join('|')}'` + : null; + + case FILTER_TYPE.DATE_IN_RANGE_EPOCH: + return `(${column} >= ${item.filter[0]} AND ${column} <= ${item.filter[1]})`; + + case FILTER_TYPE.DATE_IN_RANGE_TIMESTAMP: + return `(${column} BETWEEN '${item.filter[0]}' AND '${item.filter[1]}')`; + + case FILTER_TYPE.TEXT_IN_RANGE: + return `(${column} >= '${item.filter[0]}' AND ${column} <= '${item.filter[1]}')`; + + // NUMBER + case FILTER_TYPE.NUMBER_EQUAL: + return `${column} = ${item.filter}`; + + case FILTER_TYPE.NUMBER_NOT_EQUAL: + return `${column} != ${item.filter}`; + + case FILTER_TYPE.NUMBER_GREATER_THAN: + return `${column} > ${item.filter}`; + + case FILTER_TYPE.NUMBER_GREATER_THAN_OR_EQUAL: + return `${column} >= ${item.filter}`; + + case FILTER_TYPE.NUMBER_LESS_THAN: + return `${column} < ${item.filter}`; + + case FILTER_TYPE.NUMBER_LESS_THAN_OR_EQUAL: + return `${column} <= ${item.filter}`; + + case FILTER_TYPE.NUMBER_IN_RANGE: + return `(${column} >= ${item.filter[0]} AND ${column} <= ${item.filter[1]})`; + + default: + console.log('UNKNOWN FILTER TYPE:', item.type); + return 'true'; + } + } + + createWhereSql(queryModel, reportConfig: ReportConfigEntity) { + // const configColumns = reportConfig.column_configs ?? []; + const configFilters = reportConfig.filter_configs ?? []; + const ignoreFilterKeys = reportConfig.ignore_filter_keys ?? []; + const ignoreFilter = configFilters + .filter((el) => el.hide_field) + .map((el) => el.filter_column); + + const ignoreFilterKey = [...ignoreFilter, ...ignoreFilterKeys]; + + const whereCondition = reportConfig?.whereCondition + ? reportConfig.whereCondition(queryModel.filterModel) + : []; + + const rowGroupCols = queryModel.rowGroupCols; + const groupKeys = queryModel.groupKeys; + const filterModel = queryModel.filterModel; + + // eslint-disable-next-line @typescript-eslint/no-this-alias + const thisSelf = this; + + const whereParts = []; + if (groupKeys.length > 0) { + groupKeys.forEach(function (key, index) { + const colName = rowGroupCols[index].field; + // whereParts.push(colName + ' = "' + key + '"'); + whereParts.push( + `${thisSelf.findQueryConfig(reportConfig, colName)} = '${key}'`, + ); + }); + } + + if (filterModel) { + const keySet = Object.keys(filterModel); + keySet.forEach(function (key) { + if (!ignoreFilterKey.includes(key)) { + const item = filterModel[key]; + const newKey = thisSelf.findQueryConfig(reportConfig, key); + whereParts.push(thisSelf.createFilterSql(newKey, item)); + } + }); + } + + // set default where conditions + const defaultConditions = reportConfig.whereDefaultConditions; + const defaultWhereOptions = []; + if (defaultConditions) { + defaultConditions.forEach((condition) => { + defaultWhereOptions.push( + this.createFilterSql(condition.column, { + type: condition.filter_type, + filter: condition.values, + }), + ); + }); + } + + const tableWhereConditions = [...whereCondition, ...whereParts]; + const defaultWhereConditions = defaultWhereOptions; + + if (tableWhereConditions.length > 0) { + return `WHERE (${ + defaultWhereConditions.length + ? defaultWhereConditions?.filter(Boolean).join(' AND ') + ' AND ' + : ' ' + } ${tableWhereConditions.filter(Boolean).join(' AND ')})`; + } else if (defaultWhereConditions.length) { + return `WHERE (${defaultWhereConditions.filter(Boolean).join(' AND ')})`; + } else { + return ''; + } + } + + createOrderBySql(queryModel, reportConfig: ReportConfigEntity) { + const mainTableAlias = reportConfig.main_table_alias ?? 'main'; + const defaultOrderBy = reportConfig.defaultOrderBy ?? []; + const lowLevelOrderBy = reportConfig.lowLevelOrderBy ?? []; + + const rowGroupCols = queryModel.rowGroupCols; + const groupKeys = queryModel.groupKeys; + const sortModel = queryModel.sortModel; + + const grouping = this.isDoingGrouping(rowGroupCols, groupKeys); + + const sortParts = []; + if (sortModel) { + const groupColIds = rowGroupCols + .map((groupCol) => groupCol.id) + .slice(0, groupKeys.length + 1); + + sortModel.forEach(function (item) { + if (grouping && groupColIds.indexOf(item.colId) < 0) { + // ignore + } else { + sortParts.push(item.colId + ' ' + item.sort); + } + }); + } + + const defaultOrder = defaultOrderBy[0] + ? defaultOrderBy + : [`${mainTableAlias}.created_at DESC`]; + + const lowLevelOrder = lowLevelOrderBy[0] + ? lowLevelOrderBy + : [`${mainTableAlias}.id DESC`]; + + if (sortParts.length > 0) { + if (rowGroupCols?.length > 0) { + if (groupKeys.length > 0) { + const sortBy = sortParts[groupKeys.length]; + if (sortBy) return ' ORDER BY ' + sortBy; + // return ''; + } + + return ' ORDER BY ' + sortParts.join(', '); + } + return ( + ' ORDER BY ' + sortParts.join(', ') + `, ${lowLevelOrder.join(', ')}` + ); + } else { + if (rowGroupCols?.length > 0) { + const sortGroupData = rowGroupCols[groupKeys.length]; + if (sortGroupData) return ' ORDER BY ' + `${sortGroupData['id']} DESC`; + // return ''; + } + + return ( + ` ORDER BY ` + defaultOrder.join(', ') + `, ${lowLevelOrder.join(', ')}` + ); + } + } + + createOrderBySqlExport(queryModel, reportConfig: ReportConfigEntity) { + const mainTableAlias = reportConfig.main_table_alias ?? 'main'; + const defaultOrderBy = reportConfig.defaultOrderBy ?? []; + const lowLevelOrderBy = reportConfig.lowLevelOrderBy ?? []; + + const rowGroupCols = queryModel.rowGroupCols; + const sortModel = queryModel.sortModel; + + const defaultOrder = defaultOrderBy[0] + ? defaultOrderBy + : [`${mainTableAlias}.created_at DESC`]; + + const lowLevelOrder = lowLevelOrderBy[0] + ? lowLevelOrderBy + : [`${mainTableAlias}.id DESC`]; + + if (sortModel.length > 0) { + const sortParts = sortModel.map((i) => `${i.colId} ${i.sort}`); + return ' ORDER BY ' + `${sortParts.join(', ')}`; + } else { + if (rowGroupCols?.length > 0) { + const sortParts = rowGroupCols.map((i) => `${i.id} DESC`); + return ' ORDER BY ' + `${sortParts.join(', ')}`; + } + + return ( + ` ORDER BY ` + defaultOrder.join(', ') + `, ${lowLevelOrder.join(', ')}` + ); + } + } + + createGroupBySql(queryModel, reportConfig: ReportConfigEntity) { + // const configColumns = reportConfig.column_configs; + const rowGroupCols = queryModel.rowGroupCols; + const groupKeys = queryModel.groupKeys; + + if (this.isDoingGrouping(rowGroupCols, groupKeys)) { + const colsToGroupBy = []; + + const rowGroupCol = rowGroupCols[groupKeys.length]; + colsToGroupBy.push(this.findQueryConfig(reportConfig, rowGroupCol.field)); + + return { + groupByQuery: ' GROUP BY ' + colsToGroupBy.join(', '), + groupByColumn: colsToGroupBy.join(', '), + }; + } else { + return { + groupByQuery: '', + groupByColumn: null, + }; + } + } + + createGroupBySqlExport(queryModel, reportConfig: ReportConfigEntity) { + // const configColumns = reportConfig.column_configs; + const rowGroupCols = queryModel.rowGroupCols; + + // eslint-disable-next-line @typescript-eslint/no-this-alias + const thisSelf = this; + + if (rowGroupCols.length > 0) { + const colsToGroupBy = []; + rowGroupCols.forEach(function (rowGroupCol) { + colsToGroupBy.push( + thisSelf.findQueryConfig(reportConfig, rowGroupCol.field), + ); + }); + + return { + groupByQueryExport: ' GROUP BY ' + colsToGroupBy.join(', '), + groupByColumnExport: colsToGroupBy.join(', '), + }; + } else { + return { + groupByQueryExport: '', + groupByColumnExport: null, + }; + } + } + + findQueryConfig(config: ReportConfigEntity, column: string) { + const configColumns = config.column_configs ?? []; + + const findQuery: ReportColumnConfigEntity = configColumns.find( + (el) => el.column === column, + ); + const customQuery = config?.customQueryFilter + ? config.customQueryFilter(column) + : undefined; + + if (customQuery) return customQuery; + else if (findQuery) return findQuery.query; + + return column.replace('__', '.'); + } + + isDoingGrouping(rowGroupCols, groupKeys) { + // we are not doing grouping if at the lowest level. we are at the lowest level + // if we are grouping by more columns than we have keys for (that means the user + // has not expanded a lowest level group, OR we are not grouping at all). + return rowGroupCols.length > groupKeys.length; + } + + interpolate(str, o) { + return str.replace(/{([^{}]*)}/g, function (a, b) { + const r = o[b]; + return typeof r === 'string' || typeof r === 'number' ? r : a; + }); + } + createLimitSql(queryModel) { + const startRow = queryModel.startRow; + const endRow = queryModel.endRow; + const pageSize = endRow - startRow; + return ' LIMIT ' + (pageSize + 1) + ' OFFSET ' + startRow; + } + + getRowCount(queryModel, results) { + if (results === null || results === undefined || results.length === 0) { + return null; + } + const currentLastRow = queryModel.startRow + results.length; + return currentLastRow <= queryModel.endRow ? currentLastRow : -1; + } + + cutResultsToPageSize(queryModel, results) { + const pageSize = queryModel.endRow - queryModel.startRow; + if (results && results.length > pageSize) { + return results.splice(0, pageSize); + } else { + return results; + } + } +}