@overture-stack/lyric
Version:
Data Submission system
355 lines (354 loc) • 17.5 kB
JavaScript
import { and, count, eq, inArray, or, sql } from 'drizzle-orm/sql';
import { auditSubmittedData, submittedData, } from '@overture-stack/lyric-data-model/models';
import { ServiceUnavailable } from '../utils/errors.js';
import { AUDIT_ACTION } from '../utils/types.js';
const repository = (dependencies) => {
const LOG_MODULE = 'SUBMITTEDDATA_REPOSITORY';
const { db, logger, features } = dependencies;
const BATCH_SIZE = 500;
const auditDeleteSubmittedData = async (input, tx) => {
const { recordDeleted, diff, submissionId, username } = input;
const newAudit = {
action: AUDIT_ACTION.Values.DELETE,
dictionaryCategoryId: recordDeleted.dictionaryCategoryId,
entityName: recordDeleted.entityName,
lastValidSchemaId: recordDeleted.lastValidSchemaId,
newDataIsValid: false,
dataDiff: diff,
oldDataIsValid: recordDeleted.isValid,
organization: recordDeleted.organization,
originalSchemaId: recordDeleted.originalSchemaId,
submissionId: submissionId,
systemId: recordDeleted.systemId,
createdAt: new Date(),
createdBy: username,
};
return await (tx || db).insert(auditSubmittedData).values(newAudit);
};
const auditUpdateSubmittedData = async ({ dataDiff, oldIsValid, recordUpdated, submissionId, }, tx) => {
const newAudit = {
action: AUDIT_ACTION.Values.UPDATE,
dictionaryCategoryId: recordUpdated.dictionaryCategoryId,
entityName: recordUpdated.entityName,
lastValidSchemaId: recordUpdated.lastValidSchemaId,
newDataIsValid: recordUpdated.isValid,
dataDiff: dataDiff,
oldDataIsValid: oldIsValid,
organization: recordUpdated.organization,
originalSchemaId: recordUpdated.originalSchemaId,
submissionId: submissionId,
systemId: recordUpdated.systemId,
createdAt: new Date(),
createdBy: recordUpdated.updatedBy,
};
return await (tx || db).insert(auditSubmittedData).values(newAudit);
};
// Column name on the database used to build JSONB query
const jsonbColumnName = submittedData.data.name;
const paginatedColumns = {
entityName: true,
data: true,
organization: true,
isValid: true,
systemId: true,
};
/**
* Build a SQL object to search submitted data by entity Name
* @param {string[]} entityNameArray
* @returns {SQL<unknown> | undefined}
*/
const filterByEntityNameArray = (entityNameArray) => {
if (Array.isArray(entityNameArray)) {
return or(...entityNameArray
.filter((entity) => entity !== undefined)
.map((entity) => eq(submittedData.entityName, entity.trim())));
}
return undefined;
};
const filterByOrganizationArray = (organizationArray) => {
if (Array.isArray(organizationArray)) {
return or(...organizationArray
.filter((org) => org !== undefined)
.map((org) => eq(submittedData.organization, org.trim())));
}
return undefined;
};
return {
/**
* Deletes a submitted data record(s) by its system ID, logs the deletion, and optionally audits the deletion if auditing is enabled.
* @param params The parameters for the deletion operation.
* @param params.diff The difference between the old and new data, used for auditing
* @param params.submissionId The ID of the Submission associated with the record
* @param params.systemId The unique identifier of the record to delete
* @param params.username The name of the user performing the deletion
* @param tx The transaction to use for the operation, optional
* @returns The deleted record
*/
deleteBySystemId: async (params, tx) => {
const rows = Array.isArray(params) ? params : [params];
const result = [];
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
const batch = rows.slice(i, i + BATCH_SIZE);
const systemIds = batch.map((p) => p.systemId);
const deletedRecords = await (tx || db)
.delete(submittedData)
.where(inArray(submittedData.systemId, systemIds))
.returning();
logger.info(LOG_MODULE, `Deleted ${deletedRecords.length} SubmittedData record(s) with system ID(s): [${systemIds.join(', ')}]`);
result.push(...deletedRecords);
if (features?.audit?.enabled) {
await Promise.all(batch.map((row) => {
const record = deletedRecords.find((r) => r.systemId === row.systemId);
if (!record) {
return Promise.resolve();
}
return auditDeleteSubmittedData({ recordDeleted: record, submissionId: row.submissionId, diff: row.diff, username: row.username }, tx);
}));
}
}
return result;
},
/**
* Save new SubmittedData in Database
* @param data A SubmittedData object or array to be saved
* @param tx The transaction to use for the operation, optional
* @returns The created SubmittedData
*/
save: async (data, tx) => {
const rows = Array.isArray(data) ? data : [data];
const savedRecords = [];
try {
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
const batch = rows.slice(i, i + BATCH_SIZE);
const savedSubmittedData = await (tx || db)
.insert(submittedData)
.values(batch)
.returning({ id: submittedData.id });
savedRecords.push(...savedSubmittedData);
}
logger.debug(LOG_MODULE, `Submitted ${savedRecords.length} record(s) successfully`);
return Array.isArray(data) ? savedRecords : savedRecords[0];
}
catch (error) {
logger.error(LOG_MODULE, `Failed submitting ${rows.length} record(s)`, error);
throw error;
}
},
/**
* Returns a list of all organizations found by category ID
* @param {number} categoryId
* @returns
*/
getAllOrganizationsByCategoryId: async (categoryId) => {
try {
const resultQuery = await db
.selectDistinct({ organization: submittedData.organization })
.from(submittedData)
.where(eq(submittedData.dictionaryCategoryId, categoryId))
.orderBy(submittedData.organization);
return resultQuery.map((record) => record.organization);
}
catch (error) {
logger.error(LOG_MODULE, `Failed querying SubmittedData with categoryId '${categoryId}'`, error);
throw new ServiceUnavailable();
}
},
/**
* Find SubmittedData by category ID and organization
* @param {number} categoryId Category ID
* @param {string} organization Organization Name
* @returns The SubmittedData found
*/
getSubmittedDataByCategoryIdAndOrganization: async (categoryId, organization) => {
try {
return await db.query.submittedData.findMany({
where: and(eq(submittedData.dictionaryCategoryId, categoryId), eq(submittedData.organization, organization)),
});
}
catch (error) {
logger.error(LOG_MODULE, `Failed querying SubmittedData with categoryId '${categoryId}'`, error);
throw new ServiceUnavailable();
}
},
/**
* Find SubmittedData by category ID with pagination
* @param {number} categoryId Category ID
* @param {PaginationOptions} paginationOptions Pagination properties
* @param {object} filter Filter Options
* @param {string[] | undefined} filter.entityNames Array of entity names to filter
* @param {string[] | undefined} filter.organizations Array of organizations to filter
* @returns The SubmittedData found
*/
getSubmittedDataByCategoryIdPaginated: async (categoryId, paginationOptions, filter) => {
const { page, pageSize } = paginationOptions;
const filterEntityNameSql = filterByEntityNameArray(filter?.entityNames);
const filterOrganizationSql = filterByOrganizationArray(filter?.organizations);
try {
return await db.query.submittedData.findMany({
where: and(eq(submittedData.dictionaryCategoryId, categoryId), filterEntityNameSql, filterOrganizationSql),
columns: paginatedColumns,
orderBy: (submittedData, { asc }) => [asc(submittedData.entityName), asc(submittedData.id)],
limit: pageSize,
offset: (page - 1) * pageSize,
});
}
catch (error) {
logger.error(LOG_MODULE, `Failed querying SubmittedData with categoryId '${categoryId}'`, error);
throw new ServiceUnavailable();
}
},
/**
* Find SubmittedData by category ID and Organization with pagination
* @param {number} categoryId Category ID
* @param {string} organization Organization Name
* @param {PaginationOptions} paginationOptions Pagination properties
* @param {object} filter Filter Options
* @param {SQL | undefined} filter.sql SQL command to filter
* @param {string[] | undefined} filter.entityNames Array of entity names to filter
* @returns The SubmittedData found
*/
getSubmittedDataByCategoryIdAndOrganizationPaginated: async (categoryId, organization, paginationOptions, filter) => {
const { page, pageSize } = paginationOptions;
const filterEntityNameSql = filterByEntityNameArray(filter?.entityNames);
try {
return await db.query.submittedData.findMany({
where: and(eq(submittedData.dictionaryCategoryId, categoryId), eq(submittedData.organization, organization), filter?.sql, filterEntityNameSql),
columns: paginatedColumns,
orderBy: (submittedData, { asc }) => [asc(submittedData.entityName), asc(submittedData.id)],
limit: pageSize,
offset: (page - 1) * pageSize,
});
}
catch (error) {
logger.error(LOG_MODULE, `Failed querying SubmittedData with categoryId '${categoryId}' organization '${organization}'`, error);
throw new ServiceUnavailable();
}
},
/**
* Counts the total of records found by Category and Organization
* @param {number} categoryId Category ID
* @param {string} organization Organization Name
* @param {object} filter Filter Options
* @param {SQL | undefined} filter.sql SQL command to filter
* @param {string[] | undefined} filter.entityNames Array of entity names to filter
* @returns Total number of recourds
*/
getTotalRecordsByCategoryIdAndOrganization: async (categoryId, organization, filter) => {
const filterEntityNameSql = filterByEntityNameArray(filter?.entityNames);
try {
const resultCount = await db
.select({ total: count() })
.from(submittedData)
.where(and(eq(submittedData.dictionaryCategoryId, categoryId), eq(submittedData.organization, organization), filter?.sql, filterEntityNameSql));
return resultCount[0].total;
}
catch (error) {
logger.error(LOG_MODULE, `Failed counting SubmittedData with categoryId '${categoryId}' organization '${organization}'`, error);
throw new ServiceUnavailable();
}
},
/**
* Counts the total of records found by Category
* @param {number} categoryId Category ID
* @param {object} filter Filter options
* @param {SQL | undefined} filter.sql SQL command
* @param {string[] | undefined} filter.entityNames Array of entity names to filter
* @param {string[] | undefined} filter.organizations Organization name to filter
* @returns Total number of recourds
*/
getTotalRecordsByCategoryId: async (categoryId, filter) => {
const filterEntityNameSql = filterByEntityNameArray(filter?.entityNames);
const filterOrganizationSql = filterByOrganizationArray(filter?.organizations);
try {
const resultCount = await db
.select({ total: count() })
.from(submittedData)
.where(and(eq(submittedData.dictionaryCategoryId, categoryId), filterEntityNameSql, filterOrganizationSql));
return resultCount[0].total;
}
catch (error) {
logger.error(LOG_MODULE, `Failed counting SubmittedData with categoryId '${categoryId}'`, error);
throw new ServiceUnavailable();
}
},
/**
* Update a SubmittedData record in database
* @param submittedDataId Submitted Data ID
* @param dataDiff Difference before and after the updata
* @param newData Set fields to update
* @param oldIsValid Previous isValid value
* @param submissionId Submission ID
* @param tx The transaction to use for the operation, optional
* @returns An updated record
*/
update: async (params, tx) => {
const updates = Array.isArray(params) ? params : [params];
try {
const updatedRecords = [];
for (const u of updates) {
const updated = await (tx || db)
.update(submittedData)
.set({ ...u.newData, updatedAt: new Date() })
.where(eq(submittedData.id, u.submittedDataId))
.returning();
updatedRecords.push(updated[0]);
if (features?.audit?.enabled && Object.keys(u.dataDiff.new).length && Object.keys(u.dataDiff.old).length) {
await auditUpdateSubmittedData({
recordUpdated: updated[0],
submissionId: u.submissionId,
dataDiff: u.dataDiff,
oldIsValid: u.oldIsValid,
}, tx);
}
}
return Array.isArray(params) ? updatedRecords : updatedRecords[0];
}
catch (error) {
logger.error(LOG_MODULE, `Failed updating SubmittedData`, error);
throw new ServiceUnavailable();
}
},
/**
* Query to retrieve an unique SubmittedData record searching by System ID
* Returns a SubmittedData record if found. Otherwise returns undefined
* @param {string} systemId
* @returns {Promise<SubmittedData | undefined>}
*/
getSubmittedDataBySystemId: async (systemId) => {
try {
return await db.query.submittedData.findFirst({
where: eq(submittedData.systemId, systemId),
});
}
catch (error) {
logger.error(LOG_MODULE, `Failed querying SubmittedData by systemId '${systemId}'`, error);
throw new ServiceUnavailable();
}
},
/**
* Query to retrieve submitted data filtered by JSONB field on an organization
* Returns an array of SubmittedData records found or an empty array if there are no matching records
* @param {string} organization
* @param {Object} filterData
* @param {string} filterData.entityName
* @param {string} filterData.dataField
* @param {string | undefined} filterData.dataValue
* @returns {Promise<SubmittedData[]>}
*/
getSubmittedDataFiltered: async (organization, filterData) => {
const sqlDataFilter = filterData.map((filter) => {
return and(sql.raw(`${jsonbColumnName} ->> '${filter.dataField}' IN ('${filter.dataValue}')`), eq(submittedData.entityName, filter.entityName));
});
try {
return await db.query.submittedData.findMany({
where: and(or(...sqlDataFilter), eq(submittedData.organization, organization)),
});
}
catch (error) {
logger.error(LOG_MODULE, `Failed querying SubmittedData`, error);
throw new ServiceUnavailable();
}
},
};
};
export default repository;