@seasketch/geoprocessing
Version:
Geoprocessing and reporting framework for SeaSketch 2.0
170 lines • 8.18 kB
JavaScript
import { readDatasources } from "./datasources.js";
import { firstMatching, isRasterDatasource, isVectorDatasource, } from "../../../src/datasources/index.js";
import { readGeographies } from "../geographies/geographies.js";
import { createOrUpdatePrecalcMetrics } from "./precalc.js";
import { precalcVectorDatasource } from "./precalcVectorDatasource.js";
import { precalcRasterDatasource } from "./precalcRasterDatasource.js";
import cloneDeep from "lodash/cloneDeep.js";
/**
* Precalc one or more datasources for a project, for one or more defined geographies
* one at a time, writing them out to disk as they complete
*/
export async function precalcDatasources(projectClient, extraOptions = {}) {
const { newDatasourcePath, newGeographyPath, datasourceMatcher = ["*"], geographyMatcher = ["*"], } = extraOptions;
const allGeographies = await readGeographies(newGeographyPath);
// Start with no geographies to precalc. Matcher can specify all or some
let matchingGeographies = [];
if (geographyMatcher && geographyMatcher.length > 0) {
if (geographyMatcher.includes("*")) {
matchingGeographies = cloneDeep(allGeographies);
}
else {
matchingGeographies = cloneDeep(allGeographies).filter((geog) => geographyMatcher.includes(geog.geographyId));
if (matchingGeographies.length === 0)
throw new Error(`No matching geographies found for ${geographyMatcher}, exiting`);
}
}
const vectorDatasources = (await readDatasources(newDatasourcePath).filter((ds) => isVectorDatasource(ds)));
const datasources = cloneDeep(await readDatasources(newDatasourcePath));
// Start with no datasources to precalc. Matcher can specify all (*) or some
let matchingDatasources = [];
if (datasourceMatcher && datasourceMatcher.length > 0) {
if (datasourceMatcher.includes("*")) {
matchingDatasources = cloneDeep(datasources);
}
else {
matchingDatasources = cloneDeep(datasources).filter((ds) => datasourceMatcher.includes(ds.datasourceId));
}
}
// Process one at a time
let failed = 0;
let skipped = 0;
let successfulDs = 0;
let successfulGs = 0;
let finalMetrics = [];
const processed = {}; // Track processed datasource/geography combinations to avoid duplicates
// console.log("vector (geog) datasources", vectorDatasources);
// console.log("datasources", datasources);
// console.log("matching datasources", matchingDatasources);
// console.log("all geographies", allGeographies);
// console.log("matching geographies", matchingGeographies);
// Run precalc on matching subset of datasources for all geographies
for (const ds of matchingDatasources) {
for (const geog of allGeographies) {
// Skip if either datasource or geography has precalc set to false
if (geog.precalc === false || ds.precalc === false) {
// console.log(
// `Precalc disabled for datasource ${ds.datasourceId} for geography ${geog.geographyId}`
// );
continue;
}
// Skip if already processed
if (processed[`${ds.datasourceId}-${geog.geographyId}`] === true) {
continue;
}
try {
console.log(`Precalculating datasource ${ds.datasourceId} for geography ${geog.geographyId}`);
const geogDatasource = firstMatching(vectorDatasources, (item) => item.datasourceId === geog.datasourceId);
const metrics = await precalcMetrics(projectClient, ds, geog, geogDatasource, extraOptions);
// console.log(ds.datasourceId, geog.geographyId, metrics);
finalMetrics = finalMetrics.concat(metrics);
console.log(" ");
successfulDs += 1;
processed[`${ds.datasourceId}-${geog.geographyId}`] = true;
}
catch (error) {
if (error instanceof Error) {
console.log(error.message);
console.log(error.stack);
console.log(`Updating precalc metrics for ${ds.datasourceId} failed, moving to next`);
failed += 1;
}
}
}
}
// Also run precalc on matching subset of geographies for all datasources, for completeness
for (const geog of matchingGeographies) {
for (const ds of datasources) {
if (geog.precalc === false || ds.precalc === false) {
// console.log(
// `Precalc disabled for datasource ${ds.datasourceId} + geography ${geog.geographyId}`
// );
skipped += 1;
continue;
}
// Skip if already processed
if (processed[`${ds.datasourceId}-${geog.geographyId}`] === true) {
continue;
}
try {
console.log(`Precalculating datasource ${ds.datasourceId} + geography ${geog.geographyId}`);
const geogDatasource = firstMatching(vectorDatasources, (item) => item.datasourceId === geog.datasourceId);
const metrics = await precalcMetrics(projectClient, ds, geog, geogDatasource, extraOptions);
finalMetrics = finalMetrics.concat(metrics);
console.log(`${ds.datasourceId} precalc complete`);
console.log(" ");
successfulGs += 1;
}
catch (error) {
if (error instanceof Error) {
console.log(error.stack);
console.log(`Precalculating metrics for datasource ${ds.datasourceId} + geography ${geog.geographyId} failed, moving to next`);
failed += 1;
}
}
}
}
const successful = successfulDs + successfulGs;
if (successful > 0)
console.log(`${successfulDs} datasource/geography combinations precalculated successfully`);
if (skipped > 0)
console.log(`${successfulDs} datasource/geography combinations skipped due to precalc disabled`);
if (successfulDs === 0 && successfulGs === 0 && skipped === 0) {
console.log(`No datasources or geographies found to precalculate`);
}
if (failed > 0) {
console.log(`${failed} datasources failed to precalculate. Fix them and try again`);
}
return finalMetrics;
}
/**
* Precalculate metrics for datasource for given geography and write out to disk
*/
export const precalcMetrics = async (projectClient, ds, geog, geogDs, extraOptions) => {
const { newPrecalcPath, newDstPath, port } = extraOptions;
// precalc if possible. If external datasource, then return nothing
const curMetrics = await (async () => {
if (isVectorDatasource(ds) && ds.geo_type === "vector") {
return await precalcVectorDatasource(projectClient, ds, geog, geogDs, {
newDstPath,
port,
});
}
else if (isRasterDatasource(ds) && ds.geo_type === "raster") {
return await precalcRasterDatasource(projectClient, ds, geog, geogDs, {
newDstPath,
port,
});
}
else {
console.log(`Skipping ${ds.datasourceId}, precalc not supported`);
return [];
}
})();
const staleMetricsFilterFn = staleMetricsFilterFnFactory(ds.datasourceId, geog.geographyId);
createOrUpdatePrecalcMetrics(curMetrics, {
matcher: staleMetricsFilterFn,
filePath: newPrecalcPath,
});
return curMetrics;
};
/**
* returns a function that will filter out metrics that don't match the geographyId or don't have a classId that starts with the datasourceId (total metrics)
*/
export const staleMetricsFilterFnFactory = (datasourceId, geographyId) => {
return (m) => {
return ((!!m.classId && !m.classId.startsWith(datasourceId + "-")) ||
(!!m.geographyId && m.geographyId !== geographyId));
};
};
//# sourceMappingURL=precalcDatasources.js.map