@dpkit/ckan
Version:
Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames
74 lines • 9.62 kB
JavaScript
import { mergePackages } from "@dpkit/core";
import { makeCkanApiRequest } from "../general/index.js";
import { normalizeCkanPackage } from "./process/normalize.js";
/**
* Load a package from a CKAN instance
* @param props Object containing the URL to the CKAN package
* @returns Package object and cleanup function
*/
export async function loadPackageFromCkan(datasetUrl) {
const packageId = extractPackageId(datasetUrl);
if (!packageId) {
throw new Error(`Failed to extract package ID from URL: ${datasetUrl}`);
}
const ckanPackage = await makeCkanApiRequest({
ckanUrl: datasetUrl,
action: "package_show",
payload: { id: packageId },
});
for (const resource of ckanPackage.resources) {
const resourceId = resource.id;
if (["CSV", "XLS", "XLSX"].includes(resource.format)) {
const schema = await loadCkanSchema({ datasetUrl, resourceId });
if (schema) {
resource.schema = schema;
}
}
}
const systemPackage = normalizeCkanPackage(ckanPackage);
const userPackagePath = systemPackage.resources
.filter(resource => resource["ckan:key"] === "datapackage.json")
.map(resource => resource["ckan:url"])
.at(0);
const datapackage = await mergePackages({ systemPackage, userPackagePath });
datapackage.resources = datapackage.resources.map(resource => {
// TODO: remove these keys completely
return { ...resource, "ckan:key": undefined, "ckan:url": undefined };
});
return datapackage;
}
/**
* Extract package ID from URL
*
* Examples:
* - https://hri.fi/data/en_GB/dataset/helsingin-kaupungin-verkkosivustojen-kavijaanalytiikka
* - https://www.opendata.dk/city-of-copenhagen/parkeringszoner-information
* - https://open.africa/dataset/pib-annual-senegal
* - https://data.nhm.ac.uk/dataset/join-the-dots-collection-level-descriptions
*/
function extractPackageId(datasetUrl) {
const url = new URL(datasetUrl);
const pathParts = url.pathname.split("/").filter(Boolean);
return pathParts.at(-1);
}
/**
* Fetch resource schema data from CKAN datastore
*/
async function loadCkanSchema(options) {
try {
// For some reason, datastore_info doesn't work
// So we use data fetching endpoint that also returns the schema
const result = await makeCkanApiRequest({
ckanUrl: options.datasetUrl,
action: "datastore_search",
payload: { resource_id: options.resourceId, limit: 0 },
});
// @ts-ignore
const fields = result.fields.filter((field) => field.id !== "_id" && field.id !== "_full_text");
return { fields };
}
catch (error) {
return undefined;
}
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibG9hZC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3BhY2thZ2UvbG9hZC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsYUFBYSxFQUFFLE1BQU0sYUFBYSxDQUFBO0FBQzNDLE9BQU8sRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHFCQUFxQixDQUFBO0FBRXhELE9BQU8sRUFBRSxvQkFBb0IsRUFBRSxNQUFNLHdCQUF3QixDQUFBO0FBRTdEOzs7O0dBSUc7QUFDSCxNQUFNLENBQUMsS0FBSyxVQUFVLG1CQUFtQixDQUFDLFVBQWtCO0lBQzFELE1BQU0sU0FBUyxHQUFHLGdCQUFnQixDQUFDLFVBQVUsQ0FBQyxDQUFBO0lBQzlDLElBQUksQ0FBQyxTQUFTLEVBQUUsQ0FBQztRQUNmLE1BQU0sSUFBSSxLQUFLLENBQUMsMENBQTBDLFVBQVUsRUFBRSxDQUFDLENBQUE7SUFDekUsQ0FBQztJQUVELE1BQU0sV0FBVyxHQUFHLE1BQU0sa0JBQWtCLENBQWM7UUFDeEQsT0FBTyxFQUFFLFVBQVU7UUFDbkIsTUFBTSxFQUFFLGNBQWM7UUFDdEIsT0FBTyxFQUFFLEVBQUUsRUFBRSxFQUFFLFNBQVMsRUFBRTtLQUMzQixDQUFDLENBQUE7SUFFRixLQUFLLE1BQU0sUUFBUSxJQUFJLFdBQVcsQ0FBQyxTQUFTLEVBQUUsQ0FBQztRQUM3QyxNQUFNLFVBQVUsR0FBRyxRQUFRLENBQUMsRUFBRSxDQUFBO1FBQzlCLElBQUksQ0FBQyxLQUFLLEVBQUUsS0FBSyxFQUFFLE1BQU0sQ0FBQyxDQUFDLFFBQVEsQ0FBQyxRQUFRLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQztZQUNyRCxNQUFNLE1BQU0sR0FBRyxNQUFNLGNBQWMsQ0FBQyxFQUFFLFVBQVUsRUFBRSxVQUFVLEVBQUUsQ0FBQyxDQUFBO1lBQy9ELElBQUksTUFBTSxFQUFFLENBQUM7Z0JBQ1gsUUFBUSxDQUFDLE1BQU0sR0FBRyxNQUFNLENBQUE7WUFDMUIsQ0FBQztRQUNILENBQUM7SUFDSCxDQUFDO0lBRUQsTUFBTSxhQUFhLEdBQUcsb0JBQW9CLENBQUMsV0FBVyxDQUFDLENBQUE7SUFDdkQsTUFBTSxlQUFlLEdBQUcsYUFBYSxDQUFDLFNBQVM7U0FDNUMsTUFBTSxDQUFDLFFBQVEsQ0FBQyxFQUFFLENBQUMsUUFBUSxDQUFDLFVBQVUsQ0FBQyxLQUFLLGtCQUFrQixDQUFDO1NBQy9ELEdBQUcsQ0FBQyxRQUFRLENBQUMsRUFBRSxDQUFDLFFBQVEsQ0FBQyxVQUFVLENBQUMsQ0FBQztTQUNyQyxFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUE7SUFFUixNQUFNLFdBQVcsR0FBRyxNQUFNLGFBQWEsQ0FBQyxFQUFFLGFBQWEsRUFBRSxlQUFlLEVBQUUsQ0FBQyxDQUFBO0lBQzNFLFdBQVcsQ0FBQyxTQUFTLEdBQUcsV0FBVyxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsUUFBUSxDQUFDLEVBQUU7UUFDM0QscUNBQXFDO1FBQ3JDLE9BQU8sRUFBRSxHQUFHLFFBQVEsRUFBRSxVQUFVLEVBQUUsU0FBUyxFQUFFLFVBQVUsRUFBRSxTQUFTLEVBQUUsQ0FBQTtJQUN0RSxDQUFDLENBQUMsQ0FBQTtJQUVGLE9BQU8sV0FBVyxDQUFBO0FBQ3BCLENBQUM7QUFFRDs7Ozs7Ozs7R0FRRztBQUNILFNBQVMsZ0JBQWdCLENBQUMsVUFBa0I7SUFDMUMsTUFBTSxHQUFHLEdBQUcsSUFBSSxHQUFHLENBQUMsVUFBVSxDQUFDLENBQUE7SUFDL0IsTUFBTSxTQUFTLEdBQUcsR0FBRyxDQUFDLFFBQVEsQ0FBQyxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUMsTUFBTSxDQUFDLE9BQU8sQ0FBQyxDQUFBO0lBQ3pELE9BQU8sU0FBUyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFBO0FBQ3pCLENBQUM7QUFFRDs7R0FFRztBQUNILEtBQUssVUFBVSxjQUFjLENBQUMsT0FHN0I7SUFDQyxJQUFJLENBQUM7UUFDSCwrQ0FBK0M7UUFDL0MsZ0VBQWdFO1FBQ2hFLE1BQU0sTUFBTSxHQUFHLE1BQU0sa0JBQWtCLENBQUM7WUFDdEMsT0FBTyxFQUFFLE9BQU8sQ0FBQyxVQUFVO1lBQzNCLE1BQU0sRUFBRSxrQkFBa0I7WUFDMUIsT0FBTyxFQUFFLEVBQUUsV0FBVyxFQUFFLE9BQU8sQ0FBQyxVQUFVLEVBQUUsS0FBSyxFQUFFLENBQUMsRUFBRTtTQUN2RCxDQUFDLENBQUE7UUFFRixhQUFhO1FBQ2IsTUFBTSxNQUFNLEdBQUcsTUFBTSxDQUFDLE1BQU0sQ0FBQyxNQUFNLENBQ2pDLENBQUMsS0FBVSxFQUFFLEVBQUUsQ0FBQyxLQUFLLENBQUMsRUFBRSxLQUFLLEtBQUssSUFBSSxLQUFLLENBQUMsRUFBRSxLQUFLLFlBQVksQ0FDaEUsQ0FBQTtRQUVELE9BQU8sRUFBRSxNQUFNLEVBQUUsQ0FBQTtJQUNuQixDQUFDO0lBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQztRQUNmLE9BQU8sU0FBUyxDQUFBO0lBQ2xCLENBQUM7QUFDSCxDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiaW1wb3J0IHsgbWVyZ2VQYWNrYWdlcyB9IGZyb20gXCJAZHBraXQvY29yZVwiXG5pbXBvcnQgeyBtYWtlQ2thbkFwaVJlcXVlc3QgfSBmcm9tIFwiLi4vZ2VuZXJhbC9pbmRleC5qc1wiXG5pbXBvcnQgdHlwZSB7IENrYW5QYWNrYWdlIH0gZnJvbSBcIi4vUGFja2FnZS5qc1wiXG5pbXBvcnQgeyBub3JtYWxpemVDa2FuUGFja2FnZSB9IGZyb20gXCIuL3Byb2Nlc3Mvbm9ybWFsaXplLmpzXCJcblxuLyoqXG4gKiBMb2FkIGEgcGFja2FnZSBmcm9tIGEgQ0tBTiBpbnN0YW5jZVxuICogQHBhcmFtIHByb3BzIE9iamVjdCBjb250YWluaW5nIHRoZSBVUkwgdG8gdGhlIENLQU4gcGFja2FnZVxuICogQHJldHVybnMgUGFja2FnZSBvYmplY3QgYW5kIGNsZWFudXAgZnVuY3Rpb25cbiAqL1xuZXhwb3J0IGFzeW5jIGZ1bmN0aW9uIGxvYWRQYWNrYWdlRnJvbUNrYW4oZGF0YXNldFVybDogc3RyaW5nKSB7XG4gIGNvbnN0IHBhY2thZ2VJZCA9IGV4dHJhY3RQYWNrYWdlSWQoZGF0YXNldFVybClcbiAgaWYgKCFwYWNrYWdlSWQpIHtcbiAgICB0aHJvdyBuZXcgRXJyb3IoYEZhaWxlZCB0byBleHRyYWN0IHBhY2thZ2UgSUQgZnJvbSBVUkw6ICR7ZGF0YXNldFVybH1gKVxuICB9XG5cbiAgY29uc3QgY2thblBhY2thZ2UgPSBhd2FpdCBtYWtlQ2thbkFwaVJlcXVlc3Q8Q2thblBhY2thZ2U+KHtcbiAgICBja2FuVXJsOiBkYXRhc2V0VXJsLFxuICAgIGFjdGlvbjogXCJwYWNrYWdlX3Nob3dcIixcbiAgICBwYXlsb2FkOiB7IGlkOiBwYWNrYWdlSWQgfSxcbiAgfSlcblxuICBmb3IgKGNvbnN0IHJlc291cmNlIG9mIGNrYW5QYWNrYWdlLnJlc291cmNlcykge1xuICAgIGNvbnN0IHJlc291cmNlSWQgPSByZXNvdXJjZS5pZFxuICAgIGlmIChbXCJDU1ZcIiwgXCJYTFNcIiwgXCJYTFNYXCJdLmluY2x1ZGVzKHJlc291cmNlLmZvcm1hdCkpIHtcbiAgICAgIGNvbnN0IHNjaGVtYSA9IGF3YWl0IGxvYWRDa2FuU2NoZW1hKHsgZGF0YXNldFVybCwgcmVzb3VyY2VJZCB9KVxuICAgICAgaWYgKHNjaGVtYSkge1xuICAgICAgICByZXNvdXJjZS5zY2hlbWEgPSBzY2hlbWFcbiAgICAgIH1cbiAgICB9XG4gIH1cblxuICBjb25zdCBzeXN0ZW1QYWNrYWdlID0gbm9ybWFsaXplQ2thblBhY2thZ2UoY2thblBhY2thZ2UpXG4gIGNvbnN0IHVzZXJQYWNrYWdlUGF0aCA9IHN5c3RlbVBhY2thZ2UucmVzb3VyY2VzXG4gICAgLmZpbHRlcihyZXNvdXJjZSA9PiByZXNvdXJjZVtcImNrYW46a2V5XCJdID09PSBcImRhdGFwYWNrYWdlLmpzb25cIilcbiAgICAubWFwKHJlc291cmNlID0+IHJlc291cmNlW1wiY2thbjp1cmxcIl0pXG4gICAgLmF0KDApXG5cbiAgY29uc3QgZGF0YXBhY2thZ2UgPSBhd2FpdCBtZXJnZVBhY2thZ2VzKHsgc3lzdGVtUGFja2FnZSwgdXNlclBhY2thZ2VQYXRoIH0pXG4gIGRhdGFwYWNrYWdlLnJlc291cmNlcyA9IGRhdGFwYWNrYWdlLnJlc291cmNlcy5tYXAocmVzb3VyY2UgPT4ge1xuICAgIC8vIFRPRE86IHJlbW92ZSB0aGVzZSBrZXlzIGNvbXBsZXRlbHlcbiAgICByZXR1cm4geyAuLi5yZXNvdXJjZSwgXCJja2FuOmtleVwiOiB1bmRlZmluZWQsIFwiY2thbjp1cmxcIjogdW5kZWZpbmVkIH1cbiAgfSlcblxuICByZXR1cm4gZGF0YXBhY2thZ2Vcbn1cblxuLyoqXG4gKiBFeHRyYWN0IHBhY2thZ2UgSUQgZnJvbSBVUkxcbiAqXG4gKiBFeGFtcGxlczpcbiAqIC0gaHR0cHM6Ly9ocmkuZmkvZGF0YS9lbl9HQi9kYXRhc2V0L2hlbHNpbmdpbi1rYXVwdW5naW4tdmVya2tvc2l2dXN0b2plbi1rYXZpamFhbmFseXRpaWtrYVxuICogLSBodHRwczovL3d3dy5vcGVuZGF0YS5kay9jaXR5LW9mLWNvcGVuaGFnZW4vcGFya2VyaW5nc3pvbmVyLWluZm9ybWF0aW9uXG4gKiAtIGh0dHBzOi8vb3Blbi5hZnJpY2EvZGF0YXNldC9waWItYW5udWFsLXNlbmVnYWxcbiAqIC0gaHR0cHM6Ly9kYXRhLm5obS5hYy51ay9kYXRhc2V0L2pvaW4tdGhlLWRvdHMtY29sbGVjdGlvbi1sZXZlbC1kZXNjcmlwdGlvbnNcbiAqL1xuZnVuY3Rpb24gZXh0cmFjdFBhY2thZ2VJZChkYXRhc2V0VXJsOiBzdHJpbmcpIHtcbiAgY29uc3QgdXJsID0gbmV3IFVSTChkYXRhc2V0VXJsKVxuICBjb25zdCBwYXRoUGFydHMgPSB1cmwucGF0aG5hbWUuc3BsaXQoXCIvXCIpLmZpbHRlcihCb29sZWFuKVxuICByZXR1cm4gcGF0aFBhcnRzLmF0KC0xKVxufVxuXG4vKipcbiAqIEZldGNoIHJlc291cmNlIHNjaGVtYSBkYXRhIGZyb20gQ0tBTiBkYXRhc3RvcmVcbiAqL1xuYXN5bmMgZnVuY3Rpb24gbG9hZENrYW5TY2hlbWEob3B0aW9uczoge1xuICBkYXRhc2V0VXJsOiBzdHJpbmdcbiAgcmVzb3VyY2VJZDogc3RyaW5nXG59KSB7XG4gIHRyeSB7XG4gICAgLy8gRm9yIHNvbWUgcmVhc29uLCBkYXRhc3RvcmVfaW5mbyBkb2Vzbid0IHdvcmtcbiAgICAvLyBTbyB3ZSB1c2UgZGF0YSBmZXRjaGluZyBlbmRwb2ludCB0aGF0IGFsc28gcmV0dXJucyB0aGUgc2NoZW1hXG4gICAgY29uc3QgcmVzdWx0ID0gYXdhaXQgbWFrZUNrYW5BcGlSZXF1ZXN0KHtcbiAgICAgIGNrYW5Vcmw6IG9wdGlvbnMuZGF0YXNldFVybCxcbiAgICAgIGFjdGlvbjogXCJkYXRhc3RvcmVfc2VhcmNoXCIsXG4gICAgICBwYXlsb2FkOiB7IHJlc291cmNlX2lkOiBvcHRpb25zLnJlc291cmNlSWQsIGxpbWl0OiAwIH0sXG4gICAgfSlcblxuICAgIC8vIEB0cy1pZ25vcmVcbiAgICBjb25zdCBmaWVsZHMgPSByZXN1bHQuZmllbGRzLmZpbHRlcihcbiAgICAgIChmaWVsZDogYW55KSA9PiBmaWVsZC5pZCAhPT0gXCJfaWRcIiAmJiBmaWVsZC5pZCAhPT0gXCJfZnVsbF90ZXh0XCIsXG4gICAgKVxuXG4gICAgcmV0dXJuIHsgZmllbGRzIH1cbiAgfSBjYXRjaCAoZXJyb3IpIHtcbiAgICByZXR1cm4gdW5kZWZpbmVkXG4gIH1cbn1cbiJdfQ==