datajunction-ui
Version:
DataJunction UI
1,836 lines (1,720 loc) • 89.3 kB
JavaScript
// Note: MarkerType.Arrow is just the string "arrow" - we use the literal
// to avoid importing reactflow in this service (which would bloat the main bundle)
const MARKER_TYPE_ARROW = 'arrow';
const DJ_URL = process.env.REACT_APP_DJ_URL
? process.env.REACT_APP_DJ_URL
: 'http://localhost:8000';
const DJ_GQL = process.env.REACT_APP_DJ_GQL
? process.env.REACT_APP_DJ_GQL
: process.env.REACT_APP_DJ_URL + '/graphql';
// Export the base URL for components that need direct access
export const getDJUrl = () => DJ_URL;
// The namespace list is large (every namespace, with counts + git config) and
// changes rarely, but is fetched on every landing/namespace-page mount. Cache it
// for a short window so re-entering namespace pages doesn't refetch it each time.
// Mutations that add/remove namespaces should call invalidateNamespacesCache().
let _namespacesCache = null;
let _namespacesCacheAt = 0;
const NAMESPACES_CACHE_TTL_MS = 60000;
// URL builder that works with either an absolute or relative DJ_URL.
const _djURL = path => {
const base =
typeof window !== 'undefined' && window.location
? window.location.origin
: 'http://localhost';
return new URL(`${DJ_URL}${path}`, base);
};
const QUERY_END_STATES = ['FINISHED', 'CANCELED', 'FAILED'];
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
export const DataJunctionAPI = {
// Count-only node counts per type for a namespace, in a SINGLE request. Each type is
// an aliased findNodesPaginated selecting only totalCount (no edges) — so the server
// never hydrates node rows, unlike listNodesForLanding. Returns a { type: count } map.
// `types` are trusted enum names (e.g. 'metric') from a fixed constant, not user input.
nodeTypeCounts: async function (namespace, types) {
// Generic grouped-count query (one scan) instead of one count per node type.
const query = `
query NodeCounts($namespace: String) {
nodeCounts(groupBy: TYPE, namespace: $namespace) {
value
count
}
}
`;
const result = await (
await fetch(DJ_GQL, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
credentials: 'include',
body: JSON.stringify({ query, variables: { namespace } }),
})
).json();
const byType = {};
(result?.data?.nodeCounts || []).forEach(({ value, count }) => {
byType[value] = count;
});
// Preserve the requested order and default missing types to 0.
const counts = {};
types.forEach(type => {
counts[type] = byType[type.toUpperCase()] ?? 0;
});
return counts;
},
listNodesForLanding: async function (
namespace,
nodeTypes,
tags,
editedBy,
before,
after,
limit,
sortConfig,
mode,
{
ownedBy = null,
statuses = null,
missingDescription = false,
hasMaterialization = false,
orphanedDimension = false,
search = null,
} = {},
) {
const query = `
query ListNodes($namespace: String, $nodeTypes: [NodeType!], $tags: [String!], $editedBy: String, $mode: NodeMode, $before: String, $after: String, $limit: Int, $orderBy: NodeSortField, $ascending: Boolean, $ownedBy: String, $statuses: [NodeStatus!], $missingDescription: Boolean, $hasMaterialization: Boolean, $orphanedDimension: Boolean, $search: String) {
findNodesPaginated(
namespace: $namespace
nodeTypes: $nodeTypes
tags: $tags
editedBy: $editedBy
mode: $mode
limit: $limit
before: $before
after: $after
orderBy: $orderBy
ascending: $ascending
ownedBy: $ownedBy
statuses: $statuses
missingDescription: $missingDescription
hasMaterialization: $hasMaterialization
orphanedDimension: $orphanedDimension
search: $search
) {
pageInfo {
hasNextPage
endCursor
hasPrevPage
startCursor
}
totalCount
edges {
node {
name
type
currentVersion
tags {
name
tagType
}
editedBy
owners {
username
}
current {
displayName
status
mode
updatedAt
}
gitInfo {
repo
branch
defaultBranch
}
}
}
}
}
`;
const sortOrderMapping = {
name: 'NAME',
displayName: 'DISPLAY_NAME',
type: 'TYPE',
status: 'STATUS',
updatedAt: 'UPDATED_AT',
};
return await (
await fetch(DJ_GQL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
credentials: 'include',
body: JSON.stringify({
query,
variables: {
namespace: namespace,
nodeTypes: nodeTypes,
tags: tags,
editedBy: editedBy,
mode: mode || null,
before: before,
after: after,
limit: limit,
orderBy: sortOrderMapping[sortConfig.key],
ascending: sortConfig.direction === 'ascending',
ownedBy: ownedBy,
statuses: statuses,
missingDescription: missingDescription,
hasMaterialization: hasMaterialization,
orphanedDimension: orphanedDimension,
search: search,
},
}),
})
).json();
},
// Lightweight GraphQL query for listing cubes with display names (for preset dropdown)
listCubesForPreset: async function () {
const query = `
query ListCubes {
findNodes(nodeTypes: [CUBE]) {
name
current {
displayName
}
}
}
`;
try {
const result = await (
await fetch(DJ_GQL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
credentials: 'include',
body: JSON.stringify({ query }),
})
).json();
// Transform to simple array: [{name, display_name}]
const nodes = result?.data?.findNodes || [];
return nodes.map(node => ({
name: node.name,
display_name: node.current?.displayName || null,
}));
} catch (err) {
console.error('Failed to fetch cubes via GraphQL:', err);
return [];
}
},
// Lightweight GraphQL query for planner page - only fetches fields needed
// Much faster than REST /cubes/{name}/ which loads all columns, elements, etc.
cubeForPlanner: async function (name) {
const query = `
query GetCubeForPlanner($name: String!) {
findNodes(names: [$name]) {
name
current {
displayName
cubeMetrics {
name
}
cubeDimensions {
name
type
attribute
role
properties
}
availability {
catalog
schema_
table
validThroughTs
}
materializations {
name
config
schedule
strategy
}
}
}
}
`;
try {
const result = await (
await fetch(DJ_GQL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
credentials: 'include',
body: JSON.stringify({ query, variables: { name } }),
})
).json();
if (result.errors) {
console.error('GraphQL errors:', result.errors);
return null;
}
const node = result?.data?.findNodes?.[0];
if (!node) {
return null;
}
// Transform to match the shape expected by QueryPlannerPage
const current = node.current || {};
const cubeMetrics = (current.cubeMetrics || []).map(m => m.name);
// Full dimension objects (name/type/role/...) so the planner can render and
// pre-select the cube's dimensions directly, without a slow common-dimensions
// intersection over every cube metric just to validate known-good dims.
const cubeDimensionObjects = (current.cubeDimensions || []).map(d => ({
name: d.name,
type: d.type,
attribute: d.attribute,
role: d.role,
properties: d.properties || [],
}));
const cubeDimensions = cubeDimensionObjects.map(d => d.name);
// Extract druid_cube materialization if present (v3 or legacy)
const druidMat = (current.materializations || []).find(
m => m.name === 'druid_cube' || m.name === 'druid_cube_v3',
);
const cubeMaterialization = druidMat
? {
strategy: druidMat.strategy,
schedule: druidMat.schedule,
lookbackWindow: druidMat.config?.lookback_window,
druidDatasource: druidMat.config?.druid_datasource,
preaggTables: druidMat.config?.preagg_tables || [],
workflowUrls: druidMat.config?.workflow_urls || [],
timestampColumn: druidMat.config?.timestamp_column,
timestampFormat: druidMat.config?.timestamp_format,
}
: null;
return {
name: node.name,
display_name: current.displayName,
cube_node_metrics: cubeMetrics,
cube_node_dimensions: cubeDimensions,
cube_dimension_objects: cubeDimensionObjects,
cubeMaterialization, // Included so we don't need a second fetch
availability: current.availability || null,
};
} catch (err) {
console.error('Failed to fetch cube via GraphQL:', err);
return null;
}
},
whoami: async function () {
return await (
await fetch(`${DJ_URL}/whoami/`, { credentials: 'include' })
).json();
},
querySystemMetric: async function ({
metric,
dimensions = [],
filters = [],
orderby = [],
}) {
const params = new URLSearchParams();
dimensions.forEach(d => params.append('dimensions', d));
filters.forEach(f => params.append('filters', f));
orderby.forEach(o => params.append('orderby', o));
const url = `${DJ_URL}/system/data/${metric}?${params.toString()}`;
const res = await fetch(url, { credentials: 'include' });
if (!res.ok) {
throw new Error(`Failed to fetch metric data ${metric}: ${res.status}`);
}
return await res.json();
},
querySystemMetricSingleDimension: async function ({
metric,
dimension,
filters = [],
orderby = [],
}) {
const { columns, rows } = await DataJunctionAPI.querySystemMetric({
metric: metric,
dimensions: [dimension],
filters: filters,
orderby: orderby,
});
const dimIdx = columns.indexOf(dimension);
const metricIdx = columns.indexOf(metric);
return rows.map(row => ({
name:
dimIdx >= 0 && row[dimIdx] !== undefined && row[dimIdx] !== null
? String(row[dimIdx])
: 'unknown',
value: metricIdx >= 0 ? row[metricIdx] ?? 0 : 0,
}));
},
system: {
node_counts_by_active: async function () {
return DataJunctionAPI.querySystemMetricSingleDimension({
metric: 'system.dj.number_of_nodes',
dimension: 'system.dj.is_active.active_id',
});
},
node_counts_by_type: async function () {
return DataJunctionAPI.querySystemMetricSingleDimension({
metric: 'system.dj.number_of_nodes',
dimension: 'system.dj.node_type.type',
filters: ['system.dj.is_active.active_id=true'],
orderby: ['system.dj.node_type.type'],
});
},
node_counts_by_status: async function () {
return DataJunctionAPI.querySystemMetricSingleDimension({
metric: 'system.dj.number_of_nodes',
dimension: 'system.dj.nodes.status',
filters: ['system.dj.is_active.active_id=true'],
orderby: ['system.dj.nodes.status'],
});
},
nodes_without_description: async function () {
return DataJunctionAPI.querySystemMetricSingleDimension({
metric: 'system.dj.node_without_description',
dimension: 'system.dj.node_type.type',
filters: ['system.dj.is_active.active_id=true'],
orderby: ['system.dj.node_type.type'],
});
},
node_trends: async function () {
const { columns, rows } = await (
await fetch(
`${DJ_URL}/system/data/system.dj.number_of_nodes?dimensions=system.dj.nodes.created_at_week&dimensions=system.dj.node_type.type&filters=system.dj.nodes.created_at_week>=20240101&orderby=system.dj.nodes.created_at_week`,
{ credentials: 'include' },
)
).json();
const dateIdx = columns.indexOf('system.dj.nodes.created_at_week');
const typeIdx = columns.indexOf('system.dj.node_type.type');
const countIdx = columns.indexOf('system.dj.number_of_nodes');
const byDateint = {};
rows.forEach(row => {
const dateint = row[dateIdx];
const nodeType = row[typeIdx];
const count = row[countIdx];
if (!byDateint[dateint]) byDateint[dateint] = { date: dateint };
byDateint[dateint][nodeType] =
(byDateint[dateint][nodeType] || 0) + count;
});
return Object.entries(byDateint).map(([dateint, data]) => ({
date: dateint,
...data,
}));
},
materialization_counts_by_type: async function () {
return DataJunctionAPI.querySystemMetricSingleDimension({
metric: 'system.dj.number_of_materializations',
dimension: 'system.dj.node_type.type',
filters: ['system.dj.is_active.active_id=true'],
orderby: ['system.dj.node_type.type'],
});
},
dimensions: async function () {
return await (
await fetch(`${DJ_URL}/system/dimensions`, {
credentials: 'include',
})
).json();
},
list: async function () {
return await (
await fetch(`${DJ_URL}/system/metrics`, {
credentials: 'include',
})
).json();
},
},
logout: async function () {
return await fetch(`${DJ_URL}/logout/`, {
credentials: 'include',
method: 'POST',
});
},
catalogs: async function () {
return await (
await fetch(`${DJ_URL}/catalogs`, {
credentials: 'include',
})
).json();
},
engines: async function () {
return await (
await fetch(`${DJ_URL}/engines`, {
credentials: 'include',
})
).json();
},
node: async function (name) {
const data = await (
await fetch(`${DJ_URL}/nodes/${name}/`, {
credentials: 'include',
})
).json();
if (data.message !== undefined) {
return data;
}
data.primary_key = data.columns
.filter(col =>
col.attributes.some(attr => attr.attribute_type.name === 'primary_key'),
)
.map(col => col.name);
return data;
},
getNodeForEditing: async function (name) {
const query = `
query GetNodeForEditing($name: String!) {
findNodes (names: [$name]) {
name
type
current {
displayName
description
primaryKey
query
parents { name type }
isDerivedMetric
metricMetadata {
direction
unit { name }
expression
significantDigits
incompatibleDruidFunctions
}
requiredDimensions {
name
}
mode
customMetadata
}
tags {
name
displayName
}
owners {
username
}
}
}
`;
const results = await (
await fetch(DJ_GQL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
credentials: 'include',
body: JSON.stringify({
query,
variables: {
name: name,
},
}),
})
).json();
if (results.data.findNodes.length === 0) {
return null;
}
return results.data.findNodes[0];
},
// Fetch basic node info for multiple nodes by name (for Settings page)
getNodesByNames: async function (names) {
if (!names || names.length === 0) {
return [];
}
const query = `
query GetNodesByNames($names: [String!]) {
findNodes(names: $names) {
name
type
current {
displayName
status
mode
}
}
}
`;
const results = await (
await fetch(DJ_GQL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
credentials: 'include',
body: JSON.stringify({
query,
variables: { names },
}),
})
).json();
return results.data?.findNodes || [];
},
getMetric: async function (name) {
const query = `
query GetMetric($name: String!) {
findNodes (names: [$name]) {
name
current {
parents { name type }
isDerivedMetric
metricMetadata {
direction
unit { name }
expression
significantDigits
incompatibleDruidFunctions
}
requiredDimensions {
name
}
}
}
}
`;
const results = await (
await fetch(DJ_GQL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
credentials: 'include',
body: JSON.stringify({
query,
variables: {
name: name,
},
}),
})
).json();
return results.data.findNodes[0];
},
getCubeForEditing: async function (name) {
const query = `
query GetCubeForEditing($name: String!) {
findNodes(names: [$name]) {
name
type
owners {
username
}
current {
displayName
description
mode
cubeMetrics {
name
displayName
}
cubeDimensions {
name
attribute
role
properties
}
}
tags {
name
displayName
}
}
}
`;
const results = await (
await fetch(DJ_GQL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
credentials: 'include',
body: JSON.stringify({
query,
variables: {
name: name,
},
}),
})
).json();
if (results.data.findNodes.length === 0) {
return null;
}
return results.data.findNodes[0];
},
nodes: async function (prefix) {
const queryParams = prefix ? `?prefix=${prefix}` : '';
return await (
await fetch(`${DJ_URL}/nodes/${queryParams}`, {
credentials: 'include',
})
).json();
},
nodesWithType: async function (nodeType) {
return await (
await fetch(`${DJ_URL}/nodes/?node_type=${nodeType}`, {
credentials: 'include',
})
).json();
},
nodeDetails: async () => {
return await (
await fetch(`${DJ_URL}/nodes/details/`, {
credentials: 'include',
})
).json();
},
validateNode: async function (
nodeType,
name,
display_name,
description,
query,
) {
const response = await fetch(`${DJ_URL}/nodes/validate`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
name: name,
display_name: display_name,
description: description,
query: query,
type: nodeType,
mode: 'published',
}),
credentials: 'include',
});
return { status: response.status, json: await response.json() };
},
createNode: async function (
nodeType,
name,
display_name,
description,
query,
mode,
namespace,
primary_key,
metric_direction,
metric_unit,
required_dimensions,
custom_metadata,
) {
const metricMetadata =
metric_direction || metric_unit
? {
direction: metric_direction,
unit: metric_unit,
}
: null;
// Build request body, filtering out undefined values
const requestBody = {
name: name,
display_name: display_name,
description: description,
query: query,
mode: mode,
namespace: namespace,
primary_key: primary_key,
metric_metadata: metricMetadata,
required_dimensions: required_dimensions,
custom_metadata: custom_metadata,
};
// Remove undefined fields to avoid sending them to the API
Object.keys(requestBody).forEach(
key => requestBody[key] === undefined && delete requestBody[key],
);
const response = await fetch(`${DJ_URL}/nodes/${nodeType}`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(requestBody),
credentials: 'include',
});
return { status: response.status, json: await response.json() };
},
patchNode: async function (
name,
display_name,
description,
query,
mode,
primary_key,
metric_direction,
metric_unit,
significant_digits,
required_dimensions,
owners,
custom_metadata,
) {
try {
const metricMetadata =
metric_direction || metric_unit
? {
direction: metric_direction,
unit: metric_unit,
significant_digits: significant_digits || null,
}
: null;
// Build request body, filtering out undefined values
const requestBody = {
display_name: display_name,
description: description,
query: query,
mode: mode,
primary_key: primary_key,
metric_metadata: metricMetadata,
required_dimensions: required_dimensions,
owners: owners,
custom_metadata: custom_metadata,
};
// Remove undefined fields to avoid sending them to the API
Object.keys(requestBody).forEach(
key => requestBody[key] === undefined && delete requestBody[key],
);
const response = await fetch(`${DJ_URL}/nodes/${name}`, {
method: 'PATCH',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(requestBody),
credentials: 'include',
});
return { status: response.status, json: await response.json() };
} catch (error) {
return { status: 500, json: { message: 'Update failed' } };
}
},
createCube: async function (
name,
display_name,
description,
mode,
metrics,
dimensions,
filters,
) {
const response = await fetch(`${DJ_URL}/nodes/cube`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
name: name,
display_name: display_name,
description: description,
metrics: metrics,
dimensions: dimensions,
filters: filters,
mode: mode,
}),
credentials: 'include',
});
return { status: response.status, json: await response.json() };
},
patchCube: async function (
name,
display_name,
description,
mode,
metrics,
dimensions,
filters,
owners,
) {
const url = `${DJ_URL}/nodes/${name}`;
const response = await fetch(url, {
method: 'PATCH',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
display_name: display_name,
description: description,
metrics: metrics,
dimensions: dimensions,
filters: filters || [],
mode: mode,
owners: owners,
}),
credentials: 'include',
});
return { status: response.status, json: await response.json() };
},
refreshLatestMaterialization: async function (name) {
const url = `${DJ_URL}/nodes/${name}?refresh_materialization=true`;
const response = await fetch(url, {
method: 'PATCH',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({}),
credentials: 'include',
});
return { status: response.status, json: await response.json() };
},
registerTable: async function (
catalog,
schema,
table,
sourceNodeNamespace = undefined,
) {
const url = _djURL(`/register/table/${catalog}/${schema}/${table}`);
if (sourceNodeNamespace !== undefined) {
url.searchParams.set('source_node_namespace', sourceNodeNamespace);
}
const response = await fetch(url.toString(), {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
credentials: 'include',
});
return { status: response.status, json: await response.json() };
},
upstreams: async function (name) {
return await (
await fetch(`${DJ_URL}/nodes/${name}/upstream/`, {
credentials: 'include',
})
).json();
},
downstreams: async function (name) {
return await (
await fetch(`${DJ_URL}/nodes/${name}/downstream/`, {
credentials: 'include',
})
).json();
},
// GraphQL-based upstream/downstream queries - more efficient as they only fetch needed fields
upstreamsGQL: async function (nodeNames) {
const names = Array.isArray(nodeNames) ? nodeNames : [nodeNames];
const query = `
query GetUpstreamNodes($nodeNames: [String!]!) {
upstreamNodes(nodeNames: $nodeNames) {
name
type
current {
parents {
name
}
}
}
}
`;
const results = await (
await fetch(DJ_GQL, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
credentials: 'include',
body: JSON.stringify({ query, variables: { nodeNames: names } }),
})
).json();
return results.data?.upstreamNodes || [];
},
downstreamsGQL: async function (nodeNames) {
const names = Array.isArray(nodeNames) ? nodeNames : [nodeNames];
const query = `
query GetDownstreamNodes($nodeNames: [String!]!) {
downstreamNodes(nodeNames: $nodeNames) {
name
type
current {
parents {
name
}
}
}
}
`;
const results = await (
await fetch(DJ_GQL, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
credentials: 'include',
body: JSON.stringify({ query, variables: { nodeNames: names } }),
})
).json();
return results.data?.downstreamNodes || [];
},
// Batch-fetch cubes by name, returning each cube's name and its metric node names.
// Used to build the Sankey data flow graph without N individual cube fetches.
findCubesWithMetrics: async function (cubeNames) {
if (!cubeNames || cubeNames.length === 0) return [];
const query = `
query FindCubesWithMetrics($names: [String!]) {
findNodes(names: $names, nodeTypes: [CUBE]) {
name
current {
displayName
cubeMetrics {
name
}
}
}
}
`;
const results = await (
await fetch(DJ_GQL, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
credentials: 'include',
body: JSON.stringify({ query, variables: { names: cubeNames } }),
})
).json();
return (results.data?.findNodes || []).map(n => ({
name: n.name,
display_name: n.current?.displayName || n.name,
type: 'cube',
parents: (n.current?.cubeMetrics || []).map(m => ({ name: m.name })),
}));
},
node_dag: async function (name) {
return await (
await fetch(`${DJ_URL}/nodes/${name}/dag/`, {
credentials: 'include',
})
).json();
},
// Fetch node columns with partition info via GraphQL
// Used to check if a node has temporal partitions defined
getNodeColumnsWithPartitions: async function (nodeName) {
const query = `
query GetNodeColumnsWithPartitions($name: String!) {
findNodes(names: [$name]) {
name
current {
columns {
name
type
partition {
type_
format
granularity
}
}
}
}
}
`;
try {
const results = await (
await fetch(DJ_GQL, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
credentials: 'include',
body: JSON.stringify({ query, variables: { name: nodeName } }),
})
).json();
const node = results.data?.findNodes?.[0];
if (!node) return { columns: [], temporalPartitions: [] };
const columns = node.current?.columns || [];
const temporalPartitions = columns.filter(
col => col.partition?.type_ === 'TEMPORAL',
);
return {
columns,
temporalPartitions,
};
} catch (err) {
console.error('Failed to fetch node columns with partitions:', err);
return { columns: [], temporalPartitions: [] };
}
},
node_lineage: async function (name) {
return await (
await fetch(`${DJ_URL}/nodes/${name}/lineage/`, {
credentials: 'include',
})
).json();
},
metric: async function (name) {
return await (
await fetch(`${DJ_URL}/metrics/${name}/`, {
credentials: 'include',
})
).json();
},
clientCode: async function (name) {
return await (
await fetch(`${DJ_URL}/datajunction-clients/python/new_node/${name}`, {
credentials: 'include',
})
).json();
},
cube: async function (name) {
return await (
await fetch(`${DJ_URL}/cubes/${name}/`, {
credentials: 'include',
})
).json();
},
metrics: async function (name) {
return await (
await fetch(`${DJ_URL}/metrics/`, {
credentials: 'include',
})
).json();
},
getMetricsInfo: async function (names) {
if (!names || names.length === 0) return [];
const gqlQuery = `
query GetMetricsInfo($names: [String!]!) {
findNodes(names: $names) {
name
current {
displayName
}
gitInfo {
branch
isDefaultBranch
}
}
}
`;
const response = await fetch(DJ_GQL, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
credentials: 'include',
body: JSON.stringify({
query: gqlQuery,
variables: { names },
}),
});
const result = await response.json();
return (result?.data?.findNodes || []).map(node => ({
value: node.name,
label: node.current?.displayName || node.name,
name: node.name,
gitInfo: node.gitInfo,
}));
},
searchMetrics: async function (query, limit = 50) {
const gqlQuery = `
query SearchMetrics($q: String!, $limit: Int!) {
findNodes(search: $q, nodeTypes: [METRIC], limit: $limit) {
name
current {
displayName
}
gitInfo {
branch
isDefaultBranch
}
}
}
`;
const response = await fetch(DJ_GQL, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
credentials: 'include',
body: JSON.stringify({
query: gqlQuery,
variables: { q: query, limit },
}),
});
const result = await response.json();
return (result?.data?.findNodes || []).map(node => ({
value: node.name,
label: node.current?.displayName || node.name,
name: node.name,
gitInfo: node.gitInfo,
}));
},
commonDimensions: async function (metrics) {
const metricsQuery = '?' + metrics.map(m => `metric=${m}`).join('&');
return await (
await fetch(`${DJ_URL}/metrics/common/dimensions/${metricsQuery}`, {
credentials: 'include',
})
).json();
},
commonMetrics: async function (dimensions) {
// Dimension names are "node.attribute[role]" — strip role path and attribute to get node name
const stripped = [
...new Set(
dimensions
.map(d =>
d
.replace(/\[[^\]]*\]$/, '')
.split('.')
.slice(0, -1)
.join('.'),
)
.filter(Boolean),
),
];
if (stripped.length === 0) return [];
const dimsQuery =
'?' +
stripped.map(d => `dimension=${encodeURIComponent(d)}`).join('&') +
'&node_type=metric';
return await (
await fetch(`${DJ_URL}/dimensions/common/${dimsQuery}`, {
credentials: 'include',
})
).json();
},
history: async function (type, name, offset, limit) {
return await (
await fetch(
`${DJ_URL}/history?node=${name}&offset=${offset ? offset : 0}&limit=${
limit ? limit : 100
}`,
{
credentials: 'include',
},
)
).json();
},
revisions: async function (name) {
return await (
await fetch(`${DJ_URL}/nodes/${name}/revisions/`, {
credentials: 'include',
})
).json();
},
namespace: async function (nmspce, editedBy) {
return await (
await fetch(
`${DJ_URL}/namespaces/${nmspce}?edited_by=${editedBy}&with_edited_by=true`,
{
credentials: 'include',
},
)
).json();
},
namespaces: async function () {
return await (
await fetch(`${DJ_URL}/namespaces/`, {
credentials: 'include',
})
).json();
},
invalidateNamespacesCache: function () {
_namespacesCache = null;
_namespacesCacheAt = 0;
},
listNamespacesWithGit: async function ({ force = false } = {}) {
const now = Date.now();
if (
!force &&
_namespacesCache &&
now - _namespacesCacheAt < NAMESPACES_CACHE_TTL_MS
) {
return _namespacesCache;
}
const query = `
query ListNamespaces {
listNamespaces {
namespace
numNodes
git {
__typename
... on GitRootConfig {
repo
path
defaultBranch
}
... on GitBranchConfig {
branch
gitOnly
parentNamespace
root {
repo
path
defaultBranch
}
}
}
}
}
`;
const result = await (
await fetch(DJ_GQL, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
credentials: 'include',
body: JSON.stringify({ query }),
})
).json();
_namespacesCache = result?.data?.listNamespaces || [];
_namespacesCacheAt = now;
return _namespacesCache;
},
namespaceSources: async function (namespace) {
return await (
await fetch(`${DJ_URL}/namespaces/${namespace}/sources`, {
credentials: 'include',
})
).json();
},
namespaceSourcesBulk: async function (namespaces) {
return await (
await fetch(`${DJ_URL}/namespaces/sources/bulk`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ namespaces }),
credentials: 'include',
})
).json();
},
listDeployments: async function (namespace, limit = 5) {
const params = new URLSearchParams();
if (namespace) {
params.append('namespace', namespace);
}
params.append('limit', limit);
return await (
await fetch(`${DJ_URL}/deployments?${params.toString()}`, {
credentials: 'include',
})
).json();
},
sql: async function (metric_name, selection) {
const params = new URLSearchParams(selection);
for (const [key, value] of Object.entries(selection)) {
if (Array.isArray(value)) {
params.delete(key);
value.forEach(v => params.append(key, v));
}
}
return await (
await fetch(`${DJ_URL}/sql/${metric_name}?${params}`, {
credentials: 'include',
})
).json();
},
nodesWithDimension: async function (name, nodeType = null) {
const params = nodeType ? `?node_type=${nodeType}` : '';
return await (
await fetch(`${DJ_URL}/dimensions/${name}/nodes/${params}`, {
credentials: 'include',
})
).json();
},
materializations: async function (node) {
const data = await (
await fetch(
`${DJ_URL}/nodes/${node}/materializations?show_inactive=true&include_all_revisions=true`,
{
credentials: 'include',
},
)
).json();
return data;
},
availabilityStates: async function (node) {
const data = await (
await fetch(`${DJ_URL}/nodes/${node}/availability/`, {
credentials: 'include',
})
).json();
return data;
},
columns: async function (node) {
return await Promise.all(
node.columns.map(async col => {
return col;
}),
);
},
sqls: async function (metricSelection, dimensionSelection, filters) {
const params = new URLSearchParams();
metricSelection.map(metric => params.append('metrics', metric));
dimensionSelection.map(dimension => params.append('dimensions', dimension));
params.append('filters', filters);
return await (
await fetch(`${DJ_URL}/sql/?${params}`, {
credentials: 'include',
})
).json();
},
// V3 Measures SQL - returns pre-aggregations with components for materialization planning
measuresV3: async function (
metricSelection,
dimensionSelection,
filters = '',
) {
const params = new URLSearchParams();
metricSelection.forEach(metric => params.append('metrics', metric));
dimensionSelection.forEach(dimension =>
params.append('dimensions', dimension),
);
if (filters) {
params.append('filters', filters);
}
return await (
await fetch(`${DJ_URL}/sql/measures/v3/?${params}`, {
credentials: 'include',
})
).json();
},
// V3 Metrics SQL - returns final combined SQL query
metricsV3: async function (
metricSelection,
dimensionSelection,
filters = [],
useMaterialized = true,
dialect = null,
) {
const params = new URLSearchParams();
metricSelection.forEach(metric => params.append('metrics', metric));
dimensionSelection.forEach(dimension =>
params.append('dimensions', dimension),
);
if (filters && filters.length > 0) {
filters.forEach(f => params.append('filters', f));
}
params.append('use_materialized', useMaterialized ? 'true' : 'false');
if (dialect) {
params.append('dialect', dialect);
}
return await (
await fetch(`${DJ_URL}/sql/metrics/v3/?${params}`, {
credentials: 'include',
})
).json();
},
data: async function (
metricSelection,
dimensionSelection,
filters = [],
dialect = null,
onProgress = null,
) {
const params = new URLSearchParams();
metricSelection.map(metric => params.append('metrics', metric));
dimensionSelection.map(dimension => params.append('dimensions', dimension));
if (filters && filters.length > 0) {
filters.forEach(f => params.append('filters', f));
}
params.append('limit', '10000');
params.append('async_', 'true');
if (dialect) {
params.append('dialect', dialect);
}
let pollInterval = 1000;
// Submit the query once
const submitResponse = await fetch(`${DJ_URL}/data/?${params}`, {
credentials: 'include',
});
if (!submitResponse.ok) {
const errorData = await submitResponse.json().catch(() => ({}));
throw new Error(
errorData.message ||
errorData.detail ||
`Query failed: ${submitResponse.status}`,
);
}
let results = await submitResponse.json();
// Report links from the first response so they're visible during polling
if (onProgress && results.links?.length > 0) {
onProgress({ links: results.links });
}
// Poll by query ID using GET /data/query/{id} to avoid re-submitting
while (!QUERY_END_STATES.includes(results.state)) {
await sleep(pollInterval);
pollInterval = Math.min(pollInterval * 2, 10000);
const pollResponse = await fetch(`${DJ_URL}/data/query/${results.id}`, {
credentials: 'include',
});
if (!pollResponse.ok) {
const errorData = await pollResponse.json().catch(() => ({}));
throw new Error(
errorData.message ||
errorData.detail ||
`Query poll failed: ${pollResponse.status}`,
);
}
results = await pollResponse.json();
}
if (results.state === 'CANCELED') {
throw new Error('Query execution was canceled');
}
if (results.state === 'FAILED') {
throw new Error(results.errors?.[0] || 'Query execution failed');
}
// If FINISHED but results are empty, re-submit to /data/ a few times
// before giving up (mirrors the Python client's behavior).
if (!results.results?.length || !results.results[0]?.rows?.length) {
for (let attempt = 0; attempt < 3; attempt++) {
await sleep(2 ** attempt * 1000);
const retryResponse = await fetch(`${DJ_URL}/data/?${params}`, {
credentials: 'include',
});
if (retryResponse.ok) {
const retryResults = await retryResponse.json();
if (
retryResults.results?.length &&
retryResults.results[0]?.rows?.length
) {
results = retryResults;
break;
}
}
}
}
return results;
},
nodeData: async function (nodeName, selection = null) {
if (selection === null) {
selection = {
dimensions: [],
filters: [],
};
}
const params = new URLSearchParams(selection);
for (const [key, value] of Object.entries(selection)) {
if (Array.isArray(value)) {
params.delete(key);
value.forEach(v => params.append(key, v));
}
}
params.append('limit', '1000');
params.append('async_', 'true');
return await (
await fetch(`${DJ_URL}/data/${nodeName}?${params}`, {
credentials: 'include',
headers: { 'Cache-Control': 'max-age=86400' },
})
).json();
},
notebookExportCube: async function (cube) {
return await fetch(
`${DJ_URL}/datajunction-clients/python/notebook/?cube=${cube}`,
{
credentials: 'include',
},
);
},
notebookExportNamespace: async function (namespace) {
return await (
await fetch(
`${DJ_URL}/datajunction-clients/python/notebook/?namespace=${namespace}`,
{
credentials: 'include',
},
)
).json();
},
stream: async function (metricSelection, dimensionSelection, filters) {
const params = new URLSearchParams();
metricSelection.map(metric => params.append('metrics', metric));
dimensionSelection.map(dimension => params.append('dimensions', dimension));
params.append('filters', filters);
return new EventSource(
`${DJ_URL}/stream/?${params}&limit=10000&async_=true`,
{
withCredentials: true,
},
);
},
streamNodeData: async function (nodeName, selection = null) {
if (selection === null) {
selection = {
dimensions: [],
filters: [],
};
}
const params = new URLSearchParams(selection);
for (const [key, value] of Object.entries(selection)) {
if (Array.isArray(value)) {
params.delete(key);
value.forEach(v => params.append(key, v));
}
}
params.append('limit', '1000');
params.append('async_', 'true');
return new EventSource(`${DJ_URL}/stream/${nodeName}?${params}`, {
withCredentials: true,
});
},
lineage: async function (node) {},
compiledSql: async function (node) {
return await (
await fetch(`${DJ_URL}/sql/${node}/`, {
credentials: 'include',
})
).json();
},
dag: async function (namespace = 'default') {
const edges = [];
const data = await (
await fetch(`${DJ_URL}/nodes/`, {
credentials: 'include',
})
).json();
data.forEach(obj => {
obj.parents.forEach(parent => {
if (parent.name) {
edges.push({
id: obj.name + '-' + parent.name,
target: obj.name,
source: parent.name,
animated: true,
markerEnd: {
type: MARKER_TYPE_ARROW,
},
});
}
});
obj.columns.forEach(col => {
if (col.dimension) {
edges.push({
id: obj.name + '-' + col.dimension.name,
target: obj.name,
source: col.dimension.name,
draggable: true,
});
}
});
});
const namespaces = new Set(
data.flatMap(node => node.name.split('.').slice(0, -1)),
);
const namespaceNodes = Array.from(namespaces).map(namespace => {
return {
id: String(namespace),
type: 'DJNamespace',
data: {
label: String(namespace),
},
};
});
const nodes = data.map((node, index) => {
const primary_key = node.columns
.filter(col =>
col.attributes.some(
attr => attr.attribute_type.name === 'primary_key',
),
)
.map(col => col.name);
const column_names = node.columns.map(col => {
return { name: col.name, type: col.type };
});
return {
id: String(node.name),
type: 'DJNode',
data: {
label:
node.table !== null
? String(node.schema_ + '.' + node.table)
: String(node.name),
table: node.table,
name: String(node.name),
display_name: String(node.display_name),
type: node.type,
primary_key: primary_key,
column_names: column_names,
},
};
});
return { edges: edges, nodes: nodes, namespaces: namespaceNodes };
},
attributes: async function () {
return await (
await fetch(`${DJ_URL}/attributes`, {
credentials: 'include',
})
).json();
},
setAttributes: async function (nodeName, columnName, attributes) {
const response = await fetch(
`${DJ_URL}/nodes/${nodeName}/columns/${columnName}/attributes`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(
attributes.map(attribute => {
return {
namespace: 'system',
name: attribute,
};
}),
),
credentials: 'include',
},
);
return { status: response.status, json: await response.json() };
},
setColumnDescription: async function (nodeName, columnName, description) {
const response = await fetch(
`${DJ_URL}/nodes/${nodeName}/columns/${columnName}/description?description=${encodeURIComponent(
description,
)}`,
{
method: 'PATCH',
credentials: 'include',
},
);
return { status: response.status, json: await response.json() };
},
dimensions: async function () {
return await (
await fetch(`${DJ_URL}/dimensions`, {
credentials: 'include',
})
).json();
},
nodeDimensions: async function (nodeName) {
return await (
await fetch(`${DJ_URL}/nodes/${nodeName}/dimensions`, {
credentials: 'include',
})
).json();
},
dimensionDag: async function (nodeName) {
return await (
await fetch(`${DJ_URL}/nodes/${nodeName}/dimension-dag/`, {
credentials: 'include',
})
).json();
},
linkDimension: async function (nodeName, columnName, dimensionName) {
const response = await fetch(
`${DJ_URL}/nodes/${nodeName}/columns/${columnName}?dimension=${dimensionName}`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
credentials: 'include',
},
);
return { status: response.status, json: await response.json() };
},
unlinkDimension: async function (nodeName, columnName, dimensionName) {
const response = await fetch(
`${DJ_URL}/nodes/${nodeName}/columns/${columnName}?dimension=${dimensionName}`,
{
method: 'DELETE',
headers: {
'Content-Type': 'application/json',
},
credentials: 'include',
},
);
return { status: response.status, json: await response.json() };
},
addComplexDimensionLink: async function (
nodeName,
dimensionNode,
joinOn,
joinType = null,
joinCardinality = null,
role = null,
defaultValue = null,
) {
const response = await fetch(`${DJ_URL}/nodes/${nodeName}/link`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},