sfdx-hardis
Version:
Swiss-army-knife Toolbox for Salesforce. Allows you to define a complete CD/CD Pipeline. Orchestrate base commands and assist users with interactive wizards
279 lines • 11.6 kB
JavaScript
import { SfError } from "@salesforce/core";
import { uxLog } from "./index.js";
import fs from "fs-extra";
import path from "path";
export const DATA_CLOUD_QUERIES_FOLDER_ROOT = path.join(process.cwd(), 'scripts', 'data-cloud-queries');
const QUERY_CONNECT_PATH = "ssot/query-sql";
const DEFAULT_DATASPACE = "default";
const DEFAULT_WORKLOAD_NAME = "sfdx-hardis-cli";
const DEFAULT_ROW_LIMIT = 2000;
const DEFAULT_POLL_INTERVAL_MS = 5000;
const DEFAULT_POLL_TIMEOUT_MS = 120000;
const DEFAULT_OMIT_SCHEMA = false;
const SUCCESS_STATUSES = new Set(["finished", "resultsproduced"]);
const RUNNING_STATUSES = new Set(["running", "queued"]);
const FAILURE_STATUSES = new Set(["failed", "canceled", "cancelled", "error"]);
export async function listAvailableDataCloudQueries() {
// List folders in DATA_CLOUD_QUERIES_FOLDER_ROOT
const queries = [];
if (fs.existsSync(DATA_CLOUD_QUERIES_FOLDER_ROOT)) {
const files = await fs.readdir(DATA_CLOUD_QUERIES_FOLDER_ROOT);
for (const file of files) {
const fullPath = path.join(DATA_CLOUD_QUERIES_FOLDER_ROOT, file);
const stat = await fs.stat(fullPath);
if (stat.isFile() && path.extname(file).toLowerCase() === '.sql') {
queries.push(path.basename(file, '.sql'));
}
}
}
return queries;
}
export async function loadDataCloudQueryFromFile(queryName) {
const filePath = path.join(DATA_CLOUD_QUERIES_FOLDER_ROOT, `${queryName}.sql`);
if (!fs.existsSync(filePath)) {
throw new SfError(`Data Cloud query file not found: ${filePath}`);
}
const queryContent = await fs.readFile(filePath, 'utf8');
return queryContent;
}
export async function saveDataCloudQueryToFile(queryName, queryContent) {
// Ensure the folder exists
await fs.ensureDir(DATA_CLOUD_QUERIES_FOLDER_ROOT);
const filePath = path.join(DATA_CLOUD_QUERIES_FOLDER_ROOT, `${queryName}.sql`);
await fs.writeFile(filePath, queryContent, 'utf8');
return filePath;
}
export async function dataCloudSqlQuery(query, conn, options) {
if (!query || !query.trim()) {
throw new SfError("[DataCloudSqlQuery] The Data Cloud SQL query must be a non-empty string.");
}
const settings = resolveOptions(options);
try {
const initialChunk = await submitInitialQuery(query.trim(), conn, settings);
if (!initialChunk.metadata.length) {
throw new SfError(`[DataCloudSqlQuery] Data Cloud SQL query did not return column metadata.\n${JSON.stringify(initialChunk)}`);
}
if (!initialChunk.status.queryId) {
throw new SfError(`[DataCloudSqlQuery] Data Cloud SQL query did not return a queryId needed for pagination.\n${JSON.stringify(initialChunk)}`);
}
let metadata = initialChunk.metadata;
const records = [...initialChunk.records];
const rawData = [...initialChunk.rawData];
let status = initialChunk.status;
status = await waitForQueryCompletion(conn, status, settings);
const pagination = await fetchRemainingRows(conn, status.queryId, metadata, records.length, status.rowCount, settings);
if (!metadata.length && pagination.metadata.length) {
metadata = pagination.metadata;
}
if (pagination.records.length) {
records.push(...pagination.records);
rawData.push(...pagination.rawData);
}
uxLog("log", this, `[DataCloudSqlQuery] Retrieved ${records.length} records.`);
return {
queryId: status.queryId,
metadata,
status: {
...status,
rowCount: status.rowCount ?? records.length,
},
records,
rawData,
returnedRows: records.length,
hasMoreRows: false,
};
}
catch (error) {
throw wrapQueryError(error);
}
}
async function submitInitialQuery(query, conn, options) {
const endpoint = buildQueryConnectUrl(conn, options);
uxLog("log", this, `[DataCloudSqlQuery] Submitting initial query to ${endpoint}:\n${query}`);
const response = await conn.request({
method: "POST",
url: endpoint,
headers: { "content-type": "application/json" },
body: JSON.stringify({ sql: query }),
});
return normalizeQueryResponse(response);
}
async function waitForQueryCompletion(conn, status, options) {
const deadline = Date.now() + options.pollTimeoutMs;
let currentStatus = status;
while (isRunningStatus(currentStatus.completionStatus)) {
if (Date.now() > deadline) {
throw new SfError(`Timed out after ${options.pollTimeoutMs}ms while waiting for Data Cloud query ${currentStatus.queryId} to finish.`);
}
const waitTimeMs = Math.min(options.waitTimeMs, Math.max(0, deadline - Date.now()));
currentStatus = await fetchQueryStatus(conn, currentStatus.queryId, options, waitTimeMs);
if (isFailureStatus(currentStatus.completionStatus)) {
throw new SfError(`Data Cloud SQL query ${currentStatus.queryId} failed with status ${currentStatus.completionStatus}.`);
}
if (isRunningStatus(currentStatus.completionStatus) && options.pollIntervalMs > 0) {
await delay(options.pollIntervalMs);
}
}
if (!isSuccessStatus(currentStatus.completionStatus)) {
throw new SfError(`Unexpected Data Cloud SQL query status: ${currentStatus.completionStatus || "Unknown"}.`);
}
return currentStatus;
}
async function fetchRemainingRows(conn, queryId, metadata, alreadyFetched, totalRows, options) {
const rowLimit = Math.max(1, options.rowLimit);
const records = [];
const rawData = [];
let offset = alreadyFetched;
let remaining = typeof totalRows === "number" ? Math.max(totalRows - alreadyFetched, 0) : undefined;
while (remaining === undefined || remaining > 0) {
const response = await fetchRowsPage(conn, queryId, options, offset, rowLimit);
const rows = response.data ?? [];
if (response.metadata && !metadata.length) {
metadata = response.metadata;
}
if (!rows.length) {
break;
}
rawData.push(...rows);
records.push(...rows.map((row) => mapRowToRecord(row, metadata)));
offset += rows.length;
if (remaining !== undefined) {
remaining -= rows.length;
}
if (rows.length < rowLimit) {
break;
}
}
return { records, rawData, metadata };
}
async function fetchQueryStatus(conn, queryId, options, waitTimeMs) {
const url = buildQueryConnectUrl(conn, options, [queryId], {
waitTimeMs: waitTimeMs > 0 ? waitTimeMs : undefined,
});
const response = await conn.request({
method: "GET",
url,
});
return normalizeStatusResponse(response, queryId);
}
async function fetchRowsPage(conn, queryId, options, offset, rowLimit) {
const url = buildQueryConnectUrl(conn, options, [queryId, "rows"], {
rowLimit,
offset,
omitSchema: options.omitSchema,
});
return conn.request({
method: "GET",
url,
});
}
function normalizeQueryResponse(response) {
if (typeof response !== "object" || response === null) {
throw new SfError("Invalid response format received from Data Cloud SQL query.\n" + JSON.stringify(response, null, 2));
}
const metadata = response.metadata ?? [];
const dataRows = response.data ?? [];
const records = metadata.length ? dataRows.map((row) => mapRowToRecord(row, metadata)) : [];
const status = normalizeStatusResponse(response.status ?? {}, "");
return {
queryId: status.queryId,
metadata,
status,
records,
rawData: dataRows,
returnedRows: response.returnedRows ?? records.length,
hasMoreRows: hasMoreRowsPending(status, records.length),
};
}
function normalizeStatusResponse(status, fallbackQueryId) {
return {
chunkCount: status.chunkCount ?? 0,
completionStatus: status.completionStatus ?? "Unknown",
queryId: status.queryId ?? fallbackQueryId,
rowCount: status.rowCount,
expirationTime: status.expirationTime,
progress: status.progress,
};
}
function mapRowToRecord(row, metadata) {
if (!metadata.length) {
throw new SfError("Cannot map Data Cloud rows without column metadata.");
}
return metadata.reduce((acc, column, index) => {
acc[column.name] = row[index] ?? null;
return acc;
}, {});
}
function hasMoreRowsPending(status, fetchedRows) {
if (typeof status.rowCount === "number") {
return status.rowCount > fetchedRows;
}
const state = status.completionStatus?.toLowerCase();
return state === "running" || state === "resultsproduced";
}
function buildQueryConnectUrl(conn, options, pathSegments = [], queryParams = {}) {
const base = conn.baseUrl().replace(/\/$/, "");
const baseSegments = QUERY_CONNECT_PATH.split("/");
const path = [...baseSegments, ...pathSegments]
.filter((segment) => segment && segment.length)
.map((segment) => encodeURIComponent(segment))
.join("/");
const params = {
dataspace: options.dataspace,
workloadName: options.workloadName,
...queryParams,
};
const queryString = Object.entries(params)
.filter(([, value]) => value !== undefined && value !== null && value !== "")
.map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`)
.join("&");
return queryString ? `${base}/${path}?${queryString}` : `${base}/${path}`;
}
function resolveOptions(options) {
return {
dataspace: options?.dataspace || DEFAULT_DATASPACE,
workloadName: options?.workloadName || DEFAULT_WORKLOAD_NAME,
rowLimit: options?.rowLimit || DEFAULT_ROW_LIMIT,
pollIntervalMs: options?.pollIntervalMs || DEFAULT_POLL_INTERVAL_MS,
pollTimeoutMs: options?.pollTimeoutMs || DEFAULT_POLL_TIMEOUT_MS,
waitTimeMs: options?.waitTimeMs || DEFAULT_POLL_INTERVAL_MS,
omitSchema: options?.omitSchema ?? DEFAULT_OMIT_SCHEMA,
};
}
function isRunningStatus(status) {
return RUNNING_STATUSES.has(status?.toLowerCase());
}
function isSuccessStatus(status) {
return SUCCESS_STATUSES.has(status?.toLowerCase());
}
function isFailureStatus(status) {
return FAILURE_STATUSES.has(status?.toLowerCase());
}
function delay(ms) {
if (ms <= 0) {
return Promise.resolve();
}
return new Promise((resolve) => setTimeout(resolve, ms));
}
function wrapQueryError(error) {
if (isSfRequestError(error)) {
const details = [error.message];
const bodyMessage = error.body?.message;
const bodyDescription = error.body?.error_description;
if (bodyMessage) {
details.push(bodyMessage);
}
if (bodyDescription && bodyDescription !== bodyMessage) {
details.push(bodyDescription);
}
if (error.statusCode) {
details.push(`(status ${error.statusCode})`);
}
return new SfError(`[DataCloudSqlQuery] Data Cloud SQL query failed: ${details.filter(Boolean).join(" ")}`.trim());
}
return error instanceof SfError ? error : new SfError("[DataCloudSqlQuery] Unknown error while executing Data Cloud SQL query.");
}
function isSfRequestError(error) {
return Boolean(error && typeof error === "object" && "message" in error);
}
//# sourceMappingURL=dataCloudUtils.js.map