n8n-nodes-aws-athena-query
Version:
n8n community node for executing SQL queries on AWS Athena
457 lines • 21.4 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.AwsAthenaQuery = void 0;
const n8n_workflow_1 = require("n8n-workflow");
const crypto_1 = require("crypto");
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
class AWSSignatureV4 {
constructor(accessKeyId, secretAccessKey, region, service, sessionToken) {
this.accessKeyId = accessKeyId;
this.secretAccessKey = secretAccessKey;
this.sessionToken = sessionToken;
this.region = region;
this.service = service;
}
hash(data) {
return (0, crypto_1.createHash)('sha256').update(data, 'utf8').digest('hex');
}
hmac(key, data) {
return (0, crypto_1.createHmac)('sha256', key).update(data, 'utf8').digest();
}
getSignatureKey(dateStamp) {
const kDate = this.hmac('AWS4' + this.secretAccessKey, dateStamp);
const kRegion = this.hmac(kDate, this.region);
const kService = this.hmac(kRegion, this.service);
const kSigning = this.hmac(kService, 'aws4_request');
return kSigning;
}
sign(method, url, headers, payload) {
const urlObj = new URL(url);
const pathname = urlObj.pathname;
const querystring = urlObj.search.slice(1);
const now = new Date();
const amzDate = now.toISOString().replace(/[:\-]|\.\d{3}/g, '');
const dateStamp = amzDate.slice(0, 8);
const signedHeadersNames = Object.keys(headers)
.map((key) => key.toLowerCase())
.sort()
.join(';');
const canonicalHeaders = Object.keys(headers)
.sort((a, b) => a.toLowerCase().localeCompare(b.toLowerCase()))
.map((key) => `${key.toLowerCase()}:${headers[key]}`)
.join('\n') + '\n';
const payloadHash = this.hash(payload);
const canonicalRequest = [
method,
pathname,
querystring,
canonicalHeaders,
signedHeadersNames,
payloadHash,
].join('\n');
const algorithm = 'AWS4-HMAC-SHA256';
const credentialScope = `${dateStamp}/${this.region}/${this.service}/aws4_request`;
const stringToSign = [algorithm, amzDate, credentialScope, this.hash(canonicalRequest)].join('\n');
const signingKey = this.getSignatureKey(dateStamp);
const signature = this.hmac(signingKey, stringToSign).toString('hex');
const authorizationHeader = `${algorithm} Credential=${this.accessKeyId}/${credentialScope}, SignedHeaders=${signedHeadersNames}, Signature=${signature}`;
const resultHeaders = {
...headers,
Authorization: authorizationHeader,
'X-Amz-Date': amzDate,
};
if (this.sessionToken) {
resultHeaders['X-Amz-Security-Token'] = this.sessionToken;
}
return resultHeaders;
}
}
async function makeAthenaRequest(executeFunctions, region, target, payload, credentials) {
const endpoint = `https://athena.${region}.amazonaws.com/`;
const payloadString = JSON.stringify(payload);
const headers = {
'Content-Type': 'application/x-amz-json-1.1',
'X-Amz-Target': target,
Host: `athena.${region}.amazonaws.com`,
};
const signer = new AWSSignatureV4(credentials.accessKeyId, credentials.secretAccessKey, region, 'athena', credentials.sessionToken);
const signedHeaders = signer.sign('POST', endpoint, headers, payloadString);
const options = {
method: 'POST',
url: endpoint,
headers: signedHeaders,
body: payloadString,
json: true,
};
try {
const response = await executeFunctions.helpers.httpRequest(options);
return response;
}
catch (error) {
let errorMessage = 'AWS Athena API request failed';
let statusCode = 'Unknown';
let awsErrorCode = 'Unknown';
let awsErrorMessage = 'Unknown';
let responseBody = 'No response body';
if (error.response) {
statusCode = error.response.statusCode || error.response.status || 'Unknown';
responseBody = error.response.body || error.response.data || 'No response body';
try {
let errorBodyStr = '';
if (typeof responseBody === 'string') {
errorBodyStr = responseBody;
}
else if (typeof responseBody === 'object') {
errorBodyStr = JSON.stringify(responseBody);
}
else {
errorBodyStr = String(responseBody);
}
if (errorBodyStr) {
try {
const awsError = JSON.parse(errorBodyStr);
awsErrorCode = awsError.__type || awsError.Code || awsError.code || 'Unknown';
awsErrorMessage = awsError.message || awsError.Message || awsError.msg || errorBodyStr;
}
catch (jsonParseError) {
if (errorBodyStr.includes('<')) {
const codeMatch = errorBodyStr.match(/<Code>([^<]+)<\/Code>/);
const messageMatch = errorBodyStr.match(/<Message>([^<]+)<\/Message>/);
awsErrorCode = codeMatch ? codeMatch[1] : 'XMLParseError';
awsErrorMessage = messageMatch ? messageMatch[1] : errorBodyStr;
}
else {
awsErrorMessage = errorBodyStr;
}
}
}
}
catch (parseError) {
awsErrorMessage = `Parse error: ${parseError.message}`;
}
errorMessage = `AWS Athena API Error (${statusCode}): ${awsErrorCode} - ${awsErrorMessage}`;
}
else if (error.message) {
errorMessage = `Request Error: ${error.message}`;
}
const debugInfo = {
endpoint,
target,
payload: payload,
region,
statusCode,
awsErrorCode,
awsErrorMessage,
responseBody,
requestHeaders: signedHeaders,
originalError: error.message,
errorType: error.constructor.name
};
if (error.response) {
throw new n8n_workflow_1.NodeApiError(executeFunctions.getNode(), error, { message: `${errorMessage}\nDebug Info: ${JSON.stringify(debugInfo, null, 2)}` });
}
else {
throw new n8n_workflow_1.NodeOperationError(executeFunctions.getNode(), `${errorMessage}\nDebug Info: ${JSON.stringify(debugInfo, null, 2)}`);
}
}
}
class AwsAthenaQuery {
constructor() {
this.description = {
displayName: 'AWS Athena Query',
name: 'awsAthenaQuery',
icon: 'file:AwsAthenaQuery.node.svg',
group: ['transform'],
version: 1,
description: 'Execute SQL queries on AWS Athena',
defaults: {
name: 'AWS Athena Query',
},
inputs: ["main"],
outputs: ["main"],
usableAsTool: true,
credentials: [
{
name: 'aws',
required: true,
},
],
properties: [
{
displayName: 'Region',
name: 'region',
type: 'string',
default: 'us-east-1',
placeholder: 'us-east-1',
description: 'AWS region where your Athena service is located',
required: true,
},
{
displayName: 'Database Name',
name: 'database',
type: 'string',
default: '',
placeholder: 'Optional',
description: 'Name of the database to query. Leave empty to use the default database.',
},
{
displayName: 'SQL Query',
name: 'query',
type: 'string',
default: '',
noDataExpression: false,
required: true,
typeOptions: {
editor: 'sqlEditor',
rows: 5,
},
placeholder: 'SELECT * FROM my_table LIMIT 10',
description: 'The SQL query to execute',
},
{
displayName: 'S3 Output Location',
name: 's3OutputLocation',
type: 'string',
default: '',
placeholder: 's3://my-bucket/athena-results/',
description: 'S3 bucket path where Athena will save query results',
required: true,
},
{
displayName: 'Query Timeout (Seconds)',
name: 'timeout',
type: 'number',
default: 300,
description: 'Maximum time to wait for query completion. Defaults to 300 seconds.',
required: true,
},
{
displayName: 'Output Format',
name: 'outputFormat',
type: 'options',
options: [
{
name: 'Table Format',
value: 'tableFormat',
description: 'Each database row becomes a separate workflow item (best for data processing)',
},
{
name: 'Raw Format',
value: 'rawFormat',
description: 'All results in one item with additional metadata (query ID, row count, etc.)',
},
],
default: 'tableFormat',
description: 'How to structure the query results for use in your workflow',
required: true,
},
{
displayName: 'Max Rows Returned',
name: 'maxRowsMode',
type: 'options',
options: [
{ name: 'No Limit', value: 'noLimit', description: 'Return all available rows (Warning: May be slow)' },
{ name: 'Limit Applied', value: 'limitApplied', description: 'Return up to a maximum number of rows' },
],
default: 'noLimit',
description: 'Control how many rows are returned from the query',
required: true,
},
{
displayName: 'Max Rows',
name: 'maxRows',
type: 'number',
default: 10000,
description: 'Maximum number of rows to return when limit is applied',
required: true,
displayOptions: {
show: {
maxRowsMode: ['limitApplied'],
},
},
typeOptions: {
minValue: 1,
},
},
],
};
}
async execute() {
var _a, _b, _c, _d, _e, _f, _g;
const items = this.getInputData();
const resultItems = [];
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
try {
const region = this.getNodeParameter('region', itemIndex);
const database = this.getNodeParameter('database', itemIndex);
const query = this.getNodeParameter('query', itemIndex);
const s3OutputLocation = this.getNodeParameter('s3OutputLocation', itemIndex);
const outputFormat = this.getNodeParameter('outputFormat', itemIndex, 'tableFormat');
const timeout = this.getNodeParameter('timeout', itemIndex, 300);
if (!region || !region.trim()) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Region is required.');
}
if (!query || !query.trim()) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'SQL Query is required.');
}
if (!s3OutputLocation || !s3OutputLocation.trim()) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'S3 Output Location is required.');
}
if (timeout <= 0) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Timeout must be greater than 0.');
}
const credentials = await this.getCredentials('aws');
if (!credentials.accessKeyId || !credentials.secretAccessKey) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Invalid AWS credentials. Please ensure they are configured correctly.');
}
const timestamp = Date.now().toString();
const randomPart = Math.random().toString(36).substring(2, 17) + Math.random().toString(36).substring(2, 17);
const clientRequestToken = `n8n-${timestamp}-${randomPart}`.substring(0, 64);
const queryParams = {
QueryString: query,
ClientRequestToken: clientRequestToken,
ResultConfiguration: {
OutputLocation: s3OutputLocation,
},
};
if (database && database.trim() !== '') {
queryParams.QueryExecutionContext = {
Database: database,
};
}
const maxRowsMode = this.getNodeParameter('maxRowsMode', itemIndex, 'noLimit');
const maxRowsValue = maxRowsMode === 'limitApplied'
? this.getNodeParameter('maxRows', itemIndex)
: undefined;
if (maxRowsMode === 'limitApplied') {
if (!Number.isInteger(maxRowsValue) || maxRowsValue <= 0) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Max Rows must be a positive integer.');
}
}
const startResponse = await makeAthenaRequest(this, region, 'AmazonAthena.StartQueryExecution', queryParams, credentials);
const queryExecutionId = startResponse.QueryExecutionId;
if (!queryExecutionId) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Failed to start Athena query execution.');
}
let queryStatus = 'RUNNING';
const startTime = Date.now();
const timeoutMs = timeout * 1000;
while (queryStatus === 'RUNNING' || queryStatus === 'QUEUED') {
if (Date.now() - startTime > timeoutMs) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Query timed out after ${timeout} seconds.`);
}
await sleep(2000);
const statusResponse = await makeAthenaRequest(this, region, 'AmazonAthena.GetQueryExecution', { QueryExecutionId: queryExecutionId }, credentials);
queryStatus = ((_b = (_a = statusResponse.QueryExecution) === null || _a === void 0 ? void 0 : _a.Status) === null || _b === void 0 ? void 0 : _b.State) || 'FAILED';
if (queryStatus === 'FAILED' || queryStatus === 'CANCELLED') {
const reason = ((_d = (_c = statusResponse.QueryExecution) === null || _c === void 0 ? void 0 : _c.Status) === null || _d === void 0 ? void 0 : _d.StateChangeReason) || 'Unknown error';
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Query failed or was cancelled: ${reason}`);
}
if (queryStatus !== 'RUNNING' && queryStatus !== 'QUEUED' && queryStatus !== 'SUCCEEDED') {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Query ended with unexpected status: ${queryStatus}`);
}
}
let nextToken = undefined;
let pageIndex = 0;
let columns = [];
const parsedResults = [];
do {
const payload = { QueryExecutionId: queryExecutionId, MaxResults: 1000 };
if (nextToken)
payload.NextToken = nextToken;
const resultsResponse = await makeAthenaRequest(this, region, 'AmazonAthena.GetQueryResults', payload, credentials);
const rows = ((_e = resultsResponse.ResultSet) === null || _e === void 0 ? void 0 : _e.Rows) || [];
if (rows.length === 0) {
nextToken = resultsResponse.NextToken;
pageIndex += 1;
continue;
}
let dataRows = rows;
if (pageIndex === 0) {
columns = ((_g = (_f = rows[0]) === null || _f === void 0 ? void 0 : _f.Data) === null || _g === void 0 ? void 0 : _g.map((data) => data.VarCharValue || '')) || [];
dataRows = rows.slice(1);
}
for (const row of dataRows) {
const rowData = row.Data || [];
const parsedRow = {};
columns.forEach((column, index) => {
if (column) {
const cellData = rowData[index];
let value = null;
if (cellData) {
value =
cellData.VarCharValue ||
cellData.BigIntValue ||
cellData.BooleanValue ||
cellData.DateValue ||
cellData.DoubleValue ||
cellData.FloatValue ||
cellData.IntegerValue ||
cellData.TimestampValue ||
null;
}
parsedRow[column] = value;
}
});
parsedResults.push(parsedRow);
if (maxRowsMode === 'limitApplied' && (parsedResults.length >= maxRowsValue)) {
break;
}
}
if (maxRowsMode === 'limitApplied' && (parsedResults.length >= maxRowsValue)) {
nextToken = undefined;
}
else {
nextToken = resultsResponse.NextToken;
}
pageIndex += 1;
} while (nextToken);
if (parsedResults.length === 0) {
if (outputFormat === 'rawFormat') {
resultItems.push({
json: {
queryExecutionId,
rowCount: 0,
columns: [],
results: [],
},
});
}
continue;
}
if (outputFormat === 'tableFormat') {
for (const row of parsedResults) {
resultItems.push({ json: row });
}
}
else {
resultItems.push({
json: {
queryExecutionId,
rowCount: parsedResults.length,
columns,
results: parsedResults,
},
});
}
}
catch (error) {
if (this.continueOnFail()) {
resultItems.push({
json: this.getInputData(itemIndex)[0].json,
error,
pairedItem: itemIndex,
});
}
else {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), error, {
itemIndex,
});
}
}
}
return [resultItems];
}
}
exports.AwsAthenaQuery = AwsAthenaQuery;
//# sourceMappingURL=AwsAthenaQuery.node.js.map