@jsforce/jsforce-node
Version:
Salesforce API Library for JavaScript
809 lines (808 loc) • 27.1 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.IngestJobV2 = exports.QueryJobV2 = exports.BulkV2 = void 0;
const events_1 = require("events");
const stream_1 = require("stream");
const record_stream_1 = require("../record-stream");
const http_api_1 = __importDefault(require("../http-api"));
const jsforce_1 = require("../jsforce");
const logger_1 = require("../util/logger");
const stream_2 = require("../util/stream");
const is_1 = __importDefault(require("@sindresorhus/is"));
class JobPollingTimeoutError extends Error {
jobId;
/**
*
*/
constructor(message, jobId) {
super(message);
this.name = 'JobPollingTimeout';
this.jobId = jobId;
}
}
class BulkApiV2 extends http_api_1.default {
hasErrorInResponseBody(body) {
return (Array.isArray(body) &&
typeof body[0] === 'object' &&
'errorCode' in body[0]);
}
isSessionExpired(response) {
return (response.statusCode === 401 &&
response.body.includes('INVALID_SESSION_ID'));
}
parseError(body) {
return {
errorCode: body[0].errorCode,
message: body[0].message,
};
}
}
class BulkV2 {
connection;
logger;
/**
* Polling interval in milliseconds
*
* Default: 1000 (1 second)
*/
pollInterval = 1000;
/**
* Polling timeout in milliseconds
*
* Default: 30000 (30 seconds)
*/
pollTimeout = 30000;
constructor(connection) {
this.connection = connection;
this.logger = this.connection._logLevel
? (0, logger_1.getLogger)('bulk2').createInstance(this.connection._logLevel)
: (0, logger_1.getLogger)('bulk2');
}
/**
* Create an instance of an ingest job object.
*
* @params {NewIngestJobOptions} options object
* @returns {IngestJobV2} An ingest job instance
* @example
* // Upsert records to the Account object.
*
* const job = connection.bulk2.createJob({
* operation: 'insert'
* object: 'Account',
* });
*
* // create the job in the org
* await job.open()
*
* // upload data
* await job.uploadData(csvFile)
*
* // finished uploading data, mark it as ready for processing
* await job.close()
*/
createJob(options) {
return new IngestJobV2(this.connection, {
bodyParams: options,
pollingOptions: this,
});
}
job(type = 'ingest', options) {
if (type === 'ingest') {
return new IngestJobV2(this.connection, {
id: options.id,
pollingOptions: this,
});
}
else {
return new QueryJobV2(this.connection, {
id: options.id,
pollingOptions: this,
});
}
}
/**
* Create, upload, and start bulkload job
*/
async loadAndWaitForResults(options) {
if (!options.pollTimeout)
options.pollTimeout = this.pollTimeout;
if (!options.pollInterval)
options.pollInterval = this.pollInterval;
const { pollInterval, pollTimeout, input, ...createJobOpts } = options;
const job = this.createJob(createJobOpts);
try {
await job.open();
await job.uploadData(input);
await job.close();
await job.poll(pollInterval, pollTimeout);
return await job.getAllResults();
}
catch (error) {
const err = error;
this.logger.error(`bulk load failed due to: ${err.message}`);
if (err.name !== 'JobPollingTimeoutError') {
// fires off one last attempt to clean up and ignores the result | error
job.delete().catch((ignored) => ignored);
}
throw err;
}
}
/**
* Execute bulk query and get a record stream.
*
* Default timeout: 10000ms
*
* @param soql SOQL query
* @param options
*
* @returns {RecordStream} - Record stream, convertible to a CSV data stream
*/
async query(soql, options) {
const queryJob = new QueryJobV2(this.connection, {
bodyParams: {
query: soql,
operation: options?.scanAll ? 'queryAll' : 'query',
columnDelimiter: options?.columnDelimiter,
lineEnding: options?.lineEnding,
},
pollingOptions: this,
});
const recordStream = new record_stream_1.Parsable();
const dataStream = recordStream.stream('csv');
try {
await queryJob.open();
await queryJob.poll(options?.pollInterval, options?.pollTimeout);
const queryRecordsStream = await queryJob
.result()
.then((s) => s.stream());
queryRecordsStream.pipe(dataStream);
}
catch (error) {
const err = error;
this.logger.error(`bulk query failed due to: ${err.message}`);
if (err.name !== 'JobPollingTimeoutError') {
// fires off one last attempt to clean up and ignores the result | error
queryJob.delete().catch((ignored) => ignored);
}
throw err;
}
return recordStream;
}
}
exports.BulkV2 = BulkV2;
class QueryJobV2 extends events_1.EventEmitter {
connection;
logger;
_id;
bodyParams;
pollingOptions;
error;
jobInfo;
locator;
constructor(conn, options) {
super();
this.connection = conn;
this.logger = this.connection._logLevel
? (0, logger_1.getLogger)('bulk2:QueryJobV2').createInstance(this.connection._logLevel)
: (0, logger_1.getLogger)('bulk2:QueryJobV2');
if ('id' in options) {
this._id = options.id;
}
else {
this.bodyParams = options.bodyParams;
}
this.pollingOptions = options.pollingOptions;
// default error handler to keep the latest error
this.on('error', (error) => (this.error = error));
}
/**
* Get the query job ID.
*
* @returns {string} query job Id.
*/
get id() {
return this.jobInfo ? this.jobInfo.id : this._id;
}
/**
* Get the query job info.
*
* @returns {Promise<QueryJobInfoV2>} query job information.
*/
getInfo() {
if (this.jobInfo) {
return this.jobInfo;
}
throw new Error('No internal job info. Make sure to call `await job.check`.');
}
/**
* Creates a query job
*
* @returns {Promise<QueryJobInfoV2>} job information.
*/
async open() {
if (!this.bodyParams) {
throw new Error('Missing required body params to open a new query job.');
}
try {
this.jobInfo = await this.createQueryRequest({
method: 'POST',
body: JSON.stringify(this.bodyParams),
headers: {
'Content-Type': 'application/json; charset=utf-8',
},
responseType: 'application/json',
});
this.logger.debug(`Successfully created job ${this.id}`);
this.emit('open', this.jobInfo);
}
catch (err) {
this.emit('error', err);
throw err;
}
return this.jobInfo;
}
/**
* Abort the job
*
* The 'aborted' event is emitted when the job successfully aborts.
* @returns {Promise<QueryJobInfoV2>} job information.
*/
async abort() {
try {
const state = 'Aborted';
this.jobInfo = await this.createQueryRequest({
method: 'PATCH',
path: `/${this.id}`,
body: JSON.stringify({ state }),
headers: { 'Content-Type': 'application/json; charset=utf-8' },
responseType: 'application/json',
});
this.logger.debug(`Successfully aborted job ${this.id}`);
return this.jobInfo;
}
catch (err) {
this.emit('error', err);
throw err;
}
}
/**
* Poll for the state of the processing for the job.
*
* @param interval Polling interval in milliseconds
* @param timeout Polling timeout in milliseconds
* @returns {Promise<Record[]>} A promise that resolves when the job finished being processed.
*/
async poll(interval = this.pollingOptions.pollInterval, timeout = this.pollingOptions.pollTimeout) {
const jobId = this.id;
const startTime = Date.now();
const endTime = startTime + timeout;
this.logger.debug('Start polling for job status');
this.logger.debug(`Polling options: timeout:${timeout}ms | interval: ${interval}ms.`);
if (timeout === 0) {
throw new JobPollingTimeoutError(`Skipping polling because of timeout = 0ms. Job Id = ${jobId}`, jobId);
}
while (endTime > Date.now()) {
try {
const res = await this.check();
switch (res.state) {
case 'Aborted':
throw new Error('Job has been aborted');
case 'UploadComplete':
case 'InProgress':
this.emit('inProgress', res);
await delay(interval);
break;
case 'Failed':
// unlike ingest jobs, the API doesn't return an error msg:
// https://developer.salesforce.com/docs/atlas.en-us.api_asynch.meta/api_asynch/query_get_one_job.htm
this.logger.debug(res);
throw new Error('Query job failed to complete');
case 'JobComplete':
this.logger.debug(`Job ${this.id} was successfully processed.`);
this.emit('jobComplete', res);
return;
}
}
catch (err) {
this.emit('error', err);
throw err;
}
}
const timeoutError = new JobPollingTimeoutError(`Polling timed out after ${timeout}ms. Job Id = ${jobId}`, jobId);
this.emit('error', timeoutError);
throw timeoutError;
}
/**
* Check the latest job status
*
* @returns {Promise<QueryJobInfoV2>} job information.
*/
async check() {
try {
const jobInfo = await this.createQueryRequest({
method: 'GET',
path: `/${this.id}`,
responseType: 'application/json',
});
this.jobInfo = jobInfo;
return jobInfo;
}
catch (err) {
this.emit('error', err);
throw err;
}
}
/**
* Get the results for a query job as a record stream
*
* This method assumes the job finished being processed
* @returns {RecordStream} - Record stream, convertible to a CSV data stream
*/
async result() {
const resultStream = new record_stream_1.Parsable();
const resultDataStream = resultStream.stream('csv');
const resultsPath = `/${this.id}/results`;
while (this.locator !== 'null') {
const resPromise = this.createQueryRequest({
method: 'GET',
path: this.locator
// resultsPath starts with '/'
? `${resultsPath}?locator=${this.locator}`
: resultsPath,
headers: {
Accept: 'text/csv',
},
});
resPromise.stream().pipe(resultDataStream);
await resPromise;
}
return resultStream;
}
/**
* Deletes a query job.
*/
async delete() {
return this.createQueryRequest({
method: 'DELETE',
path: `/${this.id}`,
});
}
createQueryRequest(request) {
const { path, responseType } = request;
const basePath = `services/data/v${this.connection.version}/jobs/query`;
const url = new URL(path ? basePath + path : basePath, this.connection.instanceUrl).toString();
const httpApi = new BulkApiV2(this.connection, { responseType });
httpApi.on('response', (response) => {
this.locator = response.headers['sforce-locator'];
this.logger.debug(`sforce-locator: ${this.locator}`);
});
return httpApi.request({
...request,
url,
});
}
}
exports.QueryJobV2 = QueryJobV2;
/**
* Class for Bulk API V2 Ingest Job
*/
class IngestJobV2 extends events_1.EventEmitter {
connection;
logger;
_id;
bodyParams;
jobData;
pollingOptions;
bulkJobSuccessfulResults;
bulkJobFailedResults;
bulkJobUnprocessedRecords;
error;
jobInfo;
/**
*
*/
constructor(conn, options) {
super();
this.connection = conn;
this.logger = this.connection._logLevel
? (0, logger_1.getLogger)('bulk2:IngestJobV2').createInstance(this.connection._logLevel)
: (0, logger_1.getLogger)('bulk2:IngestJobV2');
this.pollingOptions = options.pollingOptions;
if ('id' in options) {
this._id = options.id;
}
else {
this.bodyParams = options.bodyParams;
}
this.jobData = new JobDataV2({
createRequest: (request) => this.createIngestRequest(request),
job: this,
});
// default error handler to keep the latest error
this.on('error', (error) => (this.error = error));
}
/**
* Get the query job ID.
*
* @returns {string} query job Id.
*/
get id() {
return this.jobInfo ? this.jobInfo.id : this._id;
}
/**
* Get the query job info.
*
* @returns {Promise<QueryJobInfoV2>} ingest job information.
*/
getInfo() {
if (this.jobInfo) {
return this.jobInfo;
}
throw new Error('No internal job info. Make sure to call `await job.check`.');
}
/**
* Create a job representing a bulk operation in the org
*
* @returns {Promise<QueryJobInfoV2>} job information.
*/
async open() {
if (!this.bodyParams) {
throw new Error('Missing required body params to open a new ingest job.');
}
try {
this.jobInfo = await this.createIngestRequest({
method: 'POST',
body: JSON.stringify(this.bodyParams),
headers: {
'Content-Type': 'application/json; charset=utf-8',
},
responseType: 'application/json',
});
this.logger.debug(`Successfully created job ${this.id}`);
this.emit('open');
}
catch (err) {
this.emit('error', err);
throw err;
}
return this.jobInfo;
}
/** Upload data for a job in CSV format
*
* @param input CSV as a string, or array of records or readable stream
*/
async uploadData(input) {
await this.jobData.execute(input).result;
this.logger.debug(`Successfully uploaded data to job ${this.id}`);
}
/**
* Close opened job
*
* This method will notify the org that the upload of job data is complete and is ready for processing.
*/
async close() {
try {
const state = 'UploadComplete';
this.jobInfo = await this.createIngestRequest({
method: 'PATCH',
path: `/${this.id}`,
body: JSON.stringify({ state }),
headers: { 'Content-Type': 'application/json; charset=utf-8' },
responseType: 'application/json',
});
this.logger.debug(`Successfully closed job ${this.id}`);
this.emit('close');
}
catch (err) {
this.emit('error', err);
throw err;
}
}
/**
* Set the status to abort
*/
async abort() {
try {
const state = 'Aborted';
this.jobInfo = await this.createIngestRequest({
method: 'PATCH',
path: `/${this.id}`,
body: JSON.stringify({ state }),
headers: { 'Content-Type': 'application/json; charset=utf-8' },
responseType: 'application/json',
});
this.logger.debug(`Successfully aborted job ${this.id}`);
this.emit('aborted');
}
catch (err) {
this.emit('error', err);
throw err;
}
}
/**
* Poll for the state of the processing for the job.
*
* This method will only throw after a timeout. To capture a
* job failure while polling you must set a listener for the
* `failed` event before calling it:
*
* job.on('failed', (err) => console.error(err))
* await job.poll()
*
* @param interval Polling interval in milliseconds
* @param timeout Polling timeout in milliseconds
* @returns {Promise<void>} A promise that resolves when the job finishes successfully
*/
async poll(interval = this.pollingOptions.pollInterval, timeout = this.pollingOptions.pollTimeout) {
const jobId = this.id;
const startTime = Date.now();
const endTime = startTime + timeout;
if (timeout === 0) {
throw new JobPollingTimeoutError(`Skipping polling because of timeout = 0ms. Job Id = ${jobId}`, jobId);
}
this.logger.debug('Start polling for job status');
this.logger.debug(`Polling options: timeout:${timeout}ms | interval: ${interval}ms.`);
while (endTime > Date.now()) {
try {
const res = await this.check();
switch (res.state) {
case 'Open':
throw new Error('Job is still open. Make sure close the job by `close` method on the job instance before polling.');
case 'Aborted':
throw new Error('Job has been aborted');
case 'UploadComplete':
case 'InProgress':
this.emit('inProgress', res);
await delay(interval);
break;
case 'Failed':
this.logger.debug(res);
throw new Error(`Ingest job failed to complete due to: ${res.errorMessage}`);
case 'JobComplete':
this.logger.debug(`Job ${this.id} was successfully processed.`);
this.emit('jobComplete', res);
return;
}
}
catch (err) {
this.emit('error', err);
throw err;
}
}
const timeoutError = new JobPollingTimeoutError(`Polling timed out after ${timeout}ms. Job Id = ${jobId}`, jobId);
this.emit('error', timeoutError);
throw timeoutError;
}
/**
* Check the latest batch status in server
*/
async check() {
try {
const jobInfo = await this.createIngestRequest({
method: 'GET',
path: `/${this.id}`,
responseType: 'application/json',
});
this.jobInfo = jobInfo;
return jobInfo;
}
catch (err) {
this.emit('error', err);
throw err;
}
}
/** Return all record results
*
* This method will return successful, failed and unprocessed records
*
* @returns Promise<IngestJobV2Results>
*/
async getAllResults() {
const [successfulResults, failedResults, unprocessedRecords,] = await Promise.all([
this.getSuccessfulResults(),
this.getFailedResults(),
this.getUnprocessedRecords(),
]);
return { successfulResults, failedResults, unprocessedRecords };
}
async getSuccessfulResults(raw) {
const reqOpts = {
method: 'GET',
path: `/${this.id}/successfulResults`,
};
if (raw) {
return this.createIngestRequest({
...reqOpts,
responseType: 'text/plain',
});
}
if (this.bulkJobSuccessfulResults) {
return this.bulkJobSuccessfulResults;
}
const results = await this.createIngestRequest({
method: 'GET',
path: `/${this.id}/successfulResults`,
responseType: 'text/csv',
});
this.bulkJobSuccessfulResults = results ?? [];
return this.bulkJobSuccessfulResults;
}
async getFailedResults(raw) {
const reqOpts = {
method: 'GET',
path: `/${this.id}/failedResults`,
};
if (raw) {
return this.createIngestRequest({
...reqOpts,
responseType: 'text/plain',
});
}
if (this.bulkJobFailedResults) {
return this.bulkJobFailedResults;
}
const results = await this.createIngestRequest({
...reqOpts,
responseType: 'text/csv',
});
this.bulkJobFailedResults = results ?? [];
return this.bulkJobFailedResults;
}
async getUnprocessedRecords(raw) {
const reqOpts = {
method: 'GET',
path: `/${this.id}/unprocessedrecords`,
};
if (raw) {
return this.createIngestRequest({
...reqOpts,
responseType: 'text/plain',
});
}
if (this.bulkJobUnprocessedRecords) {
return this.bulkJobUnprocessedRecords;
}
const results = await this.createIngestRequest({
...reqOpts,
responseType: 'text/csv',
});
this.bulkJobUnprocessedRecords = results ?? [];
return this.bulkJobUnprocessedRecords;
}
/**
* Deletes an ingest job.
*/
async delete() {
return this.createIngestRequest({
method: 'DELETE',
path: `/${this.id}`,
});
}
createIngestRequest(request) {
const { path, responseType } = request;
const basePath = `/services/data/v${this.connection.version}/jobs/ingest`;
const url = new URL(path ? basePath + path : basePath, this.connection.instanceUrl).toString();
return new BulkApiV2(this.connection, { responseType }).request({
...request,
url,
});
}
}
exports.IngestJobV2 = IngestJobV2;
class JobDataV2 extends stream_1.Writable {
job;
uploadStream;
downloadStream;
dataStream;
result;
/**
*
*/
constructor(options) {
super({ objectMode: true });
const createRequest = options.createRequest;
this.job = options.job;
this.uploadStream = new record_stream_1.Serializable();
this.downloadStream = new record_stream_1.Parsable();
const converterOptions = { nullValue: '#N/A' };
const uploadDataStream = this.uploadStream.stream('csv', converterOptions);
const downloadDataStream = this.downloadStream.stream('csv', converterOptions);
this.dataStream = (0, stream_2.concatStreamsAsDuplex)(uploadDataStream, downloadDataStream);
this.on('finish', () => this.uploadStream.end());
uploadDataStream.once('readable', () => {
try {
// pipe upload data to batch API request stream
const req = createRequest({
method: 'PUT',
path: `/${this.job.id}/batches`,
headers: {
'Content-Type': 'text/csv',
},
responseType: 'application/json',
});
(async () => {
try {
const res = await req;
this.emit('response', res);
}
catch (err) {
this.emit('error', err);
}
})();
uploadDataStream.pipe(req.stream());
}
catch (err) {
this.emit('error', err);
}
});
}
_write(record_, enc, cb) {
const { Id, type, attributes, ...rrec } = record_;
let record;
switch (this.job.getInfo().operation) {
case 'insert':
record = rrec;
break;
case 'delete':
case 'hardDelete':
record = { Id };
break;
default:
record = { Id, ...rrec };
}
this.uploadStream.write(record, enc, cb);
}
/**
* Returns duplex stream which accepts CSV data input and batch result output
*/
stream() {
return this.dataStream;
}
/**
* Execute batch operation
*/
execute(input) {
if (this.result) {
throw new Error('Data can only be uploaded to a job once.');
}
this.result = new Promise((resolve, reject) => {
this.once('response', () => resolve());
this.once('error', reject);
});
if (is_1.default.nodeStream(input)) {
// if input has stream.Readable interface
input.pipe(this.dataStream);
}
else {
const recordData = structuredClone(input);
if (Array.isArray(recordData)) {
for (const record of recordData) {
for (const key of Object.keys(record)) {
if (typeof record[key] === 'boolean') {
record[key] = String(record[key]);
}
}
this.write(record);
}
this.end();
}
else if (typeof recordData === 'string') {
this.dataStream.write(recordData, 'utf8');
this.dataStream.end();
}
}
return this;
}
}
function delay(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/*--------------------------------------------*/
/*
* Register hook in connection instantiation for dynamically adding this API module features
*/
(0, jsforce_1.registerModule)('bulk2', (conn) => new BulkV2(conn));
exports.default = BulkV2;