@cumulus/api-client
Version:
API client for working with the Cumulus archive API
870 lines (808 loc) • 27 kB
text/typescript
import pRetry from 'p-retry';
import { ApiGranuleRecord, ApiGranule, GranuleId, GranuleStatus } from '@cumulus/types/api/granules';
import { CollectionId } from '@cumulus/types/api/collections';
import Logger from '@cumulus/logger';
import { invokeApi } from './cumulusApiClient';
import { ApiGatewayLambdaHttpProxyResponse, InvokeApiFunction } from './types';
const logger = new Logger({ sender: '@api-client/granules' });
type AssociateExecutionRequest = {
granuleId: string
collectionId: string
executionArn: string
};
type BulkPatchGranuleCollection = {
apiGranules: ApiGranuleRecord[],
collectionId: string,
};
type BulkPatch = {
apiGranules: ApiGranuleRecord[],
dbConcurrency: number,
dbMaxPool: number,
};
type InvalidBehavior = 'error' | 'skip';
type CmrGranuleUrlType = 'http' | 's3' | 'both';
const encodeGranulesURIComponent = (
granuleId: string,
collectionId: string | undefined
): string =>
(collectionId
? `/granules/${encodeURIComponent(collectionId)}/${encodeURIComponent(granuleId)}`
: `/granules/${encodeURIComponent(granuleId)}`); // Fetching a granule without a collectionId is supported but deprecated
/**
* GET raw response from /granules/{granuleId} or /granules/{collectionId}/{granuleId}
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param params.granuleId - a granule ID
* @param [params.collectionId] - a collection ID
* @param [params.query] - query to pass the API lambda
* @param params.expectedStatusCodes - the statusCodes which the granule API is
* is expecting for the invokeApi Response,
* default is 200
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload,
* cumulusApiClient.invokeApifunction
* is the default to invoke the api lambda
* @returns - the granule fetched by the API
*/
export const getGranuleResponse = async (params: {
prefix: string,
granuleId: GranuleId,
collectionId?: CollectionId,
expectedStatusCodes?: number[] | number,
query?: { [key: string]: string },
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const {
prefix,
granuleId,
collectionId,
query,
expectedStatusCodes,
callback = invokeApi,
} = params;
const path = encodeGranulesURIComponent(granuleId, collectionId);
return await callback({
prefix,
payload: {
httpMethod: 'GET',
resource: '/{proxy+}',
path,
...(query && { queryStringParameters: query }),
},
expectedStatusCodes,
});
};
/**
* GET granule record from /granules/{granuleId} or /granules/{collectionId}/{granuleId}
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param params.granuleId - a granule ID
* @param [params.collectionId] - a collection ID
* @param [params.query] - query to pass the API lambda
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to
* invoke the
* api lambda
* @returns - the granule fetched by the API
*/
export const getGranule = async (params: {
prefix: string,
granuleId: GranuleId,
collectionId?: CollectionId,
query?: { [key: string]: string },
callback?: InvokeApiFunction
}): Promise<ApiGranuleRecord> => {
const response = await getGranuleResponse(params);
return JSON.parse(response.body);
};
/**
* Wait for a granule to be present in the database (using pRetry)
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param params.granuleId - granuleId to wait for
* @param [params.status] - expected granule status
* @param [params.retries] - number of times to retry
* @param [params.pRetryOptions] - options for pRetry
* @param [params.callback] - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
*/
export const waitForGranule = async (params: {
prefix: string,
granuleId: GranuleId,
status?: GranuleStatus,
retries?: number,
pRetryOptions?: pRetry.Options,
callback?: InvokeApiFunction
}) => {
const {
prefix,
granuleId,
status,
retries = 10,
pRetryOptions = {},
callback = invokeApi,
} = params;
await pRetry(
async () => {
// TODO update to use collectionId + granuleId
const apiResult = await getGranuleResponse({ prefix, granuleId, callback });
if (apiResult.statusCode === 500) {
throw new pRetry.AbortError('API misconfigured/down/etc, failing test');
}
if (apiResult.statusCode !== 200) {
throw new Error(`granule ${granuleId} not in database yet, status ${apiResult.statusCode} retrying....`);
}
if (status) {
const granuleStatus = JSON.parse(apiResult.body).status;
if (status !== granuleStatus) {
throw new Error(`Granule status ${granuleStatus} does not match requested status, retrying...`);
}
}
logger.info(`Granule ${granuleId} in database, proceeding...`); // TODO fix logging
},
{
retries,
onFailedAttempt: (e) => {
logger.error(e.message);
},
...pRetryOptions,
}
);
};
/**
* Reingest a granule from the Cumulus API
* PATCH /granules/{}
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param params.granuleId - a granule ID
* @param [params.collectionId] - a collection ID
* @param params.workflowName - Optional WorkflowName
* @param params.executionArn - Optional executionArn
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
* @returns - the granule fetched by the API
*/
export const reingestGranule = async (params: {
prefix: string,
granuleId: GranuleId,
collectionId?: CollectionId,
workflowName?: string | undefined,
executionArn?: string | undefined,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const {
prefix,
granuleId,
collectionId,
workflowName,
executionArn,
callback = invokeApi,
} = params;
const path = encodeGranulesURIComponent(granuleId, collectionId);
return await callback({
prefix: prefix,
payload: {
httpMethod: 'PATCH',
resource: '/{proxy+}',
path,
headers: {
'Content-Type': 'application/json',
'Cumulus-API-Version': '2',
},
body: JSON.stringify({
action: 'reingest',
workflowName,
executionArn,
}),
},
});
};
/**
* Removes a granule from CMR via the Cumulus API
* PATCH /granules/{granuleId}
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param params.granuleId - a granule ID
* @param [params.collectionId] - a collection ID
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
* @returns - the granule fetched by the API
*/
export const removeFromCMR = async (params: {
prefix: string,
granuleId: GranuleId,
collectionId?: CollectionId,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, granuleId, collectionId, callback = invokeApi } = params;
const path = encodeGranulesURIComponent(granuleId, collectionId);
return await callback({
prefix: prefix,
payload: {
httpMethod: 'PATCH',
resource: '/{proxy+}',
path,
headers: {
'Content-Type': 'application/json',
'Cumulus-API-Version': '2',
},
body: JSON.stringify({ action: 'removeFromCmr' }),
},
});
};
/**
* Run a workflow with the given granule as the payload
* PATCH /granules/{granuleId}
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param params.granuleId - a granule ID
* @param [params.collectionId] - a collection ID
* @param params.workflow - workflow to be run with given granule
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
* @param [params.meta] - metadata
* @returns - the granule fetched by the API
*/
export const applyWorkflow = async (params: {
prefix: string,
granuleId: GranuleId,
collectionId?: CollectionId,
workflow: string,
meta?: object,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const {
prefix,
granuleId,
collectionId,
workflow,
meta,
callback = invokeApi,
} = params;
const path = encodeGranulesURIComponent(granuleId, collectionId);
return await callback({
prefix: prefix,
payload: {
httpMethod: 'PATCH',
resource: '/{proxy+}',
headers: {
'Content-Type': 'application/json',
'Cumulus-API-Version': '2',
},
path,
body: JSON.stringify({ action: 'applyWorkflow', workflow, meta }),
},
});
};
/**
* Delete a granule from Cumulus via the API lambda
* DELETE /granules/${granuleId}
*
* @param params - params
* @param params.pRetryOptions - pRetry options object
* @param params.prefix - the prefix configured for the stack
* @param params.granuleId - a granule ID
* @param [params.collectionId] - a collection ID
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
* @returns - the delete confirmation from the API
*/
export const deleteGranule = async (params: {
prefix: string,
granuleId: GranuleId,
collectionId?: CollectionId,
pRetryOptions?: pRetry.Options,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const {
pRetryOptions,
prefix,
granuleId,
collectionId,
callback = invokeApi,
} = params;
const path = encodeGranulesURIComponent(granuleId, collectionId);
return await callback({
prefix: prefix,
payload: {
httpMethod: 'DELETE',
resource: '/{proxy+}',
path,
},
pRetryOptions,
});
};
/**
* Move a granule via the API
* PATCH /granules/{granuleId}
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param params.granuleId - a granule ID
* @param [params.collectionId] - a collection ID
* @param params.destinations - move granule destinations
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke
* the api lambda
* @returns - the move response from the API
*/
export const moveGranule = async (params: {
prefix: string,
granuleId: GranuleId,
collectionId?: CollectionId,
destinations: unknown[],
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const {
prefix,
granuleId,
collectionId,
destinations,
callback = invokeApi,
} = params;
const path = encodeGranulesURIComponent(granuleId, collectionId);
return await callback({
prefix: prefix,
payload: {
httpMethod: 'PATCH',
resource: '/{proxy+}',
headers: {
'Content-Type': 'application/json',
'Cumulus-API-Version': '2',
},
path,
body: JSON.stringify({ action: 'move', destinations }),
},
});
};
/**
* Removed a granule from CMR and delete from Cumulus via the API
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param params.granuleId - a granule ID
* @param [params.collectionId] - a collection ID
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
* @returns - the delete confirmation from the API
*/
export const removePublishedGranule = async (params: {
prefix: string,
granuleId: GranuleId,
collectionId?: CollectionId,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, granuleId, collectionId, callback = invokeApi } = params;
// pre-delete: Remove the granule from CMR
await removeFromCMR({ prefix, granuleId, collectionId, callback });
return deleteGranule({ prefix, granuleId, collectionId, callback });
};
/**
* Query granules stored in cumulus
* GET /granules
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param [params.query] - query to pass the API lambda
* @param [params.query.fields]
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
* @returns - the response from the callback
*/
export const listGranules = async (params: {
prefix: string,
query?: {
fields?: string[],
[key: string]: string | string[] | undefined
},
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, query, callback = invokeApi } = params;
return await callback({
prefix: prefix,
payload: {
httpMethod: 'GET',
resource: '/{proxy+}',
path: '/granules',
queryStringParameters: query,
},
});
};
/**
* Create granule into cumulus.
* POST /granules
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param [params.body] - granule to pass the API lambda
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
* @returns - the response from the callback
*/
export const createGranule = async (params: {
prefix: string,
body: ApiGranuleRecord,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, body, callback = invokeApi } = params;
return await callback({
prefix,
payload: {
httpMethod: 'POST',
resource: '/{proxy+}',
path: '/granules',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
},
});
};
/**
* Update/create granule in cumulus via PUT request. Existing values will
* be removed if not specified and in some cases replaced with defaults.
* Granule execution association history will be retained.
* PUT /granules/{collectionId}/{granuleId}
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param [params.body] - granule to pass the API lambda
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
* @returns - the response from the callback
*/
export const replaceGranule = async (params: {
prefix: string,
body: ApiGranuleRecord,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, body, callback = invokeApi } = params;
const path = encodeGranulesURIComponent(body.granuleId, body.collectionId);
return await callback({
prefix,
payload: {
httpMethod: 'PUT',
resource: '/{proxy+}',
path,
headers: {
'Content-Type': 'application/json',
'Cumulus-API-Version': '2',
},
body: JSON.stringify(body),
},
expectedStatusCodes: [200, 201],
});
};
/**
* Update granule in cumulus via PATCH request. Existing values will
* not be overwritten if not specified, null values will be removed and in
* some cases replaced with defaults.
* PATCH /granules/{granuleId}
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param [params.body] - granule to pass the API lambda
* @param params.granuleId - a granule ID
* @param [params.collectionId] - a collection ID
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
* @returns - the response from the callback
*/
export const updateGranule = async (params: {
prefix: string,
body: ApiGranuleRecord,
granuleId: GranuleId,
collectionId?: CollectionId,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, granuleId, collectionId, body, callback = invokeApi } = params;
const path = encodeGranulesURIComponent(granuleId, collectionId);
return await callback({
prefix,
payload: {
httpMethod: 'PATCH',
resource: '/{proxy+}',
path,
headers: { 'Content-Type': 'application/json', 'Cumulus-API-Version': '2' },
body: JSON.stringify(body),
},
expectedStatusCodes: [200, 201],
});
};
/**
* Associate an execution with a granule in cumulus.
* POST /granules/{granuleId}/execution
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param [params.body] - granule and execution info to pass the API lambda
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
* @returns - the response from the callback
*/
export const associateExecutionWithGranule = async (params: {
prefix: string,
body: AssociateExecutionRequest,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, body, callback = invokeApi } = params;
const path = encodeGranulesURIComponent(body.granuleId, undefined);
return await callback({
prefix,
payload: {
httpMethod: 'POST',
resource: '/{proxy+}',
path: `${path}/executions`,
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
},
});
};
/**
* Update a list of granules' to a new collectionId in postgres and elasticsearch
* PATCH /granules/bulkPatchGranuleCollection
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param params.body - body to pass the API lambda
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
* @returns - the response from the callback
*/
export const bulkPatchGranuleCollection = async (params: {
prefix: string,
body: BulkPatchGranuleCollection,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, body, callback = invokeApi } = params;
return await callback({
prefix: prefix,
payload: {
httpMethod: 'PATCH',
resource: '/{proxy+}',
headers: {
'Content-Type': 'application/json',
},
path: '/granules/bulkPatchGranuleCollection',
body: JSON.stringify(body),
},
expectedStatusCodes: 200,
});
};
/**
* Apply PATCH to a list of granules
* POST /granules/bulkPatch
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param params.body - body to pass the API lambda
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
* @returns - the response from the callback
*/
export const bulkPatch = async (params: {
prefix: string,
body: BulkPatch,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, body, callback = invokeApi } = params;
return await callback({
prefix: prefix,
payload: {
httpMethod: 'PATCH',
resource: '/{proxy+}',
headers: {
'Content-Type': 'application/json',
},
path: '/granules/bulkPatch',
body: JSON.stringify(body),
},
expectedStatusCodes: 200,
});
};
/**
* Bulk operations on granules stored in cumulus
* POST /granules/bulk
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param params.body - body to pass the API lambda
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
* @returns - the response from the callback
*/
export const bulkGranules = async (params: {
prefix: string,
body: unknown,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, body, callback = invokeApi } = params;
return await callback({
prefix: prefix,
payload: {
httpMethod: 'POST',
resource: '/{proxy+}',
headers: {
'Content-Type': 'application/json',
},
path: '/granules/bulk',
body: JSON.stringify(body),
},
expectedStatusCodes: 202,
});
};
/**
* Bulk delete granules stored in cumulus
* POST /granules/bulkDelete
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param params.body - body to pass the API lambda
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
* @returns - the response from the callback
*/
export const bulkDeleteGranules = async (params: {
prefix: string,
body: unknown,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, body, callback = invokeApi } = params;
return await callback({
prefix: prefix,
payload: {
httpMethod: 'POST',
resource: '/{proxy+}',
headers: {
'Content-Type': 'application/json',
},
path: '/granules/bulkDelete',
body: JSON.stringify(body),
},
expectedStatusCodes: 202,
});
};
export const getFileGranuleAndCollectionByBucketAndKey = async (params: {
prefix: string,
bucket: string,
key: string,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, bucket, key, callback = invokeApi } = params;
return await callback({
prefix: prefix,
payload: {
httpMethod: 'GET',
resource: '/{proxy+}',
path: `/granules/files/get_collection_and_granule_id/${encodeURIComponent(bucket)}/${encodeURIComponent(key)}`,
},
});
};
export const bulkReingestGranules = async (params: {
prefix: string,
body: unknown,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, body, callback = invokeApi } = params;
return await callback({
prefix: prefix,
payload: {
httpMethod: 'POST',
resource: '/{proxy+}',
headers: {
'Content-Type': 'application/json',
},
path: '/granules/bulkReingest',
body: JSON.stringify(body),
},
expectedStatusCodes: 202,
});
};
/**
* Bulk Granule Operations
* POST /granules/bulk
*
* @param params - params
* @param params.prefix - the prefix configured for the stack
* @param params.granules - the granules to have bulk operation on
* @param params.workflowName - workflowName for the bulk operation execution
* @param params.callback - async function to invoke the api lambda
* that takes a prefix / user payload. Defaults
* to cumulusApiClient.invokeApifunction to invoke the
* api lambda
* @returns - the response from the callback
*/
export const bulkOperation = async (params: {
prefix: string,
granules: ApiGranule[],
workflowName: string,
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, granules, workflowName, callback = invokeApi } = params;
return await callback({
prefix: prefix,
payload: {
httpMethod: 'POST',
resource: '/{proxy+}',
path: '/granules/bulk/',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ granules, workflowName }),
},
expectedStatusCodes: 202,
});
};
/**
* Bulk Granule Operations
* POST /granules/bulkChangeCollection
*/
export const bulkChangeCollection = async (params: {
prefix: string,
body: {
sourceCollectionId: string,
targetCollectionId: string,
batchSize?: number,
concurrency?: number,
s3Concurrency?: number,
listGranulesConcurrency?: number,
dbMaxPool?: number,
maxRequestGranules?: number,
invalidGranuleBehavior?: InvalidBehavior,
cmrGranuleUrlType?: CmrGranuleUrlType,
s3MultipartChunkSizeMb?: number,
executionName?: string,
},
callback?: InvokeApiFunction
}): Promise<ApiGatewayLambdaHttpProxyResponse> => {
const { prefix, body, callback = invokeApi } = params;
return await callback({
prefix: prefix,
payload: {
httpMethod: 'POST',
resource: '/{proxy+}',
path: '/granules/bulkChangeCollection/',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(body),
},
expectedStatusCodes: 200,
});
};