@backstage/plugin-techdocs-node
Version:
Common node.js functionalities for TechDocs, to be shared between techdocs-backend plugin and techdocs-cli
652 lines (646 loc) • 24 kB
JavaScript
'use strict';
var errors = require('@backstage/errors');
var integrationAwsNode = require('@backstage/integration-aws-node');
var clientS3 = require('@aws-sdk/client-s3');
var credentialProviders = require('@aws-sdk/credential-providers');
var nodeHttpHandler = require('@smithy/node-http-handler');
var libStorage = require('@aws-sdk/lib-storage');
var hpagent = require('hpagent');
var fs = require('fs-extra');
var JSON5 = require('json5');
var createLimiter = require('p-limit');
var path = require('node:path');
var helpers = require('./helpers.cjs.js');
var integration = require('@backstage/integration');
function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
var fs__default = /*#__PURE__*/_interopDefaultCompat(fs);
var JSON5__default = /*#__PURE__*/_interopDefaultCompat(JSON5);
var createLimiter__default = /*#__PURE__*/_interopDefaultCompat(createLimiter);
var path__default = /*#__PURE__*/_interopDefaultCompat(path);
const MAX_SINGLE_UPLOAD_BYTES = 5 * 1024 * 1024;
const streamToBuffer = (stream) => {
return new Promise((resolve, reject) => {
try {
const chunks = [];
stream.on("data", (chunk) => chunks.push(chunk));
stream.on(
"error",
(e) => reject(new errors.ForwardedError("Unable to read stream", e))
);
stream.on("end", () => resolve(Buffer.concat(chunks)));
} catch (e) {
throw new errors.ForwardedError("Unable to parse the response data", e);
}
});
};
class AwsS3Publish {
storageClient;
bucketName;
legacyPathCasing;
logger;
bucketRootPath;
sse;
maxAttempts;
constructor(options) {
this.storageClient = options.storageClient;
this.bucketName = options.bucketName;
this.legacyPathCasing = options.legacyPathCasing;
this.logger = options.logger;
this.bucketRootPath = options.bucketRootPath;
this.sse = options.sse;
this.maxAttempts = options.maxAttempts;
}
static async fromConfig(config, logger) {
let bucketName = "";
try {
bucketName = config.getString("techdocs.publisher.awsS3.bucketName");
} catch (error) {
throw new Error(
"Since techdocs.publisher.type is set to 'awsS3' in your app config, techdocs.publisher.awsS3.bucketName is required."
);
}
const bucketRootPath = helpers.normalizeExternalStorageRootPath(
config.getOptionalString("techdocs.publisher.awsS3.bucketRootPath") || ""
);
const sse = config.getOptionalString("techdocs.publisher.awsS3.sse");
const region = config.getOptionalString("techdocs.publisher.awsS3.region");
const accountId = config.getOptionalString(
"techdocs.publisher.awsS3.accountId"
);
const credentialsConfig = config.getOptionalConfig(
"techdocs.publisher.awsS3.credentials"
);
const credsManager = integrationAwsNode.DefaultAwsCredentialsManager.fromConfig(config);
const scmIntegrations = integration.ScmIntegrations.fromConfig(config);
const awsS3Integrations = scmIntegrations.awsS3.list();
const sdkCredentialProvider = await AwsS3Publish.buildCredentials(
credsManager,
logger,
awsS3Integrations,
accountId,
credentialsConfig,
region
);
const endpoint = config.getOptionalString(
"techdocs.publisher.awsS3.endpoint"
);
const httpsProxy = config.getOptionalString(
"techdocs.publisher.awsS3.httpsProxy"
);
const forcePathStyle = config.getOptionalBoolean(
"techdocs.publisher.awsS3.s3ForcePathStyle"
);
const maxAttempts = config.getOptionalNumber(
"techdocs.publisher.awsS3.maxAttempts"
);
const storageClient = new clientS3.S3Client({
customUserAgent: "backstage-aws-techdocs-s3-publisher",
credentialDefaultProvider: () => sdkCredentialProvider,
...region && { region },
...endpoint && { endpoint },
...forcePathStyle && { forcePathStyle },
// Enhanced retry configuration for better reliability
maxAttempts: maxAttempts || 5,
retryMode: "adaptive",
// Enhanced connection settings for large file uploads
requestHandler: new nodeHttpHandler.NodeHttpHandler({
...httpsProxy && {
httpsAgent: new hpagent.HttpsProxyAgent({ proxy: httpsProxy })
},
connectionTimeout: 6e4,
socketTimeout: 12e4
})
});
const legacyPathCasing = config.getOptionalBoolean(
"techdocs.legacyUseCaseSensitiveTripletPaths"
) || false;
return new AwsS3Publish({
storageClient,
bucketName,
bucketRootPath,
legacyPathCasing,
logger,
sse,
maxAttempts: maxAttempts || 5
});
}
static buildStaticCredentials(accessKeyId, secretAccessKey) {
return async () => {
return Promise.resolve({
accessKeyId,
secretAccessKey
});
};
}
static async buildCredentials(credsManager, logger, awsS3Integrations, accountId, credentialsConfig, region) {
if (accountId) {
return (await credsManager.getCredentialProvider({ accountId })).sdkCredentialProvider;
}
const explicitCredentials = await AwsS3Publish.getExplicitCredentials({
credsManager,
credentialsConfig,
awsS3Integrations,
logger
});
const roleArn = credentialsConfig?.getOptionalString("roleArn");
if (roleArn) {
return credentialProviders.fromTemporaryCredentials({
masterCredentials: explicitCredentials,
params: {
RoleSessionName: "backstage-aws-techdocs-s3-publisher",
RoleArn: roleArn
},
clientConfig: { region }
});
}
return explicitCredentials;
}
/**
* Custom retry wrapper for S3 operations with detailed error handling.
*/
async retryOperation(operation, operationName, maxAttempts = 3, shouldRetry = this.defaultShouldRetry.bind(this)) {
for (let attempt = 1; attempt < maxAttempts; attempt++) {
try {
return await operation();
} catch (error) {
const e = error;
if (!shouldRetry(e)) {
this.logger.error(`${operationName} failed: ${e.message}`);
throw e;
}
this.logger.warn(`${operationName} failed, retrying...`, {
attempt,
maxAttempts,
error: e.message,
errorCode: e.name,
httpStatusCode: e.$metadata?.httpStatusCode
});
const baseDelay = operationName.startsWith("Upload-") ? 2e3 : 1e3;
const backoffDelay = Math.min(
baseDelay * Math.pow(2, attempt - 1),
3e4
);
const jitter = Math.random() * 1e3;
await new Promise(
(resolve) => setTimeout(resolve, backoffDelay + jitter)
);
}
}
return await operation();
}
/**
* Determines if an S3 operation should be retried based on the error details.
*/
defaultShouldRetry(error) {
const httpStatusCode = error.$metadata?.httpStatusCode;
const errorCode = error.name;
const transientErrors = [
"NetworkingError",
"TimeoutError",
"ConnectionError",
"RequestTimeout",
"ServiceUnavailable",
"SlowDown",
"ThrottlingException"
];
if (httpStatusCode && httpStatusCode >= 500) {
return true;
}
if (httpStatusCode && httpStatusCode >= 400 && httpStatusCode < 500) {
const retriable4xxErrors = [
"RequestTimeout",
"RequestTimeoutException",
"PriorRequestNotComplete"
];
return retriable4xxErrors.includes(errorCode);
}
return transientErrors.some(
(retriableError) => errorCode === retriableError || error.message.includes(retriableError)
);
}
static async getExplicitCredentials({
credentialsConfig,
awsS3Integrations,
credsManager,
logger
}) {
const accessKeyId = credentialsConfig?.getOptionalString("accessKeyId");
const secretAccessKey = credentialsConfig?.getOptionalString("secretAccessKey");
if (accessKeyId && secretAccessKey) {
return AwsS3Publish.buildStaticCredentials(accessKeyId, secretAccessKey);
}
if (awsS3Integrations.length > 0) {
if (awsS3Integrations.length === 1) {
const singleAwsS3IntegrationConfig = awsS3Integrations[0].config;
const singleAwsS3IntegrationAccessKeyId = singleAwsS3IntegrationConfig.accessKeyId;
const singleAwsS3IntegrationSecretAccessKey = singleAwsS3IntegrationConfig.secretAccessKey;
if (singleAwsS3IntegrationAccessKeyId && singleAwsS3IntegrationSecretAccessKey) {
return AwsS3Publish.buildStaticCredentials(
singleAwsS3IntegrationAccessKeyId,
singleAwsS3IntegrationSecretAccessKey
);
}
} else {
if (accessKeyId) {
const targetAwsS3IntegrationConfig = awsS3Integrations.find(
(c) => c.config.accessKeyId === accessKeyId
);
if (!targetAwsS3IntegrationConfig) {
logger.warn(
`No AWS S3 integration config under integrations.awsS3 found for access key id ${accessKeyId}.`
);
}
const targetAwsS3IntegrationAccessKeyId = targetAwsS3IntegrationConfig?.config.accessKeyId;
const targetAwsS3IntegrationSecretAccessKey = targetAwsS3IntegrationConfig?.config.secretAccessKey;
if (targetAwsS3IntegrationAccessKeyId && targetAwsS3IntegrationSecretAccessKey) {
return AwsS3Publish.buildStaticCredentials(
targetAwsS3IntegrationAccessKeyId,
targetAwsS3IntegrationSecretAccessKey
);
}
}
}
}
return (await credsManager.getCredentialProvider()).sdkCredentialProvider;
}
/**
* Check if the defined bucket exists. Being able to connect means the configuration is good
* and the storage client will work.
*/
async getReadiness() {
try {
await this.storageClient.send(
new clientS3.HeadBucketCommand({ Bucket: this.bucketName })
);
this.logger.info(
`Successfully connected to the AWS S3 bucket ${this.bucketName}.`
);
return { isAvailable: true };
} catch (error) {
this.logger.error(
`Could not retrieve metadata about the AWS S3 bucket ${this.bucketName}. Make sure the bucket exists. Also make sure that authentication is setup either by explicitly defining credentials and region in techdocs.publisher.awsS3 in app config or by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage`
);
this.logger.error(
`from AWS client library`,
error instanceof Error ? error : new Error(String(error))
);
return {
isAvailable: false
};
}
}
/**
* Upload all the files from the generated `directory` to the S3 bucket.
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
*/
async publish({
entity,
directory
}) {
const objects = [];
const useLegacyPathCasing = this.legacyPathCasing;
const bucketRootPath = this.bucketRootPath;
const sse = this.sse;
const publishStartTime = Date.now();
let existingFiles = [];
try {
const remoteFolder = helpers.getCloudPathForLocalPath(
entity,
void 0,
useLegacyPathCasing,
bucketRootPath
);
const response = await this.retryOperation(
async () => {
const listCommand = new clientS3.ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: remoteFolder
});
return this.storageClient.send(listCommand);
},
"ListObjects",
this.maxAttempts
);
existingFiles = (response.Contents || []).map((f) => f.Key || "").filter((f) => !!f);
} catch (e) {
this.logger.error(
`Unable to list files for Entity ${entity.metadata.name}: ${errors.toError(e).message}`
);
}
let absoluteFilesToUpload;
try {
absoluteFilesToUpload = await helpers.getFileTreeRecursively(directory);
await helpers.bulkStorageOperation(
async (absoluteFilePath) => {
const relativeFilePath = path__default.default.relative(directory, absoluteFilePath);
const s3Key = helpers.getCloudPathForLocalPath(
entity,
relativeFilePath,
useLegacyPathCasing,
bucketRootPath
);
const params = {
Bucket: this.bucketName,
Key: s3Key,
...sse && { ServerSideEncryption: sse }
};
objects.push(params.Key);
const stats = await fs__default.default.stat(absoluteFilePath);
const fileSizeInBytes = stats.size;
if (fileSizeInBytes >= MAX_SINGLE_UPLOAD_BYTES) {
try {
await this.retryOperation(
() => {
const fileStream = fs__default.default.createReadStream(absoluteFilePath);
const uploadParams = { ...params, Body: fileStream };
const upload = new libStorage.Upload({
client: this.storageClient,
params: uploadParams,
partSize: MAX_SINGLE_UPLOAD_BYTES,
queueSize: 3,
leavePartsOnError: false
});
return upload.done();
},
`Upload-${params.Key}`,
this.maxAttempts
);
return;
} catch (multipartError) {
const s3Error = multipartError;
const errorName = s3Error?.name || "Unknown";
if (errorName === "InvalidPart" || errorName === "NoSuchUpload") {
this.logger.warn(
`Multipart upload failed for ${params.Key}, attempting simple upload fallback.`
);
} else {
this.logger.error(
`Multipart upload failed for ${params.Key}: ${multipartError instanceof Error ? multipartError.message : String(multipartError)}`
);
throw multipartError;
}
}
}
try {
const fileContent = await fs__default.default.readFile(absoluteFilePath);
const putParams = { ...params, Body: fileContent };
await this.retryOperation(
() => this.storageClient.send(new clientS3.PutObjectCommand(putParams)),
`Upload-${params.Key}`,
this.maxAttempts
);
if (fileSizeInBytes >= MAX_SINGLE_UPLOAD_BYTES) {
this.logger.info(
`Simple upload fallback succeeded for ${params.Key}`
);
}
} catch (error) {
this.logger.error(
`Upload failed for ${params.Key}: ${error instanceof Error ? error.message : String(error)}`
);
throw error;
}
},
absoluteFilesToUpload,
{ concurrencyLimit: 10 }
);
this.logger.info(
`Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${absoluteFilesToUpload.length}`
);
} catch (e) {
const errorMessage = `Unable to upload file(s) to AWS S3. ${e}`;
this.logger.error(errorMessage);
throw new Error(errorMessage);
}
try {
const relativeFilesToUpload = absoluteFilesToUpload.map(
(absoluteFilePath) => helpers.getCloudPathForLocalPath(
entity,
path__default.default.relative(directory, absoluteFilePath),
useLegacyPathCasing,
bucketRootPath
)
);
const staleFiles = helpers.getStaleFiles(relativeFilesToUpload, existingFiles);
await helpers.bulkStorageOperation(
async (relativeFilePath) => {
return this.retryOperation(
async () => {
const deleteCommand = new clientS3.DeleteObjectCommand({
Bucket: this.bucketName,
Key: relativeFilePath
});
return this.storageClient.send(deleteCommand);
},
"DeleteObject",
this.maxAttempts
);
},
staleFiles,
{ concurrencyLimit: 10 }
);
this.logger.info(
`Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: ${staleFiles.length}`
);
} catch (error) {
const errorMessage = `Unable to delete file(s) from AWS S3. ${error}`;
this.logger.error(errorMessage);
}
const publishEndTime = Date.now();
const publishDurationMs = publishEndTime - publishStartTime;
this.logger.info(
`Successfully published ${objects.length} files for ${entity.metadata.name} in ${Math.round(publishDurationMs / 1e3)}s`
);
return { objects };
}
async fetchTechDocsMetadata(entityName) {
try {
return await new Promise(async (resolve, reject) => {
const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
const entityDir = this.legacyPathCasing ? entityTriplet : helpers.lowerCaseEntityTriplet(entityTriplet);
const entityRootDir = path__default.default.posix.join(this.bucketRootPath, entityDir);
if (!helpers.isValidContentPath(this.bucketRootPath, entityRootDir)) {
this.logger.error(
`Invalid content path found while fetching TechDocs metadata: ${entityRootDir}`
);
throw new Error(`Metadata Not Found`);
}
try {
const resp = await this.retryOperation(
async () => {
const getCommand = new clientS3.GetObjectCommand({
Bucket: this.bucketName,
Key: `${entityRootDir}/techdocs_metadata.json`
});
return this.storageClient.send(getCommand);
},
"GetTechDocsMetadata",
this.maxAttempts
);
const techdocsMetadataJson = await streamToBuffer(
resp.Body
);
if (!techdocsMetadataJson) {
throw new Error(
`Unable to parse the techdocs metadata file ${entityRootDir}/techdocs_metadata.json.`
);
}
const techdocsMetadata = JSON5__default.default.parse(
techdocsMetadataJson.toString("utf-8")
);
resolve(techdocsMetadata);
} catch (err) {
this.logger.error(errors.toError(err).message);
reject(new Error(errors.toError(err).message));
}
});
} catch (e) {
throw new errors.ForwardedError("TechDocs metadata fetch failed", e);
}
}
/**
* Express route middleware to serve static files on a route in techdocs-backend.
*/
docsRouter() {
return async (req, res) => {
const decodedUri = decodeURI(req.path.replace(/^\//, ""));
const filePathNoRoot = this.legacyPathCasing ? decodedUri : helpers.lowerCaseEntityTripletInStoragePath(decodedUri);
const filePath = path__default.default.posix.join(this.bucketRootPath, filePathNoRoot);
if (!helpers.isValidContentPath(this.bucketRootPath, filePath)) {
this.logger.error(
`Attempted to fetch TechDocs content for a file outside of the bucket root: ${filePathNoRoot}`
);
res.status(404).send("File Not Found");
return;
}
const fileExtension = path__default.default.extname(filePath);
const responseHeaders = helpers.getHeadersForFileExtension(fileExtension);
try {
const resp = await this.storageClient.send(
new clientS3.GetObjectCommand({ Bucket: this.bucketName, Key: filePath })
);
for (const [headerKey, headerValue] of Object.entries(
responseHeaders
)) {
res.setHeader(headerKey, headerValue);
}
resp.Body.on("error", (err) => {
this.logger.warn(
`TechDocs S3 router failed to serve static files from bucket ${this.bucketName} at key ${filePath}: ${err.message}`
);
if (!res.headersSent) {
res.status(404).send("File Not Found");
} else {
res.destroy();
}
}).pipe(res);
} catch (err) {
this.logger.warn(
`TechDocs S3 router failed to serve static files from bucket ${this.bucketName} at key ${filePath}: ${errors.toError(err).message}`
);
res.status(404).send("File Not Found");
}
};
}
/**
* A helper function which checks if index.html of an Entity's docs site is available. This
* can be used to verify if there are any pre-generated docs available to serve.
*/
async hasDocsBeenGenerated(entity) {
try {
const entityTriplet = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const entityDir = this.legacyPathCasing ? entityTriplet : helpers.lowerCaseEntityTriplet(entityTriplet);
const entityRootDir = path__default.default.posix.join(this.bucketRootPath, entityDir);
if (!helpers.isValidContentPath(this.bucketRootPath, entityRootDir)) {
this.logger.error(
`Invalid content path found while checking if docs have been generated: ${entityRootDir}`
);
return Promise.resolve(false);
}
await this.storageClient.send(
new clientS3.HeadObjectCommand({
Bucket: this.bucketName,
Key: `${entityRootDir}/index.html`
})
);
return Promise.resolve(true);
} catch (e) {
return Promise.resolve(false);
}
}
async migrateDocsCase({
removeOriginal = false,
concurrency = 25
}) {
const allObjects = await this.getAllObjectsFromBucket();
const limiter = createLimiter__default.default(concurrency);
await Promise.all(
allObjects.map(
(f) => limiter(async (file) => {
let newPath;
try {
newPath = helpers.lowerCaseEntityTripletInStoragePath(file);
} catch (e) {
this.logger.warn(errors.toError(e).message);
return;
}
if (file === newPath) {
return;
}
try {
this.logger.debug(`Migrating ${file}`);
await this.storageClient.send(
new clientS3.CopyObjectCommand({
Bucket: this.bucketName,
CopySource: [this.bucketName, file].join("/"),
Key: newPath
})
);
if (removeOriginal) {
await this.storageClient.send(
new clientS3.DeleteObjectCommand({
Bucket: this.bucketName,
Key: file
})
);
}
} catch (e) {
this.logger.warn(
`Unable to migrate ${file}: ${errors.toError(e).message}`
);
}
}, f)
)
);
}
/**
* Returns a list of all object keys from the configured bucket.
*/
async getAllObjectsFromBucket({ prefix } = { prefix: "" }) {
const objects = [];
let nextContinuation;
let allObjects;
do {
const currentToken = nextContinuation;
allObjects = await this.retryOperation(
async () => {
const listCommand = new clientS3.ListObjectsV2Command({
Bucket: this.bucketName,
ContinuationToken: currentToken,
...prefix ? { Prefix: prefix } : {}
});
return this.storageClient.send(listCommand);
},
"GetAllObjects",
this.maxAttempts
);
objects.push(
...(allObjects.Contents || []).map((f) => f.Key || "").filter((f) => !!f)
);
nextContinuation = allObjects.NextContinuationToken;
} while (nextContinuation);
return objects;
}
}
exports.AwsS3Publish = AwsS3Publish;
//# sourceMappingURL=awsS3.cjs.js.map