UNPKG

lokalise-file-exchange

Version:
1,191 lines (1,186 loc) 48.5 kB
import crypto from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { Readable, pipeline } from "node:stream"; import { promisify } from "node:util"; import yauzl from "yauzl"; import { ApiError, LokaliseApi, LokaliseApiOAuth } from "@lokalise/node-api"; import { logWithColor, logWithLevel } from "kliedz"; //#region lib/errors/LokaliseError.ts /** * Represents a custom error. */ var LokaliseError = class extends Error { /** * The error code representing the type of Lokalise API error. */ code; /** * Additional details about the error. */ details; /** * Creates a new instance of LokaliseError. * * @param message - The error message. * @param code - The error code (optional). * @param details - Optional additional details about the error. */ constructor(message, code, details) { super(message); this.code = code; if (details) this.details = details; } /** * Returns a string representation of the error, including code and details. * * @returns The formatted error message. */ toString() { let baseMessage = `LokaliseError: ${this.message}`; if (this.code) baseMessage += ` (Code: ${this.code})`; if (this.details) { const formattedDetails = Object.entries(this.details).map(([key, value]) => `${key}: ${value}`).join(", "); baseMessage += ` | Details: ${formattedDetails}`; } return baseMessage; } }; //#endregion //#region lib/services/LokaliseFileExchange.ts /** * A utility class for exchanging files with the Lokalise API. */ var LokaliseFileExchange = class LokaliseFileExchange { /** * The Lokalise API client instance. */ apiClient; /** * The ID of the project in Lokalise. */ projectId; /** * Retry parameters for API requests. */ retryParams; /** * Logger function. */ logger; /** * Log threshold (do not print messages with severity less than the specified value). */ logThreshold; /** * Default retry parameters for API requests. */ static defaultRetryParams = { maxRetries: 3, initialSleepTime: 1e3, jitterRatio: .2, rng: Math.random }; static FINISHED_STATUSES = [ "finished", "cancelled", "failed" ]; static RETRYABLE_CODES = [408, 429]; static maxConcurrentProcesses = 6; static isPendingStatus(status) { return !LokaliseFileExchange.isFinishedStatus(status); } static isFinishedStatus(status) { return status != null && LokaliseFileExchange.FINISHED_STATUSES.includes(status); } /** * Creates a new instance of LokaliseFileExchange. * * @param clientConfig - Configuration for the Lokalise SDK. * @param exchangeConfig - The configuration object for file exchange operations. * @throws {LokaliseError} If the provided configuration is invalid. */ constructor(clientConfig, { projectId, useOAuth2 = false, retryParams, logThreshold = "info", logColor = true }) { this.projectId = projectId; this.logThreshold = logThreshold; this.logger = this.chooseLogger(logColor); this.retryParams = this.buildRetryParams(retryParams); this.validateParams(); const apiConfig = this.buildLokaliseClientConfig(clientConfig, logThreshold); this.apiClient = this.createApiClient(apiConfig, useOAuth2); } /** * Executes an asynchronous operation with exponential-backoff retry logic. * * The operation is attempted multiple times if it throws a retryable * `LokaliseApiError`. Each retry waits longer than the previous one based on * exponential backoff parameters (`base`, `factor`, optional jitter). * * Behaviour: * - If the operation succeeds — its result is returned immediately. * - If it fails with a retryable error — the function waits and retries. * - If the maximum number of retries is reached — throws a `LokaliseError` * with the original error details. * - If the error is non-retryable — it is immediately wrapped into * `LokaliseError` and rethrown. * - Any non-Lokalise errors are rethrown as-is. * * @param operation - A function that performs the async action to be retried. * @returns The successful result of the operation. * @throws LokaliseError After all retries fail or on non-retryable errors. */ async withExponentialBackoff(operation) { const { maxRetries } = this.retryParams; this.logMsg("debug", `Running operation with exponential backoff; max retries: ${maxRetries}`); for (let attempt = 1; attempt <= maxRetries + 1; attempt++) try { this.logMsg("debug", `Attempt #${attempt}...`); return await operation(); } catch (error) { if (error instanceof ApiError && this.isRetryable(error)) { this.logMsg("debug", `Retryable error caught: ${error.message}`); if (attempt === maxRetries + 1) throw new LokaliseError(`Maximum retries reached: ${error.message}`, error.code, error.details); const sleepMs = this.calculateSleepMs(this.retryParams, attempt); this.logMsg("debug", `Waiting ${sleepMs}ms before retry...`); await LokaliseFileExchange.sleep(sleepMs); } else if (error instanceof ApiError) throw new LokaliseError(error.message, error.code, error.details); else throw error; } throw new LokaliseError("Unexpected error during operation.", 500); } /** * Polls the status of queued processes until they are marked as "finished" * or until the maximum wait time is exceeded. * * Uses batched polling with limited concurrency and exponential backoff-like * wait times between iterations. Performs an initial status snapshot, then * repeatedly refreshes pending processes until either: * - all of them reach a finished state, or * - the time budget (`maxWaitTime`) is exhausted. * * A final refresh is performed for any still-pending processes before returning. * * @param processes - List of queued processes to poll. * @param initialWaitTime - Initial delay (in ms) between polling iterations. * @param maxWaitTime - Maximum total time (in ms) allowed for polling. * @param concurrency - Maximum number of processes to refresh per batch. * @returns A list of processes with their latest known statuses. */ async pollProcesses(processes, initialWaitTime, maxWaitTime, concurrency = LokaliseFileExchange.maxConcurrentProcesses) { this.logMsg("debug", `Start polling processes. Total processes count: ${processes.length}`); const startTime = Date.now(); const { processMap, pendingProcessIds } = this.initializePollingState(processes); await this.runPollingLoop(processMap, pendingProcessIds, startTime, initialWaitTime, maxWaitTime, concurrency); if (pendingProcessIds.size > 0) await this.refreshRemainingProcesses(processMap, pendingProcessIds, concurrency); return Array.from(processMap.values()); } /** * Builds internal tracking structures for polling: a map of process IDs * to their last known state and a set of IDs that are still pending. * * Also logs the initial status of each process. * * @param processes - Initial list of queued processes. * @returns A map of processes keyed by ID and a set of pending process IDs. */ initializePollingState(processes) { this.logMsg("debug", "Initial processes check..."); const processMap = /* @__PURE__ */ new Map(); const pendingProcessIds = /* @__PURE__ */ new Set(); for (const p of processes) { if (p.status) this.logMsg("debug", `Process ID: ${p.process_id}, status: ${p.status}`); else this.logMsg("debug", `Process ID: ${p.process_id}, status is missing`); processMap.set(p.process_id, p); if (LokaliseFileExchange.isPendingStatus(p.status)) pendingProcessIds.add(p.process_id); } return { processMap, pendingProcessIds }; } /** * Runs the main polling loop for the given processes. * * Repeatedly fetches updated process statuses in batches while: * - there are still pending IDs, and * - the elapsed time is below the configured maximum. * * Includes a small "fast-follow" recheck when some processes have missing * status on the first iterations, and uses a growing wait time between * iterations (capped by the remaining time budget). * * @param processMap - Map of process IDs to their last known state. * @param pendingProcessIds - Set of IDs that are not finished yet. * @param startTime - Timestamp (ms) when polling started. * @param initialWaitTime - Initial delay (ms) between polling iterations. * @param maxWaitTime - Maximum total polling duration (ms). * @param concurrency - Maximum number of processes to refresh per batch. */ async runPollingLoop(processMap, pendingProcessIds, startTime, initialWaitTime, maxWaitTime, concurrency) { let waitTime = initialWaitTime; let didFastFollow = false; while (pendingProcessIds.size > 0 && Date.now() - startTime < maxWaitTime) { this.logMsg("debug", `Polling... Pending IDs: ${pendingProcessIds.size}`); if (!didFastFollow && [...processMap.values()].some((p) => p.status == null)) { this.logMsg("debug", "Fast-follow: some statuses missing, quick recheck in 200ms"); await LokaliseFileExchange.sleep(200); didFastFollow = true; } const ids = [...pendingProcessIds]; const batch = await this.fetchProcessesBatch(ids, concurrency); for (const { id, process } of batch) { if (!process) continue; processMap.set(id, process); if (LokaliseFileExchange.isFinishedStatus(process.status)) { this.logMsg("debug", `Process ${id} completed with status=${process.status}.`); pendingProcessIds.delete(id); } } if (pendingProcessIds.size === 0) { this.logMsg("debug", "Finished polling. Pending processes IDs: 0"); break; } const remaining = maxWaitTime - (Date.now() - startTime); if (remaining <= 0) { this.logMsg("debug", "Time budget exhausted, stopping polling without extra sleep."); break; } const sleepMs = Math.min(waitTime, remaining); this.logMsg("debug", `Waiting ${sleepMs}...`); await LokaliseFileExchange.sleep(sleepMs); waitTime = Math.min(waitTime * 2, Math.max(0, maxWaitTime - (Date.now() - startTime))); } } /** * Performs a final status refresh for any processes that are still marked * as pending after the main polling loop. * * This gives one last chance to capture terminal statuses right before * returning the result to the caller. * * @param processMap - Map of process IDs to their last known state. * @param pendingProcessIds - Set of IDs that are still considered pending. * @param concurrency - Maximum number of processes to refresh per batch. */ async refreshRemainingProcesses(processMap, pendingProcessIds, concurrency) { this.logMsg("debug", `Final refresh for ${pendingProcessIds.size} pending processes before return...`); const finalBatch = await this.fetchProcessesBatch([...pendingProcessIds], concurrency); for (const { id, process } of finalBatch) if (process) processMap.set(id, process); } /** * Determines whether the given Lokalise API error should trigger a retry attempt. * * An error is considered retryable if its `code` matches one of the predefined * retryable status codes. * * @param error - The `LokaliseApiError` instance to evaluate. * @returns `true` if the error is retryable, otherwise `false`. */ isRetryable(error) { return LokaliseFileExchange.RETRYABLE_CODES.includes(error.code); } /** * Logs a message using the configured logger, respecting the current log threshold. * * Wraps the raw logger call by attaching metadata such as: * - `level` — severity of the log entry, * - `threshold` — active log level threshold used to filter messages, * - `withTimestamp` — instructs the logger to prepend a timestamp. * * All variadic `args` are forwarded directly to the logger. * * @param level - Log level of the message being emitted. * @param args - Additional values to pass to the logger. */ logMsg(level, ...args) { this.logger({ level, threshold: this.logThreshold, withTimestamp: true }, ...args); } /** * Fetches the most recent state of a queued process from the Lokalise API. * * Sends a GET request for the process identified by `processId` and logs * both the request and the received status. Used during polling to refresh * the status of long-running async operations. * * @param processId - The unique identifier of the queued process to retrieve. * @returns A promise resolving to the updated `QueuedProcess` object. */ async getUpdatedProcess(processId) { this.logMsg("debug", `Requesting update for process ID: ${processId}`); const updatedProcess = await this.apiClient.queuedProcesses().get(processId, { project_id: this.projectId }); if (updatedProcess.status) this.logMsg("debug", `Process ID: ${updatedProcess.process_id}, status: ${updatedProcess.status}`); else this.logMsg("debug", `Process ID: ${updatedProcess.process_id}, status is missing`); return updatedProcess; } /** * Validates essential client configuration parameters before any operations run. * * Ensures that: * - `projectId` is present and is a non-empty string, * - retry settings (`maxRetries`, `initialSleepTime`, `jitterRatio`) * fall within acceptable ranges. * * Throws a `LokaliseError` if any configuration parameter is missing, * malformed, or outside allowed bounds. */ validateParams() { if (!this.projectId || typeof this.projectId !== "string") throw new LokaliseError("Invalid or missing Project ID."); const { maxRetries, initialSleepTime, jitterRatio } = this.retryParams; if (maxRetries < 0) throw new LokaliseError("maxRetries must be greater than or equal to zero."); if (initialSleepTime <= 0) throw new LokaliseError("initialSleepTime must be a positive value."); if (jitterRatio < 0 || jitterRatio > 1) throw new LokaliseError("jitterRatio must be between 0 and 1."); } /** * Executes asynchronous work over a list of items with a fixed concurrency limit. * * Spawns up to `limit` parallel worker loops. Each loop pulls the next * unprocessed item index in a thread-safe manner (via shared counter `i`), * runs the provided async `worker` function for that item, and stores the * resulting value in the corresponding position of the `results` array. * * Processing stops when all items have been consumed. If any worker throws, * the error propagates and the whole operation rejects. * * @param items - The list of items to process. * @param limit - Maximum number of concurrent async operations. * @param worker - Async handler executed for each item. * @returns A promise resolving to an array of results, preserving input order. */ async runWithConcurrencyLimit(items, limit, worker) { const results = new Array(items.length); let i = 0; const workers = new Array(Math.min(limit, items.length)).fill(null).map(async () => { while (true) { const idx = i++; if (idx >= items.length) break; const item = items[idx]; if (item === void 0) throw new Error(`Missing item at index ${idx}`); results[idx] = await worker(item, idx); } }); await Promise.all(workers); return results; } /** * Fetches updated process states for a list of process IDs in parallel, * respecting a maximum concurrency limit. * * Each process ID is resolved via `getUpdatedProcess()`. If the fetch * succeeds, the returned object includes both `id` and the updated * `process`. If an error occurs, a warning is logged and the result * contains only the `id`, allowing polling to continue without failing * the entire batch. * * Internally uses `runWithConcurrencyLimit` to enforce controlled parallelism. * * @param processIds - The list of queued process IDs to refresh. * @param concurrency - Maximum number of simultaneous requests. * @returns A list of objects mapping each ID to its latest fetched state * (or `undefined` if the fetch failed). */ async fetchProcessesBatch(processIds, concurrency = LokaliseFileExchange.maxConcurrentProcesses) { return this.runWithConcurrencyLimit(processIds, concurrency, async (id) => { try { return { id, process: await this.getUpdatedProcess(id) }; } catch (error) { this.logMsg("warn", `Failed to fetch process ${id}:`, error); return { id }; } }); } /** * Delays execution for a given duration. * * Creates a Promise that resolves after the specified number of milliseconds, * allowing async workflows to pause without blocking the event loop. * * @param ms - Number of milliseconds to wait. * @returns A promise that resolves after the delay. */ static sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } /** * Computes the exponential-backoff delay for a retry attempt, * optionally adding jitter to avoid synchronized retries. * * @param retryParams - Backoff settings (initial delay, jitter, RNG). * @param attempt - Retry attempt number (1-based). * @returns Calculated sleep time in milliseconds. */ calculateSleepMs(retryParams, attempt) { const { initialSleepTime, jitterRatio, rng } = retryParams; const base = initialSleepTime * 2 ** (attempt - 1); const maxJitter = Math.floor(base * jitterRatio); return base + (maxJitter > 0 ? Math.floor(rng() * maxJitter) : 0); } /** * Builds the final Lokalise client configuration, * enabling silent mode when the log threshold is `"silent"`. * * @param clientConfig - Base client parameters. * @param logThreshold - Active logging threshold. * @returns The adjusted client configuration. */ buildLokaliseClientConfig(clientConfig, logThreshold) { if (logThreshold === "silent") return { ...clientConfig, silent: true }; return { ...clientConfig }; } /** * Creates the appropriate Lokalise API client instance, * choosing between OAuth2 and token-based authentication. * * @param lokaliseApiConfig - Configuration passed to the client. * @param useOAuth2 - Whether OAuth2 authentication should be used. * @returns A Lokalise API client instance. */ createApiClient(lokaliseApiConfig, useOAuth2) { if (useOAuth2) { this.logMsg("debug", "Using OAuth 2 Lokalise API client"); return new LokaliseApiOAuth(lokaliseApiConfig); } this.logMsg("debug", "Using regular (token-based) Lokalise API client"); return new LokaliseApi(lokaliseApiConfig); } /** * Merges user-provided retry settings with default retry parameters. * * @param retryParams - Optional overrides. * @returns Fully resolved retry configuration. */ buildRetryParams(retryParams) { return { ...LokaliseFileExchange.defaultRetryParams, ...retryParams }; } /** * Selects the logger implementation based on whether color output is enabled. * * @param logColor - If true, uses the colorized logger. * @returns The chosen log function. */ chooseLogger(logColor) { return logColor ? logWithColor : logWithLevel; } }; //#endregion //#region lib/services/LokaliseDownload.ts /** * Handles downloading and extracting translation files from Lokalise. */ var LokaliseDownload = class LokaliseDownload extends LokaliseFileExchange { static defaultProcessParams = { asyncDownload: false, pollInitialWaitTime: 1e3, pollMaximumWaitTime: 12e4, bundleDownloadTimeout: 0 }; streamPipeline = promisify(pipeline); /** * Downloads translations from Lokalise, optionally using async polling, and extracts them to disk. * * @param downloadTranslationParams - Full configuration for the download process, extraction destination, and optional polling or timeout settings. * @throws {LokaliseError} If the download, polling, or extraction fails. */ async downloadTranslations({ downloadFileParams, extractParams = {}, processDownloadFileParams }) { this.logMsg("debug", "Downloading translations from Lokalise..."); const processParams = this.buildProcessParams(processDownloadFileParams); const translationsBundleURL = await this.fetchTranslationBundleURL(downloadFileParams, processParams); const zipFilePath = await this.downloadZip(translationsBundleURL, processParams.bundleDownloadTimeout); await this.processZip(zipFilePath, path.resolve(extractParams.outputDir ?? "./")); } /** * Unpacks a ZIP file into the specified directory. * * @param zipFilePath - Path to the ZIP file. * @param outputDir - Directory to extract the files into. * @throws {LokaliseError} If extraction fails or malicious paths are detected. */ async unpackZip(zipFilePath, outputDir) { return new Promise((resolve, reject) => { yauzl.open(zipFilePath, { lazyEntries: true }, (err, zipfile) => { if (err) return reject(new LokaliseError(`Failed to open ZIP file at ${zipFilePath}: ${err.message}`)); zipfile.readEntry(); zipfile.on("entry", (entry) => { this.handleZipEntry(entry, zipfile, outputDir).then(() => zipfile.readEntry()).catch(reject); }); zipfile.on("end", resolve); zipfile.on("error", reject); }); }); } /** * Downloads a ZIP file from the given URL and stores it as a temporary file. * * Performs URL validation, optional timeout handling, fetch request execution, * response integrity checks, and writes the ZIP stream to disk. * * @param url - Direct URL to the ZIP bundle provided by Lokalise. * @param downloadTimeout - Optional timeout (in ms) for the HTTP request. `0` disables timeouts. * @returns Absolute path to the temporary ZIP file on disk. */ async downloadZip(url, downloadTimeout = 0) { this.logMsg("debug", "Downloading translation bundle..."); const bundleURL = this.assertHttpUrl(url); const tempZipPath = this.buildTempZipPath(); const signal = this.buildAbortSignal(downloadTimeout); const response = await this.fetchZipResponse(bundleURL, signal, downloadTimeout); const body = this.getZipResponseBody(response, url); await this.writeZipToDisk(body, tempZipPath); return tempZipPath; } /** * Builds a unique temporary file path for storing the downloaded ZIP bundle. * * Uses a UUID when available or falls back to a combination of PID, timestamp, and random bytes. * * @returns A full path to a temporary ZIP file in the OS temp directory. */ buildTempZipPath() { const uid = crypto.randomUUID?.() ?? `${process.pid}-${Date.now()}-${crypto.randomBytes(8).toString("hex")}`; return path.join(os.tmpdir(), `lokalise-${uid}.zip`); } /** * Creates an optional AbortSignal for enforcing request timeouts. * * Returns `undefined` when no timeout is configured, disabling abort handling. * * @param downloadTimeout - Timeout in milliseconds. `0` or negative disables the signal. * @returns An AbortSignal if timeout is enabled, otherwise `undefined`. */ buildAbortSignal(downloadTimeout) { if (downloadTimeout <= 0) return; return AbortSignal.timeout(downloadTimeout); } /** * Executes a fetch request for the ZIP bundle URL with optional timeout handling. * * Wraps network failures, timeouts, and unexpected fetch errors into `LokaliseError` * so higher-level logic receives consistent exceptions. * * @param bundleURL - Parsed URL pointing to the ZIP file. * @param signal - Optional `AbortSignal` used to enforce request timeouts. * @param downloadTimeout - Timeout duration (ms) used for error messaging. * @returns The raw `Response` object returned by `fetch` if the request succeeds. */ async fetchZipResponse(bundleURL, signal, downloadTimeout) { try { return await fetch(bundleURL, signal ? { signal } : {}); } catch (err) { if (err instanceof Error) { if (err.name === "TimeoutError") throw new LokaliseError(`Request timed out after ${downloadTimeout}ms`, 408, { reason: "timeout" }); throw new LokaliseError(err.message, 500, { reason: "network or fetch error" }); } /* v8 ignore start */ throw new LokaliseError("An unknown error occurred. This might indicate a bug.", 500, { reason: String(err) }); } } /** * Validates and extracts the readable body stream from a fetch response. * * Ensures the response is OK and has a non-null body before returning it. * * @param response - The HTTP response returned by `fetch`. * @param originalUrl - Original URL used for error diagnostics. * @returns A web ReadableStream of the ZIP file contents. * @throws {LokaliseError} If the response is not OK or body is missing. */ getZipResponseBody(response, originalUrl) { if (!response.ok) throw new LokaliseError(`Failed to download ZIP file: ${response.statusText} (${response.status})`); const body = response.body; if (!body) throw new LokaliseError(`Response body is null. Cannot download ZIP file from URL: ${originalUrl}`); return body; } /** * Streams the ZIP response body to a temporary file on disk. * * Cleans up the temporary file if the streaming pipeline fails. * * @param body - Web readable stream of the ZIP content. * @param tempZipPath - Path where the ZIP should be written. * @returns A promise that resolves once the file is fully written. * @throws {Error} Re-throws any pipeline errors after attempting cleanup. */ async writeZipToDisk(body, tempZipPath) { try { const nodeReadable = Readable.fromWeb(body); await this.streamPipeline(nodeReadable, fs.createWriteStream(tempZipPath)); } catch (e) { try { await fs.promises.unlink(tempZipPath); } catch { this.logMsg("debug", `Stream pipeline failed and unable to remove temp path ${tempZipPath}`); } throw e; } } /** * Retrieves a translation bundle from Lokalise with retries and exponential backoff. * * @param downloadFileParams - Parameters for Lokalise API file download. * @returns The downloaded bundle metadata. * @throws {LokaliseError} If retries are exhausted or an API error occurs. */ async getTranslationsBundle(downloadFileParams) { return this.withExponentialBackoff(() => this.apiClient.files().download(this.projectId, downloadFileParams)); } /** * Retrieves a translation bundle from Lokalise with retries and exponential backoff. * * @param downloadFileParams - Parameters for Lokalise API file download. * @returns The queued process. * @throws {LokaliseError} If retries are exhausted or an API error occurs. */ async getTranslationsBundleAsync(downloadFileParams) { return this.withExponentialBackoff(() => this.apiClient.files().async_download(this.projectId, downloadFileParams)); } /** * Extracts a single entry from a ZIP archive to the specified output directory. * * Creates necessary directories and streams the file content to disk. * * @param entry - The ZIP entry to extract. * @param zipfile - The open ZIP file instance. * @param outputDir - The directory where the entry should be written. * @returns A promise that resolves when the entry is fully written. */ async handleZipEntry(entry, zipfile, outputDir) { const fullPath = this.processZipEntryPath(outputDir, entry.fileName); if (entry.fileName.endsWith("/")) { await this.createDir(fullPath); return; } await this.createDir(path.dirname(fullPath)); return new Promise((response, reject) => { zipfile.openReadStream(entry, (readErr, readStream) => { if (readErr || !readStream) return reject(new LokaliseError(`Failed to read ZIP entry: ${entry.fileName}`)); const writeStream = fs.createWriteStream(fullPath); readStream.pipe(writeStream); writeStream.on("finish", response); writeStream.on("error", reject); readStream.on("error", reject); }); }); } /** * Creates a directory and all necessary parent directories. * * @param dir - The directory path to create. * @returns A promise that resolves when the directory is created. */ async createDir(dir) { await fs.promises.mkdir(dir, { recursive: true }); } /** * Resolves and validates the full output path for a ZIP entry. * * Prevents path traversal attacks by ensuring the resolved path stays within the output directory. * * @param outputDir - The base output directory. * @param entryFilename - The filename of the ZIP entry. * @returns The absolute and safe path to write the entry. * @throws {LokaliseError} If the entry path is detected as malicious. */ processZipEntryPath(outputDir, entryFilename) { const fullPath = path.resolve(outputDir, entryFilename); const relative = path.relative(outputDir, fullPath); if (relative.startsWith("..") || path.isAbsolute(relative)) throw new LokaliseError(`Malicious ZIP entry detected: ${entryFilename}`); return fullPath; } /** * Parses and validates a URL string, ensuring it uses HTTP or HTTPS protocol. * * @param value - The URL string to validate. * @returns A parsed `URL` object if valid. * @throws {LokaliseError} If the URL is invalid or uses an unsupported protocol. */ assertHttpUrl(value) { let parsed; try { parsed = new URL(value); } catch { throw new LokaliseError(`Invalid URL: ${value}`); } if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new LokaliseError(`Unsupported protocol in URL: ${value}`); return parsed; } /** * Builds effective process parameters for the download workflow. * * Merges caller-provided overrides with the default settings. * * @param overrides - Partial process configuration to override defaults. * @returns Fully resolved process parameters. */ buildProcessParams(overrides) { return { ...LokaliseDownload.defaultProcessParams, ...overrides }; } /** * Unpacks the downloaded ZIP archive into the target directory and * removes the temporary archive file afterwards. * * Logs progress and always attempts to delete the temporary file. * * @param zipFilePath - Path to the temporary ZIP file. * @param unpackTo - Destination directory for extracted files. */ async processZip(zipFilePath, unpackTo) { this.logMsg("debug", `Unpacking translations from ${zipFilePath} to ${unpackTo}`); try { await this.unpackZip(zipFilePath, unpackTo); this.logMsg("debug", "Translations unpacked!"); this.logMsg("debug", "Download successful!"); } finally { this.logMsg("debug", `Removing temp archive from ${zipFilePath}`); await fs.promises.unlink(zipFilePath); } } /** * Fetches the direct bundle URL in synchronous (non-async) mode. * * Calls the standard download endpoint without polling. * * @param downloadFileParams - Parameters for Lokalise API file download. * @returns Direct bundle URL returned by Lokalise. */ async fetchBundleURLSync(downloadFileParams) { this.logMsg("debug", "Async download mode disabled."); return (await this.getTranslationsBundle(downloadFileParams)).bundle_url; } /** * Polls an async download process until it completes or the maximum wait time is reached. * * Validates the final status and throws if the process did not finish properly. * * @param downloadProcess - The initially queued async process. * @param initialWait - Initial interval in ms before the first poll. * @param maxWait - Maximum total wait time in ms. * @returns The completed process object. * @throws {LokaliseError} If the process is not found or does not finish successfully. */ async pollAsyncDownload(downloadProcess, initialWait, maxWait) { this.logMsg("debug", `Waiting for download process ID ${downloadProcess.process_id} to complete...`); this.logMsg("debug", `Effective waits: initial=${initialWait}ms, max=${maxWait}ms`); const completedProcess = (await this.pollProcesses([downloadProcess], initialWait, maxWait)).find((p) => p.process_id === downloadProcess.process_id); if (!completedProcess) throw new LokaliseError(`Process ${downloadProcess.process_id} not found after polling`, 500); if (!LokaliseFileExchange.isFinishedStatus(completedProcess.status)) throw new LokaliseError(`Download process did not finish within ${maxWait}ms${completedProcess.status ? ` (last status=${completedProcess.status})` : " (status missing)"}`, 504); return completedProcess; } /** * Resolves the bundle download URL using either async or sync strategy. * * Delegates to `fetchBundleURLAsync` or `fetchBundleURLSync` * based on the `asyncDownload` flag. * * @param downloadFileParams - Parameters for Lokalise API file download. * @param processParams - Effective process parameters controlling async behavior and polling. * @returns Direct bundle URL to download. */ fetchTranslationBundleURL(downloadFileParams, processParams) { return processParams.asyncDownload ? this.fetchBundleURLAsync(downloadFileParams, processParams) : this.fetchBundleURLSync(downloadFileParams); } /** * Extracts and verifies the download URL from a finished async process. * * Ensures `details.download_url` is present and is a string. * * @param completedProcess - Process object with status `finished`. * @returns Valid download URL string. * @throws {LokaliseError} If the URL is missing or invalid. */ handleFinishedAsyncProcess(completedProcess) { const details = completedProcess.details; const url = details?.download_url; if (!url || typeof url !== "string") { this.logMsg("warn", "Process finished but details.download_url is missing or invalid", details); throw new LokaliseError("Lokalise returned finished process without a valid download_url", 502); } return url; } /** * Handles a failed or cancelled async process by throwing an error with context. * * Includes the process status and optional message from Lokalise. * * @param completedProcess - Process object with status `failed` or `cancelled`. * @throws {LokaliseError} Always throws, as the process did not succeed. */ handleFailedAsyncProcess(completedProcess) { const msg = completedProcess.message?.trim(); throw new LokaliseError(`Process ${completedProcess.process_id} ended with status=${completedProcess.status}` + (msg ? `: ${msg}` : ""), 502); } /** * Handles an unexpected async process outcome when it did not finish in time. * * Logs a warning and throws an error indicating that finalization took too long. * * @param completedProcess - Process object with unexpected status. * @param maxWait - Effective maximum wait time used during polling. * @throws {LokaliseError} Always throws to signal an unexpected async outcome. */ handleUnexpectedAsyncProcess(completedProcess, maxWait) { this.logMsg("warn", `Process ended with status=${completedProcess.status}`); throw new LokaliseError(`Download process took too long to finalize; effective=${maxWait}ms`, 500); } /** * Runs the async download flow: queues the download, polls its status, * and returns the final bundle URL once the process completes. * * Handles finished, failed/cancelled, and unexpected statuses separately. * * @param downloadFileParams - Parameters for Lokalise API async file download. * @param processParams - Effective process parameters controlling polling behavior. * @returns Direct URL to the generated ZIP bundle. * @throws {LokaliseError} If the process fails, is cancelled, or does not finalize properly. */ async fetchBundleURLAsync(downloadFileParams, processParams) { this.logMsg("debug", "Async download mode enabled."); const downloadProcess = await this.getTranslationsBundleAsync(downloadFileParams); const { pollInitialWaitTime, pollMaximumWaitTime } = processParams; const completedProcess = await this.pollAsyncDownload(downloadProcess, pollInitialWaitTime, pollMaximumWaitTime); this.logMsg("debug", `Download process status is ${completedProcess.status}`); if (completedProcess.status === "finished") return this.handleFinishedAsyncProcess(completedProcess); if (completedProcess.status === "failed" || completedProcess.status === "cancelled") this.handleFailedAsyncProcess(completedProcess); this.handleUnexpectedAsyncProcess(completedProcess, pollMaximumWaitTime); } }; //#endregion //#region lib/services/LokaliseUpload.ts /** * Handles uploading translation files to Lokalise. */ var LokaliseUpload = class LokaliseUpload extends LokaliseFileExchange { static defaultPollingParams = { pollStatuses: false, pollInitialWaitTime: 1e3, pollMaximumWaitTime: 12e4 }; /** * Collects files, uploads them to Lokalise, and optionally polls for process completion, returning both processes and errors. * * @param {UploadTranslationParams} uploadTranslationParams - Parameters for collecting and uploading files. * @returns {Promise<{ processes: QueuedProcess[]; errors: FileUploadError[] }>} A promise resolving with successful processes and upload errors. */ async uploadTranslations({ uploadFileParams, collectFileParams, processUploadFileParams } = {}) { this.logMsg("debug", "Uploading translations to Lokalise..."); const { pollStatuses, pollInitialWaitTime, pollMaximumWaitTime } = { ...LokaliseUpload.defaultPollingParams, ...processUploadFileParams }; this.logMsg("debug", "Collecting files to upload..."); const collectedFiles = await this.collectFiles(collectFileParams); this.logMsg("debug", "Collected files:", collectedFiles); this.logMsg("debug", "Performing parallel upload..."); const { processes, errors } = await this.parallelUpload(collectedFiles, uploadFileParams, processUploadFileParams); let completedProcesses = processes; this.logMsg("debug", "File uploading queued! IDs:", completedProcesses.map((p) => p.process_id)); if (pollStatuses) { this.logMsg("debug", "Polling queued processes..."); completedProcesses = await this.pollProcesses(processes, pollInitialWaitTime, pollMaximumWaitTime); this.logMsg("debug", "Polling completed!"); } this.logMsg("debug", "Upload successful!"); return { processes: completedProcesses, errors }; } /** * Collects files from the filesystem based on the given parameters. * * @param {CollectFileParams} collectFileParams - Parameters for file collection, including directories, extensions, and patterns. * @returns {Promise<string[]>} A promise resolving with the list of collected file paths. */ async collectFiles({ inputDirs = ["./locales"], extensions = [".*"], excludePatterns = [], recursive = true, fileNamePattern = ".*" } = {}) { const queue = this.makeQueue(inputDirs); const normalizedExtensions = this.normalizeExtensions(extensions); const fileNameRegex = this.makeFilenameRegexp(fileNamePattern); const excludeRegexes = this.makeExcludeRegExes(excludePatterns); return (await this.processCollectionQueue(queue, normalizedExtensions, fileNameRegex, excludeRegexes, recursive)).sort(); } /** * Uploads a single file to Lokalise. * * @param {UploadFileParams} uploadParams - Parameters for uploading the file. * @returns {Promise<QueuedProcess>} A promise resolving with the upload process details. */ async uploadSingleFile(uploadParams) { return this.withExponentialBackoff(() => this.apiClient.files().upload(this.projectId, uploadParams)); } /** * Processes a file to prepare it for upload, converting it to base64 and extracting its language code. * * @param file - The absolute path to the file. * @param projectRoot - The root directory of the project. * @param processParams - Optional processing settings including inferers. * @returns A promise resolving with the processed file details, including base64 content, relative path, and language code. */ async processFile(file, projectRoot, processParams) { const relativePath = await this.inferRelativePath(file, projectRoot, processParams); const languageCode = await this.inferLanguageCode(file, relativePath, processParams); return { data: await this.readFileAsBase64(file), filename: relativePath, lang_iso: languageCode }; } /** * Infers the relative path for an uploaded file. * * Tries a custom `filenameInferer` first; if it fails or returns empty/whitespace, * falls back to a POSIX-style relative path based on the project root. * * @param file - Absolute path to the source file. * @param projectRoot - Root directory of the project. * @param processParams - Optional processing settings including filename inferer. * @returns A promise resolving with the inferred relative path. */ async inferRelativePath(file, projectRoot, processParams) { try { const fromInferer = processParams?.filenameInferer ? await processParams.filenameInferer(file) : ""; if (!fromInferer.trim()) throw new Error("Invalid filename: empty or only whitespace"); return fromInferer; } catch { return path.posix.relative(this.toPosixPath(projectRoot), this.toPosixPath(file)); } } /** * Infers the language code for an uploaded file. * * Tries a custom `languageInferer` first; if it fails or returns empty/whitespace, * falls back to extracting the language code from the filename before the last extension. * * Example: "en.default.json" → "default" * * @param file - Absolute path to the source file. * @param relativePath - Effective relative path of the file (used for fallback parsing). * @param processParams - Optional processing settings including language inferer. * @returns A promise resolving with the inferred language code. */ async inferLanguageCode(file, relativePath, processParams) { try { const fromInferer = processParams?.languageInferer ? await processParams.languageInferer(file) : ""; if (!fromInferer.trim()) throw new Error("Invalid language code: empty or only whitespace"); return fromInferer; } catch { return path.basename(relativePath).split(".").slice(-2, -1)[0] ?? "unknown"; } } /** * Reads a file from disk and returns its content encoded as base64. * * @param file - Absolute path to the source file. * @returns A promise resolving with the file content encoded in base64. */ async readFileAsBase64(file) { return (await fs.promises.readFile(file)).toString("base64"); } /** * Uploads files in parallel with a limit on the number of concurrent uploads. * * @param {string[]} files - List of file paths to upload. * @param {Partial<UploadFileParams>} baseUploadFileParams - Base parameters for uploads. * @param {ProcessUploadFileParams} [processParams] - Optional processing settings including inferers. * @returns {Promise<{ processes: QueuedProcess[]; errors: FileUploadError[] }>} A promise resolving with successful processes and upload errors. */ async parallelUpload(files, baseUploadFileParams = {}, processParams) { const projectRoot = process.cwd(); const queuedProcesses = []; const errors = []; await this.runWithConcurrencyLimit(files, LokaliseUpload.maxConcurrentProcesses, async (file) => { try { const processedFileParams = await this.processFile(file, projectRoot, processParams); const queued = await this.uploadSingleFile({ ...baseUploadFileParams, ...processedFileParams }); queuedProcesses.push(queued); } catch (error) { errors.push({ file, error }); } }); return { processes: queuedProcesses, errors }; } /** * Normalizes an array of file extensions by ensuring each starts with a dot and is lowercase. * * @param extensions - The list of file extensions to normalize. * @returns A new array with normalized file extensions. */ normalizeExtensions(extensions) { return extensions.map((ext) => (ext.startsWith(".") ? ext : `.${ext}`).toLowerCase()); } /** * Determines whether a file should be collected based on its extension and name pattern. * * @param entry - The directory entry to evaluate. * @param normalizedExtensions - List of allowed file extensions. * @param fileNameRegex - Regular expression to match valid filenames. * @returns `true` if the file matches both extension and name pattern, otherwise `false`. */ shouldCollectFile(entry, normalizedExtensions, fileNameRegex) { const fileExt = path.extname(entry.name).toLowerCase(); const matchesExtension = normalizedExtensions.includes(".*") || normalizedExtensions.includes(fileExt); const matchesFilenamePattern = fileNameRegex.test(entry.name); return matchesExtension && matchesFilenamePattern; } /** * Creates a regular expression from a given pattern string or RegExp. * * @param fileNamePattern - The filename pattern to convert into a RegExp. * @returns A valid RegExp object. * @throws {Error} If the pattern string is invalid and cannot be compiled. */ makeFilenameRegexp(fileNamePattern) { try { return new RegExp(fileNamePattern); } catch { throw new Error(`Invalid fileNamePattern: ${fileNamePattern}`); } } /** * Converts an array of exclude patterns into an array of RegExp objects. * * @param excludePatterns - An array of strings or regular expressions to exclude. * @returns An array of compiled RegExp objects. * @throws {Error} If any pattern is invalid and cannot be compiled. */ makeExcludeRegExes(excludePatterns) { if (excludePatterns.length === 0) return []; try { return excludePatterns.map((pattern) => new RegExp(pattern)); } catch (err) { const msg = err instanceof Error ? err.message : String(err); throw new Error(`Invalid excludePatterns: ${msg}`); } } /** * Safely reads the contents of a directory, returning an empty array if access fails. * * Logs a warning if the directory cannot be read (e.g. due to permissions or non-existence). * * @param dir - The directory path to read. * @returns A promise that resolves to an array of directory entries, or an empty array on failure. */ async safeReadDir(dir) { try { return await fs.promises.readdir(dir, { withFileTypes: true }); } catch { this.logMsg("warn", `Skipping inaccessible directory: ${dir}...`); return []; } } /** * Checks if a file path matches any of the provided exclusion patterns. * * @param filePath - The path of the file to check. * @param excludeRegexes - An array of RegExp patterns to test against. * @returns `true` if the file path matches any exclude pattern, otherwise `false`. */ shouldExclude(filePath, rx) { const posix = this.toPosixPath(filePath); return rx.some((r) => r.test(filePath) || r.test(posix)); } /** * Creates a queue of absolute paths from the provided input directories. * * @param inputDirs - An array of input directory paths (relative or absolute). * @returns An array of resolved absolute directory paths. */ makeQueue(inputDirs) { return [...inputDirs.map((dir) => path.resolve(dir))]; } /** * Processes a queue of directories to collect files matching given criteria. * * Recursively reads directories (if enabled), filters files by extension, * filename pattern, and exclusion rules, and collects matching file paths. * * @param queue - The list of directories to process. * @param exts - Allowed file extensions (normalized). * @param nameRx - Regular expression to match valid filenames. * @param excludeRx - Array of exclusion patterns. * @param recursive - Whether to traverse subdirectories. * @returns A promise that resolves to an array of matched file paths. */ async processCollectionQueue(queue, exts, nameRx, excludeRx, recursive) { const found = []; while (queue.length) { const dir = queue.shift(); /* v8 ignore start */ if (!dir) { this.logMsg("debug", `collectFiles: received falsy dir entry (${String(dir)}). This is unexpected and might indicate a bug.`); continue; } /* v8 ignore stop */ const entries = await this.safeReadDir(dir); for (const entry of entries) { const fullPath = path.resolve(dir, entry.name); this.handleEntry(entry, fullPath, queue, found, { exts, nameRx, excludeRx, recursive }); } } return found; } /** * Handles a single directory entry during file collection. * * Applies exclusion rules, optionally queues directories for recursion, * and collects files that match the specified extension and filename pattern. * * @param entry - The directory entry to handle. * @param fullPath - The absolute path to the entry. * @param queue - The processing queue for directories. * @param found - The list to store matched file paths. * @param opts - Options including extensions, name pattern, exclusions, and recursion flag. */ handleEntry(entry, fullPath, queue, found, opts) { if (this.shouldExclude(fullPath, opts.excludeRx)) return; if (entry.isDirectory()) { if (opts.recursive) queue.push(fullPath); return; } if (entry.isFile() && this.shouldCollectFile(entry, opts.exts, opts.nameRx)) found.push(fullPath); } /** * Normalizes a filesystem path to POSIX format. * * Replaces platform-specific separators (e.g. `\` on Windows) * with POSIX-style `/` to ensure consistent path handling * across different operating systems. * * @param p - Original filesystem path. * @returns The same path but with POSIX separators. */ toPosixPath(p) { return p.split(path.sep).join(path.posix.sep); } }; //#endregion export { LokaliseDownload, LokaliseError, LokaliseUpload }; //# sourceMappingURL=index.mjs.map