@crowdin/app-project-module
Version:
Module that generates for you all common endpoints for serving standalone Crowdin App
376 lines (375 loc) • 20.4 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.filterSyncedData = exports.updateSyncedData = exports.filterFilesByPath = exports.filterFilesByPatterns = exports.getParentFiles = exports.getParentPaths = exports.buildPath = exports.prepareSyncFiles = exports.uniqueFileLanguages = exports.filterLanguages = exports.getExcludedTargetLanguages = exports.markUnsyncedFiles = exports.isExtendedResultType = exports.expandFilesTreeWithPagination = exports.attachFileStatus = exports.expandFilesTree = exports.skipFilesByRegex = void 0;
const types_1 = require("../types");
const types_2 = require("./types");
const storage_1 = require("../../../storage");
const util_1 = require("../../../util");
const minimatch_1 = require("minimatch");
const lodash_uniqby_1 = __importDefault(require("lodash.uniqby"));
function skipFilesByRegex(files, skipIntegrationNodes) {
if (!Array.isArray(files)) {
return [];
}
if (skipIntegrationNodes) {
files = files.filter((file) => file);
if (skipIntegrationNodes.fileNamePattern) {
const regex = new RegExp(skipIntegrationNodes.fileNamePattern);
files = files.filter((file) => !('type' in file) || !regex.test(file.name));
}
if (skipIntegrationNodes.folderNamePattern) {
const regex = new RegExp(skipIntegrationNodes.folderNamePattern);
files = files.filter((file) => 'type' in file || !regex.test(file.name));
}
}
return files;
}
exports.skipFilesByRegex = skipFilesByRegex;
function expandFilesTree(nodes, req, integration, job) {
var _a, _b, _c, _d, _e, _f, _g, _h, _j;
return __awaiter(this, void 0, void 0, function* () {
if (job && types_2.JobStatus.CANCELED === ((_a = (yield job.get())) === null || _a === void 0 ? void 0 : _a.status)) {
throw new Error('Job canceled');
}
const files = nodes.filter((file) => file.nodeType === undefined || file.nodeType === '1');
const folders = nodes.filter((folder) => folder.nodeType === '0' && !nodes.find((node) => node.parentId === folder.id));
for (const { id } of folders) {
let integrationData = [];
try {
integrationData = yield integration.getIntegrationFiles(req.integrationCredentials, req.integrationSettings, id);
}
catch (e) {
if (job) {
const currentJob = yield job.get();
const errorMessage = e instanceof Error ? e.message : String(e);
const currentErrors = (currentJob === null || currentJob === void 0 ? void 0 : currentJob.errors) ? JSON.parse(currentJob.errors) : [];
const newErrors = [...currentErrors, errorMessage];
yield job.update({ errors: newErrors });
}
continue;
}
const integrationTreeItems = isExtendedResultType(integrationData)
? integrationData.data
: integrationData;
const checkNodes = integrationTreeItems
.filter((item) => item.id !== id)
.map((item) => (Object.assign(Object.assign({}, item), { nodeType: item.nodeType || ('type' in item ? '1' : '0') })));
const expandedResult = yield expandFilesTree(checkNodes, req, integration, job);
files.push(...expandedResult);
}
const needsFileStatus = ((_c = (_b = integration.filtering) === null || _b === void 0 ? void 0 : _b.integrationFileStatus) === null || _c === void 0 ? void 0 : _c.isNew) ||
((_e = (_d = integration.filtering) === null || _d === void 0 ? void 0 : _d.integrationFileStatus) === null || _e === void 0 ? void 0 : _e.isUpdated) ||
((_g = (_f = integration.filtering) === null || _f === void 0 ? void 0 : _f.integrationFileStatus) === null || _g === void 0 ? void 0 : _g.notSynced) ||
((_j = (_h = integration.filtering) === null || _h === void 0 ? void 0 : _h.integrationFileStatus) === null || _j === void 0 ? void 0 : _j.synced) ||
integration.forcePushSources === true;
if (needsFileStatus) {
const { crowdinId, clientId } = req.crowdinContext;
return yield attachFileStatus(files, clientId, crowdinId, integration);
}
return files;
});
}
exports.expandFilesTree = expandFilesTree;
function attachFileStatus(files, clientId, crowdinId, integration) {
var _a, _b, _c, _d, _e, _f, _g, _h;
return __awaiter(this, void 0, void 0, function* () {
const needsFileStatus = ((_b = (_a = integration.filtering) === null || _a === void 0 ? void 0 : _a.integrationFileStatus) === null || _b === void 0 ? void 0 : _b.isNew) ||
((_d = (_c = integration.filtering) === null || _c === void 0 ? void 0 : _c.integrationFileStatus) === null || _d === void 0 ? void 0 : _d.isUpdated) ||
((_f = (_e = integration.filtering) === null || _e === void 0 ? void 0 : _e.integrationFileStatus) === null || _f === void 0 ? void 0 : _f.notSynced) ||
((_h = (_g = integration.filtering) === null || _g === void 0 ? void 0 : _g.integrationFileStatus) === null || _h === void 0 ? void 0 : _h.synced) ||
integration.forcePushSources === true;
if (!needsFileStatus) {
return files;
}
const syncedData = yield (0, storage_1.getStorage)().getSyncedData(clientId, crowdinId, types_1.Provider.INTEGRATION);
const syncedFiles = (syncedData === null || syncedData === void 0 ? void 0 : syncedData.files) ? JSON.parse(syncedData.files) : [];
const lastSyncTimestamp = (syncedData === null || syncedData === void 0 ? void 0 : syncedData.updatedAt) ? Number(syncedData.updatedAt) : null;
return files.map((file) => {
var _a, _b, _c, _d, _e, _f, _g, _h;
if (!file.type) {
return file;
}
const syncedFile = syncedFiles.find((syncedItem) => syncedItem.id === file.id);
const notSynced = ((_b = (_a = integration.filtering) === null || _a === void 0 ? void 0 : _a.integrationFileStatus) === null || _b === void 0 ? void 0 : _b.notSynced) === true ? !syncedFile : false;
const synced = ((_d = (_c = integration.filtering) === null || _c === void 0 ? void 0 : _c.integrationFileStatus) === null || _d === void 0 ? void 0 : _d.synced) === true ? !!syncedFile : false;
const fileSyncTimestamp = (syncedFile === null || syncedFile === void 0 ? void 0 : syncedFile.syncedAt) ? Number(syncedFile.syncedAt) : lastSyncTimestamp;
const isNew = ((_f = (_e = integration.filtering) === null || _e === void 0 ? void 0 : _e.integrationFileStatus) === null || _f === void 0 ? void 0 : _f.isNew) === true
? !syncedFile &&
fileSyncTimestamp &&
file.createdAt &&
(typeof file.createdAt === 'string' ? new Date(file.createdAt).getTime() : file.createdAt) >
fileSyncTimestamp
: false;
const shouldCalculateIsUpdated = ((_h = (_g = integration.filtering) === null || _g === void 0 ? void 0 : _g.integrationFileStatus) === null || _h === void 0 ? void 0 : _h.isUpdated) === true || integration.forcePushSources === true;
const isUpdated = file.isUpdated !== undefined
? file.isUpdated
: shouldCalculateIsUpdated
? syncedFile &&
fileSyncTimestamp &&
file.updatedAt &&
(typeof file.updatedAt === 'string' ? new Date(file.updatedAt).getTime() : file.updatedAt) >
fileSyncTimestamp
: false;
return Object.assign(Object.assign({}, file), { isNew,
notSynced,
synced,
isUpdated });
});
});
}
exports.attachFileStatus = attachFileStatus;
function expandFilesTreeWithPagination(nodes, req, integration, job) {
var _a, _b;
return __awaiter(this, void 0, void 0, function* () {
if (job && types_2.JobStatus.CANCELED === ((_a = (yield job.get())) === null || _a === void 0 ? void 0 : _a.status)) {
throw new Error('Job canceled');
}
const files = nodes.filter((file) => file.nodeType === undefined || file.nodeType === '1');
const folders = nodes.filter((folder) => folder.nodeType === '0' && !nodes.find((node) => node.parentId === folder.id));
for (const folder of folders) {
const allFolderFiles = [];
let paginationData = undefined;
let hasMore = true;
let pageCount = 0;
while (hasMore) {
if (job && types_2.JobStatus.CANCELED === ((_b = (yield job.get())) === null || _b === void 0 ? void 0 : _b.status)) {
throw new Error('Job canceled');
}
pageCount++;
try {
if (job && pageCount > 1) {
yield job.update({
info: `Loading files from ${folder.name} (page ${pageCount})`,
});
}
const result = yield integration.getIntegrationFiles(req.integrationCredentials, req.integrationSettings, folder.id, '', 0, paginationData);
let integrationTreeItems = [];
if (isExtendedResultType(result)) {
integrationTreeItems = result.data || [];
if (result.stopPagination === true) {
hasMore = false;
}
}
else {
integrationTreeItems = result || [];
}
const loadMoreElement = integrationTreeItems.find((item) => item.nodeType === 'load-more');
const actualFiles = integrationTreeItems.filter((item) => item.nodeType !== 'load-more');
allFolderFiles.push(...actualFiles);
if (loadMoreElement && loadMoreElement.pagination) {
paginationData = loadMoreElement.pagination;
}
else if (!isExtendedResultType(result) || result.stopPagination !== false) {
hasMore = false;
}
}
catch (e) {
const errorMessage = e instanceof Error ? e.message : String(e);
if (job) {
const currentJob = yield job.get();
const currentErrors = (currentJob === null || currentJob === void 0 ? void 0 : currentJob.errors) ? JSON.parse(currentJob.errors) : [];
yield job.update({ errors: [...currentErrors, errorMessage] });
}
req.logError(`Error loading page ${pageCount} for folder ${folder.id}: ${errorMessage}`);
break;
}
}
const checkNodes = allFolderFiles
.filter((item) => item.id !== folder.id)
.map((item) => (Object.assign(Object.assign({}, item), { nodeType: item.nodeType || ('type' in item ? '1' : '0') })));
const expandedResult = yield expandFilesTreeWithPagination(checkNodes, req, integration, job);
files.push(...expandedResult);
}
return files;
});
}
exports.expandFilesTreeWithPagination = expandFilesTreeWithPagination;
function isExtendedResultType(data) {
const dataTyped = data;
return !!dataTyped && !Array.isArray(dataTyped);
}
exports.isExtendedResultType = isExtendedResultType;
function isFileLeaf(file) {
return file.nodeType === '1' || !!file.type;
}
function markUnsyncedFiles({ integrationId, crowdinId, client, files, }) {
return __awaiter(this, void 0, void 0, function* () {
if (!Array.isArray(files)) {
return [];
}
const unsyncedFilesData = yield (0, storage_1.getStorage)().getUnsyncedFiles({ integrationId, crowdinId });
let unsyncedFiles = (unsyncedFilesData === null || unsyncedFilesData === void 0 ? void 0 : unsyncedFilesData.files)
? JSON.parse(unsyncedFilesData === null || unsyncedFilesData === void 0 ? void 0 : unsyncedFilesData.files)
: [];
if (unsyncedFiles.length === 0) {
return files;
}
const fileIds = new Set(files.filter((file) => isFileLeaf(file)).map((file) => `${file.id}`));
const idsToRemove = unsyncedFiles.filter((file) => !fileIds.has(`${file.id}`)).map((file) => file.id);
if (idsToRemove.length > 0) {
unsyncedFiles = unsyncedFiles.filter((file) => !idsToRemove.includes(file.id));
yield (0, storage_1.getStorage)().updateUnsyncedFiles({
integrationId,
crowdinId,
files: JSON.stringify(unsyncedFiles),
});
}
if (unsyncedFiles.length === 0) {
return files;
}
const userTimezone = (yield client.usersApi.getAuthenticatedUser()).data.timezone;
return files.map((file) => {
const unsynced = unsyncedFiles.find((f) => `${f.id}` === `${file.id}`);
if (unsynced && isFileLeaf(file)) {
const formattedDate = (0, util_1.getFormattedDate)({ date: new Date(+unsynced.updatedAt), userTimezone });
file.labels = [
{
text: `Sync failed ${formattedDate}`,
type: 'warning',
tooltip: unsynced === null || unsynced === void 0 ? void 0 : unsynced.message,
},
...(file.labels ? file.labels : []),
];
file.failed = true;
}
return file;
});
});
}
exports.markUnsyncedFiles = markUnsyncedFiles;
function getExcludedTargetLanguages({ client, projectId, languages, }) {
return __awaiter(this, void 0, void 0, function* () {
const projectData = yield client.projectsGroupsApi.getProject(projectId);
const targetLanguages = projectData.data.targetLanguageIds;
return targetLanguages.filter((language) => !languages.includes(language));
});
}
exports.getExcludedTargetLanguages = getExcludedTargetLanguages;
function filterLanguages(request, files) {
const result = {};
for (const fileId in request) {
const file = files.find((f) => f.id === parseInt(fileId, 10));
if (file) {
const excludedLanguages = new Set((file === null || file === void 0 ? void 0 : file.excludedTargetLanguages) || []);
result[fileId] = request[fileId].filter((lang) => !excludedLanguages.has(lang));
}
else {
result[fileId] = request[fileId];
}
}
return result;
}
exports.filterLanguages = filterLanguages;
function uniqueFileLanguages(files) {
const result = {};
for (const fileId in files) {
const languages = files[fileId];
result[fileId] = Array.isArray(languages) ? [...new Set(languages)] : languages;
}
return result;
}
exports.uniqueFileLanguages = uniqueFileLanguages;
function prepareSyncFiles(files) {
// If it's an array (Provider.INTEGRATION format), return as is
if (Array.isArray(files)) {
return files;
}
if (files && typeof files === 'object') {
const hasLanguageArrays = Object.values(files).some((value) => Array.isArray(value));
if (hasLanguageArrays) {
return uniqueFileLanguages(files);
}
}
return files;
}
exports.prepareSyncFiles = prepareSyncFiles;
function buildPath(file, files) {
const filePath = `/${file.name}`;
if (!file.parentId) {
return filePath;
}
const parent = files.find((f) => f.id === file.parentId);
if (!parent) {
return filePath;
}
const parentPath = buildPath(parent, files);
return `${parentPath}${filePath}`;
}
exports.buildPath = buildPath;
function getParentPaths(filePath) {
const parentPaths = new Set();
const parts = filePath.split('/');
let currentPath = '';
for (let i = 1; i < parts.length - 1; i++) {
currentPath += '/' + parts[i];
parentPaths.add(currentPath);
}
return parentPaths;
}
exports.getParentPaths = getParentPaths;
function getParentFiles(allFiles, fileCollection) {
const parentPaths = new Set();
fileCollection
.filter((file) => !!file.path)
.forEach((file) => {
const paths = getParentPaths(file.path);
paths.forEach((path) => parentPaths.add(path));
});
return allFiles.filter((file) => file.path && parentPaths.has(file.path));
}
exports.getParentFiles = getParentFiles;
function filterFilesByPatterns(files, patterns) {
return files.filter((file) => patterns.some((pattern) => (0, minimatch_1.minimatch)(file.path || '', pattern.trim())));
}
exports.filterFilesByPatterns = filterFilesByPatterns;
function filterFilesByPath(files, includePatterns, excludePatterns) {
const filesWithPaths = files.map((file) => (Object.assign(Object.assign({}, file), { path: buildPath(file, files) })));
let filteredFiles = [...filesWithPaths];
if (includePatterns === null || includePatterns === void 0 ? void 0 : includePatterns.length) {
const includedFiles = filterFilesByPatterns(filteredFiles, includePatterns);
const parentFiles = getParentFiles(filesWithPaths, includedFiles);
filteredFiles = [...new Set([...includedFiles, ...parentFiles])];
}
if (excludePatterns === null || excludePatterns === void 0 ? void 0 : excludePatterns.length) {
const excludedFiles = filterFilesByPatterns(filteredFiles, excludePatterns);
const remainingFiles = filteredFiles.filter((file) => !excludedFiles.includes(file));
const parentFiles = getParentFiles(filesWithPaths, remainingFiles);
filteredFiles = [...new Set([...remainingFiles, ...parentFiles])];
}
return filteredFiles;
}
exports.filterFilesByPath = filterFilesByPath;
function updateSyncedData(clientId, crowdinId, payload, provider = types_1.Provider.INTEGRATION) {
return __awaiter(this, void 0, void 0, function* () {
const existingSyncedData = yield (0, storage_1.getStorage)().getSyncedData(clientId, crowdinId, provider);
const currentTimestamp = Date.now().toString();
const filesWithTimestamp = (payload === null || payload === void 0 ? void 0 : payload.map((file) => (Object.assign(Object.assign({}, file), { syncedAt: currentTimestamp })))) || [];
if (existingSyncedData) {
const existingFiles = JSON.parse(existingSyncedData.files);
const mergedFiles = (0, lodash_uniqby_1.default)([...filesWithTimestamp, ...existingFiles], 'id');
yield (0, storage_1.getStorage)().updateSyncedData(JSON.stringify(mergedFiles), clientId, crowdinId, provider);
}
else {
yield (0, storage_1.getStorage)().saveSyncedData(JSON.stringify(filesWithTimestamp), clientId, crowdinId, provider);
}
});
}
exports.updateSyncedData = updateSyncedData;
function filterSyncedData(payload) {
return __awaiter(this, void 0, void 0, function* () {
return payload === null || payload === void 0 ? void 0 : payload.filter((file) => file.isUpdated !== false);
});
}
exports.filterSyncedData = filterSyncedData;