@crowdin/app-project-module
Version:
Module that generates for you all common endpoints for serving standalone Crowdin App
537 lines (536 loc) • 27.9 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.removeFinishedJobs = exports.createOrUpdateSyncSettings = exports.filterFilesFromIntegrationRequest = exports.filesCron = exports.runUpdateProviderJob = exports.runJob = void 0;
const crowdinAppFunctions = __importStar(require("@crowdin/crowdin-apps-functions"));
const storage_1 = require("../../../storage");
const connection_1 = require("../../../util/connection");
const defaults_1 = require("./defaults");
const snapshot_1 = require("./snapshot");
const logger_1 = require("../../../util/logger");
const types_1 = require("../types");
const job_1 = require("./job");
const types_2 = require("./types");
const subscription_1 = require("../../../util/subscription");
function runJob({ config, integration, job, }) {
return __awaiter(this, void 0, void 0, function* () {
(0, logger_1.log)(`Starting cron job with expression [${job.expression}]`);
const crowdinCredentialsList = yield (0, storage_1.getStorage)().getAllCrowdinCredentials();
yield Promise.all(crowdinCredentialsList.map((crowdinCredentials) => __awaiter(this, void 0, void 0, function* () {
const { token, client: crowdinClient } = yield (0, connection_1.prepareCrowdinClient)({
config,
credentials: crowdinCredentials,
autoRenew: true,
});
const { expired } = yield (0, subscription_1.checkSubscription)({
config,
token,
organization: crowdinCredentials.id,
accountType: crowdinCredentials.type,
});
if (expired) {
(0, logger_1.log)(`Subscription expired. Skipping job [${job.expression}] for organization ${crowdinCredentials.id}`);
return;
}
const integrationCredentialsList = yield (0, storage_1.getStorage)().getAllIntegrationCredentials(crowdinCredentials.id);
const allIntegrationConfigs = yield (0, storage_1.getStorage)().getAllIntegrationConfigs(crowdinCredentials.id);
for (const integrationCredentials of integrationCredentialsList) {
const integrationConfig = allIntegrationConfigs.find(({ integrationId }) => integrationId === integrationCredentials.id);
const projectId = crowdinAppFunctions.getProjectId(integrationCredentials.id);
const apiCredentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
const rootFolder = yield (0, defaults_1.getRootFolder)(config, integration, crowdinClient, projectId);
const intConfig = (integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.config) ? JSON.parse(integrationConfig.config) : undefined;
(0, logger_1.log)(`Executing task for cron job with expression [${job.expression}] for project ${projectId}`);
yield job.task(projectId, crowdinClient, apiCredentials, rootFolder, intConfig);
(0, logger_1.log)(`Task for cron job with expression [${job.expression}] for project ${projectId} completed`);
}
})));
(0, logger_1.log)(`Cron job with expression [${job.expression}] completed`);
});
}
exports.runJob = runJob;
function runUpdateProviderJob({ integrationId, crowdinId, type, title, payload, jobType, projectId, client, integration, context, credentials, rootFolder, appSettings, reRunJobId, }) {
return __awaiter(this, void 0, void 0, function* () {
try {
yield (0, job_1.runAsJob)({
integrationId,
crowdinId,
type,
title,
payload,
jobType,
projectId,
client,
reRunJobId,
jobStoreType: integration.jobStoreType,
jobCallback: (job) => __awaiter(this, void 0, void 0, function* () {
const updateParams = {
projectId,
client,
credentials,
request: payload,
rootFolder,
appSettings,
job,
};
if (type === types_2.JobType.UPDATE_TO_CROWDIN) {
yield integration.updateCrowdin(updateParams);
}
else if (type === types_2.JobType.UPDATE_TO_INTEGRATION) {
yield integration.updateIntegration(updateParams);
}
}),
});
}
catch (e) {
const action = type === types_2.JobType.UPDATE_TO_CROWDIN ? 'Auto sync files to Crowdin' : 'Auto sync files to External Service';
yield (0, logger_1.handleUserError)({
action,
error: e,
crowdinId: crowdinId,
clientId: integrationId,
});
(0, logger_1.logError)(e, context);
throw e;
}
});
}
exports.runUpdateProviderJob = runUpdateProviderJob;
function filesCron({ config, integration, period, }) {
return __awaiter(this, void 0, void 0, function* () {
(0, logger_1.log)(`Starting files cron job with period [${period}]`);
const syncSettingsList = yield (0, storage_1.getStorage)().getAllSyncSettingsByType('schedule');
const crowdinSyncSettings = syncSettingsList.filter((syncSettings) => syncSettings.provider === types_1.Provider.CROWDIN);
const integrationSyncSettings = syncSettingsList.filter((syncSettings) => syncSettings.provider === types_1.Provider.INTEGRATION);
yield Promise.all(crowdinSyncSettings.map((syncSettings) => processSyncSettings({ config, integration, period, syncSettings })));
yield Promise.all([
integrationSyncSettings.map((syncSettings) => processSyncSettings({ config, integration, period, syncSettings })),
]);
(0, logger_1.log)(`Files cron job with period [${period}] completed`);
});
}
exports.filesCron = filesCron;
function processSyncSettings({ config, integration, period, syncSettings, }) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
let projectData;
let crowdinClient;
let token;
let files = JSON.parse(syncSettings.files);
let newFiles = [];
const crowdinCredentials = yield (0, storage_1.getStorage)().getCrowdinCredentials(syncSettings.crowdinId);
const integrationCredentials = yield (0, storage_1.getStorage)().getIntegrationCredentials(syncSettings.integrationId);
const integrationConfig = yield (0, storage_1.getStorage)().getIntegrationConfig(syncSettings.integrationId);
if (!crowdinCredentials || !integrationCredentials) {
return;
}
const intConfig = (integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.config)
? JSON.parse(integrationConfig.config)
: { schedule: '0', condition: '0' };
if (period !== intConfig.schedule) {
return;
}
const projectId = crowdinAppFunctions.getProjectId(integrationCredentials.id);
const context = {
jwtPayload: {
context: {
// eslint-disable-next-line @typescript-eslint/camelcase
project_id: projectId,
// eslint-disable-next-line @typescript-eslint/camelcase
organization_id: crowdinCredentials.organizationId,
// eslint-disable-next-line @typescript-eslint/camelcase
organization_domain: crowdinCredentials.domain,
// eslint-disable-next-line @typescript-eslint/camelcase
user_id: crowdinCredentials.userId,
},
},
crowdinId: crowdinCredentials.id,
clientId: integrationCredentials.id,
};
try {
const preparedCrowdinClient = yield (0, connection_1.prepareCrowdinClient)({
config,
credentials: crowdinCredentials,
autoRenew: true,
context,
});
token = preparedCrowdinClient.token;
crowdinClient = preparedCrowdinClient.client;
}
catch (e) {
(0, logger_1.logError)(e, context);
return;
}
const { expired } = yield (0, subscription_1.checkSubscription)({
config,
token,
organization: crowdinCredentials.id,
accountType: crowdinCredentials.type,
});
if (expired) {
(0, logger_1.log)(`Subscription expired. Skipping job [${period}] for organization ${crowdinCredentials.id}`);
return;
}
try {
projectData = (yield crowdinClient.projectsGroupsApi.getProject(projectId))
.data;
}
catch (e) {
(0, logger_1.logError)(e, context);
return;
}
// eslint-disable-next-line @typescript-eslint/camelcase
context.jwtPayload.context.project_identifier = projectData.identifier;
const rootFolder = yield (0, defaults_1.getRootFolder)(config, integration, crowdinClient, projectId);
const apiCredentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
if (!integration.webhooks &&
((_a = integration.syncNewElements) === null || _a === void 0 ? void 0 : _a[syncSettings.provider]) &&
intConfig[`new-${syncSettings.provider}-files`]) {
try {
newFiles = yield getAllNewFiles({
config,
integration,
projectData,
syncSettings,
crowdinApiClient: crowdinClient,
crowdinId: crowdinCredentials.id,
integrationCredentials: apiCredentials,
integrationId: integrationCredentials.id,
integrationSettings: intConfig,
});
}
catch (e) {
(0, logger_1.logError)(e, context);
return;
}
}
if (integration.webhooks) {
const webhooks = yield (0, storage_1.getStorage)().getAllWebhooks(syncSettings.integrationId, syncSettings.crowdinId, syncSettings.provider);
const webhooksFileIds = (webhooks || []).map((webhook) => webhook.fileId);
if (syncSettings.provider === types_1.Provider.CROWDIN) {
files = webhooksFileIds.reduce((acc, fileId) => {
if (files[fileId]) {
acc[fileId] = files[fileId];
}
return acc;
}, {});
}
else {
files = files.filter((file) => webhooksFileIds.includes(file.id));
}
yield (0, storage_1.getStorage)().deleteWebhooks(webhooksFileIds, syncSettings.integrationId, syncSettings.crowdinId, syncSettings.provider);
}
if (syncSettings.provider === types_1.Provider.CROWDIN) {
const crowdinFiles = yield filterFilesFromIntegrationRequest({
config,
integration,
projectId,
crowdinFiles: Object.assign(Object.assign({}, files), newFiles),
crowdinClient,
});
const onlyTranslated = +intConfig.condition === types_1.SyncCondition.TRANSLATED;
const onlyApproved = +intConfig.condition === types_1.SyncCondition.APPROVED;
const all = +intConfig.condition === types_1.SyncCondition.ALL || intConfig.condition === undefined;
const filesToProcess = all
? crowdinFiles
: yield getOnlyTranslatedOrApprovedFiles({
projectId,
crowdinFiles,
crowdinClient,
onlyApproved,
onlyTranslated,
context,
});
if (Object.keys(filesToProcess).length <= 0) {
return;
}
(0, logger_1.log)(`Executing updateIntegration task for files cron job with period [${period}] for project ${projectId}.Files ${Object.keys(filesToProcess).length}`);
if (!all) {
if (Object.keys(filesToProcess).length === 0) {
(0, logger_1.log)(`There is no ${onlyApproved ? 'approved' : 'translated'} file`);
return;
}
}
const apiCredentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
if (!(intConfig === null || intConfig === void 0 ? void 0 : intConfig.inContext)) {
removeInContextLanguage(filesToProcess, projectData);
}
try {
yield runUpdateProviderJob({
integrationId: syncSettings.integrationId,
crowdinId: syncSettings.crowdinId,
type: types_2.JobType.UPDATE_TO_INTEGRATION,
title: `Sync files to ${config.name} [scheduled]`,
payload: filesToProcess,
jobType: types_2.JobClientType.CRON,
projectId: projectId,
client: crowdinClient,
integration,
context,
credentials: apiCredentials,
rootFolder,
appSettings: intConfig,
});
}
catch (e) {
return;
}
if (Object.keys(newFiles).length) {
yield (0, storage_1.getStorage)().updateSyncSettings(JSON.stringify(Object.assign(Object.assign({}, files), newFiles)), syncSettings.integrationId, syncSettings.crowdinId, 'schedule', syncSettings.provider);
const currentFileSnapshot = yield (0, snapshot_1.getCrowdinSnapshot)(config, integration, crowdinClient, projectId, intConfig);
yield (0, storage_1.getStorage)().updateFilesSnapshot(JSON.stringify(currentFileSnapshot), syncSettings.integrationId, syncSettings.crowdinId, syncSettings.provider);
}
(0, logger_1.log)(`updateIntegration task for files cron job with period [${period}] for project ${projectId} completed`);
}
else {
const allIntFiles = [...files, ...newFiles].map((file) => (Object.assign({ id: file.id, name: file.name, parentId: file.parent_id || file.parentId,
// eslint-disable-next-line @typescript-eslint/camelcase
parent_id: file.parent_id || file.parentId,
// eslint-disable-next-line @typescript-eslint/camelcase
node_type: file.nodeType || file.node_type }, (file.type ? { type: file.type } : {}))));
const intFiles = allIntFiles.filter((file) => 'type' in file);
if (intFiles.length <= 0) {
return;
}
(0, logger_1.log)(`Executing updateCrowdin task for files cron job with period [${period}] for project ${projectId}. Files ${intFiles.length}`);
const apiCredentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
try {
yield runUpdateProviderJob({
integrationId: syncSettings.integrationId,
crowdinId: syncSettings.crowdinId,
type: types_2.JobType.UPDATE_TO_CROWDIN,
title: 'Sync files to Crowdin [scheduled]',
payload: intFiles,
jobType: types_2.JobClientType.CRON,
projectId: projectId,
client: crowdinClient,
integration,
context,
credentials: apiCredentials,
rootFolder,
appSettings: intConfig,
});
}
catch (e) {
return;
}
if (Object.keys(newFiles).length) {
const newSyncSettingsFields = allIntFiles.map((file) => (Object.assign(Object.assign({}, file), { schedule: true, sync: false })));
yield (0, storage_1.getStorage)().updateSyncSettings(JSON.stringify(newSyncSettingsFields), syncSettings.integrationId, syncSettings.crowdinId, 'schedule', syncSettings.provider);
const currentFileSnapshot = yield (0, snapshot_1.getIntegrationSnapshot)(integration, apiCredentials, intConfig);
yield (0, storage_1.getStorage)().updateFilesSnapshot(JSON.stringify(currentFileSnapshot), syncSettings.integrationId, syncSettings.crowdinId, syncSettings.provider);
}
(0, logger_1.log)(`updateCrowdin task for files cron job with period [${period}] for project ${projectId} completed`);
}
});
}
function getFileDiff(currentFiles, savedFiles) {
return currentFiles.filter((x) => !savedFiles.some((x2) => x2.id === x.id));
}
function getAllNewFiles(args) {
return __awaiter(this, void 0, void 0, function* () {
const { config, integration, crowdinApiClient, crowdinId, integrationCredentials, integrationId, projectData, integrationSettings, syncSettings, } = args;
let currentFileSnapshot = [];
const fileSnapshotData = yield (0, storage_1.getStorage)().getFilesSnapshot(integrationId, crowdinId, syncSettings.provider);
const snapshotFiles = (fileSnapshotData === null || fileSnapshotData === void 0 ? void 0 : fileSnapshotData.files) ? JSON.parse(fileSnapshotData.files) : [];
if (syncSettings.provider === types_1.Provider.CROWDIN) {
currentFileSnapshot = yield (0, snapshot_1.getCrowdinSnapshot)(config, integration, crowdinApiClient, projectData.id, integrationSettings);
}
else {
currentFileSnapshot = yield (0, snapshot_1.getIntegrationSnapshot)(integration, integrationCredentials, integrationSettings);
}
const difference = getFileDiff(currentFileSnapshot, snapshotFiles);
const onlyFiles = difference.filter((file) => 'type' in file);
const synFiles = JSON.parse(syncSettings.files);
if (syncSettings.provider === types_1.Provider.INTEGRATION) {
if (integrationSettings[`new-${syncSettings.provider}-files`]) {
return onlyFiles;
}
const syncFolders = synFiles.filter((file) => !('type' in file));
return getNewFoldersFile(syncFolders, difference);
}
else {
const files = {};
const targetLanguages = projectData.targetLanguageIds;
if (projectData.inContext) {
targetLanguages.push(projectData.inContextPseudoLanguageId);
}
if (integrationSettings[`new-${syncSettings.provider}-files`]) {
for (const file of onlyFiles) {
files[file.id] = targetLanguages;
}
}
else {
const syncFolders = currentFileSnapshot.filter((file) => !('type' in file) && Object.keys(synFiles).includes(file.id));
const newFiles = getNewFoldersFile(syncFolders, difference);
for (const file of newFiles) {
files[file.id] = targetLanguages;
}
}
return files;
}
});
}
function getNewFoldersFile(folders, snapshotFiles) {
let files = [];
for (const folder of folders) {
const newFiles = snapshotFiles.find((file) => file.parentId === folder.id);
if (newFiles) {
files = files.concat(newFiles);
}
}
files = files.filter((file) => 'type' in file);
return files;
}
function getOnlyTranslatedOrApprovedFiles({ projectId, crowdinFiles, crowdinClient, onlyApproved, onlyTranslated, context, }) {
return __awaiter(this, void 0, void 0, function* () {
(0, logger_1.log)(`Filtering files to process only ${onlyApproved ? 'approved' : 'translated'} files`);
const filesInfo = yield Promise.all(Object.keys(crowdinFiles).map((fileId) => __awaiter(this, void 0, void 0, function* () {
try {
const res = yield crowdinClient.translationStatusApi
.withFetchAll()
.getFileProgress(projectId, Number(fileId));
return {
id: fileId,
info: res.data.map((e) => e.data),
};
}
catch (e) {
delete crowdinFiles[fileId];
(0, logger_1.logError)(e, context);
}
})));
const filteredFiles = {};
Object.keys(crowdinFiles).forEach((fileId) => {
const fileInfo = filesInfo.find((info) => (info === null || info === void 0 ? void 0 : info.id) === fileId);
if (!fileInfo) {
return;
}
const languages = crowdinFiles[fileId];
languages.forEach((language) => {
const languageInfo = fileInfo.info.find((info) => info.languageId === language);
if (!languageInfo) {
return;
}
if (onlyTranslated) {
if (languageInfo.translationProgress === 100) {
(0, logger_1.log)(`File ${fileId} is fully translated for language ${language}`);
if (!filteredFiles[fileId]) {
filteredFiles[fileId] = [];
}
filteredFiles[fileId].push(language);
}
else {
(0, logger_1.log)(`File ${fileId} is not fully translated for language ${language}, progress ${languageInfo.translationProgress}`);
}
}
if (onlyApproved) {
if (languageInfo.approvalProgress === 100) {
(0, logger_1.log)(`File ${fileId} is fully approved for language ${language}`);
if (!filteredFiles[fileId]) {
filteredFiles[fileId] = [];
}
filteredFiles[fileId].push(language);
}
else {
(0, logger_1.log)(`File ${fileId} is not fully approved for language ${language}, progress ${languageInfo.approvalProgress}`);
}
}
});
});
return filteredFiles;
});
}
function filterFilesFromIntegrationRequest({ config, integration, projectId, crowdinClient, crowdinFiles, }) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
if (integration.skipAutoSyncFoldersFilter) {
return crowdinFiles;
}
let folders;
if ((_a = config.projectIntegration) === null || _a === void 0 ? void 0 : _a.withRootFolder) {
const rootFolder = yield (0, defaults_1.getRootFolder)(config, integration, crowdinClient, projectId);
if (rootFolder) {
folders = (yield crowdinClient.sourceFilesApi.withFetchAll().listProjectDirectories(projectId, {
directoryId: rootFolder.id,
recursion: 'true',
})).data;
}
}
else {
folders = (yield crowdinClient.sourceFilesApi.withFetchAll().listProjectDirectories(projectId, { recursion: 'true' })).data;
}
if (folders) {
for (const fileId of Object.keys(crowdinFiles)) {
if (folders.find((folder) => folder.data.id === +fileId)) {
delete crowdinFiles[fileId];
}
}
}
return crowdinFiles;
});
}
exports.filterFilesFromIntegrationRequest = filterFilesFromIntegrationRequest;
function createOrUpdateSyncSettings({ req, files, provider, onlyCreate = false, }) {
return __awaiter(this, void 0, void 0, function* () {
const existingSettings = yield (0, storage_1.getStorage)().getSyncSettings(req.crowdinContext.clientId, req.crowdinContext.crowdinId, 'schedule', provider);
if (!existingSettings) {
(0, logger_1.log)(`Saving sync settings for type schedule and provider ${provider} ${JSON.stringify(files, null, 2)}`);
yield (0, storage_1.getStorage)().saveSyncSettings(JSON.stringify(files), req.crowdinContext.clientId, req.crowdinContext.crowdinId, 'schedule', provider);
}
else if (!onlyCreate) {
(0, logger_1.log)(`Updating sync settings for type schedule and provider ${provider} ${JSON.stringify(files, null, 2)}`);
yield (0, storage_1.getStorage)().updateSyncSettings(JSON.stringify(files), req.crowdinContext.clientId, req.crowdinContext.crowdinId, 'schedule', provider);
}
});
}
exports.createOrUpdateSyncSettings = createOrUpdateSyncSettings;
function removeFinishedJobs() {
return __awaiter(this, void 0, void 0, function* () {
(0, logger_1.log)('Removing all finished jobs');
yield (0, storage_1.getStorage)().deleteFinishedJobs();
(0, logger_1.log)('Removed all finished jobs');
});
}
exports.removeFinishedJobs = removeFinishedJobs;
function removeInContextLanguage(filesToProcess, projectData) {
(0, logger_1.log)('Removing in-context language from files to process');
if (!projectData.inContext) {
return;
}
for (const fileId in filesToProcess) {
filesToProcess[fileId] = filesToProcess[fileId].filter((language) => language !== projectData.inContextPseudoLanguageId);
}
(0, logger_1.log)('In-context language(' + projectData.inContextPseudoLanguageId + ') removed from files to process');
}