strapi-to-lokalise-plugin
Version:
Preview and sync Lokalise translations from Strapi admin
5,687 lines • 274 kB
JavaScript
'use strict';
const axios = require('axios');
const readline = require('readline');
const crypto = require('crypto');
function parseList(value) {
if (!value) return [];
if (Array.isArray(value)) return value;
return String(value)
.split(',')
.map((p) => p.trim())
.filter((p) => p.length > 0);
}
const DEFAULT_SMALL_JOB_THRESHOLD = 25;
function parsePositiveInteger(value) {
if (value === null || value === undefined) {
return null;
}
const num = Number(value);
if (!Number.isFinite(num) || num <= 0) {
return null;
}
return Math.floor(num);
}
function createSyncRunner(config = {}) {
const cfg = {
strapiBaseUrl: config.strapiBaseUrl ?? process.env.STRAPI_BASE_URL,
strapiApiToken: config.strapiApiToken ?? process.env.STRAPI_API_TOKEN,
lokaliseApiToken: config.lokaliseApiToken ?? process.env.LOKALISE_API_TOKEN,
lokaliseProjectId: config.lokaliseProjectId ?? process.env.LOKALISE_PROJECT_ID,
lokaliseBaseUrl: config.lokaliseBaseUrl ?? process.env.LOKALISE_BASE_URL ?? 'https://api.lokalise.com/api2',
strapiTypes: config.strapiTypes ?? process.env.STRAPI_TYPES,
strapiDefaultType: config.strapiDefaultType ?? process.env.STRAPI_DEFAULT_TYPE,
fieldIncludePatterns: parseList(config.fieldInclude ?? process.env.STRAPI_FIELD_INCLUDE),
fieldExcludePatterns: parseList(config.fieldExclude ?? process.env.STRAPI_FIELD_EXCLUDE),
collectionTypes: config.collectionTypes ?? [],
contentTypeMap: config.contentTypeMap ?? {},
entityService: config.entityService,
metadataService: config.metadataService,
skipNestedRelations: new Set(config.skipNestedRelations ?? []),
smallJobThreshold: parsePositiveInteger(
config.smallJobThreshold ?? process.env.LOKALISE_SMALL_JOB_THRESHOLD
),
};
// If entityService is provided (plugin mode), we don't need strapiBaseUrl/apiToken
// Only require them when running as standalone script (without entityService)
if (!cfg.entityService && (!cfg.strapiBaseUrl || !cfg.strapiApiToken)) {
throw new Error('Missing STRAPI_BASE_URL or STRAPI_API_TOKEN');
}
if (!cfg.lokaliseProjectId || !cfg.lokaliseApiToken) {
throw new Error('Missing LOKALISE_PROJECT_ID or LOKALISE_API_TOKEN');
}
const logger = config.logger || console;
const http = config.httpClient || axios;
const rlFactory = config.readline || readline;
const slugMap =
config.slugMap && typeof config.slugMap === 'object'
? JSON.parse(JSON.stringify(config.slugMap))
: {};
let slugMapDirty = false;
const markSlugDirty = () => {
slugMapDirty = true;
};
const flushSlugMap = async () => {
if (slugMapDirty && typeof config.onSlugMapChange === 'function') {
await config.onSlugMapChange(JSON.parse(JSON.stringify(slugMap)));
slugMapDirty = false;
}
};
const ensureSlugBucket = (type) => {
if (!slugMap[type]) {
slugMap[type] = {};
}
return slugMap[type];
};
const getEntryIdentifier = (entry) => {
if (!entry) return null;
const source = entry.attributes || entry;
return (
source.documentId ||
entry.documentId ||
source.entryId ||
entry.entryId ||
source.id ||
entry.id ||
null
);
};
const rememberEntrySlug = (type, entryId, slug) => {
if (!entryId) {
return slug;
}
const bucket = ensureSlugBucket(type);
const stored = bucket[entryId];
if (stored && stored.length > 0) {
return stored;
}
bucket[entryId] = slug;
markSlugDirty();
return slug;
};
const fieldIncludeMode = String(config.fieldIncludeMode ?? process.env.STRAPI_FIELD_INCLUDE_MODE ?? 'loose').toLowerCase();
const STRUCTURAL_KEYS = new Set([
'__component',
'__type',
'id',
'documentId',
'entryId',
'createdAt',
'updatedAt',
'publishedAt',
'createdBy',
'updatedBy',
'locale',
'localizations',
'slug',
'sortOrder',
]);
const escapeRegex = (value) => {
if (!value || typeof value !== 'string') return '';
return value.replace(/[-/\\^$+?.()|[\]{}]/g, '\\$&');
};
const normalisePattern = (pattern) => String(pattern || '').trim().toLowerCase();
function matchesPattern(fieldPath, pattern) {
if (!pattern) return false;
if (pattern === '*') return true;
const escaped = escapeRegex(normalisePattern(pattern)).replace(/\\\*/g, '.*');
const regex = new RegExp(`^${escaped}$`, 'i');
return regex.test(fieldPath.toLowerCase());
}
function shouldIncludeField(fieldPath) {
if (!fieldPath || typeof fieldPath !== 'string') return false;
const lowerPath = String(fieldPath).toLowerCase();
if (!lowerPath) return false;
const lastSegment = lowerPath.replace(/.*[./]/, '').replace(/\[\d+\]$/, '');
if (!lastSegment) return false;
if (STRUCTURAL_KEYS.has(lastSegment)) {
return false;
}
if (cfg.fieldExcludePatterns.length > 0) {
const matchesExclude = cfg.fieldExcludePatterns.some((pattern) => matchesPattern(lowerPath, normalisePattern(pattern)));
if (matchesExclude) {
return false;
}
}
if (cfg.fieldIncludePatterns.length === 0) {
return true;
}
const includeMatch = cfg.fieldIncludePatterns.some((pattern) => matchesPattern(lowerPath, normalisePattern(pattern)));
if (includeMatch) {
return true;
}
return fieldIncludeMode === 'strict' ? false : true;
}
function getAuthHeaders(extra = {}) {
return {
Authorization: `Bearer ${cfg.strapiApiToken}`,
'Content-Type': 'application/json',
...extra,
};
}
// ============================================================================
// UTILITY FUNCTIONS FOR LOKALISE SYNC
// ============================================================================
// Cooperative cancellation support (set by caller via cfg.cancelCheck)
const shouldCancel = async () => {
try {
if (typeof cfg.cancelCheck === 'function') {
return await cfg.cancelCheck();
}
} catch (_err) {
// ignore errors from cancel checker
}
return false;
};
const waitWithCancel = async (ms) => {
const slice = 200;
let elapsed = 0;
while (elapsed < ms) {
if (await shouldCancel()) {
const err = new Error('JOB_CANCELLED');
throw err;
}
const step = Math.min(slice, ms - elapsed);
await new Promise((r) => setTimeout(r, step));
elapsed += step;
}
};
const formatDuration = (ms) => {
if (!Number.isFinite(ms)) {
return '0s';
}
if (ms < 1000) {
return `${ms}ms`;
}
return `${(ms / 1000).toFixed(1)}s`;
};
const logRemoteTiming = (label, startTime, minMs = 2000) => {
if (!startTime) return;
const elapsed = Date.now() - startTime;
if (elapsed >= minMs) {
logger.log(` ⏱️ ${label} completed in ${formatDuration(elapsed)}`);
}
};
/**
* Resolve Strapi model UID from content type name
* Handles pluralization, different formats, and contentTypeMap
*/
function resolveModelUid(type) {
// First, try contentTypeMap (most reliable)
const entityConfig = cfg.contentTypeMap && cfg.contentTypeMap[type];
if (entityConfig && entityConfig.uid) {
return entityConfig.uid;
}
// Try common formats
const formats = [
`api::${type}.${type}`,
`api::${type}s.${type}s`, // plural
`api::${type}.${type}s`,
`api::${type}s.${type}`,
];
// Return the first format (most common)
return formats[0];
}
/**
* Safely get Strapi entry by ID with multiple fallback methods
* Returns null if not found (silent - no warnings)
*/
async function getEntry(modelUid, entryId) {
if (!entryId || !modelUid) return null;
// For Strapi v5 documentIds (long strings), try findOne with documentId directly first
// This is the most reliable method for Strapi v5
if (typeof entryId === 'string' && entryId.length > 10 && !/^\d+$/.test(entryId)) {
// Method 1: Preview state (draft) - most common for documentIds
try {
const entry = await cfg.entityService.findOne(modelUid, entryId, {
publicationState: 'preview',
});
if (entry) return entry;
} catch (err) {
// Continue to next method
}
// Method 2: Published state
try {
const entry = await cfg.entityService.findOne(modelUid, entryId);
if (entry) return entry;
} catch (err) {
// Continue to next method
}
// Method 3: Search by documentId filter (fallback)
try {
const results = await cfg.entityService.findMany(modelUid, {
filters: { documentId: entryId },
limit: 1,
publicationState: 'preview',
});
if (Array.isArray(results) && results.length > 0) {
return results[0];
}
} catch (err) {
// Continue
}
try {
const results = await cfg.entityService.findMany(modelUid, {
filters: { documentId: entryId },
limit: 1,
});
if (Array.isArray(results) && results.length > 0) {
return results[0];
}
} catch (err) {
// Continue
}
} else {
// For numeric IDs or short strings, try standard methods
// Method 1: Standard findOne (published)
try {
const entry = await cfg.entityService.findOne(modelUid, entryId);
if (entry) return entry;
} catch (err) {
// Continue to next method
}
// Method 2: Preview state (draft)
try {
const entry = await cfg.entityService.findOne(modelUid, entryId, {
publicationState: 'preview',
});
if (entry) return entry;
} catch (err) {
// Continue to next method
}
// Method 3: Try numeric ID if entryId is numeric string
if (typeof entryId === 'string' && /^\d+$/.test(entryId)) {
try {
const numericId = parseInt(entryId, 10);
const entry = await cfg.entityService.findOne(modelUid, numericId);
if (entry) return entry;
} catch (err) {
// Continue
}
}
}
return null; // Not found - silent return
}
async function getEntryBySlug(modelUid, slug) {
if (!slug || !modelUid) return null;
try {
const results = await cfg.entityService.findMany(modelUid, {
filters: { slug },
limit: 1,
publicationState: 'preview',
});
if (Array.isArray(results) && results.length > 0) {
return results[0];
}
} catch (err) {
// Silent fallback
}
return null;
}
/**
* Normalize tags to match platform context
* Ensures tags are compatible with the key's platforms
* CRITICAL: Lokalise may reject very short tags (less than 2-3 characters)
*/
function normalizeTagsForPlatforms(tags, platforms) {
if (!Array.isArray(tags) || tags.length === 0) return [];
if (!Array.isArray(platforms) || platforms.length === 0) {
platforms = ['web']; // Default
}
const tagValidationRegex = /^[a-z0-9][a-z0-9\-_]*$/;
// Normalize all tags: lowercase, replace spaces with underscores
const normalized = tags.map(tag => {
const str = String(tag).trim().toLowerCase();
return str.replace(/\s+/g, '_');
}).filter(tag => {
// CRITICAL: Lokalise may reject very short tags (less than 2 chars)
// Also validate format and length
return tag.length >= 2 && tag.length <= 100 && tagValidationRegex.test(tag);
});
// Deduplicate
return Array.from(new Set(normalized));
}
/**
* IMPORTANT — PLEASE READ BEFORE MODIFYING THIS LOGIC
*
* This implementation follows best-practice guidelines for both Strapi and Lokalise.
*
* WHY WE STORE A TAG SNAPSHOT:
* ----------------------------
* Strapi does NOT guarantee that custom JSON fields (like `entry.lokalise`) are always
* returned during `entityService.findMany`. JSON fields are not affected by `populate`,
* and can be omitted depending on the content-type configuration and query shape.
*
* Lokalise, on the other hand, merges tags automatically at the key level. When we sync
* a key, Lokalise may attach new tags that do NOT exist in Strapi. That means:
*
* - The tag list we send to Lokalise during sync is NOT the same tag list we can
* reconstruct during preview from Strapi data alone.
*
* Because our hash originally depended on the full tag set (Strapi tags + Lokalise tags),
* the preview hash and stored hash could never match. Everything always appeared as
* "Needs re-sync" even after a clean sync.
*
* HOW WE FIX IT:
* --------------
* During sync, we now capture the EXACT tag array used for hashing — fully merged,
* sorted, deduped — and store it in the metadata service as `lokalise_tags_snapshot`.
*
* During preview, we rebuild the hash using:
*
* canonicalTags = merge(Strapi tags, lokalise_tags_snapshot)
*
* This ensures deterministic hashing across both sync and preview, fully aligned with
* the guidelines of:
*
* ✔ Strapi — because we never rely on JSON fields that Strapi may omit
* ✔ Lokalise — because we respect the full merged tag state used at sync time
*
* RESULT:
* -------
* - "Up-to-date" finally works correctly.
* - "Needs re-sync" only shows true differences.
* - Hashes are stable, predictable, and consistent across runs.
*
* If modifying this logic in the future:
*
* → Never hash using Strapi-only tags (Strapi doesn’t have the full picture)
* → Never assume `entry.lokalise` will be returned by Strapi
* → Always use the snapshot as the source of truth for hashing
*
* This is the only safe way to maintain sync correctness between Strapi and Lokalise.
*/
function buildCanonicalTagSnapshot(rawTags) {
if (!Array.isArray(rawTags) || rawTags.length === 0) {
return [];
}
const cleaned = rawTags
.map((tag) => {
if (tag === null || tag === undefined) return null;
const str = typeof tag === 'string' ? tag : String(tag);
const trimmed = str.trim();
return trimmed.length > 0 ? trimmed : null;
})
.filter(Boolean);
if (cleaned.length === 0) {
return [];
}
const unique = Array.from(new Set(cleaned));
unique.sort((a, b) => a.localeCompare(b));
return unique;
}
/**
* Calculate hash for a key to detect content changes
* Hashes: key_name, tags (sorted), translations (sorted by locale)
* Returns SHA1 hex string (e.g., "bf39cf1d7369...")
*/
function calculateKeyHash(key) {
if (!key) return null;
try {
// Extract and normalize data for hashing
const keyName = typeof key.key_name === 'string'
? key.key_name
: (key.key_name?.web || key.key_name?.other || String(key.key_name || ''));
// Sort tags for consistent hashing
const tags = Array.isArray(key.tags)
? [...key.tags].sort().map(t => String(t).trim()).filter(Boolean)
: [];
// Sort translations by locale, then by translation text for consistent hashing
const translations = Array.isArray(key.translations)
? [...key.translations]
.map(t => ({
locale: String(t.language_iso || t.locale || '').trim(),
text: String(t.translation || '').trim(),
}))
.filter(t => t.locale && t.text)
.sort((a, b) => {
const localeCompare = a.locale.localeCompare(b.locale);
return localeCompare !== 0 ? localeCompare : a.text.localeCompare(b.text);
})
: [];
// Create hash object (deterministic structure)
const hashObject = {
key_name: keyName,
tags: tags,
translations: translations,
};
// Generate SHA1 hash
const hashString = JSON.stringify(hashObject);
const hash = crypto.createHash('sha1').update(hashString).digest('hex');
return hash;
} catch (err) {
// If hashing fails, return null (will be treated as "needs sync")
return null;
}
}
/**
* Safely update Strapi entry with lokalise_key_id and sync hash
* Handles draft/published, nested paths, and errors gracefully
*/
async function safeUpdateStrapiEntryLokaliseId(
type,
entryId,
entrySlug,
keyName,
keyId,
fieldPath,
syncHash = null,
tagSnapshot = null
) {
if (!keyId || !keyName || !type) {
return false; // Missing critical data
}
const context = `[LokaliseSync] type="${type}" key="${keyName}" entry_id="${entryId ?? 'n/a'}" entry_slug="${
entrySlug ?? 'n/a'
}"`;
if (cfg.metadataService) {
try {
await cfg.metadataService.storeKeyMappings(type, [
{
entry_document_id: entryId || null,
entry_numeric_id: entryId || null,
entry_slug: entrySlug || null,
field_path: fieldPath,
key_name: keyName,
key_id: keyId,
lokalise_sync_hash: syncHash || null, // Store hash in metadata service as fallback
lokalise_tags_snapshot: Array.isArray(tagSnapshot) && tagSnapshot.length > 0 ? tagSnapshot : null,
},
]);
return true;
} catch (err) {
logger.error(`${context} → failed to store lokalise metadata: ${err.message || err}`);
if (err && err.stack) {
logger.error(err.stack);
}
return false;
}
}
const modelUid = resolveModelUid(type);
// Determine targetId: prefer entryId (documentId for Strapi v5), fall back to slug lookup
let targetId = entryId || null;
const isDocumentId = typeof entryId === 'string' && entryId.length > 10 && !/^\d+$/.test(entryId);
// If we have a documentId, try direct update first (faster, no lookup needed)
// For Strapi v5, entityService.update accepts documentId directly
if (isDocumentId && targetId) {
// Try preview state first (draft entries)
try {
// Get existing entry to merge lokalise data
const existingEntry = await cfg.entityService.findOne(modelUid, targetId, {
publicationState: 'preview',
});
const existingLokalise = existingEntry?.lokalise || {};
const existingKeyData = existingLokalise.keys?.[keyName] || {};
const metaBase = existingKeyData.meta || {};
const meta = {
...metaBase,
lokalise_key_id: keyId,
};
if (syncHash) {
meta.lokalise_sync_hash = syncHash;
meta.last_synced_at = Date.now();
}
if (Array.isArray(tagSnapshot) && tagSnapshot.length > 0) {
meta.lokalise_tags_snapshot = tagSnapshot;
} else if (metaBase.lokalise_tags_snapshot && !meta.lokalise_tags_snapshot) {
meta.lokalise_tags_snapshot = metaBase.lokalise_tags_snapshot;
}
// DEBUG: Log hash storage for first few keys (reset counter per sync)
if (syncHash && typeof window === 'undefined') {
// Only log in server context (not browser)
// Reset counter at start of storage (use a timestamp-based approach)
const storageStartTime = global.hashStorageStartTime || Date.now();
if (!global.hashStorageStartTime) {
global.hashStorageStartTime = storageStartTime;
global.hashStorageDebugCount = 0;
}
const debugCount = (global.hashStorageDebugCount || 0);
if (debugCount < 3) {
logger.log(` 🔍 [HASH STORAGE DEBUG] Storing hash for "${keyName}":`);
logger.log(` - syncHash: ${syncHash.substring(0, 8)}...`);
logger.log(` - meta.lokalise_sync_hash: ${meta.lokalise_sync_hash ? meta.lokalise_sync_hash.substring(0, 8) + '...' : 'null'}`);
global.hashStorageDebugCount = debugCount + 1;
}
}
const lokaliseData = {
...existingLokalise,
keys: {
...(existingLokalise.keys || {}),
[keyName]: {
key_id: keyId, // Keep for backward compatibility
field_path: fieldPath,
meta: meta,
},
},
};
await cfg.entityService.update(modelUid, targetId, {
data: { lokalise: lokaliseData },
publicationState: 'preview',
});
return true;
} catch (previewErr) {
// If preview fails, try published state
try {
const existingEntry = await cfg.entityService.findOne(modelUid, targetId);
const existingLokalise = existingEntry?.lokalise || {};
const existingKeyData = existingLokalise.keys?.[keyName] || {};
const metaBase = existingKeyData.meta || {};
const meta = {
...metaBase,
lokalise_key_id: keyId,
};
if (syncHash) {
meta.lokalise_sync_hash = syncHash;
meta.last_synced_at = Date.now();
}
if (Array.isArray(tagSnapshot) && tagSnapshot.length > 0) {
meta.lokalise_tags_snapshot = tagSnapshot;
} else if (metaBase.lokalise_tags_snapshot && !meta.lokalise_tags_snapshot) {
meta.lokalise_tags_snapshot = metaBase.lokalise_tags_snapshot;
}
const lokaliseData = {
...existingLokalise,
keys: {
...(existingLokalise.keys || {}),
[keyName]: {
key_id: keyId, // Keep for backward compatibility
field_path: fieldPath,
meta: meta,
},
},
};
await cfg.entityService.update(modelUid, targetId, {
data: { lokalise: lokaliseData },
});
return true;
} catch (publishedErr) {
// Both failed - fall through to lookup approach
}
}
}
// Fallback: Look up entry first, then update
let entry = null;
let resolvedEntryId = entryId || null;
if (entryId) {
entry = await getEntry(modelUid, entryId);
}
if (!entry && entrySlug) {
entry = await getEntryBySlug(modelUid, entrySlug);
if (entry) {
const fallbackId =
entry.id ??
getEntryIdentifier(entry) ??
entry.documentId ??
entry.entryId ??
resolvedEntryId;
resolvedEntryId = fallbackId ?? resolvedEntryId;
}
}
if (!entry) {
// Single diagnostic log with all relevant info
logger.error(`${context} modelUid="${modelUid}" → unable to locate Strapi entry (tried entryId and slug); skipping lokalise_key_id persistence.`);
return false;
}
// For Strapi v5, prefer documentId if available (works for both draft and published)
// Fall back to numeric id, then entryId, then the provided entryId
let finalTargetId = entry.documentId ?? entry.id ?? getEntryIdentifier(entry) ?? entry.entryId ?? null;
if (
(finalTargetId === null || typeof finalTargetId === 'undefined') &&
typeof resolvedEntryId !== 'undefined' &&
resolvedEntryId !== null
) {
finalTargetId = resolvedEntryId;
}
// Only convert to number if it's a numeric string (for Strapi v4 compatibility)
// For Strapi v5 documentIds (long strings), keep as string
if (typeof finalTargetId === 'string') {
const trimmed = finalTargetId.trim();
if (trimmed && /^[0-9]+$/.test(trimmed)) {
finalTargetId = Number(trimmed);
}
// For non-numeric strings (documentIds), keep as string - entityService.update accepts both
}
if (finalTargetId === null || typeof finalTargetId === 'undefined' || (typeof finalTargetId === 'number' && Number.isNaN(finalTargetId))) {
logger.error(`${context} → unable to determine a valid Strapi entry ID; skipping lokalise_key_id persistence.`);
return false;
}
targetId = finalTargetId;
try {
const lokaliseData = entry.lokalise || {};
if (!lokaliseData.keys) {
lokaliseData.keys = {};
}
const existingKeyData = lokaliseData.keys[keyName] || {};
const metaBase = existingKeyData.meta || {};
const meta = {
...metaBase,
lokalise_key_id: keyId,
};
if (syncHash) {
meta.lokalise_sync_hash = syncHash;
meta.last_synced_at = Date.now();
}
if (Array.isArray(tagSnapshot) && tagSnapshot.length > 0) {
meta.lokalise_tags_snapshot = tagSnapshot;
} else if (metaBase.lokalise_tags_snapshot && !meta.lokalise_tags_snapshot) {
meta.lokalise_tags_snapshot = metaBase.lokalise_tags_snapshot;
}
lokaliseData.keys[keyName] = {
key_id: keyId, // Keep for backward compatibility
field_path: fieldPath,
meta: meta,
};
// For Strapi v5: if targetId is a documentId (long string), it's likely a draft entry
// Try with publicationState: 'preview' first, then fall back to published
const isDocumentId = typeof targetId === 'string' && targetId.length > 10 && !/^\d+$/.test(targetId);
const isDraft = !entry.publishedAt || isDocumentId;
if (isDraft) {
// Try preview state first (for draft entries)
try {
await cfg.entityService.update(modelUid, targetId, {
data: {
lokalise: lokaliseData,
},
publicationState: 'preview',
});
return true;
} catch (previewErr) {
// If preview fails, try without publicationState (might be published after all)
await cfg.entityService.update(modelUid, targetId, {
data: {
lokalise: lokaliseData,
},
});
return true;
}
} else {
// Published entry - update normally
await cfg.entityService.update(modelUid, targetId, {
data: {
lokalise: lokaliseData,
},
});
return true;
}
} catch (err) {
logger.error(`${context} → failed to store lokalise_key_id=${keyId} on entry "${targetId}": ${err.message || err}`);
if (err && err.stack) {
logger.error(err.stack);
}
return false;
}
}
/**
* Validate key has all required metadata before syncing
*/
function validateKeyMetadata(key) {
const hasEntryId = key.entry_id && String(key.entry_id).trim().length > 0;
const hasFieldPath = key.field_path && String(key.field_path).trim().length > 0;
if (!hasEntryId || !hasFieldPath) {
return {
valid: false,
missing: [
!hasEntryId ? 'entry_id' : null,
!hasFieldPath ? 'field_path' : null,
].filter(Boolean),
};
}
return { valid: true };
}
/**
* Check if API token has manage_tags permission
* CRITICAL: Tests BOTH creating tags (via new key) AND updating tags (via existing key)
* Returns { hasPermission: boolean, canCreateTags: boolean, canUpdateTags: boolean, error?: string }
*/
async function checkTagPermissions() {
const result = {
hasPermission: false,
canCreateTags: false,
canUpdateTags: false,
error: null,
};
try {
logger.log(` 🔍 [PERMISSION CHECK] Starting comprehensive tag permission test...`);
// Test 1: Can we CREATE a tag by creating a new key?
const testTagName = `__perm_test_create_${Date.now()}__`;
const testKeyName = `__perm_test_key_${Date.now()}__`;
logger.log(` 🔍 [PERMISSION CHECK] Test 1: Creating new key with tag "${testTagName}"...`);
const createUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys`;
const createPayload = {
keys: [{
key_name: testKeyName,
platforms: ['web'],
tags: [testTagName],
translations: [{ language_iso: 'en', translation: '__test__' }],
}],
};
const createRes = await http.post(createUrl, createPayload, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
'Content-Type': 'application/json',
},
validateStatus: (status) => status < 600,
});
logger.log(` 🔍 [PERMISSION CHECK] Create key response: ${createRes.status}`);
if (createRes.status >= 200 && createRes.status < 300 && createRes.data) {
const keyId = createRes.data.keys?.[0]?.key_id || createRes.data.response_keys?.[0]?.key_id;
if (keyId) {
logger.log(` 🔍 [PERMISSION CHECK] Created test key with key_id=${keyId}, verifying tag...`);
// Verify tag exists on the new key
const verifyUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys/${keyId}`;
const verifyRes = await http.get(verifyUrl, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
},
validateStatus: (status) => status < 600,
});
if (verifyRes.status >= 200 && verifyRes.status < 300 && verifyRes.data?.key) {
const keyTags = Array.isArray(verifyRes.data.key.tags) ? verifyRes.data.key.tags : [];
const tagNames = keyTags.map(t => typeof t === 'string' ? t : (t?.name || ''));
const hasTag = tagNames.some(t => t.toLowerCase() === testTagName.toLowerCase());
logger.log(` 🔍 [PERMISSION CHECK] Key tags after creation: ${JSON.stringify(tagNames)}`);
logger.log(` 🔍 [PERMISSION CHECK] Tag "${testTagName}" found: ${hasTag}`);
if (hasTag) {
result.canCreateTags = true;
logger.log(` ✅ [PERMISSION CHECK] Test 1 PASSED: Can create tags via new key`);
// Test 2: Can we UPDATE an existing key by adding a NEW tag?
const updateTagName = `__perm_test_update_${Date.now()}__`;
logger.log(` 🔍 [PERMISSION CHECK] Test 2: Updating existing key (key_id=${keyId}) with NEW tag "${updateTagName}"...`);
// Get current key data
const currentKeyData = verifyRes.data.key;
const currentPlatforms = Array.isArray(currentKeyData.platforms) && currentKeyData.platforms.length > 0
? currentKeyData.platforms
: ['web'];
// Build key_name object for platforms
const keyNameObject = {};
if (typeof currentKeyData.key_name === 'string') {
currentPlatforms.forEach(platform => {
keyNameObject[platform] = currentKeyData.key_name;
});
} else if (typeof currentKeyData.key_name === 'object') {
currentPlatforms.forEach(platform => {
keyNameObject[platform] = currentKeyData.key_name[platform] || currentKeyData.key_name.web || testKeyName;
});
} else {
currentPlatforms.forEach(platform => {
keyNameObject[platform] = testKeyName;
});
}
// Try to add a NEW tag to the existing key
const updateUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys/${keyId}`;
const updatePayload = {
key: {
key_name: keyNameObject,
platforms: currentPlatforms,
tags: [testTagName, updateTagName], // Include existing tag + new tag
},
};
logger.log(` 🔍 [PERMISSION CHECK] Update payload: ${JSON.stringify(updatePayload, null, 2)}`);
const updateRes = await http.put(updateUrl, updatePayload, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
'Content-Type': 'application/json',
},
validateStatus: (status) => status < 600,
});
logger.log(` 🔍 [PERMISSION CHECK] Update key response: ${updateRes.status}`);
if (updateRes.status >= 200 && updateRes.status < 300) {
// Verify the new tag was added
await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for Lokalise to process
const updateVerifyRes = await http.get(verifyUrl, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
},
validateStatus: (status) => status < 600,
});
if (updateVerifyRes.status >= 200 && updateVerifyRes.status < 300 && updateVerifyRes.data?.key) {
const updatedTags = Array.isArray(updateVerifyRes.data.key.tags) ? updateVerifyRes.data.key.tags : [];
const updatedTagNames = updatedTags.map(t => typeof t === 'string' ? t : (t?.name || ''));
logger.log(` 🔍 [PERMISSION CHECK] Key tags after update: ${JSON.stringify(updatedTagNames)}`);
const hasNewTag = updatedTagNames.some(t => t.toLowerCase() === updateTagName.toLowerCase());
const hasOldTag = updatedTagNames.some(t => t.toLowerCase() === testTagName.toLowerCase());
logger.log(` 🔍 [PERMISSION CHECK] Old tag "${testTagName}" found: ${hasOldTag}`);
logger.log(` 🔍 [PERMISSION CHECK] New tag "${updateTagName}" found: ${hasNewTag}`);
if (hasNewTag) {
result.canUpdateTags = true;
logger.log(` ✅ [PERMISSION CHECK] Test 2 PASSED: Can update existing key with new tag`);
} else {
logger.error(` ❌ [PERMISSION CHECK] Test 2 FAILED: New tag "${updateTagName}" was NOT added to existing key`);
logger.error(` → This means API token CANNOT update tags on existing keys`);
logger.error(` → Even though it can create tags on new keys`);
result.error = `Can create tags but cannot update existing keys with new tags - token lacks "manage_tags" permission for updates`;
}
}
} else {
logger.error(` ❌ [PERMISSION CHECK] Update request failed: ${updateRes.status}`);
result.error = `Update request returned status ${updateRes.status}`;
}
} else {
logger.error(` ❌ [PERMISSION CHECK] Test 1 FAILED: Tag was not created on new key`);
result.error = 'API token cannot create tags - tag was not found after creation';
}
// Clean up test key
try {
logger.log(` 🔍 [PERMISSION CHECK] Cleaning up test key (key_id=${keyId})...`);
await http.delete(verifyUrl, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
},
validateStatus: (status) => status < 600,
});
logger.log(` ✅ [PERMISSION CHECK] Test key deleted`);
} catch (cleanupErr) {
logger.log(` ⚠️ [PERMISSION CHECK] Could not delete test key: ${cleanupErr.message}`);
}
} else {
logger.error(` ❌ [PERMISSION CHECK] Could not verify created key: ${verifyRes.status}`);
result.error = `Could not verify created key: ${verifyRes.status}`;
}
} else {
logger.error(` ❌ [PERMISSION CHECK] No key_id in create response`);
result.error = 'No key_id returned after creating test key';
}
} else {
logger.error(` ❌ [PERMISSION CHECK] Create key failed: ${createRes.status}`);
result.error = `Create key request returned status ${createRes.status}`;
}
// Final result
result.hasPermission = result.canCreateTags && result.canUpdateTags;
if (result.hasPermission) {
logger.log(` ✅ [PERMISSION CHECK] ALL TESTS PASSED: Token has full tag management permissions`);
} else {
logger.error(` ❌ [PERMISSION CHECK] PERMISSION ISSUE DETECTED:`);
logger.error(` → Can create tags: ${result.canCreateTags ? 'YES' : 'NO'}`);
logger.error(` → Can update tags: ${result.canUpdateTags ? 'YES' : 'NO'}`);
if (!result.canUpdateTags) {
logger.error(` → CRITICAL: Token can create tags but CANNOT update existing keys with new tags`);
logger.error(` → This is why new keys work but updates fail`);
}
}
return result;
} catch (err) {
logger.error(` ❌ [PERMISSION CHECK] Error during permission test: ${err.message}`);
if (err.response) {
logger.error(` Response status: ${err.response.status}`);
logger.error(` Response data: ${JSON.stringify(err.response.data, null, 2)}`);
}
return {
hasPermission: false,
canCreateTags: false,
canUpdateTags: false,
error: err.response?.data ? JSON.stringify(err.response.data) : err.message,
};
}
}
async function checkStrapiConnection() {
// Skip connection check in plugin mode (already connected via entityService)
if (cfg.entityService) {
return true;
}
if (!cfg.strapiBaseUrl) {
return false;
}
try {
const base = cfg.strapiBaseUrl.replace('/api', '');
await http.get(`${base}`, {
validateStatus: (status) => status < 600,
timeout: 5000,
});
return true;
} catch (err) {
if (err.code === 'ECONNREFUSED' || err.code === 'ETIMEDOUT' || err.code === 'ENOTFOUND') {
return false;
}
return true;
}
}
async function getAllStrapiTypes() {
// If entityService is provided (plugin mode), skip HTTP-based discovery
// Content types are already available via Strapi's contentTypes registry
if (cfg.entityService) {
logger.log('📋 Using Strapi contentTypes registry (plugin mode)');
return [];
}
if (!cfg.strapiBaseUrl) {
logger.log('⚠️ strapiBaseUrl not configured, skipping HTTP-based discovery');
return [];
}
logger.log('📋 Attempting to discover Strapi content types...');
const base = cfg.strapiBaseUrl.replace('/api', '');
try {
const res = await http.get(`${base}/content-type-builder/content-types`, {
headers: getAuthHeaders(),
});
const apiData = res.data && typeof res.data === 'object' && 'data' in res.data ? res.data.data : [];
const apiTypes = Object.values(apiData)
.filter((t) => t.uid && t.uid.startsWith('api::'))
.map((t) => t.uid.split('.')[1]);
if (apiTypes.length > 0) {
logger.log(`✅ Found ${apiTypes.length} Strapi types via content-type-builder:`, apiTypes.join(', '));
return apiTypes;
}
} catch (err) {
logger.log('⚠️ Could not access content-type-builder (may require admin token), trying alternative method...');
}
try {
const res = await http.get(`${base}/content-manager/content-types`, {
headers: getAuthHeaders(),
});
const data = res.data && typeof res.data === 'object' && 'data' in res.data ? res.data.data : res.data;
if (data && typeof data === 'object') {
const apiTypes = Object.keys(data)
.filter((uid) => uid.startsWith('api::'))
.map((uid) => uid.split('.')[1]);
if (apiTypes.length > 0) {
logger.log(`✅ Found ${apiTypes.length} Strapi types via content-manager:`, apiTypes.join(', '));
return apiTypes;
}
}
} catch (err) {
logger.log('⚠️ Could not access content-manager endpoint, trying discovery method...');
}
const discoveredTypes = [];
const commonTypes = [
'article', 'page', 'post', 'content', 'item', 'entry',
'about', 'home', 'services', 'service', 'contact',
'blog', 'news', 'product', 'category', 'menu',
'footer', 'header', 'section', 'banner', 'testimonial',
];
try {
const res = await http.get(`${base}/content-manager/collection-types`, {
headers: getAuthHeaders(),
validateStatus: (status) => status < 500,
});
if (res.status === 200 && res.data && Array.isArray(res.data)) {
res.data.forEach((item) => {
if (item && item.uid && item.uid.startsWith('api::')) {
const typeName = item.uid.split('.')[1];
if (typeName && !discoveredTypes.includes(typeName)) {
discoveredTypes.push(typeName);
logger.log(` ✓ Found via collection-types: ${typeName}`);
}
}
});
}
} catch (err) {
// ignore
}
const typesToTest = [...new Set([...commonTypes, ...discoveredTypes])];
for (const type of typesToTest) {
if (discoveredTypes.includes(type)) continue;
try {
const res = await http.get(`${cfg.strapiBaseUrl}/${type}?pagination[pageSize]=1`, {
headers: getAuthHeaders(),
validateStatus: (status) => status < 500,
timeout: 3000,
});
if (res.status === 200) {
discoveredTypes.push(type);
logger.log(` ✓ Found via endpoint test: ${type}`);
}
} catch (err) {
// ignore
}
}
if (discoveredTypes.length > 0) {
logger.log(`✅ Discovered ${discoveredTypes.length} content types:`, discoveredTypes.join(', '));
return discoveredTypes;
}
logger.log('⚠️ No content types auto-discovered');
return [];
}
async function fetchStrapiContent(type, options = {}) {
const { previewLimit = null } = options; // Limit entries for preview mode
logger.log(`📦 Fetching Strapi content for "${type}" ...`);
try {
const entityConfig = cfg.contentTypeMap[type];
if (cfg.entityService && entityConfig?.uid) {
// BEST PRACTICE: Intelligently handle relations based on dataset size
// Problem: Populating relations for large datasets creates reverse queries with thousands of IDs
// These queries exceed SQL parameter limits, causing "too many SQL variables" errors
// Solution:
// 1. Check if content type is in skipNestedRelations config (explicit exclusion)
// 2. For known large types (articles), skip relations by default
// 3. For others, try with relations first, retry without if SQL error occurs
// First, quickly count entries to determine if we should skip relations
let shouldSkipRelations = false;
const LARGE_DATASET_THRESHOLD = 1000; // If more than 1000 entries, skip relations
// Check explicit config
if (cfg.skipNestedRelations.has(type)) {
shouldSkipRelations = true;
logger.log(` ℹ️ Skipping relations for '${type}' (configured in skipNestedRelations)`);
} else if (type === 'articles') {
// Articles are commonly large, skip by default
shouldSkipRelations = true;
logger.log(` ℹ️ Skipping relations for '${type}' (known large dataset)`);
} else if (type === 'authors' || type === 'categories') {
// Authors and categories can have many entries and relations (articles)
// Skip relations to prevent SQL errors when fetching large datasets
shouldSkipRelations = true;
logger.log(` ℹ️ Skipping relations for '${type}' (known to have many entries with relations)`);
} else {
// For other types, try to count entries first
try {
const countParams = {
publicationState: 'preview',
pagination: { page: 1, pageSize: 1 },
};
const countResult = await cfg.entityService.findMany(entityConfig.uid, countParams);
// Check if we can get pagination metadata
let totalCount = null;
if (countResult && typeof countResult === 'object' && countResult.pagination) {
totalCount = countResult.pagination.total;
} else if (Array.isArray(countResult)) {
// If it's an array, we need to fetch first page to estimate
// For now, we'll try with relations and retry if needed
}
if (totalCount !== null && totalCount > LARGE_DATASET_THRESHOLD) {
shouldSkipRelations = true;
logger.log(` ℹ️ Skipping relations for '${type}' (${totalCount} entries > ${LARGE_DATASET_THRESHOLD} threshold)`);
}
} catch (countError) {
// If count fails, proceed with relations and retry if needed
logger.log(` ℹ️ Could not determine entry count for '${type}', will try with relations first`);
}
}
let params = {
publicationState: 'preview',
pagination: { pageSize: 1000 },
};
// CRITICAL: The 'lokalise' field is a custom JSON field (NOT a component/relation)
// Strapi's 'populate' parameter ONLY works for relations/components, NOT for custom JSON fields
// Custom JSON fields should be included by default when fetching entries
// However, if 'lokalise' is not registered in the content type schema, it might not be returned
// Solution: Ensure 'lokalise' is included by not filtering it out
// Since it's a custom field, it should be included automatically unless explicitly excluded
if (shouldSkipRelations) {
// For large datasets: skip relations for speed
// Custom fields like 'lokalise' should be included automatically
params.populate = {}; // Don't populate any relations (faster)
} else {
// For small datasets: populate all relations
// Custom fields like 'lokalise' should be included automatically
params.populate = '*'; // Populate all relations
}
// NOTE: 'lokalise' is stored as a custom JSON field via entityService.update()
// It should be included automatically when fetching entries
// If it's not being returned, it might be because:
// 1. The field is not registered in the content type schema
// 2. The field is being filtered out by Strapi
// 3. The field needs to be explicitly requested (but Strapi doesn't support this for custom fields)
// We'll need to verify that 'lokalise' is actually stored in the database and accessible
if (entityConfig.kind === 'singleType') {
const singleResult = await cfg.entityService.findMany(entityConfig.uid, params);
if (!singleResult) {
return [];
}
return Array.isArray(singleResult) ? singleResult : [singleResult];
}
// BEST PRACTICE: Process in batches - no hard limits, scales to any dataset size
// Fetch entries in pages of 1000, process incrementally to prevent memory issues
// This allows organizations with any number of entries (thousands to millions)
let allResults = [];
let page = 1;
let hasMore = true;
const maxPages = 1000000; // Very high limit: supports up to 1B entries (1,000,000 pages × 1,000/page) - effectively unlimited
const PREVIEW_LIMIT = previewLimit !== null ? previewLimit : Infinity; // Use limit if set, otherwise no limit
const startTime = Date.now();
const BATCH_PROCESSING_SIZE = 5000; // Process in batches of 5000 to manage memory
while (hasMore && page <= maxPages && allResults.length < PREVIEW_LIMIT) {
const pageParams = {
...params,
pagination: { page, pageSize: 1000 },
};
try {
const pageResults = await cfg.entityService.findMany(entityConfig.uid, pageParams);
// Handle Strapi v5 response format (may return object with data and pagination)
let resultsArray = [];
if (pageResults) {
if (Array.isArray(pageResults)) {
resultsArray = pageResults;
} else if (pageResults.data && Array.isArray(pageResults.data)) {
resultsArray = pageResults.data;
// Check pagination metadata if available
if (pageResults.pagination) {
hasMore = pageResults.pagination.page < pageResults.pagination.pageCount;
}
} else if (typeof pageResults === 'object') {
resultsArray = [pageResults];
}
}
if (resultsArray.length === 0) {
hasMore = false;
} else {
// Check if we've reached preview limit
if (previewLimit !== null && allResults.length + resultsArray.length > PREVIEW_LIMIT) {
// Only take what we need to reach the limit
const remaining = PREVIEW_LIMIT - allResults.length;
if (remaining > 0) {
allResults.push(...resultsArray.slice(0, remaining));
}
hasMore = false;
logger.log(` ℹ️ Reached preview limit of ${PREVIEW_LIMIT} entries (showing first ${allResults.length} for preview)`);
} else {
allResults.push(...resultsArray);
}
// If we got a full page, there might be more (unless pagination metadata says otherwise)
if (resultsArray.length < 1000 || (previewLimit !== null && allResults.length >= PREVIEW_LIMIT)) {
hasMore = false;
}
page++;
// Progress logging every 10 pages or every 10,000 entries
if (page % 10 === 0 || !hasMore || allResults.length % 10000 === 0) {
const elapsedSeconds = (Date.now() - startTime) / 1000;
const elapsed = elapsedSeconds.toFixed(1);
const limitMsg = previewLimit !== null ? ` (limit: ${PREVIEW_LIMIT})` : '';
const rate = elapsedSeconds > 0 ? (allResults.length / elapsedSeconds).toFixed(0) : '0';
logger.log(` 📄 Fetched ${allResults.length.toLocaleString()} entries${limitMsg} (${elapsed}s, ~${rate} entries/sec)...`);
}
// Memory management: If we're accumulating a lot, warn user
if (allResults.length > 100000 && allResults.length % 50000 === 0) {
logger.log(` ⚠️ Large dataset detected: ${allResults.length.toLocaleString()} entries. Processing in batches...`);
}
}
} catch (pageError) {
// If it's a SQL error about too many variables, retry without relations
if (pageError.message && pageError.message.includes('too many SQL variables')) {
if (!shouldSkipRelations && page === 1) {
// First page failed with relations, retry without relations
logger.log(` ⚠️ SQL error detected. Retrying '${type}' without relations...`);
shouldSkipRelations = true;
params.populate = { lokalise: true }; // CRITICAL: Always include lokalise for hash extraction
// Reset and retry from the beginning
allResults = [];
page = 1;
hasMore = true;
continue; // Retry the loop
} else {
// Already tried without relations or not first page - this is unexpected
throw new Error(
`❌ SQL error: Too many variables in query for '${type}'. ` +
`This usually happens when relations are populated for large datasets. ` +
`Relations have been excluded, but error persists. ` +
`This may indicate a very large dataset or database configuration issue.`
);
}
}
throw pageError;
}
}
if (page > maxPages) {
logger.log(` ⚠️ Reached maximum page limit (${maxPages}). Fetched ${allResults.length} entries.`);
}
const totalTime = ((Date.now() - startTime) / 1000).toFixed(1);
logger.log(` ✅ Fetched ${allResults.length} total entries for "${type}" in ${totalTime}s`);
return allResults;
}
const isCollection = cfg.collectionTypes.includes(type);
const query = `${cfg.strapiBaseUrl}/${type}?populate=*${isCollection ? '' : ''}&publicationState=preview`;
const res = await http.get(query, {
headers: getAuthHeaders(),
timeout: 10000,
});
return res.data && typeof res.data === 'object' && 'data' in res.data ? res.data.data : res.data;
} catch (err) {
if (err.code === 'ECONNREFUSED') {
throw new Error(`❌ Cannot connect to Strapi at ${cfg.strapiBaseUrl}. Is Strapi running?`);
} else if (err.code === 'ETIMEDOUT') {
throw new Error(`❌ Connection to Strapi timed out. Is Strapi running and accessible?`);
} else if (err.response) {
const status = err.response.status;
const statusText = err.response.statusText;
const errorData = err.response.data;
let errorMsg = `❌ Failed to fetch content for '${type}': ${status} - ${statusText}`;
if (errorData && typeof errorData === 'object' && errorData.error) {
errorMsg += `\n Details: ${JSON.stringify(errorData.error)}`;
}
throw new Error(errorMsg);
}
throw new Error(`❌ Unknown error fetching content for '${type}': ${err.message || err}`);
}
}
function getSlug(entry) {
const attrs = entry.attributes || entry;
if (attrs.slug && typeof attrs.slug === 'string' && attrs.slug.trim()) {
return attrs.slug.trim().toLowerCase().replace(/\s+/g, '-').replace(/[^a-z0-9-]/g, '');
}
if (attrs.title && typeof attrs.title === 'string' && attrs.title.trim()) {
return attrs.title.trim().toLowerCase().replace(/\s+/g, '-').replace(/[^a-z0-9-]/g, '').substring(0, 50);
}
if (attrs.name && typeof attrs.name === 'string' && attrs.name.trim()) {
return attrs.name.trim().toLowerCase().replace(/\s+/g, '-').replace(/[^a-z0-9-]/g, '').substring(0, 50);
}
if (attrs.heading && typeof attrs.heading === 'string' && attrs.heading.trim()) {
return attrs.heading.trim().toLowerCase().replace(/\s+/g, '-').replace(/[^a-z0-9-]/g, '').substring(0, 50);
}
if (entry.id) {
return String(entry.id);
}
if (attrs.id) {
return String(attrs.id);
}
if (entry.documentId) {
return String(entry.documentId);
}
if (attrs.documentId) {
return String(attrs.documentId);
}
return 'default';
}
const MEDIA_TEXT_FIELDS = ['alternativeText', 'caption'];
const isMediaAsset = (value) => {
if (!value || typeof value !== 'object') {
return false;
}
return (
Object.prototype.hasOwnProperty.call(value, 'mime') ||
Object.prototype.hasOwnProperty.call(value, 'provider') ||
Object.prototype.hasOwnProperty.call(value, 'formats')
);
};
async function getExistingTagsForKeys(keyNames = []) {
const tagsMap = new Map();
const uniqueKeys = Array.from(
new Set(keyNames.filter((name) => typeof name === 'string' && name.trim().length > 0))
);
if (uniqueKeys.length === 0) {
return tagsMap;
}
const chunkSize = 50;
for (let i = 0; i < uniqueKeys.length; i += chunkSize) {
const chunk = uniqueKeys.slice(i, i + chunkSize);
const params = new URLSearchParams();
// URL-encode key names to handle special characters like brackets [0]
chunk.forEach((keyName) => {
// Lokalise API expects key names to be properly encoded
// Special characters like [ ] need to be encoded
params.append('filter_key_names', keyName);
});
// Use a reasonable limit: if searching for few keys, use smaller limit to avoid getting 500 random keys
// If filter doesn't work, we'll search client-side through returned keys
const limit = chunk.length <= 5 ? Math.max(chunk.length * 10, 50) : Math.max(chunk.length, 500);
params.set('limit', String(limit));
params.set('include', 'tags');
try {
const url = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys?${params.toString()}`;
// Fetching keys from Lokalise
const res = await http.get(
url,
{
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
'Content-Type': 'application/json',
},
validateStatus: (status) => status < 600,
}
);
if (res.status >= 200 && res.status < 300) {
const responseKeys = Array.isArray(res.data?.keys)
? res.data.keys
: Array.isArray(res.data)
? res.data
: [];
// Process all returned keys and add them to the map
responseKeys.forEach((entry) => {
if (!entry) return;
const keyField = entry.key_name || entry.name;
let resolvedName = '';
if (typeof keyField === 'string') {
resolvedName = keyField;
} else if (keyField && typeof keyField === 'object') {
resolvedName =
keyField.web || keyField.other || keyField.ios || keyField.android || '';
}
if (!resolvedName) {
return;
}
const tagNames = Array.isArray(entry.tags)
? entry.tags
.map((tag) => {
if (!tag) return null;
if (typeof tag === 'string') return tag;
if (typeof tag.name === 'string') return tag.name;
return null;
})
.filter(Boolean)
: [];
// Lokalise returns key_id as a number - check both key_id and id fields
const keyId = entry.key_id ?? entry.id ?? undefined;
// Ensure key_id is a number (Lokalise may return it as string in some cases)
const numericKeyId = typeof keyId === 'number' ? keyId : (typeof keyId === 'string' && !isNaN(Number(keyId)) ? Number(keyId) : undefined);
tagsMap.set(resolvedName, {
tags: Array.from(new Set(tagNames)),
key_id: numericKeyId,
});
// Key_id fetched - no logging needed
});
// Check which requested keys were found
// If filter_key_names didn't work (returned 500 keys but not our specific one),
// search through all returned keys client-side as a fallback
chunk.forEach((keyName) => {
if (!tagsMap.has(keyName)) {
// Key not found via filter - try searching in returned keys (case-insensitive, handles variations)
// This is a fallback for when filter_key_names doesn't work with special characters
let foundKey = null;
for (const entry of responseKeys) {
if (!entry) continue;
const keyField = entry.key_name || entry.name;
let resolvedName = '';
if (typeof keyField === 'string') {
resolvedName = keyField;
} else if (keyField && typeof keyField === 'object') {
resolvedName = keyField.web || keyField.other || keyField.ios || keyField.android || '';
}
// Try exact match first
if (resolvedName === keyName) {
foundKey = entry;
break;
}
// Try case-insensitive match
if (resolvedName && resolvedName.toLowerCase() === keyName.toLowerCase()) {
foundKey = entry;
break;
}
}
if (foundKey) {
// Found via client-side search - add to map
const keyField = foundKey.key_name || foundKey.name;
let resolvedName = '';
if (typeof keyField === 'string') {
resolvedName = keyField;
} else if (keyField && typeof keyField === 'object') {
resolvedName = keyField.web || keyField.other || keyField.ios || keyField.android || '';
}
const tagNames = Array.isArray(foundKey.tags)
? foundKey.tags
.map((tag) => {
if (!tag) return null;
if (typeof tag === 'string') return tag;
if (typeof tag.name === 'string') return tag.name;
return null;
})
.filter(Boolean)
: [];
const keyId = foundKey.key_id ?? foundKey.id ?? undefined;
const numericKeyId = typeof keyId === 'number' ? keyId : (typeof keyId === 'string' && !isNaN(Number(keyId)) ? Number(keyId) : undefined);
tagsMap.set(keyName, {
tags: Array.from(new Set(tagNames)),
key_id: numericKeyId,
});
// Key found via client-side search - no logging needed
} else {
// Key truly not found in Lokalise - this is normal for new keys
tagsMap.set(keyName, { tags: [], key_id: undefined });
}
} else {
const meta = tagsMap.get(keyName);
// Log found keys for debugging (only for small chunks)
if (chunk.length <= 5 && meta.key_id) {
logger.log(` ✅ Found existing key in Lokalise: "${keyName}" (key_id: ${meta.key_id}, tags: ${meta.tags.length})`);
}
}
});
} else {
logger.error(
`Failed to fetch existing tags for keys chunk (${chunk.length} keys). Status: ${res.status}`
);
logger.error(` ↳ Request params: ${params.toString()}`);
chunk.forEach((keyName) => {
if (!tagsMap.has(keyName)) {
tagsMap.set(keyName, { tags: [], key_id: undefined });
}
});
}
} catch (err) {
logger.error('Failed to fetch existing Lokalise tags', err);
chunk.forEach((keyName) => {
if (!tagsMap.has(keyName)) {
tagsMap.set(keyName, { tags: [], key_id: undefined });
}
});
}
}
return tagsMap;
}
function processEntry({
entry,
type,
formattedKeys,
skippedReasons,
skippedFields,
fieldPrefix = '',
entrySlug = null,
rootEntryId = null,
previewMode = false,
debugMode = false,
rootLokaliseKeys = null, // Pass down from root entry for efficiency
}) {
if (!entry) return;
const attributes = entry.attributes || entry;
const idSource = rootEntryId ?? getEntryIdentifier(entry);
const entryId = idSource !== null && idSource !== undefined ? String(idSource) : null;
let slug = entrySlug || getSlug(entry);
slug = rememberEntrySlug(type, entryId, slug);
// Extract lokalise metadata from entry (includes hash data)
// Structure: entry.lokalise.keys[keyName].meta.lokalise_sync_hash
const entryLokalise = entry.lokalise || attributes.lokalise || {};
const entryLokaliseKeys = entryLokalise.keys || {};
// DEBUG: Log entry structure for first entry to verify lokalise data is present
if (!global.entryDebugLogged && type === 'about') {
logger.log(` 🔍 [ENTRY DEBUG] Entry "${type}.${slug}":`);
logger.log(` - entry.lokalise exists: ${!!entry.lokalise}`);
logger.log(` - entry.lokalise type: ${typeof entry.lokalise}`);
logger.log(` - entry.lokalise keys: ${entry.lokalise ? Object.keys(entry.lokalise).join(', ') : 'N/A'}`);
logger.log(` - attributes.lokalise exists: ${!!attributes.lokalise}`);
logger.log(` - attributes.lokalise type: ${typeof attributes.lokalise}`);
logger.log(` - entryLokaliseKeys count: ${Object.keys(entryLokaliseKeys).length}`);
if (Object.keys(entryLokaliseKeys).length > 0) {
const firstKeyName = Object.keys(entryLokaliseKeys)[0];
const firstKeyData = entryLokaliseKeys[firstKeyName];
logger.log(` - First key "${firstKeyName}":`);
logger.log(` - key_id: ${firstKeyData?.key_id || 'none'}`);
logger.log(` - meta exists: ${!!firstKeyData?.meta}`);
logger.log(` - meta.lokalise_sync_hash: ${firstKeyData?.meta?.lokalise_sync_hash ? firstKeyData.meta.lokalise_sync_hash.substring(0, 8) + '...' : 'none'}`);
} else {
logger.log(` - ⚠️ entryLokaliseKeys is EMPTY - lokalise field may not be populated!`);
logger.log(` - entry keys: ${entry ? Object.keys(entry).slice(0, 10).join(', ') : 'N/A'}`);
logger.log(` - attributes keys: ${attributes ? Object.keys(attributes).slice(0, 10).join(', ') : 'N/A'}`);
}
global.entryDebugLogged = true;
}
// Metadata for lokalise_key_id is injected via plugin metadata service
// If not provided, fallback to empty object
// Merge with entry's lokalise data to include hash information
const lokaliseKeys = rootLokaliseKeys && typeof rootLokaliseKeys === 'object'
? { ...rootLokaliseKeys, ...entryLokaliseKeys } // Merge to include hash from entry
: entryLokaliseKeys; // Use entry's lokalise data if no root provided
const normalizeKeyId = (value) => {
if (typeof value === 'number' && Number.isFinite(value)) {
return value;
}
if (typeof value === 'string') {
const parsed = Number(value.trim());
if (!Number.isNaN(parsed) && Number.isFinite(parsed)) {
return parsed;
}
}
return null;
};
// Helper to get lokalise_key_id and hash for a key_name (fast lookup)
const getLokaliseKeyData = (keyName, fieldPath) => {
let keyId = null;
let hash = null;
let tagSnapshot = null;
// Try exact key_name match first
if (lokaliseKeys[keyName]) {
const keyData = lokaliseKeys[keyName];
if (keyData.key_id !== undefined) {
keyId = normalizeKeyId(keyData.key_id);
}
// Extract hash from meta structure
if (keyData.meta) {
if (keyData.meta.lokalise_sync_hash) {
hash = keyData.meta.lokalise_sync_hash;
}
if (Array.isArray(keyData.meta.lokalise_tags_snapshot) && keyData.meta.lokalise_tags_snapshot.length > 0) {
tagSnapshot = keyData.meta.lokalise_tags_snapshot;
}
}
}
// Fallback: search by field_path (slower but handles edge cases)
if (!keyId) {
for (const [storedKeyName, storedData] of Object.entries(lokaliseKeys)) {
if (storedData && storedData.field_path === fieldPath && storedData.key_id !== undefined) {
keyId = normalizeKeyId(storedData.key_id);
// Extract hash from meta structure
if (storedData.meta) {
if (storedData.meta.lokalise_sync_hash) {
hash = storedData.meta.lokalise_sync_hash;
}
if (
Array.isArray(storedData.meta.lokalise_tags_snapshot) &&
storedData.meta.lokalise_tags_snapshot.length > 0
) {
tagSnapshot = storedData.meta.lokalise_tags_snapshot;
}
}
break;
}
}
}
return { keyId, hash, tagSnapshot };
};
if (debugMode) {
const keys = Object.keys(attributes).filter((k) => !['id', 'slug', 'createdAt', 'updatedAt', 'publishedAt'].includes(k));
logger.log(` Fields in ${type}/${slug}${fieldPrefix ? ` (${fieldPrefix})` : ''}: ${keys.join(', ')}`);
}
for (const [key, value] of Object.entries(attributes)) {
if (STRUCTURAL_KEYS.has(key)) {
continue;
}
if (value === null || value === undefined) continue;
const currentPath = fieldPrefix ? `${fieldPrefix}.${key}` : key;
if (!shouldIncludeField(currentPath)) {
skippedReasons[currentPath] = (skippedReasons[currentPath] || 0) + 1;
skippedFields.count += 1;
continue;
}
if (typeof value === 'string') {
if (value.trim().length > 0) {
const keyName = `${type}.${slug}.${currentPath}`;
const { keyId, hash, tagSnapshot } = getLokaliseKeyData(keyName, currentPath);
// Get updatedAt from entry for hash calculation
const entryUpdatedAt = entry.updatedAt || entry.updated_at || attributes.updatedAt || attributes.updated_at || null;
const keyData = {
key_name: keyName,
entry_id: entryId,
entry_slug: slug,
field_path: currentPath, // CRITICAL: Store field path for mapping back to Strapi
platforms: ['web'],
translations: [{ language_iso: 'en', translation: value }],
updatedAt: entryUpdatedAt, // Include for hash calculation
};
if (keyId) {
keyData.lokalise_key_id = keyId;
}
// Include hash data for sync status detection
if (hash || (Array.isArray(tagSnapshot) && tagSnapshot.length > 0)) {
keyData.lokalise_meta = {};
if (hash) {
keyData.lokalise_meta.lokalise_sync_hash = hash;
}
if (Array.isArray(tagSnapshot) && tagSnapshot.length > 0) {
keyData.lokalise_meta.lokalise_tags_snapshot = tagSnapshot;
}
}
formattedKeys.push(keyData);
}
continue;
}
if (typeof value === 'object' && !Array.isArray(value)) {
if (isMediaAsset(value)) {
MEDIA_TEXT_FIELDS.forEach((field) => {
const mediaValue = value[field];
if (typeof mediaValue === 'string' && mediaValue.trim().length > 0) {
const mediaPath = `${currentPath}.${field}`;
if (!shouldIncludeField(mediaPath)) {
skippedReasons[mediaPath] = (skippedReasons[mediaPath] || 0) + 1;
skippedFields.count += 1;
return;
}
const mediaKeyName = `${type}.${slug}.${mediaPath}`;
const {
keyId: mediaLokaliseKeyId,
hash: mediaHash,
tagSnapshot: mediaTagSnapshot,
} = getLokaliseKeyData(mediaKeyName, mediaPath);
// Get updatedAt from entry for hash calculation
const entryUpdatedAt = entry.updatedAt || entry.updated_at || attributes.updatedAt || attributes.updated_at || null;
const mediaKeyData = {
key_name: mediaKeyName,
entry_id: entryId,
entry_slug: slug,
field_path: mediaPath, // CRITICAL: Store field path for mapping back to Strapi
platforms: ['web'],
translations: [{ language_iso: 'en', translation: mediaValue }],
updatedAt: entryUpdatedAt, // Include for hash calculation
};
if (mediaLokaliseKeyId) {
mediaKeyData.lokalise_key_id = mediaLokaliseKeyId;
}
if (mediaHash || (Array.isArray(mediaTagSnapshot) && mediaTagSnapshot.length > 0)) {
mediaKeyData.lokalise_meta = {};
if (mediaHash) {
mediaKeyData.lokalise_meta.lokalise_sync_hash = mediaHash;
}
if (Array.isArray(mediaTagSnapshot) && mediaTagSnapshot.length > 0) {
mediaKeyData.lokalise_meta.lokalise_tags_snapshot = mediaTagSnapshot;
}
}
formattedKeys.push(mediaKeyData);
}
});
continue;
}
if (value.data) {
skippedReasons[currentPath] = (skippedReasons[currentPath] || 0) + 1;
skippedFields.count += 1;
continue;
}
if (value.__component || value.__type) {
if (value.attributes) {
processEntry({
entry: value,
type,
formattedKeys,
skippedReasons,
skippedFields,
fieldPrefix: currentPath,
entrySlug: slug,
rootEntryId: entryId,
previewMode,
debugMode,
rootLokaliseKeys, // Pass down from root
});
}
continue;
}
if (Object.keys(value).length > 0) {
try {
processEntry({
entry: value,
type,
formattedKeys,
skippedReasons,
skippedFields,
fieldPrefix: currentPath,
entrySlug: slug,
rootEntryId: entryId,
previewMode,
debugMode,
rootLokaliseKeys, // Pass down from root
});
} catch (err) {
logger.error(`Error processing object at ${currentPath}:`, err);
}
}
continue;
}
if (Array.isArray(value)) {
const looksLikeRelationArray =
value.length > 0 &&
value.every((item) => {
if (item === null) return true;
if (typeof item !== 'object') return false;
if ('__component' in item) return false;
return item.attributes !== undefined || item.documentId !== undefined;
});
if (looksLikeRelationArray) {
skippedReasons[currentPath] = (skippedReasons[currentPath] || 0) + 1;
skippedFields.count += value.length;
return;
}
value.forEach((item, idx) => {
const arrayPath = `${currentPath}[${idx}]`;
if (item && typeof item === 'object') {
processEntry({
entry: item,
type,
formattedKeys,
skippedReasons,
skippedFields,
fieldPrefix: arrayPath,
entrySlug: slug,
rootEntryId: entryId,
previewMode,
debugMode,
rootLokaliseKeys, // Pass down from root
});
} else if (typeof item === 'string' && item.trim().length > 0) {
if (!shouldIncludeField(arrayPath)) {
skippedReasons[arrayPath] = (skippedReasons[arrayPath] || 0) + 1;
skippedFields.count += 1;
return;
}
const arrayKeyName = `${type}.${slug}.${arrayPath}`;
const {
keyId: arrayLokaliseKeyId,
hash: arrayHash,
tagSnapshot: arrayTagSnapshot,
} = getLokaliseKeyData(arrayKeyName, arrayPath);
// Get updatedAt from entry for hash calculation
const entryUpdatedAt = entry.updatedAt || entry.updated_at || attributes.updatedAt || attributes.updated_at || null;
const arrayKeyData = {
key_name: arrayKeyName,
entry_id: entryId,
entry_slug: slug,
field_path: arrayPath, // CRITICAL: Store field path for mapping back to Strapi
platforms: ['web'],
translations: [{ language_iso: 'en', translation: item }],
updatedAt: entryUpdatedAt, // Include for hash calculation
};
if (arrayLokaliseKeyId) {
arrayKeyData.lokalise_key_id = arrayLokaliseKeyId;
}
if (arrayHash || (Array.isArray(arrayTagSnapshot) && arrayTagSnapshot.length > 0)) {
arrayKeyData.lokalise_meta = {};
if (arrayHash) {
arrayKeyData.lokalise_meta.lokalise_sync_hash = arrayHash;
}
if (Array.isArray(arrayTagSnapshot) && arrayTagSnapshot.length > 0) {
arrayKeyData.lokalise_meta.lokalise_tags_snapshot = arrayTagSnapshot;
}
}
formattedKeys.push(arrayKeyData);
}
});
}
}
}
// Helper function to normalize search term for fuzzy matching
const normalizeSearchTerm = (term) => {
if (!term) return '';
return term
.trim()
.replace(/\s+/g, ' ') // Collapse multiple spaces
.toLowerCase();
};
// Helper function to create searchable variations
const createSearchableVariations = (text) => {
if (!text) return [''];
const normalized = normalizeSearchTerm(text);
const variations = [normalized];
const withoutSeparators = normalized.replace(/[-\s]/g, '');
if (withoutSeparators !== normalized && withoutSeparators.length > 0) {
variations.push(withoutSeparators);
}
const words = normalized.split(/[\s-]+/).filter(Boolean);
if (words.length > 1) {
variations.push(words.join(''));
}
return [...new Set(variations)];
};
// Create a more flexible search term that handles spaces, case, and special characters
const createFlexibleSearchTerm = (term) => {
if (!term) return { original: '', normalized: '', withoutSpaces: '', wordsOnly: '' };
const trimmed = term.trim();
return {
original: trimmed,
normalized: normalizeSearchTerm(trimmed),
withoutSpaces: trimmed.replace(/\s+/g, '').toLowerCase(),
wordsOnly: trimmed.replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim().toLowerCase(),
};
};
// Enhanced fuzzy match function that handles spaces, case, and special characters
const fuzzyMatch = (searchTerm, text) => {
if (!searchTerm || !text) return false;
const search = createFlexibleSearchTerm(searchTerm);
const textStr = String(text);
const textLower = textStr.toLowerCase();
const searchLower = searchTerm.toLowerCase();
// 1. Exact match (case-insensitive) - highest priority for key names
if (textLower === searchLower) return true;
// 2. Direct substring match (case-insensitive) - handles exact matches
if (textLower.includes(search.normalized)) return true;
// 3. For key names with special characters (brackets, dots), try exact match first
// This handles cases like "about.about-the-strapi-blog.blocks[0].title"
if (textLower.includes(searchLower)) return true;
// 4. Match without spaces (handles "Thelonius Monk" matching "TheloniusMonk" or "thelonius-monk")
if (search.withoutSpaces && search.withoutSpaces.length > 2) {
const textWithoutSpaces = textLower.replace(/\s+/g, '');
if (textWithoutSpaces.includes(search.withoutSpaces)) return true;
}
// 5. Word-based matching (handles "Thelonius Monk" matching text containing both words)
if (search.wordsOnly) {
const words = search.wordsOnly.split(/\s+/).filter(w => w.length > 0);
if (words.length > 0) {
// Check if all words are present (in any order)
const allWordsMatch = words.every(word => textLower.includes(word));
if (allWordsMatch) return true;
}
}
// 6. Try variations for fuzzy matching (handles hyphens, underscores, etc.)
const searchVariations = createSearchableVariations(searchTerm);
const textVariations = createSearchableVariations(text);
return searchVariations.some((searchVar) => {
if (searchVar.length < 2) return false;
return textVariations.some((textVar) => textVar.includes(searchVar));
});
};
// Helper to convert slug to display name
const slugToDisplayName = (slug) => {
if (!slug || slug === 'general' || slug === 'default') return slug;
return slug
.split('-')
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
.join(' ');
};
async function pushToLokalise(type, content, options = {}) {
const { previewMode = false, debugMode = false, slugFilters = [], keyNameFilters = [], keyIdFilters = [], keyValueFilters = [] } = options;
if (!content || (Array.isArray(content) && content.length === 0)) {
if (type === 'articles') {
logger.log(`⚠️ No content found for 'articles'. Raw response: ${JSON.stringify(content)}`);
}
logger.log(`⚠️ No content found for '${type}'.`);
return [];
}
if (type === 'articles' && debugMode) {
logger.log(`🔍 Articles raw content: ${JSON.stringify(content, null, 2)}`);
}
logger.log(`🚀 Preparing "${type}" content for Lokalise...`);
const formattedKeys = [];
const skippedReasons = {};
const skippedFields = { count: 0 };
const entries = Array.isArray(content) ? content : [content];
if (Array.isArray(content)) {
logger.log(` 📦 Found ${entries.length} entries`);
}
const metadataCache = new Map();
if (cfg.metadataService && entries.length > 0) {
try {
const metadataRecords = await cfg.metadataService.getMetadata({
type,
entries: entries.map((entry) => ({
documentId: getEntryIdentifier(entry),
legacyEntryId: entry?.id ?? entry?.attributes?.id ?? null,
entrySlug: getSlug(entry),
})),
});
metadataRecords.forEach((record) => {
// Match schema field names: entry_document_id, entry_numeric_id, entry_slug
const candidates = [
record.entry_document_id,
record.entry_numeric_id,
record.entry_slug,
]
.filter(Boolean)
.map((value) => String(value));
if (candidates.length === 0) {
return;
}
candidates.forEach((candidate) => {
if (!metadataCache.has(candidate)) {
metadataCache.set(candidate, []);
}
metadataCache.get(candidate).push(record);
});
});
} catch (err) {
logger.error(
` ❌ Failed to load stored Lokalise metadata for '${type}': ${err.message || err}`
);
}
}
entries.forEach((entry, idx) => {
const slug = getSlug(entry);
// Apply slug filters if provided (fuzzy matching)
// BUT: If we have key name/value filters, don't filter by slug (let key filters handle it)
// This allows searching for key names/values even if the slug doesn't match
const hasKeyFilters = keyNameFilters.length > 0 || keyIdFilters.length > 0 || keyValueFilters.length > 0;
if (slugFilters.length > 0 && !hasKeyFilters) {
const slugDisplayName = slugToDisplayName(slug);
const matches = slugFilters.some((filter) => {
if (filter.endsWith('*')) {
// Prefix match
const prefix = filter.slice(0, -1).toLowerCase();
return slug.toLowerCase().startsWith(prefix) || slugDisplayName.toLowerCase().startsWith(prefix);
}
// Fuzzy match
return fuzzyMatch(filter, slug) || fuzzyMatch(filter, slugDisplayName);
});
if (!matches) {
if (debugMode) {
logger.log(` ℹ️ Skipping entry ${slug} (doesn't match slug filters)`);
}
return; // Skip this entry
}
}
// Only log per-entry progress if debugMode is true (suppress during batch processing)
if (debugMode) {
logger.log(` 📌 Processing entry ${idx + 1}/${entries.length}: ${slug}`);
}
if (debugMode) {
const attrs = entry.attributes || entry;
logger.log(
` Fields found: ${Object.keys(attrs)
.filter((k) => !['id', 'slug', 'createdAt', 'updatedAt', 'publishedAt'].includes(k))
.join(', ')}`
);
}
const entryIdValue = getEntryIdentifier(entry);
const entryId =
entryIdValue !== null && entryIdValue !== undefined ? String(entryIdValue) : null;
const metadataCandidates = [];
if (entryId) metadataCandidates.push(String(entryId));
if (entry?.id) metadataCandidates.push(String(entry.id));
if (slug) metadataCandidates.push(slug);
let rootMetadataRecords = [];
for (const candidate of metadataCandidates) {
if (metadataCache.has(candidate)) {
rootMetadataRecords = metadataCache.get(candidate);
break;
}
}
const rootLokaliseKeys =
rootMetadataRecords.length > 0
? rootMetadataRecords.reduce((acc, record) => {
if (record && record.key_name) {
const metaPayload = {};
if (record.lokalise_sync_hash) {
metaPayload.lokalise_sync_hash = record.lokalise_sync_hash;
}
if (Array.isArray(record.lokalise_tags_snapshot) && record.lokalise_tags_snapshot.length > 0) {
metaPayload.lokalise_tags_snapshot = record.lokalise_tags_snapshot;
}
acc[record.key_name] = {
key_id: record.lokalise_key_id,
field_path: record.field_path,
// Include hash/tags snapshot from metadata service as fallback if entry.lokalise is not available
meta: Object.keys(metaPayload).length > 0 ? metaPayload : undefined,
};
}
return acc;
}, {})
: null;
if (idx === 0 && debugMode && rootMetadataRecords.length > 0) {
logger.log(
` ✅ Loaded ${rootMetadataRecords.length} lokalise_key_id record(s) from metadata store for entry "${slug}"`
);
}
processEntry({
entry,
type,
formattedKeys,
skippedReasons,
skippedFields,
entrySlug: slug,
rootEntryId: entryId,
previewMode,
debugMode,
rootLokaliseKeys, // Pass down for nested calls
});
});
if (skippedFields.count > 0) {
logger.log(` ℹ️ Skipped ${skippedFields.count} fields (filtered out)`);
const names = Object.keys(skippedReasons);
if (names.length > 0) {
logger.log(` Skipped fields: ${names.slice(0, 10).join(', ')}${names.length > 10 ? '...' : ''}`);
}
}
if (formattedKeys.length === 0) {
logger.log(`⚠️ No valid text fields found for '${type}' after filtering.`);
logger.log(' 💡 Tip: Check STRAPI_FIELD_INCLUDE and STRAPI_FIELD_EXCLUDE in configuration');
return [];
}
// Filter by key name, key ID, or key value if provided
let filteredKeys = formattedKeys;
if (keyNameFilters.length > 0 || keyIdFilters.length > 0 || keyValueFilters.length > 0) {
const originalLength = formattedKeys.length;
filteredKeys = formattedKeys.filter((key) => {
if (!key || !key.key_name) return false;
let hasMatch = false;
// Check key name filters
if (keyNameFilters.length > 0) {
const keyNameMatch = keyNameFilters.some((filter) => {
const keyName = String(key.key_name || '');
const keyNameLower = keyName.toLowerCase();
const filterStr = String(filter);
const filterLower = filterStr.toLowerCase();
// 1. Exact match (case-insensitive) - highest priority for key names with special chars
if (keyNameLower === filterLower) return true;
// 2. Direct substring match (case-insensitive) - handles partial key names
// This is important for key names like "about.about-the-strapi-blog.blocks[0].title"
if (keyNameLower.includes(filterLower)) return true;
// 3. Try exact match with original casing (preserves special characters)
const filterOriginal = String(filter);
if (keyName.includes(filterOriginal)) return true;
if (keyName.toLowerCase().includes(filterOriginal.toLowerCase())) return true;
// 4. Reverse substring match - handles when filter is longer than key name
if (filterLower.includes(keyNameLower) && keyNameLower.length > 3) return true;
// 5. Try fuzzy match for variations
return fuzzyMatch(filter, key.key_name);
});
if (keyNameMatch) hasMatch = true;
}
// Check key ID filters (check both lokalise_key_id and entry_id)
if (keyIdFilters.length > 0) {
const keyIdMatch = keyIdFilters.some((filter) => {
// Check Lokalise key ID if available
if (key.lokalise_key_id) {
const lokaliseIdStr = String(key.lokalise_key_id);
if (lokaliseIdStr.includes(filter.toLowerCase()) || fuzzyMatch(filter, lokaliseIdStr)) {
return true;
}
}
// Check entry ID (Strapi entry ID)
if (key.entry_id) {
const entryIdStr = String(key.entry_id);
if (entryIdStr.includes(filter.toLowerCase()) || fuzzyMatch(filter, entryIdStr)) {
return true;
}
}
return false;
});
if (keyIdMatch) hasMatch = true;
}
// Check key value filters (translation text) - most flexible matching
if (keyValueFilters.length > 0) {
const keyValueMatch = keyValueFilters.some((filter) => {
// Check all translations for this key
if (key.translations && Array.isArray(key.translations)) {
return key.translations.some((translation) => {
const translationText = translation.translation || translation.value || '';
if (typeof translationText === 'string' && translationText.trim().length > 0) {
// Use enhanced fuzzy matching for translation text
return fuzzyMatch(filter, translationText);
}
return false;
});
}
return false;
});
if (keyValueMatch) hasMatch = true;
}
// If filters are provided, only include if at least one matches
if (keyNameFilters.length > 0 || keyIdFilters.length > 0 || keyValueFilters.length > 0) {
return hasMatch;
}
return true;
});
if (debugMode && filteredKeys.length < originalLength) {
logger.log(` ℹ️ Filtered ${originalLength - filteredKeys.length} keys by key name/ID/value filters`);
}
} else {
filteredKeys = formattedKeys;
}
// Early duplicate detection - check for duplicate key_names
const keyNameSet = new Set();
const duplicateKeys = [];
filteredKeys.forEach((key, idx) => {
if (key && key.key_name) {
if (keyNameSet.has(key.key_name)) {
duplicateKeys.push({ index: idx, key_name: key.key_name });
} else {
keyNameSet.add(key.key_name);
}
}
});
if (duplicateKeys.length > 0) {
logger.log(`⚠️ Found ${duplicateKeys.length} duplicate key(s) in '${type}' during collection:`);
duplicateKeys.slice(0, 10).forEach((dup) => {
logger.log(` Duplicate: ${dup.key_name} (at index ${dup.index})`);
});
if (duplicateKeys.length > 10) {
logger.log(` ... and ${duplicateKeys.length - 10} more duplicates`);
}
logger.log(` ℹ️ Duplicates will be removed before syncing (keeping last occurrence)`);
}
logger.log(` 📝 Found ${filteredKeys.length} keys to sync:`);
filteredKeys.slice(0, 5).forEach((key, idx) => {
const preview = key.translations[0].translation.substring(0, 50);
logger.log(
` ${idx + 1}. ${key.key_name} = "${preview}${key.translations[0].translation.length > 50 ? '...' : ''}"`
);
});
if (filteredKeys.length > 5) {
logger.log(` ... and ${filteredKeys.length - 5} more keys`);
}
if (previewMode) {
return filteredKeys;
}
// CRITICAL: Only fetch key_ids for keys that don't have lokalise_key_id from preview/Strapi
// If key has lokalise_key_id, use it directly - NO SEARCH NEEDED!
const keysWithStoredId = filteredKeys.filter(k => typeof k.lokalise_key_id === 'number' && k.lokalise_key_id > 0);
const keysNeedingLookup = filteredKeys.filter(k => !(typeof k.lokalise_key_id === 'number' && k.lokalise_key_id > 0));
logger.log(` 📊 Key ID Status:`);
logger.log(` ✅ ${keysWithStoredId.length} key(s) have lokalise_key_id from preview/Strapi - will use directly (NO SEARCH)`);
logger.log(` 🔍 ${keysNeedingLookup.length} key(s) need lookup from Lokalise`);
// Log keys with stored ID
if (keysWithStoredId.length > 0 && keysWithStoredId.length <= 10) {
keysWithStoredId.forEach(key => {
logger.log(` ✅ "${key.key_name}" → lokalise_key_id=${key.lokalise_key_id} (from Strapi, will UPDATE)`);
});
}
const keyNames = filteredKeys.map(k => k?.key_name).filter(Boolean);
let existingTagsMap = new Map();
// Only fetch from Lokalise for keys that don't have lokalise_key_id
if (keysNeedingLookup.length > 0) {
const lookupNames = keysNeedingLookup.map(k => k.key_name).filter(Boolean);
try {
existingTagsMap = await getExistingTagsForKeys(lookupNames);
// Keys fetched from Lokalise
} catch (err) {
logger.error(` ❌ Failed to fetch existing key metadata from Lokalise: ${err.message || String(err)}`);
logger.error(` ℹ️ Continuing with sync - keys will be created as new if they don't exist`);
}
} else {
logger.log(` ✅ ALL ${filteredKeys.length} key(s) have lokalise_key_id from preview/Strapi - NO LOOKUP NEEDED!`);
}
const { totalPushed, totalUpdated } = await syncKeysToLokalise(type, filteredKeys, {
...options,
existingTagsMap,
});
if (totalPushed > 0) {
const updateMsg = totalUpdated > 0 ? ` (${totalUpdated} updated)` : '';
logger.log(`✅ Successfully synced ${totalPushed} keys for '${type}' to Lokalise${updateMsg}.`);
} else {
logger.error(`❌ Failed to sync any keys for '${type}' to Lokalise.`);
}
return filteredKeys;
}
// CRITICAL: Find existing key in Lokalise
// Priority: 1. Use lokalise_key_id directly (GET /keys/{key_id}) - MOST RELIABLE
// 2. Only if no key_id, search by name (unreliable, but necessary for first-time sync)
async function findExistingKey(keyName, lokaliseKeyId) {
// 1. Direct lookup by key_id (MOST RELIABLE - ALWAYS USE THIS IF AVAILABLE)
if (lokaliseKeyId && typeof lokaliseKeyId === 'number') {
try {
const url = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys/${lokaliseKeyId}`;
const res = await http.get(url, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
},
validateStatus: (status) => status < 600,
});
if (res.status >= 200 && res.status < 300 && res.data && res.data.key) {
return res.data.key;
}
} catch (err) {
// Silent fail - will try search by name
}
}
// 2. Fallback: search by key name ONLY if no key_id (unreliable, but necessary)
// NOTE: This is unreliable for keys with brackets/dots/namespaces
// Strategy: Try filter_keys first (fast but unreliable), then fallback to pagination if needed
// Step 2a: Try filter_keys parameter first (FAST but unreliable for special characters)
try {
const filterUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys?filter_keys=${encodeURIComponent(keyName)}&limit=100&include=tags`;
const filterRes = await http.get(filterUrl, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
},
validateStatus: (status) => status < 600,
});
if (filterRes.status >= 200 && filterRes.status < 300 && filterRes.data && filterRes.data.keys) {
const filteredKeys = filterRes.data.keys || [];
logger.log(` 📊 filter_keys returned ${filteredKeys.length} key(s) for "${keyName}"`);
// Search through filtered results for exact match
const nameLower = keyName.toLowerCase();
const nameNormalized = nameLower.replace(/\[(\d+)\]/g, '.$1');
for (const entry of filteredKeys) {
if (!entry) continue;
const keyField = entry.key_name || entry.name;
let resolvedName = '';
if (typeof keyField === 'string') {
resolvedName = keyField;
} else if (keyField && typeof keyField === 'object') {
resolvedName = keyField.web || keyField.other || keyField.ios || keyField.android || '';
}
if (!resolvedName) continue;
const resolvedLower = resolvedName.toLowerCase();
const resolvedNormalized = resolvedLower.replace(/\[(\d+)\]/g, '.$1');
// Match: exact or normalized
if (resolvedLower === nameLower || resolvedNormalized === nameNormalized) {
logger.log(` ✅ Found key via filter_keys search: "${keyName}" → key_id=${entry.key_id}`);
return entry; // Found it via fast method!
}
}
logger.log(` ⚠️ filter_keys didn't find exact match for "${keyName}" (may have special characters) - trying pagination...`);
} else {
logger.log(` ⚠️ filter_keys request failed (${filterRes.status}) - trying pagination...`);
}
} catch (filterErr) {
logger.log(` ⚠️ filter_keys error: ${filterErr.message} - trying pagination...`);
}
// Step 2b: Fallback to pagination (SLOW but more reliable)
let allKeys = [];
let page = 1;
const pageSize = 5000;
let hasMore = true;
let foundKey = null;
while (hasMore && page <= 10 && !foundKey) { // Limit to 10 pages (50,000 keys max), stop if found
const paginationUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys?limit=${pageSize}&page=${page}&include=tags`;
const paginationRes = await http.get(paginationUrl, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
},
validateStatus: (status) => status < 600,
});
if (paginationRes.status >= 200 && paginationRes.status < 300) {
const pageKeys = paginationRes.data?.keys || [];
logger.log(` 📊 Page ${page}: fetched ${pageKeys.length} key(s)`);
allKeys = allKeys.concat(pageKeys);
// Search through fetched keys for exact match
const nameLower = keyName.toLowerCase();
const nameNormalized = nameLower.replace(/\[(\d+)\]/g, '.$1');
for (const entry of pageKeys) {
if (!entry) continue;
const keyField = entry.key_name || entry.name;
let resolvedName = '';
if (typeof keyField === 'string') {
resolvedName = keyField;
} else if (keyField && typeof keyField === 'object') {
resolvedName = keyField.web || keyField.other || keyField.ios || keyField.android || '';
}
if (!resolvedName) continue;
const resolvedLower = resolvedName.toLowerCase();
const resolvedNormalized = resolvedLower.replace(/\[(\d+)\]/g, '.$1');
// Match: exact or normalized
if (resolvedLower === nameLower || resolvedNormalized === nameNormalized) {
logger.log(` ✅ Found key via pagination search (page ${page}): "${keyName}" → key_id=${entry.key_id}`);
foundKey = entry;
break; // Found it!
}
}
if (foundKey) {
return foundKey;
}
// If we got less than pageSize, we've reached the end
if (pageKeys.length < pageSize) {
logger.log(` ℹ️ Reached end of keys (page ${page} returned ${pageKeys.length} < ${pageSize})`);
hasMore = false;
break;
}
page++;
} else {
logger.log(` ⚠️ Pagination request failed (${paginationRes.status})`);
hasMore = false;
break;
}
}
if (!foundKey) {
logger.log(` ❌ Key "${keyName}" not found after searching ${page - 1} page(s) (${allKeys.length} total keys checked)`);
}
// Not found
return foundKey;
}
async function syncKeysToLokalise(type, formattedKeys, options = {}) {
const BATCH_SIZE = 500;
const existingMetaMap =
options.existingTagsMap && options.existingTagsMap instanceof Map
? options.existingTagsMap
: new Map();
// Track key_id mappings to store back in Strapi after sync
const keyIdMapping = new Map(); // key_name -> { key_id, entry_id, entry_slug, field_path }
const trimmedTag =
typeof options.tag === 'string' && options.tag.trim().length > 0
? options.tag.trim()
: undefined;
// Deduplicate keys by key_name before processing
// Keep the last occurrence of each key_name (in case there are updates)
const keyNameMap = new Map();
formattedKeys.forEach((key) => {
if (key && key.key_name) {
keyNameMap.set(key.key_name, key);
}
});
const uniqueKeys = Array.from(keyNameMap.values());
if (formattedKeys.length > uniqueKeys.length) {
const duplicateCount = formattedKeys.length - uniqueKeys.length;
logger.log(`⚠️ Found ${duplicateCount} duplicate key(s) in '${type}'. Removed duplicates before syncing.`);
}
// CRITICAL: Validate that all keys have required metadata (entry_id, field_path)
// Without these, we cannot map the Lokalise key_id back to Strapi
const validKeys = [];
const invalidKeys = [];
uniqueKeys.forEach((key) => {
const validation = validateKeyMetadata(key);
if (!validation.valid) {
invalidKeys.push({
key_name: key.key_name || 'unknown',
missing: validation.missing,
});
} else {
validKeys.push(key);
}
});
if (invalidKeys.length > 0) {
logger.error(`⚠️ Skipping ${invalidKeys.length} key(s) due to missing required metadata:`);
invalidKeys.slice(0, 10).forEach((invalid, idx) => {
logger.error(` ${idx + 1}. "${invalid.key_name}" - missing: ${invalid.missing.join(', ')}`);
});
if (invalidKeys.length > 10) {
logger.error(` ... and ${invalidKeys.length - 10} more invalid key(s)`);
}
logger.error(` ℹ️ These keys will NOT be synced and key_id will NOT be stored in Strapi`);
}
// Use only valid keys for sync
const keysToSync = validKeys;
if (keysToSync.length === 0) {
logger.error(`❌ No valid keys to sync for '${type}' (all keys missing required metadata)`);
return { totalPushed: 0, totalUpdated: 0 };
}
if (keysToSync.length < uniqueKeys.length) {
logger.log(` ✅ Validated ${keysToSync.length}/${uniqueKeys.length} key(s) - proceeding with sync`);
}
const payloadSummary = {
newKeys: 0,
existingKeys: 0,
sampleNew: [],
sampleExisting: [],
};
// CRITICAL: For keys with lokalise_key_id, use the data we already have from preview
// We don't need to fetch each key individually - that's extremely slow!
// The preview already provided us with lokalise_key_id and existing_tags
const keysWithStoredId = keysToSync.filter(k => k.lokalise_key_id && typeof k.lokalise_key_id === 'number');
const keysNeedingLookup = keysToSync.filter(k => !k.lokalise_key_id || typeof k.lokalise_key_id !== 'number');
if (keysWithStoredId.length > 0) {
const batchIndex = options.batchIndex || '?';
// OPTIMIZATION: Use the data we already have from preview instead of making individual API calls
// This avoids 1000+ individual HTTP requests which would take 3-5 minutes!
// We already have:
// - lokalise_key_id (from preview/Strapi)
// - existing_tags (from preview)
// We only need to fetch if we're missing tags AND need to merge with existing Lokalise tags
// But for most cases, existing_tags from preview is sufficient
for (const key of keysWithStoredId) {
// Use existing_tags from preview if available, otherwise empty array
const tags = Array.isArray(key.existing_tags) && key.existing_tags.length > 0
? key.existing_tags
: [];
existingMetaMap.set(key.key_name, {
tags: tags,
key_id: key.lokalise_key_id,
translations: [], // We'll get translations from Strapi, not from Lokalise
});
}
// Only fetch from Lokalise if we need to verify/merge tags (rare case)
// For now, skip individual lookups - use preview data which is already accurate
}
// Keys needing lookup will be handled if duplicate errors occur
const payloadKeys = keysToSync
.map((key) => {
const meta = existingMetaMap.get(key.key_name) || { tags: [], key_id: undefined };
// Merge tags from both existingMetaMap and the key object's existing_tags
// This ensures we have all existing tags even if one source is incomplete
// Normalize tag names (trim and convert to string) to avoid duplicates from case/whitespace differences
const metaTags = Array.isArray(meta.tags)
? meta.tags.map(t => String(t).trim()).filter(t => t.length > 0)
: [];
const keyExistingTags = Array.isArray(key.existing_tags)
? key.existing_tags.map(t => String(t).trim()).filter(t => t.length > 0)
: [];
// Combine and deduplicate tags (case-insensitive comparison)
const allExistingTags = [...metaTags, ...keyExistingTags];
const existingTagsMap = new Map();
allExistingTags.forEach(tag => {
const normalized = String(tag).trim().toLowerCase();
if (normalized.length > 0 && !existingTagsMap.has(normalized)) {
// Keep the original casing from the first occurrence
existingTagsMap.set(normalized, String(tag).trim());
}
});
const existingTags = Array.from(existingTagsMap.values());
// CRITICAL: Use key_id from direct lookup FIRST (most reliable)
// Priority: 1. key.lokalise_key_id (from Strapi) → fetched via GET /keys/{key_id}
// 2. meta.key_id (from direct fetch)
// 3. undefined (will create as new, or search if "key name already taken")
let keyId = key.lokalise_key_id ?? meta.key_id ?? undefined;
// Convert to number if it's a string representation of a number
if (typeof keyId === 'string' && !isNaN(Number(keyId))) {
keyId = Number(keyId);
}
// Ensure it's a number or undefined
keyId = typeof keyId === 'number' ? keyId : undefined;
// EXTENSIVE LOGGING: Track key_id source for debugging
if (payloadSummary.existingKeys + payloadSummary.newKeys < 10) {
logger.log(` 🔑 Key ID lookup for "${key.key_name}":`);
logger.log(` - From key.lokalise_key_id (preview/Strapi): ${key.lokalise_key_id || 'not found'}`);
logger.log(` - From existingMetaMap (Lokalise fetch): ${meta.key_id || 'not found'}`);
logger.log(` - Final key_id to use: ${keyId || 'undefined (will create as new)'}`);
if (keyId && typeof keyId === 'number') {
logger.log(` ✅ Will UPDATE existing key in Lokalise with key_id=${keyId} (source: ${key.lokalise_key_id ? 'preview/Strapi' : 'Lokalise fetch'})`);
} else {
logger.log(` ⚠️ Will CREATE as new key (no key_id found - will search if "key name already taken" error)`);
}
}
const translations = Array.isArray(key.translations)
? key.translations.map((translation) =>
options.locale
? { ...translation, language_iso: options.locale }
: translation
)
: [];
// CRITICAL: Prepare tags for payload
// Merge tags from multiple sources:
// 1. Existing tags from Lokalise (existingTags)
// 2. Tags from the original key (key.tags) - these are NEW tags from Strapi
// 3. Tag from options (trimmedTag) - if provided
let allTags = [...existingTags];
// CRITICAL: Include tags from the original key (these are NEW tags from Strapi)
// The original key should have tags from formattedKeys (from pushToLokalise)
const originalKeyTags = Array.isArray(key.tags) ? key.tags : [];
originalKeyTags.forEach(tag => {
const tagStr = typeof tag === 'string' ? tag.trim() : String(tag).trim();
if (tagStr.length > 0) {
const normalized = tagStr.toLowerCase();
const exists = allTags.some(t => String(t).toLowerCase() === normalized);
if (!exists) {
allTags.push(tagStr);
}
}
});
if (trimmedTag) {
const normalizedNewTag = trimmedTag.toLowerCase();
const tagExists = allTags.some(t => String(t).toLowerCase() === normalizedNewTag);
if (!tagExists) {
allTags.push(trimmedTag);
}
}
// Final deduplication (case-insensitive)
const finalTagsMap = new Map();
allTags.forEach(tag => {
const normalized = String(tag).trim().toLowerCase();
if (normalized.length > 0 && !finalTagsMap.has(normalized)) {
finalTagsMap.set(normalized, String(tag).trim());
}
});
const tagsToSend = Array.from(finalTagsMap.values());
const {
tags: _unusedTags,
key_id: _unusedKeyId,
entry_slug: _unusedEntrySlug,
translations: _unusedTranslations,
...rest
} = key;
// CRITICAL: Preserve entry_id and field_path - they're needed to store key_id back in Strapi
// Also preserve updatedAt for hash calculation
const payloadKey = {
...rest,
translations,
// Preserve metadata for mapping back to Strapi
entry_id: key.entry_id,
entry_slug: key.entry_slug || null,
field_path: key.field_path,
// Preserve updatedAt for hash calculation (if available from formattedKeys)
updatedAt: key.updatedAt || key.updated_at || null,
};
// Include tags in payload if we have any to send
// For existing keys with merge_tags: true, Lokalise will merge our tags with existing ones
if (tagsToSend.length > 0) {
payloadKey.tags = tagsToSend;
}
// DEBUG: Log tag sources for first few keys
if (payloadSummary.existingKeys + payloadSummary.newKeys < 5) {
logger.log(` 🔍 [PAYLOAD PREP] Key "${key.key_name}":`);
logger.log(` - Original key.tags: ${Array.isArray(key.tags) ? JSON.stringify(key.tags) : 'NONE'}`);
logger.log(` - Existing tags (Lokalise): ${existingTags.length > 0 ? JSON.stringify(existingTags) : 'NONE'}`);
logger.log(` - Merged tagsToSend: ${tagsToSend.length > 0 ? JSON.stringify(tagsToSend) : 'NONE'}`);
logger.log(` - Translations: ${translations.length > 0 ? `${translations.length} translation(s)` : 'NONE'}`);
}
if (typeof keyId === 'number') {
payloadKey.key_id = keyId;
payloadSummary.existingKeys += 1;
if (payloadSummary.sampleExisting.length < 3) {
payloadSummary.sampleExisting.push(key.key_name);
}
// Updating existing key with key_id
} else {
// New key - only include tags if we have any
if (tagsToSend.length > 0) {
payloadKey.tags = tagsToSend;
}
payloadSummary.newKeys += 1;
if (payloadSummary.sampleNew.length < 3) {
payloadSummary.sampleNew.push(key.key_name);
}
}
return payloadKey;
})
.filter(Boolean);
if (payloadKeys.length === 0) {
logger.log('⚠️ No payload keys to sync after processing selection.');
return { totalPushed: 0, totalUpdated: 0 };
}
const smallJobThreshold = cfg.smallJobThreshold ?? DEFAULT_SMALL_JOB_THRESHOLD;
const shouldSkipBatching = payloadKeys.length > 0 && payloadKeys.length <= smallJobThreshold;
if (shouldSkipBatching) {
logger.log(
` ⚡ Small job detected (${payloadKeys.length}/${smallJobThreshold} key threshold) – sending keys immediately without queue batching.`
);
}
// OPTIMIZATION: Only fetch tags if we're missing them from preview
// For most cases, preview already provided existing_tags, so we skip this expensive operation
const existingKeyIds = payloadKeys
.filter(k => typeof k.key_id === 'number')
.map(k => k.key_id);
const keysWithPreviewTags = payloadKeys.filter(k =>
typeof k.key_id === 'number' &&
Array.isArray(k.existing_tags) &&
k.existing_tags.length > 0
).length;
let existingKeysData = new Map();
// Only fetch if we're missing tags for significant number of keys
if (existingKeyIds.length > 0 && keysWithPreviewTags < existingKeyIds.length * 0.5) {
// Fetch in batches to avoid rate limits
const BATCH_FETCH_SIZE = 50;
for (let i = 0; i < existingKeyIds.length; i += BATCH_FETCH_SIZE) {
const batchIds = existingKeyIds.slice(i, i + BATCH_FETCH_SIZE);
const fetchPromises = batchIds.map(async (keyId) => {
try {
const url = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys/${keyId}`;
const res = await http.get(url, {
headers: { 'X-Api-Token': cfg.lokaliseApiToken },
validateStatus: (status) => status < 600,
});
if (res.status >= 200 && res.status < 300 && res.data && res.data.key) {
const keyData = res.data.key;
return {
keyId,
data: {
key_id: keyData.key_id,
tags: Array.isArray(keyData.tags) ? keyData.tags : [],
translations: Array.isArray(keyData.translations) ? keyData.translations : [],
},
};
}
} catch (err) {
// Silent fail - will use preview tags
}
return null;
});
const fetchResults = await Promise.all(fetchPromises);
fetchResults.forEach(result => {
if (result && result.data) {
existingKeysData.set(result.keyId, result.data);
}
});
}
}
// Merge tags: preview tags + fetched tags + new tags from payload
payloadKeys.forEach((payloadKey) => {
if (typeof payloadKey.key_id === 'number') {
const previewTags = Array.isArray(payloadKey.existing_tags) ? payloadKey.existing_tags : [];
const currentPayloadTags = Array.isArray(payloadKey.tags) ? payloadKey.tags : [];
let allTags = [...previewTags];
// Add fetched tags if available
if (existingKeysData.has(payloadKey.key_id)) {
const fetchedData = existingKeysData.get(payloadKey.key_id);
const fetchedTags = Array.isArray(fetchedData.tags) ? fetchedData.tags : [];
fetchedTags.forEach(tag => {
const normalized = String(tag).trim().toLowerCase();
if (!allTags.some(t => String(t).trim().toLowerCase() === normalized)) {
allTags.push(tag);
}
});
}
// Add new tags from payload
currentPayloadTags.forEach(tag => {
const normalized = String(tag).trim().toLowerCase();
if (!allTags.some(t => String(t).trim().toLowerCase() === normalized)) {
allTags.push(tag);
}
});
// Final deduplication
const finalTagsMap = new Map();
allTags.forEach(tag => {
const normalized = String(tag).trim().toLowerCase();
if (normalized.length > 0 && !finalTagsMap.has(normalized)) {
finalTagsMap.set(normalized, String(tag).trim());
}
});
payloadKey.tags = Array.from(finalTagsMap.values());
}
});
const existingKeys = [];
const newKeys = [];
payloadKeys.forEach((item) => {
if (typeof item.key_id === 'number') {
existingKeys.push(item);
} else {
newKeys.push(item);
}
});
let totalPushed = 0;
let totalUpdated = 0;
let noopDetected = false;
// Summary logging removed for performance - only log on errors
// CRITICAL: Process existing keys (with key_id) using PUT /keys/{key_id} BEFORE POST
// Lokalise bulk POST doesn't reliably update when key_id is in payload - it still tries to CREATE
// We must use PUT for updates to avoid "key name already taken" errors
// This avoids the slow POST → error → PUT path
if (existingKeys.length > 0) {
const batchIndex = options.batchIndex || '?';
logger.log(` 🔄 [BATCH ${batchIndex}] Processing ${existingKeys.length} existing key(s) with PUT /keys/{key_id} (skipping POST)...`);
// Process existing keys directly with PUT (same logic as error recovery, but proactive)
// This is much faster than POST → error → PUT path
// We'll trigger the error recovery PUT logic by simulating the duplicate error
// But with stored key_id mapping, it will use the fast path
// Store key_id mapping for fast error recovery
const existingKeyIdMap = new Map();
existingKeys.forEach(key => {
const keyName = typeof key.key_name === 'string' ? key.key_name : (key.key_name?.web || key.key_name?.other || '');
if (typeof key.key_id === 'number' && keyName) {
existingKeyIdMap.set(keyName, key.key_id);
}
});
// These will be processed via error recovery with fast path
// But we need to ensure they're NOT in the POST batch
}
const syncBatch = async (batch, method, batchNumber, totalBatches) => {
const batchIndex = options.batchIndex || '?';
const batchStartTime = Date.now();
const url = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys`;
let existingKeysProcessed = false;
let suppressDuplicateErrorLog = false;
const keyDataMap = new Map();
// CRITICAL: Separate existing keys (with key_id) from new keys
// Existing keys will be processed with PUT directly (fast!)
// New keys will go through POST (normal flow)
const existingInBatch = batch.filter(k => typeof k.key_id === 'number');
const newKeysInBatch = batch.filter(k => typeof k.key_id !== 'number');
// Store key_id mapping for error recovery (fallback for any existing keys that slip through)
const keyIdMapForBatch = new Map();
existingInBatch.forEach(k => {
const keyName = typeof k.key_name === 'string' ? k.key_name : (k.key_name?.web || k.key_name?.other || '');
if (typeof k.key_id === 'number' && keyName) {
keyIdMapForBatch.set(keyName, k.key_id);
}
});
// Attach keyIdMapForBatch to batch for error recovery (fast path)
batch.keyIdMapForBatch = keyIdMapForBatch;
// Count how many keys have key_id (existing) vs new
const existingCount = existingInBatch.length;
const newCount = newKeysInBatch.length;
// CRITICAL: Declare keysToUpdateWithPut in broader scope so it's accessible in error handling
let keysToUpdateWithPut = [];
// CRITICAL: Process existing keys with PUT directly when batch contains ONLY existing keys
// This avoids the slow POST → error → PUT path completely for pure-update batches
// For mixed batches, we still prepare keysToUpdateWithPut for error recovery
if (existingInBatch.length > 0) {
logger.log(` 🔄 [BATCH ${batchIndex}] Processing ${existingInBatch.length} existing key(s) with PUT (fast path)...`);
// Prepare existing keys for PUT processing (same format as error recovery)
// We already have key_id stored, so we can use it directly (fast path!)
// CRITICAL: Always prepare keysToUpdateWithPut for existing keys, even in mixed batches
// This ensures error recovery can use them if POST fails for existing keys
keysToUpdateWithPut = existingInBatch.map(key => {
const keyName = typeof key.key_name === 'string' ? key.key_name : (key.key_name?.web || key.key_name?.other || '');
// DEBUG: Verify tags and translations are in the key object
const hasTags = Array.isArray(key.tags) && key.tags.length > 0;
const hasTranslations = Array.isArray(key.translations) && key.translations.length > 0;
if (!hasTags && !hasTranslations) {
logger.log(` ⚠️ [BATCH ${batchIndex}] Key "${keyName}" has NO tags (${hasTags}) and NO translations (${hasTranslations}) - key keys: ${Object.keys(key).join(', ')}`);
} else {
logger.log(` ✅ [BATCH ${batchIndex}] Key "${keyName}" has tags: ${hasTags} (${hasTags ? key.tags.length : 0}), translations: ${hasTranslations} (${hasTranslations ? key.translations.length : 0})`);
}
return {
key_id: key.key_id,
key_name: keyName,
batchKey: key, // CRITICAL: This must preserve tags and translations from the original payload
existingTags: Array.isArray(key.existing_tags) ? key.existing_tags : [],
};
});
// If this batch contains ONLY existing keys, we can skip POST entirely
if (newKeysInBatch.length === 0) {
existingKeysProcessed = true;
suppressDuplicateErrorLog = true;
} else {
// Mixed batch: existing keys will be included in POST and will likely fail
// We've prepared keysToUpdateWithPut for error recovery
logger.log(` 🔄 [BATCH ${batchIndex}] Mixed batch: ${existingInBatch.length} existing + ${newKeysInBatch.length} new keys - existing keys prepared for error recovery`);
}
}
// Construct payload
// - For new keys: include as-is (no key_id)
// - For mixed batches: include existing keys without key_id to trigger fast error recovery
const batchForPost = [
...newKeysInBatch,
];
if (!existingKeysProcessed && existingInBatch.length > 0) {
batchForPost.push(...existingInBatch.map(k => {
const { key_id, ...rest } = k;
if (k.lokalise_key_id && typeof k.lokalise_key_id === 'number') {
rest.lokalise_key_id = k.lokalise_key_id;
}
return rest;
}));
}
const payload = {
keys: batchForPost,
options: {
merge_tags: true,
replace_modified: true,
},
};
// Use POST for bulk operations (new keys + mixed batches)
// For batches containing only existing keys, skip POST and simulate duplicate errors
let res;
if (existingKeysProcessed && existingInBatch.length > 0) {
logger.log(` ⚡ [BATCH ${batchIndex}] Skipping POST for ${existingInBatch.length} existing key(s) - forcing direct PUT flow`);
res = {
status: 200,
data: {
inserted: 0,
updated: 0,
errors: existingInBatch.map(key => {
const keyName = typeof key.key_name === 'string'
? key.key_name
: (key.key_name?.web || key.key_name?.other || key.key_name?.ios || key.key_name?.android || '');
return {
message: 'This key name is already taken',
code: 400,
key_name: keyName, // Use string keyName, not key.key_name (which might be an object)
meta: {
key_name: keyName,
},
};
}),
},
};
suppressDuplicateErrorLog = true;
// CRITICAL: When skipping POST, we must process keysToUpdateWithPut directly
// This is the fast path - we already have key_id, so process PUT immediately
if (keysToUpdateWithPut.length > 0) {
// Process keysToUpdateWithPut directly (same logic as error recovery path)
// Step 1: Fetch current key data from Lokalise
if (keysToUpdateWithPut.length === 1) {
// Fast path: Single key - direct fetch
const { key_id } = keysToUpdateWithPut[0];
try {
const getUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys/${key_id}`;
const getRes = await http.get(getUrl, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
},
validateStatus: (status) => status < 600,
});
if (getRes.status >= 200 && getRes.status < 300 && getRes.data && getRes.data.key) {
keyDataMap.set(key_id, getRes.data.key);
}
} catch (err) {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to fetch key_id=${key_id}: ${err.message || String(err)}`);
}
} else {
// Parallel path: Multiple keys - fetch all in parallel
const fetchPromises = keysToUpdateWithPut.map(async ({ key_id }) => {
try {
const getUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys/${key_id}`;
const getRes = await http.get(getUrl, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
},
validateStatus: (status) => status < 600,
});
if (getRes.status >= 200 && getRes.status < 300 && getRes.data && getRes.data.key) {
keyDataMap.set(key_id, getRes.data.key);
}
} catch (err) {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to fetch key_id=${key_id}: ${err.message || String(err)}`);
}
});
await Promise.all(fetchPromises);
}
// Step 2: Process updates (will be handled by the existing code path below)
// We'll set a flag to indicate we need to process these keys
res.data.keysToUpdateWithPut = keysToUpdateWithPut;
}
} else {
const maxAttempts = 5;
let attempt = 0;
// eslint-disable-next-line no-constant-condition
while (true) {
attempt++;
try {
res = await http.post(
url,
payload,
{
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
'Content-Type': 'application/json',
},
validateStatus: (status) => status < 600,
}
);
} catch (postErr) {
// Treat network resets/timeouts as retryable
const msg = postErr?.message || '';
const retryable = /ECONNRESET|ETIMEDOUT|ENETUNREACH|EAI_AGAIN/i.test(msg);
if (!retryable || attempt >= maxAttempts) {
throw postErr;
}
const backoff = Math.min(15000, 1000 * Math.pow(2, attempt - 1)) + Math.floor(Math.random() * 300);
logger.log(` ⏳ Retryable network error, attempt ${attempt}/${maxAttempts}. Waiting ${backoff}ms...`);
await new Promise(r => setTimeout(r, backoff));
continue;
}
// Handle rate limits/transient server errors with backoff and retry
if ((res.status === 429 || res.status === 420 || res.status === 503) && attempt < maxAttempts) {
const retryAfterHeader = Number(res.headers?.['retry-after']) || 0;
const backoff = retryAfterHeader > 0
? retryAfterHeader * 1000
: Math.min(20000, 1500 * Math.pow(2, attempt - 1)) + Math.floor(Math.random() * 500);
logger.log(` ⏳ Rate limited (status ${res.status}). Attempt ${attempt}/${maxAttempts}. Waiting ${backoff}ms...`);
await waitWithCancel(backoff);
continue;
}
break;
}
}
if (res.status >= 200 && res.status < 300) {
const responseData = res.data || {};
const inserted = responseData.inserted ?? 0;
const updated = responseData.updated ?? 0;
const errors = responseData.errors || [];
// Extract key_ids from Lokalise response to store back in Strapi
// Lokalise returns created/updated keys with their key_ids
if (responseData.keys && Array.isArray(responseData.keys)) {
responseData.keys.forEach(lokaliseKey => {
if (!lokaliseKey || !lokaliseKey.key_id) return;
// Find matching key in batch by key_name
const keyName = typeof lokaliseKey.key_name === 'string'
? lokaliseKey.key_name
: (lokaliseKey.key_name?.web || lokaliseKey.key_name?.other || lokaliseKey.key_name?.ios || lokaliseKey.key_name?.android || '');
if (keyName) {
const batchKey = batch.find(k => {
const batchKeyName = typeof k.key_name === 'string' ? k.key_name : (k.key_name?.web || k.key_name?.other || '');
return batchKeyName === keyName ||
batchKeyName?.toLowerCase() === keyName.toLowerCase() ||
batchKeyName?.replace(/\[(\d+)\]/g, '.$1') === keyName.replace(/\[(\d+)\]/g, '.$1');
});
if (batchKey && batchKey.field_path && (batchKey.entry_id || batchKey.entry_slug)) {
const tagSnapshot = buildCanonicalTagSnapshot(batchKey.tags);
// Store mapping: we'll use this to update Strapi entries after sync
// Include key data for hash calculation
keyIdMapping.set(keyName, {
key_id: typeof lokaliseKey.key_id === 'number' ? lokaliseKey.key_id : Number(lokaliseKey.key_id),
entry_id: batchKey.entry_id || null,
entry_slug: batchKey.entry_slug || null,
field_path: batchKey.field_path,
key_name: keyName,
// Store key data for hash calculation
key_data: {
key_name: keyName,
tags: tagSnapshot,
translations: Array.isArray(batchKey.translations) ? batchKey.translations : [],
updatedAt: batchKey.updatedAt || batchKey.updated_at || null,
},
tag_snapshot: tagSnapshot,
});
}
}
});
}
// Check for errors in response
const shouldLogLokaliseErrors = errors.length > 0 && !suppressDuplicateErrorLog;
// CRITICAL: When we skip POST (fast path), we MUST process keysToUpdateWithPut directly
// Even if suppressDuplicateErrorLog is true, we still need to update the keys with PUT
// This ensures tags and translations are updated when we skip POST
// Also check if res.data.keysToUpdateWithPut exists (set when POST is skipped)
const hasKeysToUpdateFromSkip = res.data && res.data.keysToUpdateWithPut && Array.isArray(res.data.keysToUpdateWithPut) && res.data.keysToUpdateWithPut.length > 0;
// If we have keysToUpdateWithPut from skipped POST, use them
if (hasKeysToUpdateFromSkip && keysToUpdateWithPut.length === 0) {
keysToUpdateWithPut = res.data.keysToUpdateWithPut;
logger.log(` ⚡ [BATCH ${batchIndex}] Using keysToUpdateWithPut from skipped POST (${keysToUpdateWithPut.length} key(s))`);
}
// CRITICAL: forceProcessFastPath should be true when:
// 1. POST was skipped (existingKeysProcessed) AND we have keys to update
// 2. OR we have keysToUpdateWithPut from skipped POST response
const forceProcessFastPath = (existingKeysProcessed && keysToUpdateWithPut.length > 0) || hasKeysToUpdateFromSkip;
// DEBUG: Log the conditions
logger.log(` 🔍 [BATCH ${batchIndex}] Error processing conditions:`);
logger.log(` - existingKeysProcessed: ${existingKeysProcessed}`);
logger.log(` - keysToUpdateWithPut.length: ${keysToUpdateWithPut.length}`);
logger.log(` - hasKeysToUpdateFromSkip: ${hasKeysToUpdateFromSkip}`);
logger.log(` - forceProcessFastPath: ${forceProcessFastPath}`);
logger.log(` - shouldLogLokaliseErrors: ${shouldLogLokaliseErrors}`);
logger.log(` - errors.length: ${errors.length}`);
// Process errors OR fast path keys (both need PUT processing)
// CRITICAL: Even if suppressDuplicateErrorLog is true, we MUST process PUT requests when POST is skipped
if (shouldLogLokaliseErrors || forceProcessFastPath) {
logger.log(` ✅ [BATCH ${batchIndex}] Entering error/fast-path processing block`);
// Log errors concisely (only once)
const errorSummary = errors.slice(0, 3).map((err, idx) => {
const errorMsg = typeof err === 'object' ? JSON.stringify(err) : String(err);
return `Error ${idx + 1}: ${errorMsg}`;
}).join('; ');
logger.error(`[BATCH ${batchIndex}] ERROR: Lokalise returned ${errors.length} error(s) in batch ${batchNumber}/${totalBatches}${errorSummary ? ` - ${errorSummary}` : ''}`);
if (errors.length > 3) {
logger.error(`[BATCH ${batchIndex}] ... and ${errors.length - 3} more error(s)`);
}
// Handle "key name already taken" errors - this means the key exists but we couldn't find it
// CRITICAL: Lokalise search by name is UNRELIABLE for keys with brackets/dots/namespaces
// The ONLY reliable way is to use key_id directly via GET /keys/{key_id}
// If we don't have key_id stored, we need to fetch ALL keys and search client-side
const duplicateKeyErrors = errors.filter(err =>
err.message && (
err.message.includes('already taken') ||
err.message.includes('duplicate') ||
err.code === 400
)
);
// CRITICAL: Declare keysToUpdateWithPutFromErrors here so it's accessible after the if block
let keysToUpdateWithPutFromErrors = [];
// CRITICAL: Declare retryTagsMap and foundKeys in broader scope so they're accessible after try-catch
// They will be populated either by fast path (keysToUpdateWithPut) or by lookup
const retryTagsMap = new Map();
let foundKeys = [];
logger.log(` 🔍 [BATCH ${batchIndex}] DEBUG: duplicateKeyErrors.length=${duplicateKeyErrors.length}`);
if (duplicateKeyErrors.length > 0) {
logger.log(` 🔍 [BATCH ${batchIndex}] DEBUG: Processing ${duplicateKeyErrors.length} duplicate error(s)`);
// Extract key names from errors
const duplicateKeyNames = duplicateKeyErrors
.map(err => {
if (err.key_name) {
// Handle both string and object formats
if (typeof err.key_name === 'string') {
return err.key_name;
} else if (typeof err.key_name === 'object') {
return err.key_name.web || err.key_name.ios || err.key_name.android || err.key_name.other || null;
}
}
return null;
})
.filter(Boolean);
logger.log(` 🔍 [BATCH ${batchIndex}] DEBUG: duplicateKeyNames.length=${duplicateKeyNames.length}, duplicateKeyNames=${JSON.stringify(duplicateKeyNames)}`);
if (duplicateKeyNames.length > 0) {
logger.log(` 🔍 [BATCH ${batchIndex}] DEBUG: Entered if (duplicateKeyNames.length > 0) block`);
try {
logger.log(` 🔍 [BATCH ${batchIndex}] DEBUG: Entered try block inside duplicateKeyNames check`);
// CRITICAL: If we have keysToUpdateWithPut prepared (existing keys with key_id), use them directly
// This applies to both fast path (POST skipped) and mixed batches (POST failed for existing keys)
// We already have key_id stored, so no need for slow lookups!
const matchedKeyNames = new Set(); // Track which keys were matched to avoid duplicate lookups
if (keysToUpdateWithPut.length > 0) {
// Match duplicate key names to keysToUpdateWithPut by key_name
const matchedKeys = duplicateKeyNames
.map(keyName => {
const matched = keysToUpdateWithPut.find(k => {
const kName = typeof k.key_name === 'string' ? k.key_name : (k.key_name?.web || k.key_name?.other || '');
return kName === keyName;
});
if (matched) {
matchedKeyNames.add(keyName);
}
return matched;
})
.filter(Boolean);
if (matchedKeys.length > 0) {
logger.log(` ⚡ [BATCH ${batchIndex}] Using pre-prepared keysToUpdateWithPut for ${matchedKeys.length} existing key(s) - skipping slow lookups for these`);
keysToUpdateWithPutFromErrors = matchedKeys;
logger.log(` ⚡ [BATCH ${batchIndex}] Using fast-path keysToUpdateWithPut (${matchedKeys.length} key(s)) with tags/translations preserved`);
logger.log(` 🔍 [BATCH ${batchIndex}] DEBUG: After fast path assignment, keysToUpdateWithPutFromErrors.length=${keysToUpdateWithPutFromErrors.length}`);
logger.log(` 🔍 [BATCH ${batchIndex}] DEBUG: Will still lookup ${duplicateKeyNames.length - matchedKeys.length} unmatched key(s)`);
} else {
// No matches found in keysToUpdateWithPut, fall through to normal lookup
logger.log(` ⚠️ [BATCH ${batchIndex}] No matches found in keysToUpdateWithPut, falling back to lookup`);
// Fall through to normal error recovery path
}
}
// CRITICAL: Do lookups for keys that weren't matched in keysToUpdateWithPut
// This handles mixed batches where some keys have key_id (matched) and others need lookup
const unmatchedDuplicateKeyNames = duplicateKeyNames.filter(keyName => !matchedKeyNames.has(keyName));
if (unmatchedDuplicateKeyNames.length > 0) {
logger.log(` 🔍 [BATCH ${batchIndex}] Looking up ${unmatchedDuplicateKeyNames.length} unmatched duplicate key(s) that need key_id lookup`);
// Normal error recovery path - do lookups for unmatched keys only
// CRITICAL: First check keyIdMapForBatch (stored before removing key_id from payload)
// This avoids slow lookups for keys we already know the key_id for
const keyIdMapForBatch = batch.keyIdMapForBatch || new Map();
const fastPathCount = unmatchedDuplicateKeyNames.filter(keyName => {
const storedKeyId = keyIdMapForBatch.get(keyName);
return storedKeyId && typeof storedKeyId === 'number';
}).length;
if (fastPathCount > 0) {
logger.log(` ⚡ [BATCH ${batchIndex}] Using FAST PATH for ${fastPathCount}/${unmatchedDuplicateKeyNames.length} unmatched duplicate key(s) (key_id already stored, no lookup needed)`);
}
unmatchedDuplicateKeyNames.forEach(keyName => {
const storedKeyId = keyIdMapForBatch.get(keyName);
if (storedKeyId && typeof storedKeyId === 'number') {
// We have the key_id - use it directly (fast path)
const batchKey = batch.find(k => {
const batchKeyName = typeof k.key_name === 'string' ? k.key_name : (k.key_name?.web || k.key_name?.other || '');
return batchKeyName === keyName;
});
if (batchKey) {
retryTagsMap.set(keyName, {
key_id: storedKeyId,
tags: Array.isArray(batchKey.existing_tags) ? batchKey.existing_tags : [],
translations: [],
});
}
}
});
// OPTIMIZATION: Batch lookup using getExistingTagsForKeys instead of individual calls
try {
const keysNeedingLookup = unmatchedDuplicateKeyNames.filter(name => !retryTagsMap.has(name));
if (keysNeedingLookup.length > 0) {
const batchLookupMap = await getExistingTagsForKeys(keysNeedingLookup);
batchLookupMap.forEach((meta, keyName) => {
if (meta && meta.key_id) {
retryTagsMap.set(keyName, meta);
}
});
}
// For keys not found in batch lookup, try individual lookup
// CRITICAL: Try even without lokalise_key_id - use findExistingKey to search by name
// OPTIMIZATION: Pre-compute batchKey mapping and process lookups in parallel
// Only lookup unmatched keys (not already matched in fast path)
const notFoundKeys = unmatchedDuplicateKeyNames.filter(name => !retryTagsMap.has(name));
if (notFoundKeys.length > 0) {
// Pre-compute batchKey mapping (more efficient than finding in loop)
const batchKeyMap = new Map();
batch.forEach(k => {
const batchKeyName = typeof k.key_name === 'string' ? k.key_name : (k.key_name?.web || k.key_name?.other || '');
if (batchKeyName) {
batchKeyMap.set(batchKeyName, k);
}
});
// Process lookups in parallel (much faster than sequential)
const lookupPromises = notFoundKeys.map(async (keyName) => {
// Check for cancellation before processing
if (await shouldCancel()) {
const err = new Error('JOB_CANCELLED');
throw err;
}
const batchKey = batchKeyMap.get(keyName);
// Check keyIdMapForBatch first, then batchKey properties
const storedKeyId = keyIdMapForBatch.get(keyName);
const lokaliseKeyId = storedKeyId || batchKey?.lokalise_key_id || batchKey?.key_id || undefined;
// Try to find the key - use lokalise_key_id if available, otherwise search by name
try {
const existingKey = await findExistingKey(keyName, lokaliseKeyId);
if (existingKey && existingKey.key_id) {
const tagNames = Array.isArray(existingKey.tags)
? existingKey.tags.map(t => typeof t === 'string' ? t : (t?.name || '')).filter(Boolean)
: [];
return {
keyName,
meta: {
tags: Array.from(new Set(tagNames)),
key_id: existingKey.key_id,
translations: Array.isArray(existingKey.translations) ? existingKey.translations : [],
}
};
}
} catch (lookupErr) {
// Log but don't throw - continue with other lookups
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to lookup key "${keyName}": ${lookupErr.message || String(lookupErr)}`);
}
return null;
});
// Wait for all lookups to complete
const lookupResults = await Promise.all(lookupPromises);
// Add found keys to retryTagsMap
lookupResults.forEach(result => {
if (result && result.meta) {
retryTagsMap.set(result.keyName, result.meta);
}
});
}
} catch (err) {
// Check if job was cancelled - re-throw to stop processing
if (err.message === 'JOB_CANCELLED') {
throw err;
}
logger.error(`[BATCH ${batchIndex}] ERROR: Batch lookup failed for duplicates: ${err.message || String(err)}`);
}
// Calculate foundKeys from retryTagsMap (only for unmatched keys in error recovery path)
// Matched keys are already in keysToUpdateWithPutFromErrors
foundKeys = unmatchedDuplicateKeyNames.filter(name => {
const meta = retryTagsMap.get(name);
return meta && meta.key_id;
});
// If we found some keys, proceed with updates for those
// If we didn't find any, log warning but don't throw - let the batch continue
// The key exists in Lokalise (we got "already taken" error), so it will be skipped
// CRITICAL: Only check unmatched keys (matched keys are already in keysToUpdateWithPutFromErrors)
const unmatchedFoundKeys = foundKeys.filter(keyName => !matchedKeyNames.has(keyName));
if (unmatchedFoundKeys.length === 0 && unmatchedDuplicateKeyNames.length > 0) {
logger.error(`[BATCH ${batchIndex}] ERROR: Could not find key_ids for ${unmatchedDuplicateKeyNames.length} unmatched duplicate key(s) - these keys exist in Lokalise but cannot be updated without key_id`);
logger.error(`[BATCH ${batchIndex}] ERROR: Keys: ${unmatchedDuplicateKeyNames.join(', ')}`);
logger.error(`[BATCH ${batchIndex}] ERROR: These keys will be skipped. To fix: ensure keys have lokalise_key_id stored in Strapi or manually find key_id in Lokalise`);
// Don't throw - just skip these keys and continue with the batch
// The keys exist in Lokalise, they just can't be updated without key_id
}
// Build keysToUpdateWithPutFromErrors from error recovery (only for unmatched keys)
// Matched keys are already in keysToUpdateWithPutFromErrors from fast path
if (unmatchedFoundKeys.length > 0) {
logger.log(` 🔍 [BATCH ${batchIndex}] Adding ${unmatchedFoundKeys.length} looked-up key(s) to keysToUpdateWithPutFromErrors`);
for (const keyName of unmatchedFoundKeys) {
const meta = retryTagsMap.get(keyName);
if (meta && meta.key_id && typeof meta.key_id === 'number') {
const batchKey = batch.find(k => {
const batchKeyName = typeof k.key_name === 'string' ? k.key_name : (k.key_name?.web || k.key_name?.other || '');
return batchKeyName === keyName;
});
if (batchKey) {
keysToUpdateWithPutFromErrors.push({
key_id: meta.key_id,
key_name: keyName,
batchKey: batchKey,
existingTags: Array.isArray(meta.tags) ? meta.tags : [],
});
} else {
logger.error(`[BATCH ${batchIndex}] ERROR: Could not find batchKey for "${keyName}" in batch`);
}
}
}
}
}
} catch (err) {
// Check if job was cancelled - re-throw to stop processing
if (err.message === 'JOB_CANCELLED') {
throw err;
}
logger.error(`[BATCH ${batchIndex}] ERROR: Exception in duplicate key processing: ${err.message || String(err)}`);
logger.error(`[BATCH ${batchIndex}] ERROR: Stack: ${err.stack || 'No stack trace'}`);
// Don't throw - continue with PUT processing if we have keysToUpdateWithPutFromErrors
if (keysToUpdateWithPutFromErrors.length === 0 && keysToUpdateWithPut.length > 0) {
logger.log(` ⚡ [BATCH ${batchIndex}] Exception occurred, but using keysToUpdateWithPut as fallback`);
keysToUpdateWithPutFromErrors = keysToUpdateWithPut;
}
}
logger.log(` 🔍 [BATCH ${batchIndex}] DEBUG: Exited try-catch block, continuing to PUT processing...`);
logger.log(` 🔍 [BATCH ${batchIndex}] DEBUG: keysToUpdateWithPutFromErrors.length=${keysToUpdateWithPutFromErrors.length}, keysToUpdateWithPut.length=${keysToUpdateWithPut.length}`);
// CRITICAL: For keys we found, update them using PUT /keys/{key_id}
// This follows Lokalise best practices: fetch, merge tags, then PUT update
// Only process keys we successfully found key_ids for
// NOTE: keysToUpdateWithPutFromErrors is already declared above (line 2915)
// If we already have keysToUpdateWithPutFromErrors (from fast path), skip building from error recovery
// Only build from error recovery if we don't have keysToUpdateWithPutFromErrors and foundKeys was populated
if (keysToUpdateWithPutFromErrors.length === 0 && foundKeys && foundKeys.length > 0 && retryTagsMap && retryTagsMap.size > 0) {
logger.log(` 🔍 [BATCH ${batchIndex}] DEBUG: Building from error recovery, foundKeys.length=${foundKeys.length}`);
// Create new keysToUpdateWithPut from error recovery
for (const keyName of foundKeys) {
const meta = retryTagsMap.get(keyName);
if (meta && meta.key_id && typeof meta.key_id === 'number') {
const batchKey = batch.find(k => {
const batchKeyName = typeof k.key_name === 'string' ? k.key_name : (k.key_name?.web || k.key_name?.other || '');
return batchKeyName === keyName;
});
if (batchKey) {
keysToUpdateWithPutFromErrors.push({
key_id: meta.key_id,
key_name: keyName,
batchKey: batchKey,
existingTags: Array.isArray(meta.tags) ? meta.tags : [],
});
} else {
logger.error(`[BATCH ${batchIndex}] ERROR: Could not find batchKey for "${keyName}" in batch`);
}
}
}
}
} // End of if (duplicateKeyNames.length > 0) block
// ========================================
// COMPREHENSIVE LOGGING: PUT Processing Start
// ========================================
logger.log(` 📊 [BATCH ${batchIndex}] ========== PUT PROCESSING START ==========`);
logger.log(` 📊 [BATCH ${batchIndex}] State check:`);
logger.log(` - keysToUpdateWithPutFromErrors.length: ${keysToUpdateWithPutFromErrors.length}`);
logger.log(` - keysToUpdateWithPut.length: ${keysToUpdateWithPut.length}`);
logger.log(` - forceProcessFastPath: ${forceProcessFastPath}`);
logger.log(` - existingKeysProcessed: ${existingKeysProcessed}`);
// Use the appropriate keysToUpdateWithPut (from fast path or error recovery)
const keysToProcess = keysToUpdateWithPutFromErrors.length > 0 ? keysToUpdateWithPutFromErrors : keysToUpdateWithPut;
logger.log(` 📊 [BATCH ${batchIndex}] Calculated keysToProcess.length: ${keysToProcess.length}`);
// DEBUG: Log what we're about to process
if (keysToProcess.length > 0) {
logger.log(` 📊 [BATCH ${batchIndex}] ✅ About to process ${keysToProcess.length} key(s) with PUT requests`);
keysToProcess.forEach((item, idx) => {
const hasTags = Array.isArray(item.batchKey?.tags) && item.batchKey.tags.length > 0;
const hasTranslations = Array.isArray(item.batchKey?.translations) && item.batchKey.translations.length > 0;
logger.log(` 📊 [BATCH ${batchIndex}] Key ${idx + 1}/${keysToProcess.length}:`);
logger.log(` - key_id: ${item.key_id}`);
logger.log(` - key_name: "${item.key_name}"`);
logger.log(` - has tags: ${hasTags ? `YES (${item.batchKey.tags.length})` : 'NO'}`);
logger.log(` - has translations: ${hasTranslations ? `YES (${item.batchKey.translations.length})` : 'NO'}`);
if (hasTags) {
logger.log(` - tags: ${JSON.stringify(item.batchKey.tags.slice(0, 3))}${item.batchKey.tags.length > 3 ? '...' : ''}`);
}
});
} else {
logger.error(` ❌ [BATCH ${batchIndex}] ERROR: keysToProcess.length is 0 - no keys to process!`);
logger.error(` ❌ [BATCH ${batchIndex}] This means PUT requests will NOT be sent!`);
logger.error(` ❌ [BATCH ${batchIndex}] Check: keysToUpdateWithPutFromErrors=${keysToUpdateWithPutFromErrors.length}, keysToUpdateWithPut=${keysToUpdateWithPut.length}`);
}
// Update keys using PUT /keys/{key_id} (proper Lokalise API method)
// OPTIMIZATION: Fast path for single key (skip parallel overhead), parallel for multiple keys
if (keysToProcess.length > 0) {
logger.log(` 🚀 [BATCH ${batchIndex}] ========== STARTING PUT REQUESTS ==========`);
logger.log(` 🚀 [BATCH ${batchIndex}] Processing ${keysToProcess.length} key(s) with PUT /keys/{key_id}...`);
// OPTIMIZATION: For single key, use direct sequential path (faster, no Promise.all overhead)
// For multiple keys, use parallel fetching
if (keysToProcess.length === 1) {
// Fast path: Single key - direct fetch (no parallel overhead)
const { key_id, key_name } = keysToProcess[0];
try {
const getUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys/${key_id}`;
const getRes = await http.get(getUrl, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
},
validateStatus: (status) => status < 600,
});
if (getRes.status >= 200 && getRes.status < 300 && getRes.data && getRes.data.key) {
keyDataMap.set(key_id, getRes.data.key);
}
} catch (err) {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to fetch key_id=${key_id}: ${err.message || String(err)}`);
}
} else {
// Parallel path: Multiple keys - fetch all in parallel
const fetchPromises = keysToProcess.map(async ({ key_id, key_name }) => {
try {
const getUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys/${key_id}`;
const getRes = await http.get(getUrl, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
},
validateStatus: (status) => status < 600,
});
if (getRes.status >= 200 && getRes.status < 300 && getRes.data && getRes.data.key) {
keyDataMap.set(key_id, getRes.data.key);
}
} catch (err) {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to fetch key_id=${key_id}: ${err.message || String(err)}`);
}
});
await Promise.all(fetchPromises);
}
// Step 2: Process updates
// OPTIMIZATION: Fast path for single key (direct processing), parallel for multiple keys
logger.log(` 🔍 [BATCH ${batchIndex}] Step 2: About to process ${keysToProcess.length} key(s) with PUT...`);
logger.log(` 🔍 [BATCH ${batchIndex}] DEBUG: keysToProcess.length=${keysToProcess.length}, entering if/else block...`);
if (keysToProcess.length === 1) {
logger.log(` 🔍 [BATCH ${batchIndex}] DEBUG: Entering SINGLE key path`);
// Fast path: Single key - direct processing (no Promise.all overhead)
const { key_id, key_name, batchKey, existingTags } = keysToProcess[0];
logger.log(` 🔍 [BATCH ${batchIndex}] Single key path: key_id=${key_id}, key_name="${key_name}"`);
logger.log(` 🔍 [BATCH ${batchIndex}] batchKey structure: ${JSON.stringify(Object.keys(batchKey || {}))}`);
logger.log(` 🔍 [BATCH ${batchIndex}] batchKey.tags: ${Array.isArray(batchKey?.tags) ? `${batchKey.tags.length} tags` : 'NOT ARRAY'}`);
logger.log(` 🔍 [BATCH ${batchIndex}] batchKey.translations: ${Array.isArray(batchKey?.translations) ? `${batchKey.translations.length} translations` : 'NOT ARRAY'}`);
// Check for cancellation
if (await shouldCancel()) {
const err = new Error('JOB_CANCELLED');
throw err;
}
// Process single key directly (inline, no async map overhead)
let updateSuccess = false;
let updateResponse = null;
try {
logger.log(` 🔍 [BATCH ${batchIndex}] Processing PUT for key_id=${key_id} "${key_name}"...`);
// Get pre-fetched key data
let currentKey = keyDataMap.get(key_id);
if (!currentKey) {
logger.error(`[BATCH ${batchIndex}] ERROR: Key data not found for key_id=${key_id} - fetching now...`);
// Fetch it now if not in map
try {
const getUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys/${key_id}`;
const getRes = await http.get(getUrl, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
},
validateStatus: (status) => status < 600,
});
if (getRes.status >= 200 && getRes.status < 300 && getRes.data && getRes.data.key) {
keyDataMap.set(key_id, getRes.data.key);
currentKey = getRes.data.key;
logger.log(` ✅ [BATCH ${batchIndex}] Fetched key data for key_id=${key_id} - tags: ${Array.isArray(currentKey.tags) ? currentKey.tags.length : 0}, translations: ${Array.isArray(currentKey.translations) ? currentKey.translations.length : 0}`);
} else {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to fetch key_id=${key_id}: ${getRes.status}`);
return;
}
} catch (fetchErr) {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to fetch key_id=${key_id}: ${fetchErr.message || String(fetchErr)}`);
return;
}
}
if (!currentKey) {
logger.error(`[BATCH ${batchIndex}] ERROR: Key data still not found for key_id=${key_id} after fetch - skipping`);
return;
}
// Use pre-fetched key data
const currentTagsFromLokalise = Array.isArray(currentKey.tags) ? currentKey.tags : [];
const currentTranslations = Array.isArray(currentKey.translations) ? currentKey.translations : [];
// CRITICAL: Define actualPlatforms and keyNameObject early in broader scope
// so they're accessible in both tag update and translation update sections
const keyPlatforms = Array.isArray(currentKey.platforms) && currentKey.platforms.length > 0
? currentKey.platforms
: ['web']; // Default to web if not specified
const actualPlatforms = Array.isArray(keyPlatforms) && keyPlatforms.length > 0
? keyPlatforms
: ['web']; // Default fallback
// Build key_name object early so it's accessible in translation update section
const keyNameObject = {};
if (currentKey.key_name) {
if (typeof currentKey.key_name === 'string') {
actualPlatforms.forEach(platform => {
keyNameObject[platform] = currentKey.key_name;
});
} else if (typeof currentKey.key_name === 'object') {
actualPlatforms.forEach(platform => {
if (currentKey.key_name[platform]) {
keyNameObject[platform] = currentKey.key_name[platform];
} else if (currentKey.key_name.web && actualPlatforms.includes('web')) {
keyNameObject[platform] = currentKey.key_name.web;
} else {
keyNameObject[platform] = key_name;
}
});
Object.keys(keyNameObject).forEach(platform => {
if (!actualPlatforms.includes(platform)) {
delete keyNameObject[platform];
}
});
}
} else {
actualPlatforms.forEach(platform => {
keyNameObject[platform] = key_name;
});
}
// Extract and normalize tag names from Lokalise (tags might be objects with 'name' property or strings)
// CRITICAL: Normalize existing tags the same way we normalize new tags (spaces → underscores, lowercase)
const normalizeTagForComparison = (tag) => {
const str = typeof tag === 'string'
? tag
: (tag && typeof tag === 'object' && tag.name ? String(tag.name) : String(tag));
return str.trim().toLowerCase().replace(/\s+/g, '_');
};
const existingTagNames = currentTagsFromLokalise.map(tag => {
if (typeof tag === 'string') {
return tag.trim();
} else if (tag && typeof tag === 'object' && tag.name) {
return String(tag.name).trim();
}
return String(tag).trim();
}).filter(Boolean);
// Also create normalized versions for comparison (handles spaces vs underscores)
const normalizedExistingTags = existingTagNames.map(tag => normalizeTagForComparison(tag));
// Step 3: Extract new tags from Strapi
// CRITICAL: batchKey should have tags from the original formatted key
logger.log(` 📋 [BATCH ${batchIndex}] ========== TAG EXTRACTION ==========`);
logger.log(` 📋 [BATCH ${batchIndex}] Key: "${key_name}" (key_id=${key_id})`);
logger.log(` 📋 [BATCH ${batchIndex}] batchKey structure: ${JSON.stringify(Object.keys(batchKey || {}))}`);
logger.log(` 📋 [BATCH ${batchIndex}] batchKey.tags type: ${Array.isArray(batchKey.tags) ? 'ARRAY' : typeof batchKey.tags}`);
logger.log(` 📋 [BATCH ${batchIndex}] batchKey.tags value: ${JSON.stringify(batchKey.tags || 'NONE')}`);
const newTagsFromStrapi = Array.isArray(batchKey.tags) ? batchKey.tags : [];
logger.log(` 📋 [BATCH ${batchIndex}] Extracted newTagsFromStrapi.length: ${newTagsFromStrapi.length}`);
if (newTagsFromStrapi.length === 0 && batchKey.key_name) {
logger.log(` ⚠️ [BATCH ${batchIndex}] No tags found in batchKey for "${batchKey.key_name}" - batchKey keys: ${Object.keys(batchKey).join(', ')}`);
}
const newTagNames = newTagsFromStrapi.map(tag => {
if (typeof tag === 'string') {
return tag.trim();
} else if (tag && typeof tag === 'object' && tag.name) {
return String(tag.name).trim();
}
return String(tag).trim();
}).filter(Boolean);
// Step 4: Merge tags (deduplicate by normalized name - handles spaces vs underscores)
// CRITICAL: Compare normalized versions to handle "test syn interruption" vs "test_syn_interruption"
const allTagNames = [...existingTagNames];
newTagNames.forEach(newTag => {
const normalizedNew = normalizeTagForComparison(newTag);
// Check if normalized version already exists
const exists = normalizedExistingTags.some(existingNormalized => existingNormalized === normalizedNew);
if (!exists && normalizedNew.length > 0) {
allTagNames.push(newTag);
}
});
// Final deduplication (case-insensitive, space-insensitive)
const finalTagsMap = new Map();
allTagNames.forEach(tagName => {
const normalized = normalizeTagForComparison(tagName);
if (normalized.length > 0 && !finalTagsMap.has(normalized)) {
// Keep the original format from Lokalise if it exists, otherwise use the new one
finalTagsMap.set(normalized, tagName);
}
});
// CRITICAL: Use 'let' instead of 'const' because we need to reassign after normalization
let mergedTags = Array.from(finalTagsMap.values());
// Tag merging complete - proceed with update
// Step 5: Prepare tags for Lokalise
// CRITICAL: Keep existing tags in their original format from Lokalise
// Only normalize NEW tags that don't exist yet
// Note: keyPlatforms, actualPlatforms, and keyNameObject are already defined above
// Separate existing tags (keep original format) and new tags (normalize)
const tagsToSend = [];
const newTagsToCreate = [];
mergedTags.forEach(tag => {
const normalizedTag = normalizeTagForComparison(tag);
const exists = normalizedExistingTags.some(existingNormalized => existingNormalized === normalizedTag);
if (exists) {
// Keep original format from Lokalise for existing tags
const originalTag = existingTagNames.find(et =>
normalizeTagForComparison(et) === normalizedTag
);
if (originalTag) {
tagsToSend.push(originalTag);
} else {
tagsToSend.push(tag); // Fallback
}
} else {
// Send new tags exactly as user typed them (preserve spaces, case, underscores)
const trimmedTag = String(tag).trim();
if (trimmedTag.length > 0) {
tagsToSend.push(trimmedTag);
newTagsToCreate.push(trimmedTag);
}
}
});
// Deduplicate
mergedTags = Array.from(new Set(tagsToSend));
// Step 5b: Determine missing tags (tags that need to be created)
const missingTags = newTagsToCreate;
// Warn about very short tags (Lokalise may reject tags < 3 characters)
const veryShortTags = missingTags.filter(tag => tag.length < 3);
if (veryShortTags.length > 0) {
logger.error(`[BATCH ${batchIndex}] ERROR: ${veryShortTags.length} tag(s) are too short (< 3 chars): ${JSON.stringify(veryShortTags)}`);
}
// Step 6: Create missing tags using temporary keys (CRITICAL: POST /projects/{project_id}/tags returns 404)
// Lokalise does NOT have a direct tag creation endpoint - we must create temporary keys with tags
// This forces Lokalise to register the tags in the project
// CRITICAL: Declare tempKeyIds in broader scope so we can clean up after main key update
const tempKeyIds = [];
// Create missing tags using temporary keys
if (missingTags.length > 0) {
// Ensure we only create each missing tag once per process
if (!globalThis.__lokaliseEnsuredTags) {
globalThis.__lokaliseEnsuredTags = new Set();
}
const ensuredSet = globalThis.__lokaliseEnsuredTags;
const tagsToEnsure = missingTags.filter(t => !ensuredSet.has(String(t).toLowerCase()));
if (tagsToEnsure.length > 0) {
// Get platforms from the current key to use for temporary keys
const tempKeyPlatforms = Array.isArray(currentKey.platforms) && currentKey.platforms.length > 0
? currentKey.platforms
: ['web']; // Default to web if not specified
if (tagsToEnsure.length > 0) {
try {
const tempKeyName = `__temp_tags_${Date.now()}__`;
const tempKeyUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys`;
const tempKeyPayload = {
keys: [
{
key_name: tempKeyName,
platforms: tempKeyPlatforms, // Use same platforms as the actual key
tags: tagsToEnsure,
translations: [
{
language_iso: 'en',
translation: '__temp__',
},
],
},
],
};
const tempKeyRes = await http.post(tempKeyUrl, tempKeyPayload, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
'Content-Type': 'application/json',
},
validateStatus: (status) => status < 600,
});
if (tempKeyRes.status >= 200 && tempKeyRes.status < 300 && tempKeyRes.data && tempKeyRes.data.keys && tempKeyRes.data.keys.length > 0) {
const tempKeyId = tempKeyRes.data.keys[0].key_id;
// Track this single temp key for cleanup
tagsToEnsure.forEach(tag => tempKeyIds.push({ key_id: tempKeyId, tag }));
// Mark ensured to avoid re-creation on subsequent keys
tagsToEnsure.forEach(tag => ensuredSet.add(String(tag).toLowerCase()));
// Temporary key created - tags should now exist
} else {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to create temporary key for tags: ${tempKeyRes.status} ${JSON.stringify(tagsToEnsure)}`);
}
} catch (tagErr) {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to create temporary key: ${tagErr.message || String(tagErr)}`);
if (tagErr.stack) {
logger.error(`[BATCH ${batchIndex}] ERROR STACK: ${tagErr.stack}`);
}
}
}
// OPTIMIZATION: No wait needed - tags are created synchronously via temp key
// Lokalise registers tags immediately when temp key is created
// Removing wait saves 200ms per key (significant for large batches)
}
// Step 7: Update the key with all tags using PUT /keys/{key_id}
// CRITICAL: Lokalise requires platforms to be included in PUT requests for tag updates to work
// Without platforms, Lokalise silently ignores tag updates
// Note: actualPlatforms and keyNameObject are already defined above in broader scope
// CRITICAL: Verify keyNameObject only contains platforms from actualPlatforms
const keyNamePlatforms = Object.keys(keyNameObject);
const invalidPlatforms = keyNamePlatforms.filter(p => !actualPlatforms.includes(p));
if (invalidPlatforms.length > 0) {
logger.error(`[BATCH ${batchIndex}] ERROR: Invalid platforms in key_name: ${JSON.stringify(invalidPlatforms)}`);
invalidPlatforms.forEach(platform => {
delete keyNameObject[platform];
});
}
const putUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys/${key_id}`;
// CRITICAL: Final validation before building payload
// 1. Ensure keyNameObject only contains platforms from actualPlatforms
// 2. Ensure platforms array matches keyNameObject keys exactly
const keyNameObjectKeys = Object.keys(keyNameObject);
const platformMismatch = keyNameObjectKeys.some(p => !actualPlatforms.includes(p)) ||
actualPlatforms.some(p => !keyNameObjectKeys.includes(p));
if (platformMismatch) {
logger.error(`[BATCH ${batchIndex}] ERROR: Platform mismatch - fixing...`);
const fixedKeyNameObject = {};
actualPlatforms.forEach(platform => {
if (keyNameObject[platform]) {
fixedKeyNameObject[platform] = keyNameObject[platform];
} else if (keyNameObject.web) {
fixedKeyNameObject[platform] = keyNameObject.web;
} else {
fixedKeyNameObject[platform] = key_name;
}
});
Object.assign(keyNameObject, fixedKeyNameObject);
Object.keys(keyNameObject).forEach(k => {
if (!actualPlatforms.includes(k)) {
delete keyNameObject[k];
}
});
}
// CRITICAL: Lokalise API requires FLAT payload structure (NOT wrapped in "key" object)
// Structure: { "tags": [...] } OR { "key_name": "...", "platforms": [...], "tags": [...] }
// If we're only updating tags, we can send just tags
// If we're also updating key_name/platforms, include them in the flat structure
// Determine if we need to include key_name and platforms
// Only include them if we're actually changing them (for now, we'll include them to be safe)
const putPayload = {};
// If key_name is a simple string (single platform), use string format
// Otherwise, use object format
if (actualPlatforms.length === 1 && keyNameObject[actualPlatforms[0]]) {
// Single platform - use string format for key_name
putPayload.key_name = keyNameObject[actualPlatforms[0]];
putPayload.platforms = actualPlatforms;
} else {
// Multiple platforms or object format - use object format
putPayload.key_name = keyNameObject;
putPayload.platforms = actualPlatforms;
}
// Always include tags
putPayload.tags = mergedTags;
// DEBUG: Log FULL payload structure to verify everything is included
logger.log(` 📋 [BATCH ${batchIndex}] ========================================`);
logger.log(` 📋 [BATCH ${batchIndex}] PUT PAYLOAD FOR key_id=${key_id} "${key_name}":`);
logger.log(` 📋 [BATCH ${batchIndex}] - key_name: ${JSON.stringify(putPayload.key_name)}`);
logger.log(` 📋 [BATCH ${batchIndex}] - platforms: ${JSON.stringify(putPayload.platforms)}`);
logger.log(` 📋 [BATCH ${batchIndex}] - tags: ${mergedTags.length > 0 ? JSON.stringify(mergedTags) : 'NONE'}`);
logger.log(` 📋 [BATCH ${batchIndex}] - tags count: ${mergedTags.length}`);
logger.log(` 📋 [BATCH ${batchIndex}] - Full payload: ${JSON.stringify(putPayload, null, 2)}`);
logger.log(` 📋 [BATCH ${batchIndex}] ========================================`);
// Also log what's in batchKey for comparison
logger.log(` 🔍 [BATCH ${batchIndex}] batchKey.tags: ${Array.isArray(batchKey.tags) ? JSON.stringify(batchKey.tags) : 'NOT AN ARRAY'}`);
logger.log(` 🔍 [BATCH ${batchIndex}] batchKey.translations: ${Array.isArray(batchKey.translations) ? `${batchKey.translations.length} translation(s)` : 'NOT AN ARRAY'}`);
// CRITICAL: Final validation - platforms must match key_name keys (if key_name is object)
if (typeof putPayload.key_name === 'object' && !Array.isArray(putPayload.key_name)) {
const payloadPlatforms = putPayload.platforms;
const payloadKeyNameKeys = Object.keys(putPayload.key_name);
if (JSON.stringify(payloadPlatforms.sort()) !== JSON.stringify(payloadKeyNameKeys.sort())) {
logger.error(`[BATCH ${batchIndex}] ERROR: Platform mismatch in PUT payload`);
throw new Error(`Platform mismatch: platforms=${JSON.stringify(payloadPlatforms)} but key_name has keys=${JSON.stringify(payloadKeyNameKeys)}`);
}
}
// CRITICAL: Verify platforms are included before sending
if (!putPayload.platforms || !Array.isArray(putPayload.platforms) || putPayload.platforms.length === 0) {
logger.error(`[BATCH ${batchIndex}] ERROR: Platforms array missing in PUT payload`);
putPayload.platforms = ['web'];
if (typeof putPayload.key_name === 'object' && !Array.isArray(putPayload.key_name)) {
putPayload.key_name = { web: key_name };
} else {
putPayload.key_name = key_name;
}
}
// Variables already declared above - just use them here
updateSuccess = false;
updateResponse = null;
try {
logger.log(` 🚀 [BATCH ${batchIndex}] Sending PUT request to: ${putUrl}`);
logger.log(` 🚀 [BATCH ${batchIndex}] PUT request payload: ${JSON.stringify(putPayload, null, 2)}`);
const putStartedAt = Date.now();
const putRes = await http.put(putUrl, putPayload, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
'Content-Type': 'application/json',
},
validateStatus: (status) => status < 600,
});
logRemoteTiming(`[BATCH ${batchIndex}] PUT key_id=${key_id} (tags)`, putStartedAt);
logger.log(` 📥 [BATCH ${batchIndex}] PUT response status: ${putRes.status}`);
logger.log(` 📥 [BATCH ${batchIndex}] PUT response data: ${JSON.stringify(putRes.data || {}, null, 2)}`);
if (putRes.status >= 200 && putRes.status < 300) {
updateSuccess = true;
updateResponse = putRes;
// Check if tags were actually updated
if (putRes.data && putRes.data.key && Array.isArray(putRes.data.key.tags)) {
const responseTags = putRes.data.key.tags.map(t => typeof t === 'string' ? t : (t?.name || ''));
const sentSet = new Set(mergedTags.map(t => normalizeTagForComparison(t)));
const receivedSet = new Set(responseTags.map(t => normalizeTagForComparison(t)));
const missingInResponse = mergedTags.filter(t => !receivedSet.has(normalizeTagForComparison(t)));
if (missingInResponse.length > 0) {
logger.error(`[BATCH ${batchIndex}] ERROR: ${missingInResponse.length} tag(s) missing in response: ${JSON.stringify(missingInResponse)}`);
}
}
} else {
logger.error(`[BATCH ${batchIndex}] ERROR: PUT /keys/{key_id} failed: ${putRes.status} ${JSON.stringify(putRes.data || {})}`);
}
} catch (putErr) {
logger.error(`[BATCH ${batchIndex}] ERROR: PUT /keys/{key_id} failed: ${putErr.message || String(putErr)}`);
if (putErr.stack) {
logger.error(`[BATCH ${batchIndex}] ERROR STACK: ${putErr.stack}`);
}
if (putErr.response && putErr.response.data) {
logger.error(`[BATCH ${batchIndex}] ERROR RESPONSE: ${JSON.stringify(putErr.response.data)}`);
}
throw putErr;
}
if (updateSuccess && updateResponse) {
// OPTIMIZATION: Skip detailed verification for performance - just check response
// For large batches, detailed verification adds significant time
const responseTags = Array.isArray(updateResponse?.data?.key?.tags)
? updateResponse.data.key.tags.map(t => (typeof t === 'string' ? t : (t?.name || '')))
: [];
// Quick check: if response has fewer tags than sent, log error
const sentCount = mergedTags.length;
const receivedCount = responseTags.length;
if (receivedCount < sentCount) {
logger.error(`[BATCH ${batchIndex}] ERROR: Sent ${sentCount} tag(s) but received ${receivedCount} - some tags may have been rejected`);
}
totalUpdated += 1;
logger.log(` ✅ [BATCH ${batchIndex}] Successfully updated tags for key_id=${key_id} "${key_name}" - totalUpdated=${totalUpdated}`);
// CRITICAL: Clean up temporary keys AFTER main key is successfully updated
// This ensures tags are "anchored" to the main key before we delete temp keys
if (tempKeyIds.length > 0 && updateSuccess) {
// Clean up temporary keys
for (const { key_id: tempKeyId } of tempKeyIds) {
try {
const deleteUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys/${tempKeyId}`;
await http.delete(deleteUrl, {
headers: { 'X-Api-Token': cfg.lokaliseApiToken },
validateStatus: (status) => status < 600,
});
} catch (deleteErr) {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to delete temp key ${tempKeyId}: ${deleteErr.message || String(deleteErr)}`);
}
}
}
// CRITICAL: Store key_id in keyIdMapping so it gets saved back to Strapi
// This ensures we don't have to search for this key again in future syncs
if (batchKey && batchKey.field_path && (batchKey.entry_id || batchKey.entry_slug)) {
const tagSnapshot = buildCanonicalTagSnapshot(batchKey.tags);
keyIdMapping.set(key_name, {
key_id: key_id,
entry_id: batchKey.entry_id || null,
entry_slug: batchKey.entry_slug || null,
field_path: batchKey.field_path,
key_name: key_name,
// Store key data for hash calculation
key_data: {
key_name: key_name,
tags: tagSnapshot,
translations: Array.isArray(batchKey.translations) ? batchKey.translations : [],
updatedAt: batchKey.updatedAt || batchKey.updated_at || null,
},
tag_snapshot: tagSnapshot,
});
// Key ID stored for Strapi
} else {
logger.error(`[BATCH ${batchIndex}] ERROR: Cannot store key_id - missing metadata for "${key_name}"`);
}
} else {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to update tags for key_id=${key_id} - updateSuccess=${updateSuccess}, updateResponse=${updateResponse ? 'exists' : 'null'}`);
}
// Step 6: Update translations using PUT /keys/{key_id} (NOT PUT /translations/{translation_id})
// CRITICAL: For Strapi keys with "web" platform, we MUST update translations via PUT /keys/{key_id}
// Using PUT /translations/{translation_id} fails with 400 for segment-based keys
logger.log(` 📋 [BATCH ${batchIndex}] ========== TRANSLATION UPDATE CHECK ==========`);
logger.log(` 📋 [BATCH ${batchIndex}] Key: "${key_name}" (key_id=${key_id})`);
logger.log(` 📋 [BATCH ${batchIndex}] updateSuccess: ${updateSuccess}, updateResponse: ${updateResponse ? 'exists' : 'null'}`);
if (updateSuccess && updateResponse) {
// CRITICAL: batchKey should have translations from the original formatted key
logger.log(` 📋 [BATCH ${batchIndex}] batchKey.translations type: ${Array.isArray(batchKey.translations) ? 'ARRAY' : typeof batchKey.translations}`);
logger.log(` 📋 [BATCH ${batchIndex}] batchKey.translations value: ${JSON.stringify(batchKey.translations || 'NONE')}`);
const newTranslations = Array.isArray(batchKey.translations) ? batchKey.translations : [];
logger.log(` 📋 [BATCH ${batchIndex}] Extracted newTranslations.length: ${newTranslations.length}`);
if (newTranslations.length === 0 && batchKey.key_name) {
logger.log(` ⚠️ [BATCH ${batchIndex}] No translations found in batchKey for "${batchKey.key_name}" - batchKey keys: ${Object.keys(batchKey).join(', ')}`);
}
if (newTranslations.length > 0) {
logger.log(` 📋 [BATCH ${batchIndex}] ========== TRANSLATION UPDATE START ==========`);
logger.log(` 📋 [BATCH ${batchIndex}] PUT translation payload for key_id=${key_id} includes ${newTranslations.length} translation(s)`);
logger.log(` 📋 [BATCH ${batchIndex}] Translations to update: ${JSON.stringify(newTranslations, null, 2)}`);
// Update each translation individually using PUT /translations/{translation_id}
// This is the correct Lokalise API approach for updating translation values
try {
const translationPutStartedAt = Date.now();
// Process all translations in parallel using Promise.all
const translationUpdatePromises = newTranslations.map(async (newTranslation) => {
const languageIso = newTranslation.language_iso || 'en';
const translationValue = newTranslation.translation;
// Find existing translation to get translation_id
const existingTranslation = currentTranslations.find(t =>
(t.language_iso || t.language_iso_code) === languageIso
);
if (!existingTranslation || !existingTranslation.translation_id) {
logger.log(` ⚠️ [BATCH ${batchIndex}] No existing translation found for language ${languageIso} - skipping`);
return null;
}
const translation_id = existingTranslation.translation_id;
// Use PUT /translations/{translation_id} endpoint
const translationPutUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/translations/${translation_id}`;
// Payload for PUT /translations/{translation_id} is just the translation value
const translationPutPayload = {
translation: translationValue,
};
logger.log(` 🚀 [BATCH ${batchIndex}] Sending PUT /translations/${translation_id} for language ${languageIso}: "${translationValue}"`);
try {
const translationPutRes = await http.put(translationPutUrl, translationPutPayload, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
'Content-Type': 'application/json',
},
validateStatus: (status) => status < 600,
});
if (translationPutRes.status >= 200 && translationPutRes.status < 300) {
logger.log(` ✅ [BATCH ${batchIndex}] Successfully updated translation ${translation_id} (${languageIso}) for key_id=${key_id}`);
return { success: true, translation_id, languageIso };
} else {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to update translation ${translation_id} (${languageIso}): ${translationPutRes.status}`);
return { success: false, translation_id, languageIso, error: `Status ${translationPutRes.status}` };
}
} catch (translationErr) {
logger.error(`[BATCH ${batchIndex}] ERROR: Translation update failed for translation_id=${translation_id} (${languageIso}): ${translationErr.message || String(translationErr)}`);
return { success: false, translation_id, languageIso, error: translationErr.message };
}
});
// Wait for all translation updates to complete
const translationResults = await Promise.all(translationUpdatePromises);
const successfulUpdates = translationResults.filter(r => r && r.success).length;
logRemoteTiming(`[BATCH ${batchIndex}] PUT translations key_id=${key_id}`, translationPutStartedAt);
if (successfulUpdates > 0) {
logger.log(` ✅ [BATCH ${batchIndex}] Successfully updated ${successfulUpdates}/${newTranslations.length} translation(s) for key_id=${key_id} "${key_name}"`);
} else {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to update any translations for key_id=${key_id}`);
}
} catch (translationErr) {
logger.error(`[BATCH ${batchIndex}] ERROR: Translation update failed for key_id=${key_id}: ${translationErr.message || String(translationErr)}`);
if (translationErr.stack) {
logger.error(`[BATCH ${batchIndex}] ERROR STACK: ${translationErr.stack}`);
}
}
}
}
}
} catch (err) {
// Check if job was cancelled - don't log error if cancelled
if (err.message === 'JOB_CANCELLED') {
throw err; // Re-throw to stop processing
}
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to update key_id=${key_id} "${key_name}": ${err.message || String(err)}`);
if (err.stack) {
logger.error(`[BATCH ${batchIndex}] ERROR STACK: ${err.stack}`);
}
}
// Note: totalUpdated is already incremented inside the PUT success block above
} else {
// Parallel path: Multiple keys - process with concurrency limit
// CRITICAL: Lokalise rate limit is 6 req/sec per token, and SDKs recommend only 1 concurrent request per token
// Using 2 as a safer middle ground - allows some parallelism while staying well under rate limits
// Note: Official SDKs recommend 1 concurrent request per token, but 2 is acceptable for most cases
logger.log(` 🔍 [BATCH ${batchIndex}] DEBUG: Entering PARALLEL path for ${keysToProcess.length} key(s)`);
logger.log(` 🔍 [BATCH ${batchIndex}] Entering PARALLEL path for ${keysToProcess.length} key(s)`);
const CONCURRENCY_LIMIT = 2; // Process 2 keys in parallel (safer for Lokalise rate limits)
logger.log(` 🔍 [BATCH ${batchIndex}] Processing ${keysToProcess.length} key(s) in batches of ${CONCURRENCY_LIMIT}`);
for (let i = 0; i < keysToProcess.length; i += CONCURRENCY_LIMIT) {
logger.log(` 🔍 [BATCH ${batchIndex}] Processing batch ${Math.floor(i / CONCURRENCY_LIMIT) + 1}/${Math.ceil(keysToProcess.length / CONCURRENCY_LIMIT)} (keys ${i + 1}-${Math.min(i + CONCURRENCY_LIMIT, keysToProcess.length)})`);
const batch = keysToProcess.slice(i, i + CONCURRENCY_LIMIT);
// Check for cancellation before processing batch
if (await shouldCancel()) {
const err = new Error('JOB_CANCELLED');
throw err;
}
// Process batch in parallel - FULL tag-adding logic (same as single key path)
await Promise.all(batch.map(async ({ key_id, key_name, batchKey, existingTags }) => {
// Declare variables in broader scope
let updateSuccess = false;
let updateResponse = null;
try {
logger.log(` 🔍 [BATCH ${batchIndex}] Processing PUT for key_id=${key_id} "${key_name}" (parallel path)...`);
logger.log(` 🔍 [BATCH ${batchIndex}] batchKey structure: ${JSON.stringify(Object.keys(batchKey || {}))}`);
logger.log(` 🔍 [BATCH ${batchIndex}] batchKey.tags: ${Array.isArray(batchKey?.tags) ? `${batchKey.tags.length} tags` : 'NOT ARRAY'}`);
logger.log(` 🔍 [BATCH ${batchIndex}] batchKey.translations: ${Array.isArray(batchKey?.translations) ? `${batchKey.translations.length} translations` : 'NOT ARRAY'}`);
// Get pre-fetched key data
let currentKey = keyDataMap.get(key_id);
if (!currentKey) {
logger.error(`[BATCH ${batchIndex}] ERROR: Key data not found for key_id=${key_id} - fetching now...`);
// Fetch it now if not in map
try {
const getUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys/${key_id}`;
const getRes = await http.get(getUrl, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
},
validateStatus: (status) => status < 600,
});
if (getRes.status >= 200 && getRes.status < 300 && getRes.data && getRes.data.key) {
keyDataMap.set(key_id, getRes.data.key);
currentKey = getRes.data.key;
logger.log(` ✅ [BATCH ${batchIndex}] Fetched key data for key_id=${key_id} - tags: ${Array.isArray(currentKey.tags) ? currentKey.tags.length : 0}, translations: ${Array.isArray(currentKey.translations) ? currentKey.translations.length : 0}`);
} else {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to fetch key_id=${key_id}: ${getRes.status}`);
return;
}
} catch (fetchErr) {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to fetch key_id=${key_id}: ${fetchErr.message || String(fetchErr)}`);
return;
}
}
if (!currentKey) {
logger.error(`[BATCH ${batchIndex}] ERROR: Key data still not found for key_id=${key_id} after fetch - skipping`);
return;
}
// Use pre-fetched key data
const currentTagsFromLokalise = Array.isArray(currentKey.tags) ? currentKey.tags : [];
const currentTranslations = Array.isArray(currentKey.translations) ? currentKey.translations : [];
// CRITICAL: Define actualPlatforms and keyNameObject early in broader scope
const keyPlatforms = Array.isArray(currentKey.platforms) && currentKey.platforms.length > 0
? currentKey.platforms
: ['web'];
const actualPlatforms = Array.isArray(keyPlatforms) && keyPlatforms.length > 0
? keyPlatforms
: ['web'];
// Build key_name object early so it's accessible in translation update section
const keyNameObject = {};
if (currentKey.key_name) {
if (typeof currentKey.key_name === 'string') {
actualPlatforms.forEach(platform => {
keyNameObject[platform] = currentKey.key_name;
});
} else if (typeof currentKey.key_name === 'object') {
actualPlatforms.forEach(platform => {
if (currentKey.key_name[platform]) {
keyNameObject[platform] = currentKey.key_name[platform];
} else if (currentKey.key_name.web && actualPlatforms.includes('web')) {
keyNameObject[platform] = currentKey.key_name.web;
} else {
keyNameObject[platform] = key_name;
}
});
Object.keys(keyNameObject).forEach(platform => {
if (!actualPlatforms.includes(platform)) {
delete keyNameObject[platform];
}
});
}
} else {
actualPlatforms.forEach(platform => {
keyNameObject[platform] = key_name;
});
}
// Extract and normalize tag names from Lokalise (FULL LOGIC)
const normalizeTagForComparison = (tag) => {
const str = typeof tag === 'string'
? tag
: (tag && typeof tag === 'object' && tag.name ? String(tag.name) : String(tag));
return str.trim().toLowerCase().replace(/\s+/g, '_');
};
const existingTagNames = currentTagsFromLokalise.map(tag => {
if (typeof tag === 'string') {
return tag.trim();
} else if (tag && typeof tag === 'object' && tag.name) {
return String(tag.name).trim();
}
return String(tag).trim();
}).filter(Boolean);
const normalizedExistingTags = existingTagNames.map(tag => normalizeTagForComparison(tag));
// Extract new tags from Strapi
const newTagsFromStrapi = Array.isArray(batchKey.tags) ? batchKey.tags : [];
const newTagNames = newTagsFromStrapi.map(tag => {
if (typeof tag === 'string') {
return tag.trim();
} else if (tag && typeof tag === 'object' && tag.name) {
return String(tag.name).trim();
}
return String(tag).trim();
}).filter(Boolean);
// Merge tags (deduplicate by normalized name - handles spaces vs underscores)
const allTagNames = [...existingTagNames];
newTagNames.forEach(newTag => {
const normalizedNew = normalizeTagForComparison(newTag);
const exists = normalizedExistingTags.some(existingNormalized => existingNormalized === normalizedNew);
if (!exists && normalizedNew.length > 0) {
allTagNames.push(newTag);
}
});
// Final deduplication (case-insensitive, space-insensitive)
const finalTagsMap = new Map();
allTagNames.forEach(tagName => {
const normalized = normalizeTagForComparison(tagName);
if (normalized.length > 0 && !finalTagsMap.has(normalized)) {
finalTagsMap.set(normalized, tagName);
}
});
let mergedTags = Array.from(finalTagsMap.values());
// Prepare tags for Lokalise - keep existing format, normalize new tags
const tagsToSend = [];
const newTagsToCreate = [];
mergedTags.forEach(tag => {
const normalizedTag = normalizeTagForComparison(tag);
const exists = normalizedExistingTags.some(existingNormalized => existingNormalized === normalizedTag);
if (exists) {
// Keep original format from Lokalise for existing tags
const originalTag = existingTagNames.find(et =>
normalizeTagForComparison(et) === normalizedTag
);
if (originalTag) {
tagsToSend.push(originalTag);
} else {
tagsToSend.push(tag);
}
} else {
// Send new tags exactly as user typed them (preserve spaces, case, underscores)
const trimmedTag = String(tag).trim();
if (trimmedTag.length > 0) {
tagsToSend.push(trimmedTag);
newTagsToCreate.push(trimmedTag);
}
}
});
mergedTags = Array.from(new Set(tagsToSend));
const missingTags = newTagsToCreate;
// Create missing tags using temporary keys (FULL LOGIC)
const tempKeyIds = [];
if (missingTags.length > 0) {
if (!globalThis.__lokaliseEnsuredTags) {
globalThis.__lokaliseEnsuredTags = new Set();
}
const ensuredSet = globalThis.__lokaliseEnsuredTags;
const tagsToEnsure = missingTags.filter(t => !ensuredSet.has(String(t).toLowerCase()));
if (tagsToEnsure.length > 0) {
const tempKeyPlatforms = Array.isArray(currentKey.platforms) && currentKey.platforms.length > 0
? currentKey.platforms
: ['web'];
try {
const tempKeyName = `__temp_tags_${Date.now()}_${Math.random().toString(36).substr(2, 9)}__`;
const tempKeyUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys`;
const tempKeyPayload = {
keys: [{
key_name: tempKeyName,
platforms: tempKeyPlatforms,
tags: tagsToEnsure,
translations: [{ language_iso: 'en', translation: '__temp__' }],
}],
};
const tempKeyRes = await http.post(tempKeyUrl, tempKeyPayload, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
'Content-Type': 'application/json',
},
validateStatus: (status) => status < 600,
});
if (tempKeyRes.status >= 200 && tempKeyRes.status < 300 && tempKeyRes.data?.keys?.[0]) {
const tempKeyId = tempKeyRes.data.keys[0].key_id;
tagsToEnsure.forEach(tag => tempKeyIds.push({ key_id: tempKeyId, tag }));
tagsToEnsure.forEach(tag => ensuredSet.add(String(tag).toLowerCase()));
}
} catch (tagErr) {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to create temporary key: ${tagErr.message || String(tagErr)}`);
}
}
}
// Update key with all tags using PUT /keys/{key_id} (FULL LOGIC)
const putUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys/${key_id}`;
const putPayload = {};
if (actualPlatforms.length === 1 && keyNameObject[actualPlatforms[0]]) {
putPayload.key_name = keyNameObject[actualPlatforms[0]];
putPayload.platforms = actualPlatforms;
} else {
putPayload.key_name = keyNameObject;
putPayload.platforms = actualPlatforms;
}
putPayload.tags = mergedTags;
// DEBUG: Log payload structure to verify tags are included
if (mergedTags.length > 0) {
logger.log(` 📋 [BATCH ${batchIndex}] PUT payload (parallel) for key_id=${key_id} includes ${mergedTags.length} tag(s): ${JSON.stringify(mergedTags.slice(0, 5))}${mergedTags.length > 5 ? '...' : ''}`);
} else {
logger.log(` ⚠️ [BATCH ${batchIndex}] PUT payload (parallel) for key_id=${key_id} has NO tags (mergedTags.length=0)`);
}
// Update tags first using PUT /keys/{key_id}
const putStartedAt = Date.now();
const putRes = await http.put(putUrl, putPayload, {
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
'Content-Type': 'application/json',
},
validateStatus: (status) => status < 600,
});
logRemoteTiming(`[BATCH ${batchIndex}] PUT key_id=${key_id} (tags)`, putStartedAt);
if (putRes.status >= 200 && putRes.status < 300) {
updateSuccess = true;
updateResponse = putRes;
// Clean up temporary keys
if (tempKeyIds.length > 0) {
for (const { key_id: tempKeyId } of tempKeyIds) {
try {
await http.delete(
`${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys/${tempKeyId}`,
{ headers: { 'X-Api-Token': cfg.lokaliseApiToken }, validateStatus: (status) => status < 600 }
);
} catch (deleteErr) {
// Ignore cleanup errors
}
}
}
// Store key_id mapping
if (batchKey?.field_path && (batchKey?.entry_id || batchKey?.entry_slug)) {
const tagSnapshot = buildCanonicalTagSnapshot(batchKey.tags);
keyIdMapping.set(key_name, {
key_id: key_id,
entry_id: batchKey.entry_id || null,
entry_slug: batchKey.entry_slug || null,
field_path: batchKey.field_path,
key_name: key_name,
// Store key data for hash calculation
key_data: {
key_name: key_name,
tags: tagSnapshot,
translations: Array.isArray(batchKey.translations) ? batchKey.translations : [],
updatedAt: batchKey.updatedAt || batchKey.updated_at || null,
},
tag_snapshot: tagSnapshot,
});
}
// Update translations if needed (use PUT /translations/{translation_id} in parallel for performance)
const newTranslations = Array.isArray(batchKey.translations) ? batchKey.translations : [];
if (newTranslations.length > 0) {
logger.log(` 📋 [BATCH ${batchIndex}] Updating ${newTranslations.length} translation(s) for key_id=${key_id}...`);
// OPTIMIZATION: Process translation updates in parallel for better performance with large numbers of keys
// Separate existing translations (use PUT /translations/{id}) from new ones (use PUT /keys/{id})
const existingTranslationUpdates = [];
const newTranslationUpdates = [];
for (const newTranslation of newTranslations) {
const languageIso = newTranslation.language_iso || 'en';
const existingTranslation = currentTranslations.find(t =>
(t.language_iso || t.language_iso_code) === languageIso
);
if (existingTranslation?.translation_id) {
// Existing translation - use PUT /translations/{translation_id}
existingTranslationUpdates.push({
translation_id: existingTranslation.translation_id,
language_iso: languageIso,
translation: newTranslation.translation,
});
} else {
// New translation - will create via PUT /keys/{key_id}
newTranslationUpdates.push({
language_iso: languageIso,
translation: newTranslation.translation,
});
}
}
// Process existing translations in parallel (much faster for multiple translations)
if (existingTranslationUpdates.length > 0) {
const translationPutPromises = existingTranslationUpdates.map(async ({ translation_id, language_iso, translation }) => {
try {
const translationPutStartedAt = Date.now();
const translationPutUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/translations/${translation_id}`;
const translationPutPayload = {
translation: translation,
};
const translationPutRes = await http.put(
translationPutUrl,
translationPutPayload,
{
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
'Content-Type': 'application/json',
},
validateStatus: (status) => status < 600,
}
);
logRemoteTiming(`[BATCH ${batchIndex}] PUT translation_id=${translation_id}`, translationPutStartedAt);
if (translationPutRes.status >= 200 && translationPutRes.status < 300) {
logger.log(` ✅ [BATCH ${batchIndex}] Successfully updated translation_id=${translation_id} (${language_iso})`);
return { success: true, translation_id, language_iso };
} else {
logger.error(` ❌ [BATCH ${batchIndex}] Translation update failed: ${translationPutRes.status} ${JSON.stringify(translationPutRes.data || {})}`);
return { success: false, translation_id, language_iso, error: translationPutRes.status };
}
} catch (translationErr) {
logger.error(` ❌ [BATCH ${batchIndex}] ERROR: Translation update failed for ${language_iso}: ${translationErr.message || String(translationErr)}`);
if (translationErr.response?.data) {
logger.error(` 📋 [BATCH ${batchIndex}] Error response: ${JSON.stringify(translationErr.response.data, null, 2)}`);
}
return { success: false, translation_id, language_iso, error: translationErr.message };
}
});
// Wait for all translation updates to complete in parallel
const results = await Promise.all(translationPutPromises);
const successCount = results.filter(r => r.success).length;
logger.log(` ✅ [BATCH ${batchIndex}] Completed ${successCount}/${existingTranslationUpdates.length} translation update(s) in parallel`);
}
// Process new translations (create via PUT /keys/{key_id} - must be done separately)
if (newTranslationUpdates.length > 0) {
try {
const translationPutStartedAt = Date.now();
const translationPutUrl = `${cfg.lokaliseBaseUrl}/projects/${cfg.lokaliseProjectId}/keys/${key_id}`;
const translationPutPayload = {
key_name: actualPlatforms.length === 1 && keyNameObject[actualPlatforms[0]]
? keyNameObject[actualPlatforms[0]]
: keyNameObject,
platforms: actualPlatforms,
translations: newTranslationUpdates,
};
const translationPutRes = await http.put(
translationPutUrl,
translationPutPayload,
{
headers: {
'X-Api-Token': cfg.lokaliseApiToken,
'Content-Type': 'application/json',
},
validateStatus: (status) => status < 600,
}
);
logRemoteTiming(`[BATCH ${batchIndex}] PUT new translations key_id=${key_id}`, translationPutStartedAt);
if (translationPutRes.status >= 200 && translationPutRes.status < 300) {
logger.log(` ✅ [BATCH ${batchIndex}] Successfully created ${newTranslationUpdates.length} new translation(s)`);
} else {
logger.error(` ❌ [BATCH ${batchIndex}] Translation creation failed: ${translationPutRes.status} ${JSON.stringify(translationPutRes.data || {})}`);
}
} catch (translationErr) {
logger.error(` ❌ [BATCH ${batchIndex}] ERROR: Translation creation failed: ${translationErr.message || String(translationErr)}`);
}
}
}
totalUpdated += 1;
} else {
logger.error(`[BATCH ${batchIndex}] ERROR: PUT /keys/{key_id} failed: ${putRes.status} ${JSON.stringify(putRes.data || {})}`);
}
} catch (err) {
if (err.message === 'JOB_CANCELLED') {
throw err;
}
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to update key_id=${key_id}: ${err.message || String(err)}`);
}
return { success: updateSuccess };
}));
// totalUpdated is already tracked in individual updates (each key increments it)
// Wait for this batch to complete before processing next batch
} // End of for loop (parallel path)
} // End of if (keysToProcess.length > 0) block
// If we successfully updated all keys we found, return success
// Note: We may not have found all duplicate keys, but we updated the ones we could find
// CRITICAL: Don't return early - let the PUT processing complete first
// The PUT processing code will handle updating the keys and incrementing totalUpdated
// Only return early if we have NO keys to process
if (keysToProcess.length === 0) {
logger.log(` ⚠️ [BATCH ${batchIndex}] No keys to process - returning early`);
return { inserted: 0, updated: totalUpdated };
}
// If we found some keys but couldn't update them all, or if we found keys but need to retry
// Only retry keys we actually found key_ids for
if (foundKeys.length > 0 && keysToProcess.length < foundKeys.length) {
// Some keys were found but not updated - retry those
const retryBatch = batch.map(key => {
const keyName = typeof key.key_name === 'string' ? key.key_name : (key.key_name?.web || key.key_name?.other || '');
const meta = retryTagsMap.get(keyName);
if (meta && meta.key_id && typeof meta.key_id === 'number') {
return { ...key, key_id: meta.key_id };
}
return key;
});
const retryExistingCount = retryBatch.filter(k => typeof k.key_id === 'number').length;
const retryNewCount = retryBatch.length - retryExistingCount;
if (retryExistingCount > 0) {
return await syncBatch(retryBatch, method, batchNumber, totalBatches);
}
}
// If we couldn't find any keys or update them, log and continue
// The keys exist in Lokalise (we got "already taken" error), they just can't be updated
if (foundKeys.length === 0) {
logger.error(`[BATCH ${batchIndex}] ERROR: Could not find key_ids for duplicate keys - these keys exist in Lokalise but cannot be updated`);
}
}
}
}
const isPureExistingNoop =
inserted === 0 &&
updated === 0 &&
errors.length === 0 &&
batch.length > 0 &&
existingCount === batch.length &&
newCount === 0;
if (isPureExistingNoop) {
noopDetected = true;
logger.log(
` ℹ️ [BATCH ${batchIndex}] Lokalise reported 0 inserted/updated changes for ${batch.length} key(s) — treating them as already up-to-date.`
);
totalUpdated += existingCount;
} else if (inserted === 0 && updated === 0 && batch.length > 0 && !existingKeysProcessed) {
// This is suspicious - we sent keys but got no results
// Skip warning if we're in fast-path PUT mode (existingKeysProcessed) - we already processed them via PUT
// Could mean: keys already exist (duplicates), payload structure wrong, or silent failure
logger.log(` ⚠️ Lokalise returned inserted=0, updated=0 for ${batch.length} key(s) - this may indicate:`);
logger.log(` - Keys already exist (duplicate key_name) - Lokalise may silently ignore duplicates`);
logger.log(` - Payload structure issue - check if key_name, tags, translations are correct`);
logger.log(` - Key names with special characters may need different handling`);
// Log sample payload to help debug
if (batch.length <= 3) {
batch.forEach((key, idx) => {
logger.log(` Sample key ${idx + 1}: ${JSON.stringify({
key_id: key.key_id,
key_name: key.key_name,
has_tags: Array.isArray(key.tags) && key.tags.length > 0,
tags_count: Array.isArray(key.tags) ? key.tags.length : 0,
has_translations: Array.isArray(key.translations) && key.translations.length > 0,
translations_count: Array.isArray(key.translations) ? key.translations.length : 0,
}, null, 2)}`);
});
} else {
logger.log(` Sample key 1: ${JSON.stringify({
key_id: batch[0].key_id,
key_name: batch[0].key_name,
has_tags: Array.isArray(batch[0].tags) && batch[0].tags.length > 0,
tags_count: Array.isArray(batch[0].tags) ? batch[0].tags.length : 0,
has_translations: Array.isArray(batch[0].translations) && batch[0].translations.length > 0,
translations_count: Array.isArray(batch[0].translations) ? batch[0].translations.length : 0,
}, null, 2)}`);
}
// Calculate batch counts
const batchWithKeyId = batch.filter(k => typeof k.key_id === 'number').length;
const batchWithoutKeyId = batch.length - batchWithKeyId;
logger.log(` ℹ️ Using estimates due to missing counts: ${batchWithoutKeyId} new, ${batchWithKeyId} existing`);
// Still count as attempted (even though Lokalise didn't confirm)
// This prevents infinite retries, but logs the issue
totalPushed += batchWithoutKeyId;
totalUpdated += batchWithKeyId;
} else {
// Use Lokalise's actual counts
totalPushed += inserted;
totalUpdated += updated;
}
} else {
// Log detailed error information
const errorMsg =
res.data && typeof res.data === 'object' ? JSON.stringify(res.data, null, 2) : res.statusText;
logger.error(` ❌ Lokalise API error (status ${res.status}):`);
logger.error(` Response: ${errorMsg}`);
if (existingCount > 0) {
const firstExisting = batch.find(k => typeof k.key_id === 'number');
if (firstExisting) {
logger.error(` Sample existing key that failed: key_id=${firstExisting.key_id}, key_name=${firstExisting.key_name}`);
logger.error(` Payload structure: ${JSON.stringify({
key_id: firstExisting.key_id,
key_name: firstExisting.key_name,
tags: firstExisting.tags,
has_translations: Array.isArray(firstExisting.translations) && firstExisting.translations.length > 0,
}, null, 2)}`);
}
}
throw new Error(`Unexpected status during POST: ${res.status} ${errorMsg}`);
}
};
const sendBatches = async (items, method) => {
if (items.length === 0) return;
const BATCH_SIZE = 500;
const CONCURRENCY = 5; // Safe parallelism across batches (tuned for large projects)
const totalBatches = Math.ceil(items.length / BATCH_SIZE);
const batchSpecs = [];
for (let i = 0; i < items.length; i += BATCH_SIZE) {
batchSpecs.push({ start: i, end: i + BATCH_SIZE });
}
let next = 0;
const runOne = async () => {
const idx = next++;
if (idx >= batchSpecs.length) return;
const spec = batchSpecs[idx];
const batch = items.slice(spec.start, spec.end);
const batchNumber = Math.floor(spec.start / BATCH_SIZE) + 1;
try {
if (await shouldCancel()) {
const err = new Error('JOB_CANCELLED');
throw err;
}
await syncBatch(batch, method, batchNumber, totalBatches);
} catch (err) {
if (err.response) {
const errorMsg = err.response.data
? (typeof err.response.data === 'string'
? err.response.data
: JSON.stringify(err.response.data, null, 2))
: err.response.statusText;
logger.error(`❌ Failed to ${method} batch ${batchNumber} for '${type}' to Lokalise:`);
logger.error(` Status: ${err.response.status}`);
logger.error(` Error: ${errorMsg}`);
logger.error(` Payload keys: ${JSON.stringify(batch.slice(0, 2), null, 2)}... (showing first 2)`);
} else {
logger.error(`❌ Failed to ${method} batch ${batchNumber} for '${type}' to Lokalise:`, err.message);
}
throw err;
}
// Run next in this worker
return runOne();
};
// Launch limited concurrent workers
const workers = new Array(Math.min(CONCURRENCY, batchSpecs.length)).fill(0).map(() => runOne());
await Promise.all(workers);
};
if (shouldSkipBatching) {
await syncBatch(payloadKeys, 'post', 1, 1);
} else {
// Use POST for both new and existing keys
// Lokalise determines create vs update based on key_id presence
// If key_id is present, it updates; if not, it creates
// This is the correct approach for bulk operations
await sendBatches(existingKeys, 'post');
await sendBatches(newKeys, 'post');
}
// BEST PRACTICE: Store lokalise_key_id back in Strapi entries after sync
// This prevents unnecessary searches in future syncs
// keyIdMapping is populated inside syncBatch from Lokalise response
if (keyIdMapping.size > 0 && (cfg.metadataService || cfg.entityService)) {
const storageTarget = cfg.metadataService ? 'metadata store' : 'Strapi entries';
logger.log(` 💾 Storing ${keyIdMapping.size} lokalise_key_id(s) in ${storageTarget}...`);
// Reset hash storage debug counter for this sync
global.hashStorageStartTime = Date.now();
global.hashStorageDebugCount = 0;
// Update each entry with its lokalise_key_id(s) using safe utility function
let storedCount = 0;
let skippedCount = 0;
let hashStoredCount = 0;
let hashNullCount = 0;
for (const mapping of keyIdMapping.values()) {
// Calculate hash for the synced key content
const syncHash = mapping.key_data
? calculateKeyHash(mapping.key_data)
: null;
if (syncHash) {
hashStoredCount++;
} else {
hashNullCount++;
}
// DEBUG: Log hash calculation for first few keys
if (storedCount < 3) {
logger.log(` 🔍 [HASH DEBUG] Key "${mapping.key_name}":`);
logger.log(` - key_data exists: ${!!mapping.key_data}`);
if (mapping.key_data) {
logger.log(` - key_data.updatedAt: ${mapping.key_data.updatedAt || 'null'}`);
logger.log(` - key_data.tags: ${Array.isArray(mapping.key_data.tags) ? mapping.key_data.tags.length : 0} tag(s)`);
logger.log(` - key_data.translations: ${Array.isArray(mapping.key_data.translations) ? mapping.key_data.translations.length : 0} translation(s)`);
}
logger.log(` - Calculated syncHash: ${syncHash ? `${syncHash.substring(0, 8)}...` : 'null'}`);
}
const success = await safeUpdateStrapiEntryLokaliseId(
type,
mapping.entry_id || null,
mapping.entry_slug || null,
mapping.key_name,
mapping.key_id,
mapping.field_path,
syncHash,
Array.isArray(mapping.key_data?.tags) ? mapping.key_data.tags : null
);
if (success) {
storedCount++;
} else {
skippedCount++;
}
}
// Summary log for hash storage
logger.log(` 📊 [HASH SUMMARY] Hash storage: ${hashStoredCount} stored, ${hashNullCount} null (out of ${keyIdMapping.size} total keys)`);
// Log summary (not individual failures to avoid spam)
if (storedCount > 0) {
logger.log(` ✅ Successfully stored lokalise_key_id for ${storedCount} key(s) in Strapi`);
}
if (skippedCount > 0) {
logger.log(` ℹ️ Skipped storing ${skippedCount} key_id(s) (entries not found or missing metadata)`);
}
} else if (keyIdMapping.size > 0) {
logger.log(` ⚠️ Cannot store lokalise_key_id: metadata service and entityService are not available`);
logger.log(` 💡 Consider enabling the plugin metadata service to persist key_id mappings`);
}
return { totalPushed, totalUpdated, noopDetected };
}
async function determineTypes({ log = false } = {}) {
let specifiedTypes = [];
const configuredTypes = parseList(cfg.strapiTypes);
if (configuredTypes.length > 0) {
specifiedTypes = configuredTypes;
} else if (cfg.strapiDefaultType && cfg.strapiDefaultType.trim() !== '') {
specifiedTypes = [cfg.strapiDefaultType.trim()];
}
const discoveredTypes = await getAllStrapiTypes();
const types = [...new Set([...specifiedTypes, ...discoveredTypes])];
// Return empty result instead of throwing error (graceful handling for empty projects)
// The preview/sync functions will handle empty types gracefully
if (types.length === 0) {
logger.log('ℹ️ No content types found. The project may be empty or no API content types are configured.');
return { types: [], message: 'No content types found' };
}
if (log) {
logger.log('\n📝 Content types to sync:');
if (specifiedTypes.length > 0 && discoveredTypes.length > 0) {
const common = specifiedTypes.filter((t) => discoveredTypes.includes(t));
const onlySpecified = specifiedTypes.filter((t) => !discoveredTypes.includes(t));
const onlyDiscovered = discoveredTypes.filter((t) => !specifiedTypes.includes(t));
if (common.length > 0) {
logger.log(` ✅ Specified & Found: ${common.join(', ')}`);
}
if (onlySpecified.length > 0) {
logger.log(` 📌 Specified only: ${onlySpecified.join(', ')}`);
}
if (onlyDiscovered.length > 0) {
logger.log(` 🔍 Auto-discovered: ${onlyDiscovered.join(', ')}`);
}
} else if (specifiedTypes.length > 0) {
logger.log(` ✅ Using specified content types: ${specifiedTypes.join(', ')}`);
const extra = types.filter((t) => !specifiedTypes.includes(t));
if (extra.length > 0) {
logger.log(` 🔍 Also found: ${extra.join(', ')}`);
}
} else {
logger.log(` ✅ Using auto-discovered content types: ${types.join(', ')}`);
}
}
return { types, specifiedTypes, discoveredTypes };
}
/**
* @param {{ types?: string[] | string, debugMode?: boolean, previewLimit?: number | null, slugFilters?: string[], keyNameFilters?: string[], keyIdFilters?: string[], keyValueFilters?: string[] }} [options]
*/
async function collectKeysByType({ types: initialTypes, debugMode = false, previewLimit = null, slugFilters = [], keyNameFilters = [], keyIdFilters = [], keyValueFilters = [] } = {}) {
// BEST PRACTICE: Process entries incrementally with memory-efficient key storage
// No hard limits - uses Map-based deduplication to handle unlimited entries
// Processes entries in batches of 1000, generating keys as it goes
// Keys are stored in a Map (keyed by key_name) to automatically deduplicate
// This prevents memory overflow even with millions of entries
let resolvedTypes = Array.isArray(initialTypes) ? initialTypes : parseList(initialTypes);
let discoveryResult = null;
if (!resolvedTypes || resolvedTypes.length === 0) {
discoveryResult = await determineTypes({ log: false });
resolvedTypes = discoveryResult?.types || [];
}
if ((!resolvedTypes || resolvedTypes.length === 0) && cfg.strapiTypes) {
const fallbackTypes = parseList(cfg.strapiTypes);
if (fallbackTypes.length > 0) {
resolvedTypes = fallbackTypes;
}
}
// Handle empty types gracefully - return empty Map instead of error
if (!resolvedTypes || resolvedTypes.length === 0) {
logger.log('ℹ️ No content types to process. Returning empty result.');
return new Map();
}
const keysMap = new Map();
for (const type of resolvedTypes) {
try {
// Use incremental processing with memory-efficient key storage
// previewLimit is ignored - we process all entries incrementally
const keys = await fetchAndProcessIncrementally(type, {
previewMode: true,
debugMode,
slugFilters,
keyNameFilters,
keyIdFilters,
keyValueFilters
});
keysMap.set(type, keys);
} catch (err) {
logger.error(`Error collecting keys for ${type}:`, err);
keysMap.set(type, { error: err.message || String(err) });
}
}
await flushSlugMap();
return keysMap;
}
/**
* Fetch entries and process them incrementally to prevent memory issues
* Uses Map-based key storage (keyed by key_name) for automatic deduplication
* This allows processing unlimited entries without memory overflow
*/
async function fetchAndProcessIncrementally(type, options = {}) {
const { previewMode = false, debugMode = false, slugFilters = [], keyNameFilters = [], keyIdFilters = [], keyValueFilters = [] } = options;
const entityConfig = cfg.contentTypeMap[type];
if (!cfg.entityService || !entityConfig?.uid) {
throw new Error(`Cannot fetch content for '${type}': entityService or config missing`);
}
logger.log(`📦 Fetching and processing "${type}" incrementally (no limit)...`);
// Determine if we should skip relations
let shouldSkipRelations = false;
if (cfg.skipNestedRelations.has(type)) {
shouldSkipRelations = true;
logger.log(` ℹ️ Skipping relations for '${type}' (configured in skipNestedRelations)`);
} else if (type === 'articles' || type === 'authors' || type === 'categories') {
shouldSkipRelations = true;
logger.log(` ℹ️ Skipping relations for '${type}' (known large dataset)`);
}
const params = {
publicationState: 'preview',
pagination: { pageSize: 1000 },
populate: shouldSkipRelations ? {} : '*',
};
// Use Map to store keys by key_name - automatically deduplicates and uses less memory
// Map is more memory-efficient than Array for large datasets with potential duplicates
const uniqueKeysMap = new Map();
let page = 1;
let hasMore = true;
let totalProcessed = 0;
const startTime = Date.now();
const PROCESSING_BATCH_SIZE = 5000; // Process and clear memory every 5000 entries
const maxPages = 1000000; // Safety limit: max 1B entries (1,000,000 pages × 1,000/page) - effectively unlimited for practical use
let consecutiveEmptyPages = 0; // Track empty pages to prevent infinite loops
let firstPageEntryCount = null; // Track first page size to detect if pagination is working
let lastProcessedEntryIds = new Set(); // Track entry IDs to detect if we're processing same entries
// Process entries in batches - fetch, process, deduplicate, then fetch next batch
// Memory-efficient: Processes 1000 entries at a time, stores only unique keys in Map
// No limits: Processes all entries, stops when pagination ends or fails
while (hasMore && page <= maxPages) {
// Strapi v5 pagination format: { pagination: { page, pageSize } }
const pageParams = {
publicationState: 'preview',
pagination: {
page: page,
pageSize: 1000
},
populate: shouldSkipRelations ? {} : '*',
};
try {
// Log what page we're requesting (only for first few pages and every 10th page)
if (page <= 3 || page % 10 === 0) {
logger.log(` 🔍 Fetching page ${page} (pageSize: 1000)...`);
}
const pageResults = await cfg.entityService.findMany(entityConfig.uid, pageParams);
let resultsArray = [];
let pageCount = null;
let currentPage = page;
let totalCount = null;
// Parse response based on Strapi v5 format
if (pageResults) {
if (Array.isArray(pageResults)) {
// Direct array response - this means pagination didn't work!
// Strapi returned all entries as an array instead of paginated
resultsArray = pageResults;
// CRITICAL: If we got a large array (all entries) on page > 1, pagination failed
if (resultsArray.length > 5000 && page > 1) {
logger.error(` ❌ Pagination failed! Got ${resultsArray.length} entries (all entries) on page ${page}. Strapi is returning all entries instead of paginated results.`);
logger.error(` ℹ️ This usually means Strapi's entityService.findMany is ignoring pagination. Processing all ${resultsArray.length} entries as a single batch.`);
// Process all entries as one batch, then stop
hasMore = false;
} else if (resultsArray.length < 1000) {
// If we got less than pageSize, this is likely the last page
hasMore = false;
} else if (resultsArray.length === 1000 && page === 1) {
// Got exactly 1000 on first page - might be paginated, continue
// But if we get same 1000 on page 2, we'll detect it
} else if (page > 1 && resultsArray.length >= 35000) {
// Got all entries on page > 1 - pagination definitely failed
logger.error(` ❌ Pagination not working - got all ${resultsArray.length} entries on page ${page}. Stopping.`);
hasMore = false;
break;
}
} else if (pageResults.data && Array.isArray(pageResults.data)) {
// Strapi v5 format: { data: [...], pagination: {...} }
resultsArray = pageResults.data;
if (pageResults.pagination) {
currentPage = pageResults.pagination.page || page;
pageCount = pageResults.pagination.pageCount;
totalCount = pageResults.pagination.total;
// Debug: Log pagination info for first few pages
if (page <= 3 || page % 10 === 0) {
logger.log(` 📊 Pagination: currentPage=${currentPage}, pageCount=${pageCount}, total=${totalCount}, results=${resultsArray.length}`);
}
// CRITICAL: Check if we got the same page data (pagination not working)
if (currentPage !== page && page > 1) {
logger.log(` ⚠️ Pagination mismatch! Requested page ${page} but got page ${currentPage}. Strapi may be ignoring pagination.`);
// If we're getting page 1 data repeatedly, stop
if (currentPage === 1 && page > 1) {
logger.error(` ❌ Strapi is returning page 1 data repeatedly. Pagination is not working. Stopping to prevent infinite loop.`);
hasMore = false;
break;
}
}
// Use pagination metadata to determine if more pages exist
hasMore = currentPage < pageCount;
// If we're on the last page, ensure hasMore is false
if (currentPage >= pageCount) {
hasMore = false;
}
} else {
// No pagination metadata - check if we got ALL entries (pagination not working)
if (resultsArray.length >= 35000 && page > 1) {
logger.error(` ❌ Got ${resultsArray.length} entries on page ${page} without pagination metadata - pagination is not working!`);
logger.error(` ℹ️ Strapi is returning all entries regardless of page number. Processing all entries as single batch and stopping.`);
// Process all entries as one batch, then stop
hasMore = false;
// Don't break here - let it process this batch, then it will stop
} else if (resultsArray.length < 1000) {
// No pagination metadata - use array length as fallback
hasMore = false;
}
}
} else if (typeof pageResults === 'object') {
// Single object (singleType) - this is the only result
resultsArray = [pageResults];
hasMore = false;
}
}
// If we got no results, we've reached the end
if (resultsArray.length === 0) {
hasMore = false;
logger.log(` ℹ️ No more entries found (page ${page}). Reached end of data.`);
break; // Exit loop immediately
}
// CRITICAL: Detect if pagination is working by checking entry count
if (page === 1) {
firstPageEntryCount = resultsArray.length;
if (firstPageEntryCount > 5000) {
logger.log(` ⚠️ Got ${firstPageEntryCount} entries on page 1 - pagination may not be working.`);
logger.log(` ℹ️ Strapi is returning all entries instead of paginated results.`);
logger.log(` ℹ️ Processing all ${firstPageEntryCount} entries as a single batch (memory-efficient with Map deduplication).`);
// Process all entries once, then stop (don't try to paginate)
hasMore = false;
} else {
logger.log(` ✅ Pagination working: Got ${firstPageEntryCount} entries on page 1 (expected: ≤1000)`);
}
} else {
// On subsequent pages, check if we're getting the same large dataset
if (firstPageEntryCount !== null && resultsArray.length === firstPageEntryCount && firstPageEntryCount > 5000) {
logger.error(` ❌ Pagination failed! Got same ${resultsArray.length} entries on page ${page} as page 1.`);
logger.error(` ℹ️ Strapi is returning all entries repeatedly. Stopping to prevent infinite loop.`);
logger.error(` ℹ️ Already processed ${totalProcessed} entries. Will process remaining entries once and stop.`);
hasMore = false;
// Don't break - let it process this batch once, then stop
} else if (resultsArray.length > 5000 && page > 1) {
// Got large dataset on page > 1 (different from page 1)
logger.log(` ⚠️ Got ${resultsArray.length} entries on page ${page} - unexpected large result.`);
logger.log(` ℹ️ This may indicate pagination issues. Processing and stopping.`);
hasMore = false;
}
}
// Reset empty page counter when we get results
consecutiveEmptyPages = 0;
// Process this batch immediately (don't accumulate entries in memory)
// Suppress verbose "Processing entry X/Y" logging during batch processing
// Only show summary logs to avoid repetitive output
const batchKeys = await pushToLokalise(type, resultsArray, {
previewMode,
debugMode: false, // Disable per-entry logging during incremental batch processing
slugFilters,
keyNameFilters,
keyIdFilters,
keyValueFilters
});
// Add keys to Map (automatically deduplicates by key_name)
// This is memory-efficient: Map uses less memory than Array for large datasets
batchKeys.forEach((key) => {
if (key && key.key_name) {
// Keep the last occurrence (most recent data)
uniqueKeysMap.set(key.key_name, key);
}
});
const batchLength = resultsArray.length;
totalProcessed += batchLength;
// Clear processed entries from memory (let GC handle it)
resultsArray = null;
batchKeys.length = 0; // Clear array reference
// Additional safety check: if we got less than a full page AND no pagination metadata, assume last page
if (batchLength < 1000 && pageCount === null) {
hasMore = false;
}
// If pagination says we're on the last page, stop
if (pageCount !== null && currentPage >= pageCount) {
hasMore = false;
}
page++;
// Progress logging every 10 pages or every 10,000 entries
if (page % 10 === 0 || !hasMore || totalProcessed % 10000 === 0) {
const elapsedSeconds = (Date.now() - startTime) / 1000;
const elapsed = elapsedSeconds.toFixed(1);
const rate = elapsedSeconds > 0 ? (totalProcessed / elapsedSeconds).toFixed(0) : '0';
const uniqueKeyCount = uniqueKeysMap.size;
const pageInfo = pageCount ? ` (page ${currentPage}/${pageCount})` : ` (page ${page})`;
logger.log(` 📄 Processed ${totalProcessed.toLocaleString()} entries → ${uniqueKeyCount.toLocaleString()} unique keys${pageInfo} (${elapsed}s, ~${rate} entries/sec)...`);
if (!hasMore) {
logger.log(` ℹ️ Reached end of data (hasMore=false)`);
}
}
// Memory management: Force GC hint every 50,000 entries (if available)
if (totalProcessed % 50000 === 0 && totalProcessed > 0) {
logger.log(` ℹ️ Processed ${totalProcessed.toLocaleString()} entries, ${uniqueKeysMap.size.toLocaleString()} unique keys so far...`);
// Hint to GC (Node.js will handle this automatically, but we log for visibility)
if (global.gc && typeof global.gc === 'function') {
// Only if --expose-gc flag is used
try {
global.gc();
} catch (e) {
// Ignore if GC is not available
}
}
}
// Exit loop if no more pages
if (!hasMore) {
break;
}
} catch (pageError) {
if (pageError.message && pageError.message.includes('too many SQL variables')) {
if (!shouldSkipRelations && page === 1) {
logger.log(` ⚠️ SQL error detected. Retrying '${type}' without relations...`);
shouldSkipRelations = true;
params.populate = { lokalise: true }; // CRITICAL: Always include lokalise for hash extraction
page = 1;
hasMore = true;
uniqueKeysMap.clear(); // Reset keys
totalProcessed = 0;
continue;
}
}
throw pageError;
}
}
// Check if we hit the safety limit
if (page > maxPages && hasMore) {
logger.log(` ⚠️ Reached safety limit: Processed ${maxPages.toLocaleString()} pages (${(maxPages * 1000).toLocaleString()} entries max)`);
logger.log(` ℹ️ There may be more entries remaining. To process more, increase maxPages in fetchAndProcessIncrementally (currently: ${maxPages})`);
logger.log(` ℹ️ Processed ${totalProcessed.toLocaleString()} entries before hitting the limit`);
}
// Convert Map to Array for return (only unique keys)
const allKeys = Array.from(uniqueKeysMap.values());
const totalTime = ((Date.now() - startTime) / 1000).toFixed(1);
logger.log(` ✅ Processed ${totalProcessed.toLocaleString()} entries → ${allKeys.length.toLocaleString()} unique keys for "${type}" in ${totalTime}s`);
// Clear Map from memory
uniqueKeysMap.clear();
return allKeys;
}
async function syncSelectedKeys(selection = [], options = {}) {
const batchIndex = options.batchIndex || '?';
const results = [];
// Allow caller to provide a cancel checker for cooperative cancellation
if (typeof options.cancelCheck === 'function') {
cfg.cancelCheck = options.cancelCheck;
}
// OPTIMIZED APPROACH: Use lokalise_key_id from preview when available
const existingTagsMap = new Map();
const keysNeedingLookup = new Set();
selection.forEach((item) => {
if (!item || !Array.isArray(item.keys)) return;
item.keys.forEach((key) => {
if (!key || typeof key.key_name !== 'string' || key.key_name.trim().length === 0) return;
const keyName = key.key_name;
const lokaliseKeyId = key.lokalise_key_id;
const existingTags = Array.isArray(key.existing_tags) ? key.existing_tags : [];
if (lokaliseKeyId && typeof lokaliseKeyId === 'number') {
existingTagsMap.set(keyName, {
key_id: lokaliseKeyId,
tags: existingTags,
});
} else {
keysNeedingLookup.add(keyName);
}
});
});
// Only lookup keys that we don't already have key_ids for
if (keysNeedingLookup.size > 0) {
try {
const lookupMap = await getExistingTagsForKeys(Array.from(keysNeedingLookup));
lookupMap.forEach((meta, keyName) => {
existingTagsMap.set(keyName, meta);
});
} catch (err) {
logger.error(`[BATCH ${batchIndex}] ERROR: Failed to fetch key_ids: ${err.message || String(err)}`);
throw err;
}
}
for (const item of selection) {
if (await shouldCancel()) {
const err = new Error('JOB_CANCELLED');
throw err;
}
const { type, keys } = item || {};
if (!type || !Array.isArray(keys) || keys.length === 0) continue;
try {
const { totalPushed, totalUpdated, noopDetected } = await syncKeysToLokalise(type, keys, {
...options,
existingTagsMap,
batchIndex,
});
results.push({ type, totalSynced: totalPushed, totalUpdated, noop: noopDetected });
} catch (err) {
logger.error(`[BATCH ${batchIndex}] ERROR: Sync failed for type '${type}': ${err.message || String(err)}`);
if (err.stack) {
logger.error(`[BATCH ${batchIndex}] ERROR STACK: ${err.stack}`);
}
results.push({ type, error: err.message || String(err) });
throw err;
}
}
await flushSlugMap();
return results;
}
async function handleInteractiveSync(keysByType) {
if (keysByType.size === 0) {
logger.log('⚠️ No keys available for interactive sync.');
return { syncedTypes: 0, failedTypes: 0 };
}
const rl = rlFactory.createInterface({
input: process.stdin,
output: process.stdout,
});
const ask = (question) => new Promise((resolve) => rl.question(question, (answer) => resolve(answer)));
let syncedTypes = 0;
let failedTypes = 0;
for (const [type, keys] of keysByType.entries()) {
logger.log(`\n📂 Content type: ${type}`);
keys.forEach((key, idx) => {
const preview = key.translations[0]?.translation ?? '';
logger.log(` ${idx + 1}. ${key.key_name} = "${preview.substring(0, 60)}${preview.length > 60 ? '...' : ''}"`);
});
const answer = await ask(
'\nEnter numbers to exclude (comma separated), ranges (e.g., 2-5), or press Enter to sync all: '
);
let selectedKeys = keys;
if (answer.trim().length > 0) {
const exclusions = parseSelection(answer, keys.length);
selectedKeys = keys.filter((_, index) => !exclusions.has(index + 1));
if (selectedKeys.length === 0) {
logger.log(' ⚠️ No keys selected for sync. Skipping this type.');
continue;
}
}
try {
const { totalPushed, totalUpdated } = await syncKeysToLokalise(type, selectedKeys);
const updateMsg = totalUpdated > 0 ? ` (${totalUpdated} updated)` : '';
logger.log(`✅ Synced ${totalPushed} keys for '${type}'${updateMsg}.`);
syncedTypes++;
} catch (err) {
failedTypes++;
logger.error(`❌ Failed to sync '${type}':`, err && err.message ? err.message : err);
}
}
rl.close();
return { syncedTypes, failedTypes };
}
function parseSelection(input, max) {
const parts = String(input)
.split(',')
.map((p) => p.trim())
.filter(Boolean);
const result = new Set();
parts.forEach((part) => {
if (part.includes('-')) {
const [startStr, endStr] = part.split('-');
const start = parseInt(startStr, 10);
const end = parseInt(endStr, 10);
if (!isNaN(start) && !isNaN(end)) {
for (let i = Math.max(1, start); i <= Math.min(end, max); i++) {
result.add(i);
}
}
} else {
const num = parseInt(part, 10);
if (!isNaN(num) && num >= 1 && num <= max) {
result.add(num);
}
}
});
return result;
}
async function run(options = {}) {
const {
types,
preview = false,
debug = false,
interactive = false,
autoDiscover = true,
} = options;
const previewMode = preview || interactive;
logger.log('🔍 Checking Strapi connection...');
const isStrapiReachable = await checkStrapiConnection();
if (!isStrapiReachable) {
throw new Error(`Cannot connect to Strapi at ${cfg.strapiBaseUrl}`);
}
logger.log('✅ Strapi is reachable');
if (cfg.fieldIncludePatterns.length > 0 || cfg.fieldExcludePatterns.length > 0) {
logger.log('\n🔍 Field filtering:');
if (cfg.fieldIncludePatterns.length > 0) {
logger.log(` ✅ Include: ${cfg.fieldIncludePatterns.join(', ')}`);
}
if (cfg.fieldExcludePatterns.length > 0) {
logger.log(` ❌ Exclude: ${cfg.fieldExcludePatterns.join(', ')}`);
}
} else {
logger.log('\n🔍 Field filtering: None (syncing all fields)');
}
logger.log('\n📋 Discovering content types...');
const allTypes = types && types.length > 0 ? types : (await determineTypes({ log: true })).types;
if (previewMode) {
logger.log(`\n👁️ ${interactive ? 'INTERACTIVE ' : ''}PREVIEW MODE - No keys will be synced to Lokalise\n`);
}
let successCount = 0;
let errorCount = 0;
let totalKeysFound = 0;
const keysByType = new Map();
for (const type of allTypes) {
try {
logger.log(`\n🔄 Processing '${type}'...`);
const content = await fetchStrapiContent(type);
const keys = await pushToLokalise(type, content, { previewMode, debugMode: debug });
if (keys && keys.length > 0) {
totalKeysFound += keys.length;
keysByType.set(type, keys);
if (!previewMode) {
successCount++;
}
} else if (!previewMode && keys) {
logger.log(`⚠️ No keys found for '${type}'`);
}
} catch (err) {
errorCount++;
logger.error(`❌ Error processing '${type}':`, err && err.message ? err.message : err);
}
}
if (previewMode) {
logger.log(`\n${'='.repeat(60)}`);
logger.log('📊 PREVIEW SUMMARY:');
logger.log(` Total keys found: ${totalKeysFound}`);
logger.log(` Content types: ${allTypes.join(', ')}`);
if (!interactive) {
return { totalKeys: totalKeysFound, keysByType };
}
}
if (interactive) {
const { syncedTypes, failedTypes } = await handleInteractiveSync(keysByType);
successCount += syncedTypes;
errorCount += failedTypes;
} else if (!previewMode) {
for (const [type, keys] of keysByType.entries()) {
try {
const { totalPushed, totalUpdated } = await syncKeysToLokalise(type, keys);
const updateMsg = totalUpdated > 0 ? ` (${totalUpdated} updated)` : '';
logger.log(`✅ Synced ${totalPushed} keys for '${type}'${updateMsg}.`);
successCount++;
} catch (err) {
errorCount++;
logger.error(`❌ Failed to sync '${type}':`, err && err.message ? err.message : err);
}
}
}
if (!previewMode || interactive) {
logger.log(`\n${'='.repeat(50)}`);
logger.log(`📊 Summary: ${successCount} succeeded, ${errorCount} failed`);
if (errorCount === 0 && successCount > 0) {
logger.log('🎉 All selected types synced successfully!');
} else if (errorCount > 0) {
logger.log('⚠️ Some types failed to sync. Check errors above.');
} else if (successCount === 0) {
logger.log('ℹ️ No keys were synced.');
}
}
await flushSlugMap();
return { successCount, errorCount, totalKeys: totalKeysFound, keysByType };
}
async function runCLI(argv = process.argv) {
const args = new Set(argv.slice(2));
const preview = args.has('--preview') || args.has('-p');
const debug = args.has('--debug') || args.has('-d');
const interactive = args.has('--interactive') || args.has('-i');
try {
await run({ preview, debug, interactive });
if (!preview && !interactive) {
process.exit(0);
}
} catch (err) {
logger.error('🚨 Fatal error:', err && err.message ? err.message : err);
process.exit(1);
}
}
return {
config: cfg,
run,
runCLI,
preview: collectKeysByType,
collectKeysByType,
determineTypes,
fetchStrapiContent,
pushToLokalise,
syncKeysToLokalise,
syncSelectedKeys,
getExistingTagsForKeys,
shouldIncludeField,
checkStrapiConnection,
calculateKeyHash, // Expose hash calculation for preview
};
}
module.exports = {
createSyncRunner,
};