import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const projectRoot = path.resolve(__dirname, ".."); const localesDirectory = path.join(projectRoot, "src", "i18n", "generated"); const activeLocales = ["da", "en", "sv", "de", "no"]; const linkTokenPattern = /@(?:\.[\p{L}]+)?:(?:\{'[^']+'\}|[\p{L}\p{N}_.-]+)/gu; const placeholderPattern = /\{[A-Za-z_][A-Za-z0-9_]*\}/g; const wordTokenPattern = /[\p{L}\p{N}]+/gu; const linkedMessagePattern = /^@(?\.[\p{L}]+)?:(?:\{'(?[^']+)'\}|(?[\p{L}\p{N}_.-]+))$/u; const mode = process.argv.includes("--apply") ? "apply" : "check"; const isPlainObject = (value) => value !== null && typeof value === "object" && !Array.isArray(value); const readJson = (filePath) => JSON.parse(fs.readFileSync(filePath, "utf8").replace(/^\uFEFF/, "")); const writeJson = (filePath, value) => { fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); }; const cloneJson = (value) => JSON.parse(JSON.stringify(value)); const flattenStringEntries = (value, prefix = "") => { if (typeof value === "string") { return [{ key: prefix, value }]; } if (!isPlainObject(value)) { return []; } return Object.entries(value).flatMap(([key, entry]) => flattenStringEntries(entry, prefix ? `${prefix}.${key}` : key) ); }; const getValueAtPath = (value, keyPath) => { let current = value; for (const segment of keyPath.split(".")) { if (!isPlainObject(current) || !(segment in current)) { return undefined; } current = current[segment]; } return current; }; const setValueAtPath = (value, keyPath, nextValue) => { const segments = keyPath.split("."); let current = value; for (const segment of segments.slice(0, -1)) { current = current[segment]; } current[segments.at(-1)] = nextValue; }; const compareStringValues = (left, right) => left.localeCompare(right, "en", { sensitivity: "base" }); const protectSegments = (value) => { const protectedRanges = []; for (const pattern of [linkTokenPattern, placeholderPattern]) { pattern.lastIndex = 0; for (const match of value.matchAll(pattern)) { protectedRanges.push({ start: match.index ?? 0, end: (match.index ?? 0) + match[0].length }); } } protectedRanges.sort((a, b) => a.start - b.start || b.end - a.end); const mergedRanges = []; for (const range of protectedRanges) { const previous = mergedRanges.at(-1); if (previous && range.start <= previous.end) { previous.end = Math.max(previous.end, range.end); } else { mergedRanges.push({ ...range }); } } const segments = []; let offset = 0; for (const range of mergedRanges) { if (offset < range.start) { segments.push({ protected: false, value: value.slice(offset, range.start) }); } segments.push({ protected: true, value: value.slice(range.start, range.end) }); offset = range.end; } if (offset < value.length) { segments.push({ protected: false, value: value.slice(offset) }); } return segments; }; const normalizeToken = (token) => token.normalize("NFKC").toLocaleLowerCase(); const isAuditedToken = (token) => /\p{L}/u.test(token); const collectTokens = (value) => { const tokens = []; for (const segment of protectSegments(value)) { if (segment.protected) { continue; } wordTokenPattern.lastIndex = 0; for (const match of segment.value.matchAll(wordTokenPattern)) { const token = match[0]; if (isAuditedToken(token)) { tokens.push(token); } } } return tokens; }; const stripMarks = (value) => value.normalize("NFKD").replace(/\p{M}/gu, ""); const slugify = (token, usedKeys) => { const base = stripMarks(token) .toLocaleLowerCase() .replace(/[^\p{L}\p{N}]+/gu, "_") .replace(/^_+|_+$/g, "") .replace(/_{2,}/g, "_") || "word"; const prefixed = /^\p{N}/u.test(base) ? `word_${base}` : base; let key = prefixed; let suffix = 2; while (usedKeys.has(key)) { key = `${prefixed}_${suffix}`; suffix += 1; } usedKeys.add(key); return key; }; const isCapitalizedToken = (token) => { const lower = token.toLocaleLowerCase(); return token === `${lower.charAt(0).toLocaleUpperCase()}${lower.slice(1)}` && token !== lower; }; const isUpperToken = (token) => token.length > 1 && token === token.toLocaleUpperCase() && token !== token.toLocaleLowerCase(); const isLowerToken = (token) => token === token.toLocaleLowerCase() && token !== token.toLocaleUpperCase(); const linkedWord = (keyPath, modifier = "") => `@${modifier}:{'${keyPath}'}`; const applyTextModifier = (value, modifier) => { switch (modifier) { case ".capitalize": return `${value.charAt(0).toLocaleUpperCase()}${value.slice(1)}`; case ".upper": return value.toLocaleUpperCase(); case ".lower": return value.toLocaleLowerCase(); default: return value; } }; const modifierForToken = (token, word) => { if (normalizeToken(token) !== normalizeToken(word.value)) { return null; } const candidateModifiers = ["", ".upper", ".capitalize", ".lower"]; for (const modifier of candidateModifiers) { if (applyTextModifier(word.value, modifier) === token) { return modifier; } } return null; }; const replaceRepeatedTokens = (value, tokenToWord) => protectSegments(value) .map((segment) => { if (segment.protected) { return segment.value; } return segment.value.replace(wordTokenPattern, (token) => { const normalized = normalizeToken(token); const word = tokenToWord.get(normalized); if (!word || !isAuditedToken(token)) { return token; } const exactVariant = word.variants?.get(token); if (exactVariant) { return linkedWord(exactVariant.path); } const modifier = modifierForToken(token, word); if (modifier !== null) { return linkedWord(word.path, modifier); } return token; }); }) .join(""); const applyLinkedModifier = (value, modifier) => { return applyTextModifier(value, modifier); }; const resolveLinkedMessage = (messages, keyPath, seen = new Set()) => { if (seen.has(keyPath)) { return undefined; } seen.add(keyPath); const value = getValueAtPath(messages, keyPath); if (typeof value !== "string") { return undefined; } const exactLink = value.trim().match(linkedMessagePattern); if (exactLink) { const resolved = resolveLinkedMessage(messages, exactLink.groups.literal ?? exactLink.groups.path, seen); return resolved === undefined ? undefined : applyLinkedModifier(resolved, exactLink.groups.modifier); } return value.replace(linkTokenPattern, (token) => { const match = token.match(linkedMessagePattern); if (!match?.groups) { return token; } const resolved = resolveLinkedMessage(messages, match.groups.literal ?? match.groups.path, new Set(seen)); if (resolved === undefined) { return token; } return applyLinkedModifier(resolved, match.groups.modifier); }); }; const reorderLocaleRoot = (localeMessages) => { const { words, templates, ...rest } = localeMessages; return { words, templates, ...rest }; }; const transformLocale = (locale, originalMessages) => { const messages = cloneJson(originalMessages); const entries = flattenStringEntries(messages).filter((entry) => !entry.key.startsWith("words.")); const tokenStats = new Map(); for (const entry of entries) { for (const token of collectTokens(entry.value)) { const normalized = normalizeToken(token); const stat = tokenStats.get(normalized) ?? { count: 0, casings: new Map() }; stat.count += 1; stat.casings.set(token, (stat.casings.get(token) ?? 0) + 1); tokenStats.set(normalized, stat); } } const repeatedTokens = [...tokenStats.entries()] .filter(([, stat]) => stat.count > 1) .sort(([left], [right]) => compareStringValues(left, right)); const existingWords = messages.words ?? {}; const generatedWords = {}; const tokenToWord = new Map(); const usedKeys = new Set(Object.keys(existingWords.generated ?? {})); for (const [normalized, stat] of repeatedTokens) { const existingWordEntry = flattenStringEntries(existingWords).find( (entry) => normalizeToken(entry.value) === normalized ); if (existingWordEntry) { tokenToWord.set(normalized, { path: `words.${existingWordEntry.key}`, value: existingWordEntry.value }); continue; } const casings = [...stat.casings.entries()].sort( ([leftToken, leftCount], [rightToken, rightCount]) => rightCount - leftCount || compareStringValues(leftToken, rightToken) ); const lowerCaseToken = casings.find(([token]) => token === normalized)?.[0]; const canonical = lowerCaseToken ?? casings[0][0]; const key = slugify(normalized, usedKeys); generatedWords[key] = canonical; const word = { path: `words.generated.${key}`, value: canonical, variants: new Map() }; for (const [casing] of casings) { if (modifierForToken(casing, word) !== null) { continue; } const variantKey = slugify(`${normalized}_${casing}`, usedKeys); generatedWords[variantKey] = casing; word.variants.set(casing, { path: `words.generated.${variantKey}`, value: casing }); } tokenToWord.set(normalized, word); } messages.words = { grammar: messages.words?.grammar ?? {}, actions: messages.words?.actions ?? {}, entities: messages.words?.entities ?? {}, statuses: messages.words?.statuses ?? {}, domain: messages.words?.domain ?? {}, services: messages.words?.services ?? {}, units: messages.words?.units ?? {}, ...(messages.words ?? {}), generated: { ...(messages.words?.generated ?? {}), ...Object.fromEntries(Object.entries(generatedWords).sort(([left], [right]) => compareStringValues(left, right))), }, }; for (const entry of entries) { const nextValue = replaceRepeatedTokens(entry.value, tokenToWord); if (nextValue !== entry.value) { setValueAtPath(messages, entry.key, nextValue); } } const mismatches = []; for (const entry of entries) { const previousResolved = resolveLinkedMessage(originalMessages, entry.key); const resolved = resolveLinkedMessage(messages, entry.key); if (resolved !== previousResolved) { mismatches.push(`${entry.key}: expected ${JSON.stringify(previousResolved)}, got ${JSON.stringify(resolved)}`); } } return { messages: reorderLocaleRoot(messages), generatedWordCount: Object.keys(generatedWords).length, repeatedTokenCount: repeatedTokens.length, mismatches, }; }; const countRepeatedTokensOutsideWords = (messages) => { const counts = new Map(); for (const entry of flattenStringEntries(messages).filter((entry) => !entry.key.startsWith("words."))) { for (const token of collectTokens(entry.value)) { const normalized = normalizeToken(token); counts.set(normalized, (counts.get(normalized) ?? 0) + 1); } } return [...counts.entries()].filter(([, count]) => count > 1); }; let hasErrors = false; for (const locale of activeLocales) { const filePath = path.join(localesDirectory, `${locale}-v2.json`); const originalMessages = readJson(filePath); const result = transformLocale(locale, originalMessages); const repeatedAfter = countRepeatedTokensOutsideWords(result.messages); if (result.mismatches.length > 0) { hasErrors = true; console.error(`${locale}: transformed output changed ${result.mismatches.length} strings`); for (const mismatch of result.mismatches.slice(0, 20)) { console.error(` ${mismatch}`); } } if (repeatedAfter.length > 0) { hasErrors = true; console.error(`${locale}: ${repeatedAfter.length} repeated token groups remain outside words`); for (const [token, count] of repeatedAfter.slice(0, 20)) { console.error(` ${token}: ${count}`); } } console.log( `${locale}: ${result.repeatedTokenCount} repeated token groups, ${result.generatedWordCount} new generated word entries` ); if (mode === "apply" && !hasErrors) { writeJson(filePath, result.messages); } } if (hasErrors) { process.exit(1); }