391 lines
12 KiB
JavaScript
391 lines
12 KiB
JavaScript
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
const projectRoot = path.resolve(__dirname, "..");
|
|
const localesDirectory = path.join(projectRoot, "src", "i18n", "generated");
|
|
const activeLocales = ["da", "en", "sv", "de", "no"];
|
|
|
|
const linkTokenPattern = /@(?:\.[\p{L}]+)?:(?:\{'[^']+'\}|[\p{L}\p{N}_.-]+)/gu;
|
|
const placeholderPattern = /\{[A-Za-z_][A-Za-z0-9_]*\}/g;
|
|
const wordTokenPattern = /[\p{L}\p{N}]+/gu;
|
|
const linkedMessagePattern = /^@(?<modifier>\.[\p{L}]+)?:(?:\{'(?<literal>[^']+)'\}|(?<path>[\p{L}\p{N}_.-]+))$/u;
|
|
|
|
const mode = process.argv.includes("--apply") ? "apply" : "check";
|
|
|
|
const isPlainObject = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
|
|
|
|
const readJson = (filePath) => JSON.parse(fs.readFileSync(filePath, "utf8").replace(/^\uFEFF/, ""));
|
|
|
|
const writeJson = (filePath, value) => {
|
|
fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
|
};
|
|
|
|
const cloneJson = (value) => JSON.parse(JSON.stringify(value));
|
|
|
|
const flattenStringEntries = (value, prefix = "") => {
|
|
if (typeof value === "string") {
|
|
return [{ key: prefix, value }];
|
|
}
|
|
|
|
if (!isPlainObject(value)) {
|
|
return [];
|
|
}
|
|
|
|
return Object.entries(value).flatMap(([key, entry]) =>
|
|
flattenStringEntries(entry, prefix ? `${prefix}.${key}` : key)
|
|
);
|
|
};
|
|
|
|
const getValueAtPath = (value, keyPath) => {
|
|
let current = value;
|
|
for (const segment of keyPath.split(".")) {
|
|
if (!isPlainObject(current) || !(segment in current)) {
|
|
return undefined;
|
|
}
|
|
current = current[segment];
|
|
}
|
|
return current;
|
|
};
|
|
|
|
const setValueAtPath = (value, keyPath, nextValue) => {
|
|
const segments = keyPath.split(".");
|
|
let current = value;
|
|
for (const segment of segments.slice(0, -1)) {
|
|
current = current[segment];
|
|
}
|
|
current[segments.at(-1)] = nextValue;
|
|
};
|
|
|
|
const compareStringValues = (left, right) => left.localeCompare(right, "en", { sensitivity: "base" });
|
|
|
|
const protectSegments = (value) => {
|
|
const protectedRanges = [];
|
|
for (const pattern of [linkTokenPattern, placeholderPattern]) {
|
|
pattern.lastIndex = 0;
|
|
for (const match of value.matchAll(pattern)) {
|
|
protectedRanges.push({ start: match.index ?? 0, end: (match.index ?? 0) + match[0].length });
|
|
}
|
|
}
|
|
|
|
protectedRanges.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
const mergedRanges = [];
|
|
for (const range of protectedRanges) {
|
|
const previous = mergedRanges.at(-1);
|
|
if (previous && range.start <= previous.end) {
|
|
previous.end = Math.max(previous.end, range.end);
|
|
} else {
|
|
mergedRanges.push({ ...range });
|
|
}
|
|
}
|
|
|
|
const segments = [];
|
|
let offset = 0;
|
|
for (const range of mergedRanges) {
|
|
if (offset < range.start) {
|
|
segments.push({ protected: false, value: value.slice(offset, range.start) });
|
|
}
|
|
segments.push({ protected: true, value: value.slice(range.start, range.end) });
|
|
offset = range.end;
|
|
}
|
|
if (offset < value.length) {
|
|
segments.push({ protected: false, value: value.slice(offset) });
|
|
}
|
|
|
|
return segments;
|
|
};
|
|
|
|
const normalizeToken = (token) => token.normalize("NFKC").toLocaleLowerCase();
|
|
|
|
const isAuditedToken = (token) => /\p{L}/u.test(token);
|
|
|
|
const collectTokens = (value) => {
|
|
const tokens = [];
|
|
for (const segment of protectSegments(value)) {
|
|
if (segment.protected) {
|
|
continue;
|
|
}
|
|
|
|
wordTokenPattern.lastIndex = 0;
|
|
for (const match of segment.value.matchAll(wordTokenPattern)) {
|
|
const token = match[0];
|
|
if (isAuditedToken(token)) {
|
|
tokens.push(token);
|
|
}
|
|
}
|
|
}
|
|
return tokens;
|
|
};
|
|
|
|
const stripMarks = (value) => value.normalize("NFKD").replace(/\p{M}/gu, "");
|
|
|
|
const slugify = (token, usedKeys) => {
|
|
const base =
|
|
stripMarks(token)
|
|
.toLocaleLowerCase()
|
|
.replace(/[^\p{L}\p{N}]+/gu, "_")
|
|
.replace(/^_+|_+$/g, "")
|
|
.replace(/_{2,}/g, "_") || "word";
|
|
const prefixed = /^\p{N}/u.test(base) ? `word_${base}` : base;
|
|
let key = prefixed;
|
|
let suffix = 2;
|
|
while (usedKeys.has(key)) {
|
|
key = `${prefixed}_${suffix}`;
|
|
suffix += 1;
|
|
}
|
|
usedKeys.add(key);
|
|
return key;
|
|
};
|
|
|
|
const isCapitalizedToken = (token) => {
|
|
const lower = token.toLocaleLowerCase();
|
|
return token === `${lower.charAt(0).toLocaleUpperCase()}${lower.slice(1)}` && token !== lower;
|
|
};
|
|
|
|
const isUpperToken = (token) => token.length > 1 && token === token.toLocaleUpperCase() && token !== token.toLocaleLowerCase();
|
|
|
|
const isLowerToken = (token) => token === token.toLocaleLowerCase() && token !== token.toLocaleUpperCase();
|
|
|
|
const linkedWord = (keyPath, modifier = "") => `@${modifier}:{'${keyPath}'}`;
|
|
|
|
const applyTextModifier = (value, modifier) => {
|
|
switch (modifier) {
|
|
case ".capitalize":
|
|
return `${value.charAt(0).toLocaleUpperCase()}${value.slice(1)}`;
|
|
case ".upper":
|
|
return value.toLocaleUpperCase();
|
|
case ".lower":
|
|
return value.toLocaleLowerCase();
|
|
default:
|
|
return value;
|
|
}
|
|
};
|
|
|
|
const modifierForToken = (token, word) => {
|
|
if (normalizeToken(token) !== normalizeToken(word.value)) {
|
|
return null;
|
|
}
|
|
|
|
const candidateModifiers = ["", ".upper", ".capitalize", ".lower"];
|
|
for (const modifier of candidateModifiers) {
|
|
if (applyTextModifier(word.value, modifier) === token) {
|
|
return modifier;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
};
|
|
|
|
const replaceRepeatedTokens = (value, tokenToWord) =>
|
|
protectSegments(value)
|
|
.map((segment) => {
|
|
if (segment.protected) {
|
|
return segment.value;
|
|
}
|
|
|
|
return segment.value.replace(wordTokenPattern, (token) => {
|
|
const normalized = normalizeToken(token);
|
|
const word = tokenToWord.get(normalized);
|
|
if (!word || !isAuditedToken(token)) {
|
|
return token;
|
|
}
|
|
|
|
const exactVariant = word.variants?.get(token);
|
|
if (exactVariant) {
|
|
return linkedWord(exactVariant.path);
|
|
}
|
|
|
|
const modifier = modifierForToken(token, word);
|
|
if (modifier !== null) {
|
|
return linkedWord(word.path, modifier);
|
|
}
|
|
|
|
return token;
|
|
});
|
|
})
|
|
.join("");
|
|
|
|
const applyLinkedModifier = (value, modifier) => {
|
|
return applyTextModifier(value, modifier);
|
|
};
|
|
|
|
const resolveLinkedMessage = (messages, keyPath, seen = new Set()) => {
|
|
if (seen.has(keyPath)) {
|
|
return undefined;
|
|
}
|
|
seen.add(keyPath);
|
|
|
|
const value = getValueAtPath(messages, keyPath);
|
|
if (typeof value !== "string") {
|
|
return undefined;
|
|
}
|
|
|
|
const exactLink = value.trim().match(linkedMessagePattern);
|
|
if (exactLink) {
|
|
const resolved = resolveLinkedMessage(messages, exactLink.groups.literal ?? exactLink.groups.path, seen);
|
|
return resolved === undefined ? undefined : applyLinkedModifier(resolved, exactLink.groups.modifier);
|
|
}
|
|
|
|
return value.replace(linkTokenPattern, (token) => {
|
|
const match = token.match(linkedMessagePattern);
|
|
if (!match?.groups) {
|
|
return token;
|
|
}
|
|
|
|
const resolved = resolveLinkedMessage(messages, match.groups.literal ?? match.groups.path, new Set(seen));
|
|
if (resolved === undefined) {
|
|
return token;
|
|
}
|
|
|
|
return applyLinkedModifier(resolved, match.groups.modifier);
|
|
});
|
|
};
|
|
|
|
const reorderLocaleRoot = (localeMessages) => {
|
|
const { words, templates, ...rest } = localeMessages;
|
|
return { words, templates, ...rest };
|
|
};
|
|
|
|
const transformLocale = (locale, originalMessages) => {
|
|
const messages = cloneJson(originalMessages);
|
|
const entries = flattenStringEntries(messages).filter((entry) => !entry.key.startsWith("words."));
|
|
const tokenStats = new Map();
|
|
|
|
for (const entry of entries) {
|
|
for (const token of collectTokens(entry.value)) {
|
|
const normalized = normalizeToken(token);
|
|
const stat = tokenStats.get(normalized) ?? { count: 0, casings: new Map() };
|
|
stat.count += 1;
|
|
stat.casings.set(token, (stat.casings.get(token) ?? 0) + 1);
|
|
tokenStats.set(normalized, stat);
|
|
}
|
|
}
|
|
|
|
const repeatedTokens = [...tokenStats.entries()]
|
|
.filter(([, stat]) => stat.count > 1)
|
|
.sort(([left], [right]) => compareStringValues(left, right));
|
|
const existingWords = messages.words ?? {};
|
|
const generatedWords = {};
|
|
const tokenToWord = new Map();
|
|
const usedKeys = new Set(Object.keys(existingWords.generated ?? {}));
|
|
|
|
for (const [normalized, stat] of repeatedTokens) {
|
|
const existingWordEntry = flattenStringEntries(existingWords).find(
|
|
(entry) => normalizeToken(entry.value) === normalized
|
|
);
|
|
if (existingWordEntry) {
|
|
tokenToWord.set(normalized, { path: `words.${existingWordEntry.key}`, value: existingWordEntry.value });
|
|
continue;
|
|
}
|
|
|
|
const casings = [...stat.casings.entries()].sort(
|
|
([leftToken, leftCount], [rightToken, rightCount]) =>
|
|
rightCount - leftCount || compareStringValues(leftToken, rightToken)
|
|
);
|
|
const lowerCaseToken = casings.find(([token]) => token === normalized)?.[0];
|
|
const canonical = lowerCaseToken ?? casings[0][0];
|
|
const key = slugify(normalized, usedKeys);
|
|
generatedWords[key] = canonical;
|
|
const word = { path: `words.generated.${key}`, value: canonical, variants: new Map() };
|
|
|
|
for (const [casing] of casings) {
|
|
if (modifierForToken(casing, word) !== null) {
|
|
continue;
|
|
}
|
|
|
|
const variantKey = slugify(`${normalized}_${casing}`, usedKeys);
|
|
generatedWords[variantKey] = casing;
|
|
word.variants.set(casing, { path: `words.generated.${variantKey}`, value: casing });
|
|
}
|
|
|
|
tokenToWord.set(normalized, word);
|
|
}
|
|
|
|
messages.words = {
|
|
grammar: messages.words?.grammar ?? {},
|
|
actions: messages.words?.actions ?? {},
|
|
entities: messages.words?.entities ?? {},
|
|
statuses: messages.words?.statuses ?? {},
|
|
domain: messages.words?.domain ?? {},
|
|
services: messages.words?.services ?? {},
|
|
units: messages.words?.units ?? {},
|
|
...(messages.words ?? {}),
|
|
generated: {
|
|
...(messages.words?.generated ?? {}),
|
|
...Object.fromEntries(Object.entries(generatedWords).sort(([left], [right]) => compareStringValues(left, right))),
|
|
},
|
|
};
|
|
|
|
for (const entry of entries) {
|
|
const nextValue = replaceRepeatedTokens(entry.value, tokenToWord);
|
|
if (nextValue !== entry.value) {
|
|
setValueAtPath(messages, entry.key, nextValue);
|
|
}
|
|
}
|
|
|
|
const mismatches = [];
|
|
for (const entry of entries) {
|
|
const previousResolved = resolveLinkedMessage(originalMessages, entry.key);
|
|
const resolved = resolveLinkedMessage(messages, entry.key);
|
|
if (resolved !== previousResolved) {
|
|
mismatches.push(`${entry.key}: expected ${JSON.stringify(previousResolved)}, got ${JSON.stringify(resolved)}`);
|
|
}
|
|
}
|
|
|
|
return {
|
|
messages: reorderLocaleRoot(messages),
|
|
generatedWordCount: Object.keys(generatedWords).length,
|
|
repeatedTokenCount: repeatedTokens.length,
|
|
mismatches,
|
|
};
|
|
};
|
|
|
|
const countRepeatedTokensOutsideWords = (messages) => {
|
|
const counts = new Map();
|
|
for (const entry of flattenStringEntries(messages).filter((entry) => !entry.key.startsWith("words."))) {
|
|
for (const token of collectTokens(entry.value)) {
|
|
const normalized = normalizeToken(token);
|
|
counts.set(normalized, (counts.get(normalized) ?? 0) + 1);
|
|
}
|
|
}
|
|
return [...counts.entries()].filter(([, count]) => count > 1);
|
|
};
|
|
|
|
let hasErrors = false;
|
|
|
|
for (const locale of activeLocales) {
|
|
const filePath = path.join(localesDirectory, `${locale}-v2.json`);
|
|
const originalMessages = readJson(filePath);
|
|
const result = transformLocale(locale, originalMessages);
|
|
const repeatedAfter = countRepeatedTokensOutsideWords(result.messages);
|
|
|
|
if (result.mismatches.length > 0) {
|
|
hasErrors = true;
|
|
console.error(`${locale}: transformed output changed ${result.mismatches.length} strings`);
|
|
for (const mismatch of result.mismatches.slice(0, 20)) {
|
|
console.error(` ${mismatch}`);
|
|
}
|
|
}
|
|
|
|
if (repeatedAfter.length > 0) {
|
|
hasErrors = true;
|
|
console.error(`${locale}: ${repeatedAfter.length} repeated token groups remain outside words`);
|
|
for (const [token, count] of repeatedAfter.slice(0, 20)) {
|
|
console.error(` ${token}: ${count}`);
|
|
}
|
|
}
|
|
|
|
console.log(
|
|
`${locale}: ${result.repeatedTokenCount} repeated token groups, ${result.generatedWordCount} new generated word entries`
|
|
);
|
|
|
|
if (mode === "apply" && !hasErrors) {
|
|
writeJson(filePath, result.messages);
|
|
}
|
|
}
|
|
|
|
if (hasErrors) {
|
|
process.exit(1);
|
|
}
|