pleno-vue/scripts/i18n-v2-template-dedupe.mjs

import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const projectRoot = path.resolve(__dirname, "..");
const localesDirectory = path.join(projectRoot, "src", "i18n", "generated");
const activeLocales = ["da", "en", "sv", "de", "no"];
const mode = process.argv.includes("--apply") ? "apply" : "check";

const templatePrefix = "templates.generated.compat";
const placeholderPattern = /\{[A-Za-z_][A-Za-z0-9_]*}/g;
const exactLinkPattern = /^@(?:\.[\p{L}]+)?:(?:\x7b'[^']+'\x7d|[\p{L}\p{N}_.-]+)$/u;

const isPlainObject = (value) => value !== null && typeof value === "object" && !Array.isArray(value);

const readJson = (filePath) => JSON.parse(fs.readFileSync(filePath, "utf8").replace(/^\uFEFF/, ""));

const writeJson = (filePath, value) => {
  fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
};

const cloneJson = (value) => JSON.parse(JSON.stringify(value));

const flattenStringEntries = (value, prefix = "") => {
  if (typeof value === "string") {
    return [{ key: prefix, value }];
  }

  if (!isPlainObject(value)) {
    return [];
  }

  return Object.entries(value).flatMap(([key, entry]) =>
    flattenStringEntries(entry, prefix ? `${prefix}.${key}` : key)
  );
};

const getValueAtPath = (value, keyPath) => {
  let current = value;
  for (const segment of keyPath.split(".")) {
    if (!isPlainObject(current) || !(segment in current)) {
      return undefined;
    }
    current = current[segment];
  }
  return current;
};

const ensureObjectAtPath = (value, keyPath) => {
  let current = value;
  if (!keyPath) {
    return current;
  }

  for (const segment of keyPath.split(".")) {
    if (!isPlainObject(current[segment])) {
      current[segment] = {};
    }
    current = current[segment];
  }
  return current;
};

const setValueAtPath = (value, keyPath, nextValue) => {
  const segments = keyPath.split(".");
  const parent = ensureObjectAtPath(value, segments.slice(0, -1).join("."));
  parent[segments.at(-1)] = nextValue;
};

const deleteValueAtPath = (value, keyPath) => {
  const segments = keyPath.split(".");
  const parents = [];
  let current = value;

  for (const segment of segments.slice(0, -1)) {
    if (!isPlainObject(current) || !(segment in current)) {
      return false;
    }
    parents.push([current, segment]);
    current = current[segment];
  }

  if (!isPlainObject(current) || !(segments.at(-1) in current)) {
    return false;
  }

  delete current[segments.at(-1)];

  for (let index = parents.length - 1; index >= 0; index -= 1) {
    const [parent, segment] = parents[index];
    const child = parent[segment];
    if (isPlainObject(child) && Object.keys(child).length === 0) {
      delete parent[segment];
    } else {
      break;
    }
  }

  return true;
};

const normalizePhrase = (value) =>
  value
    .normalize("NFKC")
    .toLocaleLowerCase()
    .replace(placeholderPattern, "{}")
    .replace(/[\p{P}\p{S}]+/gu, " ")
    .replace(/\s+/g, " ")
    .trim();

const linkToTemplate = (suffix) => `@:{'${templatePrefix}.${suffix}'}`;

const canonicalPriority = (suffix) => {
  if (suffix.startsWith("common.")) return 0;
  if (suffix.startsWith("global.")) return 1;
  if (suffix.startsWith("nav.")) return 2;
  if (suffix.startsWith("tables.common.")) return 3;
  if (suffix.startsWith("objects.columns.")) return 4;
  if (suffix.startsWith("auth.")) return 5;
  if (suffix.startsWith("products.")) return 6;
  return 10;
};

const chooseCanonicalSuffix = (suffixes) =>
  [...suffixes].sort(
    (left, right) =>
      canonicalPriority(left) - canonicalPriority(right) ||
      left.split(".").length - right.split(".").length ||
      left.length - right.length ||
      left.localeCompare(right, "en", { sensitivity: "base" })
  )[0];

const createDisjointSet = (items) => {
  const parent = new Map(items.map((item) => [item, item]));

  const find = (item) => {
    const currentParent = parent.get(item);
    if (currentParent === item) {
      return item;
    }
    const root = find(currentParent);
    parent.set(item, root);
    return root;
  };

  const union = (left, right) => {
    const leftRoot = find(left);
    const rightRoot = find(right);
    if (leftRoot !== rightRoot) {
      parent.set(rightRoot, leftRoot);
    }
  };

  const groups = () => {
    const grouped = new Map();
    for (const item of items) {
      const root = find(item);
      const group = grouped.get(root) ?? [];
      group.push(item);
      grouped.set(root, group);
    }
    return [...grouped.values()];
  };

  return { union, groups };
};

const replaceTemplatePathReferences = (value, replacements) => {
  if (typeof value === "string") {
    let nextValue = value;
    for (const [fromSuffix, toSuffix] of replacements.entries()) {
      nextValue = nextValue.replaceAll(`${templatePrefix}.${fromSuffix}`, `${templatePrefix}.${toSuffix}`);
    }
    return nextValue;
  }

  if (Array.isArray(value)) {
    return value.map((entry) => replaceTemplatePathReferences(entry, replacements));
  }

  if (!isPlainObject(value)) {
    return value;
  }

  return Object.fromEntries(
    Object.entries(value).map(([key, entry]) => [key, replaceTemplatePathReferences(entry, replacements)])
  );
};

const readRawLocales = () =>
  Object.fromEntries(
    activeLocales.map((locale) => [locale, readJson(path.join(localesDirectory, `${locale}-v2.json`))])
  );

const collectTemplateSuffixes = (rawLocales) => {
  const suffixes = new Set();
  for (const locale of activeLocales) {
    for (const entry of flattenStringEntries(getValueAtPath(rawLocales[locale], templatePrefix) ?? {})) {
      suffixes.add(entry.key);
    }
  }
  return [...suffixes].sort((left, right) => left.localeCompare(right, "en", { sensitivity: "base" }));
};

const transformCatalogs = (originalGlobalMessages, originalRawLocales) => {
  const rawLocales = cloneJson(originalRawLocales);
  let globalMessages = cloneJson(originalGlobalMessages);
  const suffixes = collectTemplateSuffixes(rawLocales);
  const disjointSet = createDisjointSet(suffixes);

  for (const locale of activeLocales) {
    const groupedByLocalePhrase = new Map();
    for (const suffix of suffixes) {
      const value = getValueAtPath(rawLocales[locale], `${templatePrefix}.${suffix}`);
      if (typeof value !== "string") {
        continue;
      }

      const signature = normalizePhrase(value);
      if (signature.length < 4) {
        continue;
      }

      const group = groupedByLocalePhrase.get(signature) ?? [];
      group.push(suffix);
      groupedByLocalePhrase.set(signature, group);
    }

    for (const group of groupedByLocalePhrase.values()) {
      if (group.length <= 1) {
        continue;
      }

      const [first, ...rest] = group;
      for (const suffix of rest) {
        disjointSet.union(first, suffix);
      }
    }
  }

  const duplicateGroups = disjointSet.groups().filter((group) => group.length > 1);
  const replacements = new Map();

  for (const group of duplicateGroups) {
    const canonicalSuffix = chooseCanonicalSuffix(group);
    for (const suffix of group) {
      if (suffix !== canonicalSuffix) {
        replacements.set(suffix, canonicalSuffix);
      }
    }
  }

  for (const [fromSuffix, toSuffix] of [...replacements.entries()]) {
    const seen = new Set([fromSuffix]);
    let finalSuffix = toSuffix;
    while (replacements.has(finalSuffix) && !seen.has(finalSuffix)) {
      seen.add(finalSuffix);
      finalSuffix = replacements.get(finalSuffix);
    }
    replacements.set(fromSuffix, finalSuffix);
  }

  if (replacements.size > 0) {
    globalMessages = replaceTemplatePathReferences(globalMessages, replacements);
    for (const locale of activeLocales) {
      rawLocales[locale] = replaceTemplatePathReferences(rawLocales[locale], replacements);
    }
  }

  let removedTemplateCount = 0;
  let repointedAliasCount = 0;
  for (const [fromSuffix, toSuffix] of replacements.entries()) {
    setValueAtPath(globalMessages.shared, fromSuffix, linkToTemplate(toSuffix));
    repointedAliasCount += 1;

    for (const locale of activeLocales) {
      if (deleteValueAtPath(rawLocales[locale], `${templatePrefix}.${fromSuffix}`)) {
        removedTemplateCount += 1;
      }
    }
  }

  const remainingDuplicateGroups = [];
  for (const locale of activeLocales) {
    const nonLinkedTemplates = flattenStringEntries(getValueAtPath(rawLocales[locale], templatePrefix) ?? {}).filter(
      (entry) => !exactLinkPattern.test(entry.value.trim())
    );
    const byPhrase = new Map();
    for (const entry of nonLinkedTemplates) {
      const signature = normalizePhrase(entry.value);
      const group = byPhrase.get(signature) ?? [];
      group.push(entry.key);
      byPhrase.set(signature, group);
    }
    for (const [signature, group] of byPhrase.entries()) {
      if (signature.length >= 4 && group.length > 1) {
        remainingDuplicateGroups.push(`${locale}: ${signature} <- ${group.join(", ")}`);
      }
    }
  }

  return {
    globalMessages,
    rawLocales,
    duplicateGroupCount: duplicateGroups.length,
    replacementCount: replacements.size,
    repointedAliasCount,
    removedTemplateCount,
    remainingDuplicateGroups,
  };
};

const globalPath = path.join(localesDirectory, "global-v2.json");
const originalGlobalMessages = readJson(globalPath);
const originalRawLocales = readRawLocales();
const result = transformCatalogs(originalGlobalMessages, originalRawLocales);

console.log(
  [
    `duplicate generated template groups: ${result.duplicateGroupCount}`,
    `template replacements: ${result.replacementCount}`,
    `repointed shared aliases: ${result.repointedAliasCount}`,
    `removed locale template entries: ${result.removedTemplateCount}`,
  ].join("\n")
);

if (result.remainingDuplicateGroups.length > 0) {
  console.error(`near-duplicate generated templates remain: ${result.remainingDuplicateGroups.length}`);
  for (const group of result.remainingDuplicateGroups.slice(0, 30)) {
    console.error(`  ${group}`);
  }
  process.exit(1);
}

if (mode === "apply") {
  writeJson(globalPath, result.globalMessages);
  for (const locale of activeLocales) {
    writeJson(path.join(localesDirectory, `${locale}-v2.json`), result.rawLocales[locale]);
  }
}