subcommands for i18n tooling, unique message IDs

3 files changed, 163 insertions, 103 deletions

diff --git a/packages/pogen/src/po2ts.ts b/packages/pogen/src/po2ts.ts
index d0f4ed34d..ac85b9b76 100644
--- a/packages/pogen/src/po2ts.ts
+++ b/packages/pogen/src/po2ts.ts
@@ -23,38 +23,44 @@ import * as po2json from "po2json";
 import * as fs from "fs";
 import * as path from "path";
 
-const files = fs
-  .readdirSync("./src/i18n")
-  .filter((x) => x.endsWith(".po"))
-  .map((x) => path.join("./src/i18n/", x));
-
-if (files.length === 0) {
-  console.error("no .po files found in src/i18n/");
-  process.exit(1);
-}
+export function po2ts(): void {
+  const files = fs
+    .readdirSync("./src/i18n")
+    .filter((x) => x.endsWith(".po"))
+    .map((x) => path.join("./src/i18n/", x));
 
-console.log(files);
+  if (files.length === 0) {
+    console.error("no .po files found in src/i18n/");
+    process.exit(1);
+  }
 
-const chunks: string[] = [];
+  console.log(files);
 
-for (const filename of files) {
-  const m = filename.match(/([a-zA-Z0-9-_]+).po/);
+  const chunks: string[] = [];
 
-  if (!m) {
-    console.error("error: unexpected filename (expected <lang>.po)");
-    process.exit(1);
-  }
+  for (const filename of files) {
+    const m = filename.match(/([a-zA-Z0-9-_]+).po/);
 
-  const lang = m[1];
-  const pojson = po2json.parseFileSync(filename, {
-    format: "jed1.x",
-    fuzzy: true,
-  });
-  const s =
-    "strings['" + lang + "'] = " + JSON.stringify(pojson, null, "  ") + ";\n\n";
-  chunks.push(s);
-}
+    if (!m) {
+      console.error("error: unexpected filename (expected <lang>.po)");
+      process.exit(1);
+    }
+
+    const lang = m[1];
+    const pojson = po2json.parseFileSync(filename, {
+      format: "jed1.x",
+      fuzzy: true,
+    });
+    const s =
+      "strings['" +
+      lang +
+      "'] = " +
+      JSON.stringify(pojson, null, "  ") +
+      ";\n\n";
+    chunks.push(s);
+  }
 
-const tsContents = chunks.join("");
+  const tsContents = chunks.join("");
 
-fs.writeFileSync("src/i18n/strings.ts", tsContents);
+  fs.writeFileSync("src/i18n/strings.ts", tsContents);
+}
diff --git a/packages/pogen/src/pogen.ts b/packages/pogen/src/pogen.ts
new file mode 100644
index 000000000..7d128ce7a
--- /dev/null
+++ b/packages/pogen/src/pogen.ts
@@ -0,0 +1,21 @@
+import { potextract } from "./potextract.js";
+
+function usage(): never {
+  console.log("usage: pogen <extract|merge|emit>");
+  process.exit(1);
+}
+
+export function main() {
+  const subcommand = process.argv[2];
+  if (process.argv.includes("--help") || !subcommand) {
+    usage();
+  }
+  switch (subcommand) {
+    case "extract":
+      potextract();
+      break;
+    default:
+      console.error(`unknown subcommand '${subcommand}'`);
+      usage();
+  }
+}
diff --git a/packages/pogen/src/potextract.ts b/packages/pogen/src/potextract.ts
index 5999d9e1c..5cc085ef8 100644
--- a/packages/pogen/src/potextract.ts
+++ b/packages/pogen/src/potextract.ts
@@ -14,21 +14,27 @@
  GNU Taler; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
  */
 
-
 /**
  * Imports.
  */
 import * as ts from "typescript";
+import * as fs from "fs";
+import * as os from "os";
+import path = require("path/posix");
 
 function wordwrap(str: string, width: number = 80): string[] {
   var regex = ".{1," + width + "}(\\s|$)|\\S+(\\s|$)";
   return str.match(RegExp(regex, "g"));
 }
 
-export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) {
-  processNode(sourceFile);
+function processFile(
+  sourceFile: ts.SourceFile,
+  outChunks: string[],
+  knownMessageIds: Set<string>,
+) {
   let lastTokLine = 0;
   let preLastTokLine = 0;
+  processNode(sourceFile);
 
   function getTemplate(node: ts.Node): string {
     switch (node.kind) {
@@ -140,7 +146,8 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) {
         outChunks.push(`#. ${cl}\n`);
       }
     }
-    outChunks.push(`#: ${sourceFile.fileName}:${line + 1}\n`);
+    const fn = path.relative(process.cwd(), sourceFile.fileName);
+    outChunks.push(`#: ${fn}:${line + 1}\n`);
     outChunks.push(`#, c-format\n`);
   }
 
@@ -148,7 +155,7 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) {
     // Do escaping, wrap break at newlines
     let parts = msg
       .match(/(.*\n|.+$)/g)
-      .map((x) => x.replace(/\n/g, "\\n"))
+      .map((x) => x.replace(/\n/g, "\\n").replace(/"/g, '\\"'))
       .map((p) => wordwrap(p))
       .reduce((a, b) => a.concat(b));
     if (parts.length == 1) {
@@ -188,7 +195,7 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) {
     }
   }
 
-  function trim(s) {
+  function trim(s: string) {
     return s.replace(/^[ \n\t]*/, "").replace(/[ \n\t]*$/, "");
   }
 
@@ -284,10 +291,13 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) {
           let content = getJsxContent(node);
           let { line } = ts.getLineAndCharacterOfPosition(sourceFile, node.pos);
           let comment = getComment(node);
-          formatMsgComment(line, comment);
-          formatMsgLine("msgid", content);
-          outChunks.push(`msgstr ""\n`);
-          outChunks.push("\n");
+          if (!knownMessageIds.has(content)) {
+            knownMessageIds.add(content);
+            formatMsgComment(line, comment);
+            formatMsgLine("msgid", content);
+            outChunks.push(`msgstr ""\n`);
+            outChunks.push("\n");
+          }
           return;
         }
         if (arrayEq(path, ["i18n", "TranslateSwitch"])) {
@@ -304,11 +314,14 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) {
             console.error("plural form missing");
             process.exit(1);
           }
-          formatMsgLine("msgid", singularForm);
-          formatMsgLine("msgid_plural", pluralForm);
-          outChunks.push(`msgstr[0] ""\n`);
-          outChunks.push(`msgstr[1] ""\n`);
-          outChunks.push(`\n`);
+          if (!knownMessageIds.has(singularForm)) {
+            knownMessageIds.add(singularForm);
+            formatMsgLine("msgid", singularForm);
+            formatMsgLine("msgid_plural", pluralForm);
+            outChunks.push(`msgstr[0] ""\n`);
+            outChunks.push(`msgstr[1] ""\n`);
+            outChunks.push(`\n`);
+          }
           return;
         }
         break;
@@ -333,13 +346,16 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) {
           <ts.TaggedTemplateExpression>ce.arguments[1],
         );
         let comment = getComment(ce);
-
-        formatMsgComment(line, comment);
-        formatMsgLine("msgid", t1.template);
-        formatMsgLine("msgid_plural", t2.template);
-        outChunks.push(`msgstr[0] ""\n`);
-        outChunks.push(`msgstr[1] ""\n`);
-        outChunks.push("\n");
+        const msgid = t1.template;
+        if (!knownMessageIds.has(msgid)) {
+          knownMessageIds.add(msgid);
+          formatMsgComment(line, comment);
+          formatMsgLine("msgid", t1.template);
+          formatMsgLine("msgid_plural", t2.template);
+          outChunks.push(`msgstr[0] ""\n`);
+          outChunks.push(`msgstr[1] ""\n`);
+          outChunks.push("\n");
+        }
 
         // Important: no processing for child i18n expressions here
         return;
@@ -351,10 +367,14 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) {
         if (path[0] != "i18n") {
           break;
         }
-        formatMsgComment(line, comment);
-        formatMsgLine("msgid", template);
-        outChunks.push(`msgstr ""\n`);
-        outChunks.push("\n");
+        const msgid = template;
+        if (!knownMessageIds.has(msgid)) {
+          knownMessageIds.add(msgid);
+          formatMsgComment(line, comment);
+          formatMsgLine("msgid", template);
+          outChunks.push(`msgstr ""\n`);
+          outChunks.push("\n");
+        }
         break;
       }
     }
@@ -363,51 +383,48 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) {
   }
 }
 
-const configPath = ts.findConfigFile(
-  /*searchPath*/ "./",
-  ts.sys.fileExists,
-  "tsconfig.json",
-);
-if (!configPath) {
-  throw new Error("Could not find a valid 'tsconfig.json'.");
-}
-
-console.log(configPath);
-
-const cmdline = ts.getParsedCommandLineOfConfigFile(
-  configPath,
-  {},
-  {
-    fileExists: ts.sys.fileExists,
-    getCurrentDirectory: ts.sys.getCurrentDirectory,
-    onUnRecoverableConfigFileDiagnostic: (e) => console.log(e),
-    readDirectory: ts.sys.readDirectory,
-    readFile: ts.sys.readFile,
-    useCaseSensitiveFileNames: true,
-  },
-);
-
-console.log(cmdline);
-
-const prog = ts.createProgram({
-  options: cmdline.options,
-  rootNames: cmdline.fileNames,
-});
-
-const allFiles = prog.getSourceFiles();
-
-const ownFiles = allFiles.filter(
-  (x) =>
-    !x.isDeclarationFile &&
-    !prog.isSourceFileFromExternalLibrary(x) &&
-    !prog.isSourceFileDefaultLibrary(x),
-);
-
-console.log(ownFiles.map((x) => x.fileName));
-
-const chunks = [];
+export function potextract() {
+  const configPath = ts.findConfigFile(
+    /*searchPath*/ "./",
+    ts.sys.fileExists,
+    "tsconfig.json",
+  );
+  if (!configPath) {
+    throw new Error("Could not find a valid 'tsconfig.json'.");
+  }
 
-chunks.push(`# SOME DESCRIPTIVE TITLE.
+  const cmdline = ts.getParsedCommandLineOfConfigFile(
+    configPath,
+    {},
+    {
+      fileExists: ts.sys.fileExists,
+      getCurrentDirectory: ts.sys.getCurrentDirectory,
+      onUnRecoverableConfigFileDiagnostic: (e) => console.log(e),
+      readDirectory: ts.sys.readDirectory,
+      readFile: ts.sys.readFile,
+      useCaseSensitiveFileNames: true,
+    },
+  );
+
+  const prog = ts.createProgram({
+    options: cmdline.options,
+    rootNames: cmdline.fileNames,
+  });
+
+  const allFiles = prog.getSourceFiles();
+
+  const ownFiles = allFiles.filter(
+    (x) =>
+      !x.isDeclarationFile &&
+      !prog.isSourceFileFromExternalLibrary(x) &&
+      !prog.isSourceFileDefaultLibrary(x),
+  );
+
+  //console.log(ownFiles.map((x) => x.fileName));
+
+  const chunks = [];
+
+  chunks.push(`# SOME DESCRIPTIVE TITLE.
 # Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
 # This file is distributed under the same license as the PACKAGE package.
 # FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
@@ -424,10 +441,26 @@ msgstr ""
 "Language: \\n"
 "MIME-Version: 1.0\\n"
 "Content-Type: text/plain; charset=UTF-8\\n"
-"Content-Transfer-Encoding: 8bit\\n"`);
+"Content-Transfer-Encoding: 8bit\\n"\n\n`);
 
-for (const f of ownFiles) {
-  processFile(f, chunks);
-}
+  const knownMessageIds = new Set<string>();
+
+  for (const f of ownFiles) {
+    processFile(f, chunks, knownMessageIds);
+  }
+
+  const pot = chunks.join("");
 
-console.log(chunks.join(""));
+  //console.log(pot);
+
+  const packageJson = JSON.parse(
+    fs.readFileSync("./package.json", { encoding: "utf-8" }),
+  );
+
+  const poDomain = packageJson.pogen?.domain;
+  if (!poDomain) {
+    console.error("missing 'pogen.domain' field in package.json");
+    process.exit(1);
+  }
+  fs.writeFileSync(`./src/i18n/${poDomain}.pot`, pot);
+}