From 465ccdaa06d30a995235fd6438172eb125243321 Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Tue, 15 Feb 2022 17:45:26 +0100 Subject: subcommands for i18n tooling, unique message IDs --- packages/pogen/src/po2ts.ts | 62 +++++++------ packages/pogen/src/pogen.ts | 21 +++++ packages/pogen/src/potextract.ts | 183 +++++++++++++++++++++++---------------- 3 files changed, 163 insertions(+), 103 deletions(-) create mode 100644 packages/pogen/src/pogen.ts (limited to 'packages/pogen') diff --git a/packages/pogen/src/po2ts.ts b/packages/pogen/src/po2ts.ts index d0f4ed34d..ac85b9b76 100644 --- a/packages/pogen/src/po2ts.ts +++ b/packages/pogen/src/po2ts.ts @@ -23,38 +23,44 @@ import * as po2json from "po2json"; import * as fs from "fs"; import * as path from "path"; -const files = fs - .readdirSync("./src/i18n") - .filter((x) => x.endsWith(".po")) - .map((x) => path.join("./src/i18n/", x)); - -if (files.length === 0) { - console.error("no .po files found in src/i18n/"); - process.exit(1); -} +export function po2ts(): void { + const files = fs + .readdirSync("./src/i18n") + .filter((x) => x.endsWith(".po")) + .map((x) => path.join("./src/i18n/", x)); -console.log(files); + if (files.length === 0) { + console.error("no .po files found in src/i18n/"); + process.exit(1); + } -const chunks: string[] = []; + console.log(files); -for (const filename of files) { - const m = filename.match(/([a-zA-Z0-9-_]+).po/); + const chunks: string[] = []; - if (!m) { - console.error("error: unexpected filename (expected .po)"); - process.exit(1); - } + for (const filename of files) { + const m = filename.match(/([a-zA-Z0-9-_]+).po/); - const lang = m[1]; - const pojson = po2json.parseFileSync(filename, { - format: "jed1.x", - fuzzy: true, - }); - const s = - "strings['" + lang + "'] = " + JSON.stringify(pojson, null, " ") + ";\n\n"; - chunks.push(s); -} + if (!m) { + console.error("error: unexpected filename (expected .po)"); + process.exit(1); + } + + const lang = m[1]; + const pojson = po2json.parseFileSync(filename, { + format: "jed1.x", + fuzzy: true, + }); + const s = + "strings['" + + lang + + "'] = " + + JSON.stringify(pojson, null, " ") + + ";\n\n"; + chunks.push(s); + } -const tsContents = chunks.join(""); + const tsContents = chunks.join(""); -fs.writeFileSync("src/i18n/strings.ts", tsContents); + fs.writeFileSync("src/i18n/strings.ts", tsContents); +} diff --git a/packages/pogen/src/pogen.ts b/packages/pogen/src/pogen.ts new file mode 100644 index 000000000..7d128ce7a --- /dev/null +++ b/packages/pogen/src/pogen.ts @@ -0,0 +1,21 @@ +import { potextract } from "./potextract.js"; + +function usage(): never { + console.log("usage: pogen "); + process.exit(1); +} + +export function main() { + const subcommand = process.argv[2]; + if (process.argv.includes("--help") || !subcommand) { + usage(); + } + switch (subcommand) { + case "extract": + potextract(); + break; + default: + console.error(`unknown subcommand '${subcommand}'`); + usage(); + } +} diff --git a/packages/pogen/src/potextract.ts b/packages/pogen/src/potextract.ts index 5999d9e1c..5cc085ef8 100644 --- a/packages/pogen/src/potextract.ts +++ b/packages/pogen/src/potextract.ts @@ -14,21 +14,27 @@ GNU Taler; see the file COPYING. If not, see */ - /** * Imports. */ import * as ts from "typescript"; +import * as fs from "fs"; +import * as os from "os"; +import path = require("path/posix"); function wordwrap(str: string, width: number = 80): string[] { var regex = ".{1," + width + "}(\\s|$)|\\S+(\\s|$)"; return str.match(RegExp(regex, "g")); } -export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) { - processNode(sourceFile); +function processFile( + sourceFile: ts.SourceFile, + outChunks: string[], + knownMessageIds: Set, +) { let lastTokLine = 0; let preLastTokLine = 0; + processNode(sourceFile); function getTemplate(node: ts.Node): string { switch (node.kind) { @@ -140,7 +146,8 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) { outChunks.push(`#. ${cl}\n`); } } - outChunks.push(`#: ${sourceFile.fileName}:${line + 1}\n`); + const fn = path.relative(process.cwd(), sourceFile.fileName); + outChunks.push(`#: ${fn}:${line + 1}\n`); outChunks.push(`#, c-format\n`); } @@ -148,7 +155,7 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) { // Do escaping, wrap break at newlines let parts = msg .match(/(.*\n|.+$)/g) - .map((x) => x.replace(/\n/g, "\\n")) + .map((x) => x.replace(/\n/g, "\\n").replace(/"/g, '\\"')) .map((p) => wordwrap(p)) .reduce((a, b) => a.concat(b)); if (parts.length == 1) { @@ -188,7 +195,7 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) { } } - function trim(s) { + function trim(s: string) { return s.replace(/^[ \n\t]*/, "").replace(/[ \n\t]*$/, ""); } @@ -284,10 +291,13 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) { let content = getJsxContent(node); let { line } = ts.getLineAndCharacterOfPosition(sourceFile, node.pos); let comment = getComment(node); - formatMsgComment(line, comment); - formatMsgLine("msgid", content); - outChunks.push(`msgstr ""\n`); - outChunks.push("\n"); + if (!knownMessageIds.has(content)) { + knownMessageIds.add(content); + formatMsgComment(line, comment); + formatMsgLine("msgid", content); + outChunks.push(`msgstr ""\n`); + outChunks.push("\n"); + } return; } if (arrayEq(path, ["i18n", "TranslateSwitch"])) { @@ -304,11 +314,14 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) { console.error("plural form missing"); process.exit(1); } - formatMsgLine("msgid", singularForm); - formatMsgLine("msgid_plural", pluralForm); - outChunks.push(`msgstr[0] ""\n`); - outChunks.push(`msgstr[1] ""\n`); - outChunks.push(`\n`); + if (!knownMessageIds.has(singularForm)) { + knownMessageIds.add(singularForm); + formatMsgLine("msgid", singularForm); + formatMsgLine("msgid_plural", pluralForm); + outChunks.push(`msgstr[0] ""\n`); + outChunks.push(`msgstr[1] ""\n`); + outChunks.push(`\n`); + } return; } break; @@ -333,13 +346,16 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) { ce.arguments[1], ); let comment = getComment(ce); - - formatMsgComment(line, comment); - formatMsgLine("msgid", t1.template); - formatMsgLine("msgid_plural", t2.template); - outChunks.push(`msgstr[0] ""\n`); - outChunks.push(`msgstr[1] ""\n`); - outChunks.push("\n"); + const msgid = t1.template; + if (!knownMessageIds.has(msgid)) { + knownMessageIds.add(msgid); + formatMsgComment(line, comment); + formatMsgLine("msgid", t1.template); + formatMsgLine("msgid_plural", t2.template); + outChunks.push(`msgstr[0] ""\n`); + outChunks.push(`msgstr[1] ""\n`); + outChunks.push("\n"); + } // Important: no processing for child i18n expressions here return; @@ -351,10 +367,14 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) { if (path[0] != "i18n") { break; } - formatMsgComment(line, comment); - formatMsgLine("msgid", template); - outChunks.push(`msgstr ""\n`); - outChunks.push("\n"); + const msgid = template; + if (!knownMessageIds.has(msgid)) { + knownMessageIds.add(msgid); + formatMsgComment(line, comment); + formatMsgLine("msgid", template); + outChunks.push(`msgstr ""\n`); + outChunks.push("\n"); + } break; } } @@ -363,51 +383,48 @@ export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) { } } -const configPath = ts.findConfigFile( - /*searchPath*/ "./", - ts.sys.fileExists, - "tsconfig.json", -); -if (!configPath) { - throw new Error("Could not find a valid 'tsconfig.json'."); -} - -console.log(configPath); - -const cmdline = ts.getParsedCommandLineOfConfigFile( - configPath, - {}, - { - fileExists: ts.sys.fileExists, - getCurrentDirectory: ts.sys.getCurrentDirectory, - onUnRecoverableConfigFileDiagnostic: (e) => console.log(e), - readDirectory: ts.sys.readDirectory, - readFile: ts.sys.readFile, - useCaseSensitiveFileNames: true, - }, -); - -console.log(cmdline); - -const prog = ts.createProgram({ - options: cmdline.options, - rootNames: cmdline.fileNames, -}); - -const allFiles = prog.getSourceFiles(); - -const ownFiles = allFiles.filter( - (x) => - !x.isDeclarationFile && - !prog.isSourceFileFromExternalLibrary(x) && - !prog.isSourceFileDefaultLibrary(x), -); - -console.log(ownFiles.map((x) => x.fileName)); - -const chunks = []; +export function potextract() { + const configPath = ts.findConfigFile( + /*searchPath*/ "./", + ts.sys.fileExists, + "tsconfig.json", + ); + if (!configPath) { + throw new Error("Could not find a valid 'tsconfig.json'."); + } -chunks.push(`# SOME DESCRIPTIVE TITLE. + const cmdline = ts.getParsedCommandLineOfConfigFile( + configPath, + {}, + { + fileExists: ts.sys.fileExists, + getCurrentDirectory: ts.sys.getCurrentDirectory, + onUnRecoverableConfigFileDiagnostic: (e) => console.log(e), + readDirectory: ts.sys.readDirectory, + readFile: ts.sys.readFile, + useCaseSensitiveFileNames: true, + }, + ); + + const prog = ts.createProgram({ + options: cmdline.options, + rootNames: cmdline.fileNames, + }); + + const allFiles = prog.getSourceFiles(); + + const ownFiles = allFiles.filter( + (x) => + !x.isDeclarationFile && + !prog.isSourceFileFromExternalLibrary(x) && + !prog.isSourceFileDefaultLibrary(x), + ); + + //console.log(ownFiles.map((x) => x.fileName)); + + const chunks = []; + + chunks.push(`# SOME DESCRIPTIVE TITLE. # Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER # This file is distributed under the same license as the PACKAGE package. # FIRST AUTHOR , YEAR. @@ -424,10 +441,26 @@ msgstr "" "Language: \\n" "MIME-Version: 1.0\\n" "Content-Type: text/plain; charset=UTF-8\\n" -"Content-Transfer-Encoding: 8bit\\n"`); +"Content-Transfer-Encoding: 8bit\\n"\n\n`); -for (const f of ownFiles) { - processFile(f, chunks); -} + const knownMessageIds = new Set(); + + for (const f of ownFiles) { + processFile(f, chunks, knownMessageIds); + } + + const pot = chunks.join(""); -console.log(chunks.join("")); + //console.log(pot); + + const packageJson = JSON.parse( + fs.readFileSync("./package.json", { encoding: "utf-8" }), + ); + + const poDomain = packageJson.pogen?.domain; + if (!poDomain) { + console.error("missing 'pogen.domain' field in package.json"); + process.exit(1); + } + fs.writeFileSync(`./src/i18n/${poDomain}.pot`, pot); +} -- cgit v1.2.3