From e6c1294c910e6b54d24d62981632cf5e5f79d33f Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Mon, 14 Feb 2022 16:51:13 +0100 Subject: pogen: read files from tsconfig, import po2ts --- packages/pogen/src/dumpTree.ts | 49 +++++ packages/pogen/src/po2ts.ts | 60 ++++++ packages/pogen/src/potextract.ts | 433 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 542 insertions(+) create mode 100644 packages/pogen/src/dumpTree.ts create mode 100644 packages/pogen/src/po2ts.ts create mode 100644 packages/pogen/src/potextract.ts (limited to 'packages/pogen/src') diff --git a/packages/pogen/src/dumpTree.ts b/packages/pogen/src/dumpTree.ts new file mode 100644 index 000000000..af25caf32 --- /dev/null +++ b/packages/pogen/src/dumpTree.ts @@ -0,0 +1,49 @@ +/* + This file is part of TALER + (C) 2016 GNUnet e.V. + + TALER is free software; you can redistribute it and/or modify it under the + terms of the GNU General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + TALER; see the file COPYING. If not, see + */ + + +/** + * Print the syntax tree of a TypeScript program. + * + * @author Florian Dold + */ + +"use strict"; + +import { readFileSync } from "fs"; +import { execSync } from "child_process"; +import * as ts from "typescript"; + + +export function processFile(sourceFile: ts.SourceFile) { + processNode(sourceFile); + + function processNode(node: ts.Node, level=0) { + let indent = ""; + for (let i = 0; i < level; i++) { + indent = indent + " "; + } + console.log(indent + ts.SyntaxKind[node.kind]); + ts.forEachChild(node, (n) => processNode(n, level+1)); + } +} + +const fileNames = process.argv.slice(2); + +fileNames.forEach(fileName => { + let sourceFile = ts.createSourceFile(fileName, readFileSync(fileName).toString(), ts.ScriptTarget.ES2016, /*setParentNodes */ true); + processFile(sourceFile); +}); diff --git a/packages/pogen/src/po2ts.ts b/packages/pogen/src/po2ts.ts new file mode 100644 index 000000000..d0f4ed34d --- /dev/null +++ b/packages/pogen/src/po2ts.ts @@ -0,0 +1,60 @@ +/* + This file is part of GNU Taler + (C) 2020 Taler Systems S.A. + + GNU Taler is free software; you can redistribute it and/or modify it under the + terms of the GNU General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + GNU Taler; see the file COPYING. If not, see + */ + +/** + * Convert a .po file into a JavaScript / TypeScript expression. + */ + +// @ts-ignore +import * as po2json from "po2json"; +import * as fs from "fs"; +import * as path from "path"; + +const files = fs + .readdirSync("./src/i18n") + .filter((x) => x.endsWith(".po")) + .map((x) => path.join("./src/i18n/", x)); + +if (files.length === 0) { + console.error("no .po files found in src/i18n/"); + process.exit(1); +} + +console.log(files); + +const chunks: string[] = []; + +for (const filename of files) { + const m = filename.match(/([a-zA-Z0-9-_]+).po/); + + if (!m) { + console.error("error: unexpected filename (expected .po)"); + process.exit(1); + } + + const lang = m[1]; + const pojson = po2json.parseFileSync(filename, { + format: "jed1.x", + fuzzy: true, + }); + const s = + "strings['" + lang + "'] = " + JSON.stringify(pojson, null, " ") + ";\n\n"; + chunks.push(s); +} + +const tsContents = chunks.join(""); + +fs.writeFileSync("src/i18n/strings.ts", tsContents); diff --git a/packages/pogen/src/potextract.ts b/packages/pogen/src/potextract.ts new file mode 100644 index 000000000..5999d9e1c --- /dev/null +++ b/packages/pogen/src/potextract.ts @@ -0,0 +1,433 @@ +/* + This file is part of GNU Taler + (C) 2019-2022 Taler Systems S.A. + + GNU Taler is free software; you can redistribute it and/or modify it under the + terms of the GNU General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + GNU Taler; see the file COPYING. If not, see + */ + + +/** + * Imports. + */ +import * as ts from "typescript"; + +function wordwrap(str: string, width: number = 80): string[] { + var regex = ".{1," + width + "}(\\s|$)|\\S+(\\s|$)"; + return str.match(RegExp(regex, "g")); +} + +export function processFile(sourceFile: ts.SourceFile, outChunks: string[]) { + processNode(sourceFile); + let lastTokLine = 0; + let preLastTokLine = 0; + + function getTemplate(node: ts.Node): string { + switch (node.kind) { + case ts.SyntaxKind.FirstTemplateToken: + return (node).text; + case ts.SyntaxKind.TemplateExpression: + let te = node; + let textFragments = [te.head.text]; + for (let tsp of te.templateSpans) { + textFragments.push(`%${(textFragments.length - 1) / 2 + 1}$s`); + textFragments.push(tsp.literal.text.replace(/%/g, "%%")); + } + return textFragments.join(""); + default: + return "(pogen.ts: unable to parse)"; + } + } + + function getComment(node: ts.Node): string { + let lc = ts.getLineAndCharacterOfPosition(sourceFile, node.pos); + let lastComments; + for (let l = preLastTokLine; l < lastTokLine; l++) { + let pos = ts.getPositionOfLineAndCharacter(sourceFile, l, 0); + let comments = ts.getTrailingCommentRanges(sourceFile.text, pos); + if (comments) { + lastComments = comments; + } + } + if (!lastComments) { + return; + } + let candidate = lastComments[lastComments.length - 1]; + let candidateEndLine = ts.getLineAndCharacterOfPosition( + sourceFile, + candidate.end, + ).line; + if (candidateEndLine != lc.line - 1) { + return; + } + let text = sourceFile.text.slice(candidate.pos, candidate.end); + switch (candidate.kind) { + case ts.SyntaxKind.SingleLineCommentTrivia: + // Remove comment leader + text = text.replace(/^[/][/]\s*/, ""); + break; + case ts.SyntaxKind.MultiLineCommentTrivia: + // Remove comment leader and trailer, + // handling white space just like xgettext. + text = text + .replace(/^[/][*](\s*?\n|\s*)?/, "") + .replace(/(\n[ \t]*?)?[*][/]$/, ""); + break; + } + return text; + } + + function getPath(node: ts.Node): string[] { + switch (node.kind) { + case ts.SyntaxKind.PropertyAccessExpression: + let pae = node; + return Array.prototype.concat(getPath(pae.expression), [pae.name.text]); + case ts.SyntaxKind.Identifier: + let id = node; + return [id.text]; + } + return ["(other)"]; + } + + function arrayEq(a1: T[], a2: T[]) { + if (a1.length != a2.length) { + return false; + } + for (let i = 0; i < a1.length; i++) { + if (a1[i] != a2[i]) { + return false; + } + } + return true; + } + + interface TemplateResult { + comment: string; + path: string[]; + template: string; + line: number; + } + + function processTaggedTemplateExpression( + tte: ts.TaggedTemplateExpression, + ): TemplateResult { + let lc = ts.getLineAndCharacterOfPosition(sourceFile, tte.pos); + if (lc.line != lastTokLine) { + preLastTokLine = lastTokLine; + lastTokLine = lc.line; + } + let path = getPath(tte.tag); + let res: TemplateResult = { + path, + line: lc.line, + comment: getComment(tte), + template: getTemplate(tte.template).replace(/"/g, '\\"'), + }; + return res; + } + + function formatMsgComment(line: number, comment?: string) { + if (comment) { + for (let cl of comment.split("\n")) { + outChunks.push(`#. ${cl}\n`); + } + } + outChunks.push(`#: ${sourceFile.fileName}:${line + 1}\n`); + outChunks.push(`#, c-format\n`); + } + + function formatMsgLine(head: string, msg: string) { + // Do escaping, wrap break at newlines + let parts = msg + .match(/(.*\n|.+$)/g) + .map((x) => x.replace(/\n/g, "\\n")) + .map((p) => wordwrap(p)) + .reduce((a, b) => a.concat(b)); + if (parts.length == 1) { + outChunks.push(`${head} "${parts[0]}"\n`); + } else { + outChunks.push(`${head} ""\n`); + for (let p of parts) { + outChunks.push(`"${p}"\n`); + } + } + } + + function getJsxElementPath(node: ts.Node) { + let path; + let process = (childNode) => { + switch (childNode.kind) { + case ts.SyntaxKind.JsxOpeningElement: { + let e = childNode as ts.JsxOpeningElement; + return (path = getPath(e.tagName)); + } + default: + break; + } + }; + ts.forEachChild(node, process); + return path; + } + + function translateJsxExpression(node: ts.Node, h) { + switch (node.kind) { + case ts.SyntaxKind.StringLiteral: { + let e = node as ts.StringLiteral; + return e.text; + } + default: + return `%${h[0]++}$s`; + } + } + + function trim(s) { + return s.replace(/^[ \n\t]*/, "").replace(/[ \n\t]*$/, ""); + } + + function getJsxContent(node: ts.Node) { + let fragments = []; + let holeNum = [1]; + let process = (childNode) => { + switch (childNode.kind) { + case ts.SyntaxKind.JsxText: { + let e = childNode as ts.JsxText; + let s = e.text; + let t = s.split("\n").map(trim).join(" "); + if (s[0] === " ") { + t = " " + t; + } + if (s[s.length - 1] === " ") { + t = t + " "; + } + fragments.push(t); + } + case ts.SyntaxKind.JsxOpeningElement: + break; + case ts.SyntaxKind.JsxElement: + fragments.push(`%${holeNum[0]++}$s`); + break; + case ts.SyntaxKind.JsxExpression: { + let e = childNode as ts.JsxExpression; + fragments.push(translateJsxExpression(e.expression, holeNum)); + break; + } + case ts.SyntaxKind.JsxClosingElement: + break; + default: + let lc = ts.getLineAndCharacterOfPosition( + childNode.getSourceFile(), + childNode.getStart(), + ); + console.error( + `unrecognized syntax in JSX Element ${ + ts.SyntaxKind[childNode.kind] + } (${childNode.getSourceFile().fileName}:${lc.line + 1}:${ + lc.character + 1 + }`, + ); + break; + } + }; + ts.forEachChild(node, process); + return fragments.join("").trim().replace(/ +/g, " "); + } + + function getJsxSingular(node: ts.Node) { + let res; + let process = (childNode) => { + switch (childNode.kind) { + case ts.SyntaxKind.JsxElement: { + let path = getJsxElementPath(childNode); + if (arrayEq(path, ["i18n", "TranslateSingular"])) { + res = getJsxContent(childNode); + } + } + default: + break; + } + }; + ts.forEachChild(node, process); + return res; + } + + function getJsxPlural(node: ts.Node) { + let res; + let process = (childNode) => { + switch (childNode.kind) { + case ts.SyntaxKind.JsxElement: { + let path = getJsxElementPath(childNode); + if (arrayEq(path, ["i18n", "TranslatePlural"])) { + res = getJsxContent(childNode); + } + } + default: + break; + } + }; + ts.forEachChild(node, process); + return res; + } + + function processNode(node: ts.Node) { + switch (node.kind) { + case ts.SyntaxKind.JsxElement: + let path = getJsxElementPath(node); + if (arrayEq(path, ["i18n", "Translate"])) { + let content = getJsxContent(node); + let { line } = ts.getLineAndCharacterOfPosition(sourceFile, node.pos); + let comment = getComment(node); + formatMsgComment(line, comment); + formatMsgLine("msgid", content); + outChunks.push(`msgstr ""\n`); + outChunks.push("\n"); + return; + } + if (arrayEq(path, ["i18n", "TranslateSwitch"])) { + let { line } = ts.getLineAndCharacterOfPosition(sourceFile, node.pos); + let comment = getComment(node); + formatMsgComment(line, comment); + let singularForm = getJsxSingular(node); + if (!singularForm) { + console.error("singular form missing"); + process.exit(1); + } + let pluralForm = getJsxPlural(node); + if (!pluralForm) { + console.error("plural form missing"); + process.exit(1); + } + formatMsgLine("msgid", singularForm); + formatMsgLine("msgid_plural", pluralForm); + outChunks.push(`msgstr[0] ""\n`); + outChunks.push(`msgstr[1] ""\n`); + outChunks.push(`\n`); + return; + } + break; + case ts.SyntaxKind.CallExpression: { + // might be i18n.plural(i18n[.X]`...`, i18n[.X]`...`) + let ce = node; + let path = getPath(ce.expression); + if (!arrayEq(path, ["i18n", "plural"])) { + break; + } + if (ce.arguments[0].kind != ts.SyntaxKind.TaggedTemplateExpression) { + break; + } + if (ce.arguments[1].kind != ts.SyntaxKind.TaggedTemplateExpression) { + break; + } + let { line } = ts.getLineAndCharacterOfPosition(sourceFile, ce.pos); + let t1 = processTaggedTemplateExpression( + ce.arguments[0], + ); + let t2 = processTaggedTemplateExpression( + ce.arguments[1], + ); + let comment = getComment(ce); + + formatMsgComment(line, comment); + formatMsgLine("msgid", t1.template); + formatMsgLine("msgid_plural", t2.template); + outChunks.push(`msgstr[0] ""\n`); + outChunks.push(`msgstr[1] ""\n`); + outChunks.push("\n"); + + // Important: no processing for child i18n expressions here + return; + } + case ts.SyntaxKind.TaggedTemplateExpression: { + let tte = node; + let { comment, template, line, path } = + processTaggedTemplateExpression(tte); + if (path[0] != "i18n") { + break; + } + formatMsgComment(line, comment); + formatMsgLine("msgid", template); + outChunks.push(`msgstr ""\n`); + outChunks.push("\n"); + break; + } + } + + ts.forEachChild(node, processNode); + } +} + +const configPath = ts.findConfigFile( + /*searchPath*/ "./", + ts.sys.fileExists, + "tsconfig.json", +); +if (!configPath) { + throw new Error("Could not find a valid 'tsconfig.json'."); +} + +console.log(configPath); + +const cmdline = ts.getParsedCommandLineOfConfigFile( + configPath, + {}, + { + fileExists: ts.sys.fileExists, + getCurrentDirectory: ts.sys.getCurrentDirectory, + onUnRecoverableConfigFileDiagnostic: (e) => console.log(e), + readDirectory: ts.sys.readDirectory, + readFile: ts.sys.readFile, + useCaseSensitiveFileNames: true, + }, +); + +console.log(cmdline); + +const prog = ts.createProgram({ + options: cmdline.options, + rootNames: cmdline.fileNames, +}); + +const allFiles = prog.getSourceFiles(); + +const ownFiles = allFiles.filter( + (x) => + !x.isDeclarationFile && + !prog.isSourceFileFromExternalLibrary(x) && + !prog.isSourceFileDefaultLibrary(x), +); + +console.log(ownFiles.map((x) => x.fileName)); + +const chunks = []; + +chunks.push(`# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\\n" +"Report-Msgid-Bugs-To: \\n" +"POT-Creation-Date: 2016-11-23 00:00+0100\\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n" +"Last-Translator: FULL NAME \\n" +"Language-Team: LANGUAGE \\n" +"Language: \\n" +"MIME-Version: 1.0\\n" +"Content-Type: text/plain; charset=UTF-8\\n" +"Content-Transfer-Encoding: 8bit\\n"`); + +for (const f of ownFiles) { + processFile(f, chunks); +} + +console.log(chunks.join("")); -- cgit v1.2.3