harness,util: implement writing changed config back to the file system, support multiple projects

4 files changed, 197 insertions, 76 deletions

diff --git a/packages/taler-harness/src/harness/harness.ts b/packages/taler-harness/src/harness/harness.ts
index 68c0744fc..b27eaa371 100644
--- a/packages/taler-harness/src/harness/harness.ts
+++ b/packages/taler-harness/src/harness/harness.ts
@@ -651,7 +651,7 @@ export class FakebankService
     config.setString("bank", "max_debt", bc.maxDebt ?? `${bc.currency}:100`);
     config.setString("bank", "ram_limit", `${1024}`);
     const cfgFilename = testDir + "/bank.conf";
-    config.write(cfgFilename, { excludeDefaults: true });
+    config.writeTo(cfgFilename, { excludeDefaults: true });
 
     return new FakebankService(gc, bc, cfgFilename);
   }
@@ -680,7 +680,7 @@ export class FakebankService
     }
     const config = Configuration.load(this.configFile);
     config.setString("bank", "suggested_exchange", e.baseUrl);
-    config.write(this.configFile, { excludeDefaults: true });
+    config.writeTo(this.configFile, { excludeDefaults: true });
   }
 
   get baseUrl(): string {
@@ -790,7 +790,7 @@ export class LibeufinBankService
       `${bc.currency}:100`,
     );
     const cfgFilename = testDir + "/bank.conf";
-    config.write(cfgFilename, { excludeDefaults: true });
+    config.writeTo(cfgFilename, { excludeDefaults: true });
 
     return new LibeufinBankService(gc, bc, cfgFilename);
   }
@@ -828,7 +828,7 @@ export class LibeufinBankService
       "suggested_withdrawal_exchange",
       e.baseUrl,
     );
-    config.write(this.configFile, { excludeDefaults: true });
+    config.writeTo(this.configFile, { excludeDefaults: true });
   }
 
   get baseUrl(): string {
@@ -1052,7 +1052,7 @@ export class ExchangeService implements ExchangeServiceInterface {
   changeConfig(f: (config: Configuration) => void) {
     const config = Configuration.load(this.configFilename);
     f(config);
-    config.write(this.configFilename, { excludeDefaults: true });
+    config.writeTo(this.configFilename, { excludeDefaults: true });
   }
 
   static create(gc: GlobalTestState, e: ExchangeConfig) {
@@ -1118,7 +1118,7 @@ export class ExchangeService implements ExchangeServiceInterface {
     fs.writeFileSync(masterPrivFile, Buffer.from(exchangeMasterKey.eddsaPriv));
 
     const cfgFilename = testDir + `/exchange-${e.name}.conf`;
-    config.write(cfgFilename, { excludeDefaults: true });
+    config.writeTo(cfgFilename, { excludeDefaults: true });
     return new ExchangeService(gc, e, cfgFilename, exchangeMasterKey);
   }
 
@@ -1127,13 +1127,13 @@ export class ExchangeService implements ExchangeServiceInterface {
     offeredCoins.forEach((cc) =>
       setCoin(config, cc(this.exchangeConfig.currency)),
     );
-    config.write(this.configFilename, { excludeDefaults: true });
+    config.writeTo(this.configFilename, { excludeDefaults: true });
   }
 
   addCoinConfigList(ccs: CoinConfig[]) {
     const config = Configuration.load(this.configFilename);
     ccs.forEach((cc) => setCoin(config, cc));
-    config.write(this.configFilename, { excludeDefaults: true });
+    config.writeTo(this.configFilename, { excludeDefaults: true });
   }
 
   enableAgeRestrictions(maskStr: string) {
@@ -1144,7 +1144,7 @@ export class ExchangeService implements ExchangeServiceInterface {
       "age_groups",
       maskStr,
     );
-    config.write(this.configFilename, { excludeDefaults: true });
+    config.writeTo(this.configFilename, { excludeDefaults: true });
   }
 
   get masterPub() {
@@ -1165,7 +1165,7 @@ export class ExchangeService implements ExchangeServiceInterface {
   ): Promise<void> {
     const config = Configuration.load(this.configFilename);
     await f(config);
-    config.write(this.configFilename, { excludeDefaults: true });
+    config.writeTo(this.configFilename, { excludeDefaults: true });
   }
 
   async addBankAccount(
@@ -1206,7 +1206,7 @@ export class ExchangeService implements ExchangeServiceInterface {
       "password",
       exchangeBankAccount.accountPassword,
     );
-    config.write(this.configFilename, { excludeDefaults: true });
+    config.writeTo(this.configFilename, { excludeDefaults: true });
   }
 
   exchangeHttpProc: ProcessWrapper | undefined;
@@ -1701,7 +1701,7 @@ export class MerchantService implements MerchantServiceInterface {
     config.setString("merchantdb-postgres", "config", mc.database);
     // Do not contact demo.taler.net exchange in tests
     config.setString("merchant-exchange-kudos", "disabled", "yes");
-    config.write(cfgFilename, { excludeDefaults: true });
+    config.writeTo(cfgFilename, { excludeDefaults: true });
 
     return new MerchantService(gc, mc, cfgFilename);
   }
@@ -1719,7 +1719,7 @@ export class MerchantService implements MerchantServiceInterface {
       this.merchantConfig.currency,
     );
     config.setString(`merchant-exchange-${e.name}`, "master_key", e.masterPub);
-    config.write(this.configFilename, { excludeDefaults: true });
+    config.writeTo(this.configFilename, { excludeDefaults: true });
   }
 
   async addDefaultInstance(): Promise<void> {
diff --git a/packages/taler-harness/src/harness/sync.ts b/packages/taler-harness/src/harness/sync.ts
index 64c9acaef..567a2e92d 100644
--- a/packages/taler-harness/src/harness/sync.ts
+++ b/packages/taler-harness/src/harness/sync.ts
@@ -85,7 +85,7 @@ export class SyncService {
     config.setString("syncdb-postgres", "config", sc.database);
     config.setString("sync", "payment_backend_url", sc.paymentBackendUrl);
     config.setString("sync", "upload_limit_mb", `${sc.uploadLimitMb}`);
-    config.write(cfgFilename);
+    config.writeTo(cfgFilename);
 
     return new SyncService(gc, sc, cfgFilename);
   }
diff --git a/packages/taler-harness/src/index.ts b/packages/taler-harness/src/index.ts
index 0f282e123..2b1fd9a0d 100644
--- a/packages/taler-harness/src/index.ts
+++ b/packages/taler-harness/src/index.ts
@@ -30,7 +30,6 @@ import {
   TalerAuthenticationHttpClient,
   TalerBankConversionHttpClient,
   TalerCoreBankHttpClient,
-  TalerErrorCode,
   TalerMerchantInstanceHttpClient,
   TalerMerchantManagementHttpClient,
   TransactionsResponse,
@@ -42,7 +41,6 @@ import {
   randomBytes,
   rsaBlind,
   setGlobalLogLevelFromString,
-  setPrintHttpRequestAsCurl,
   stringifyPayTemplateUri,
 } from "@gnu-taler/taler-util";
 import { clk } from "@gnu-taler/taler-util/clk";
@@ -80,7 +78,6 @@ import {
 } from "./harness/helpers.js";
 import { getTestInfo, runTests } from "./integrationtests/testrunner.js";
 import { lintExchangeDeployment } from "./lint.js";
-import { randomUUID } from "crypto";
 
 const logger = new Logger("taler-harness:index.ts");
 
@@ -356,25 +353,46 @@ advancedCli
     );
   });
 
-const configCli = testingCli.subcommand("configArgs", "config", {
-  help: "Subcommands for handling the Taler configuration.",
-});
+const configCli = testingCli
+  .subcommand("configArgs", "config", {
+    help: "Subcommands for handling the Taler configuration.",
+  })
+  .maybeOption("configEntryFile", ["-c", "--config"], clk.STRING, {
+    help: "Configuration file to use.",
+  })
+  .maybeOption("project", ["--project"], clk.STRING, {
+    help: `Selection of the project to inspect/change the config (default: taler).`,
+  });
 
-configCli.subcommand("show", "show").action(async (args) => {
-  const config = Configuration.load();
-  const cfgStr = config.stringify({
-    diagnostics: true,
+configCli
+  .subcommand("show", "show", {
+    help: "Show the current configuration.",
+  })
+  .action(async (args) => {
+    const config = Configuration.load(
+      args.configArgs.configEntryFile,
+      args.configArgs.project,
+    );
+    const cfgStr = config.stringify({
+      diagnostics: true,
+    });
+    console.log(cfgStr);
   });
-  console.log(cfgStr);
-});
 
 configCli
-  .subcommand("get", "get")
+  .subcommand("get", "get", {
+    help: "Get a configuration option.",
+  })
   .requiredArgument("section", clk.STRING)
   .requiredArgument("option", clk.STRING)
-  .flag("file", ["-f"])
+  .flag("file", ["-f"], {
+    help: "Treat the value as a filename, expanding placeholders.",
+  })
   .action(async (args) => {
-    const config = Configuration.load();
+    const config = Configuration.load(
+      args.configArgs.configEntryFile,
+      args.configArgs.project,
+    );
     let res;
     if (args.get.file) {
       res = config.getPath(args.get.section, args.get.option);
@@ -389,6 +407,35 @@ configCli
     }
   });
 
+configCli
+  .subcommand("set", "set", {
+    help: "Set a configuration option.",
+  })
+  .requiredArgument("section", clk.STRING)
+  .requiredArgument("option", clk.STRING)
+  .requiredArgument("value", clk.STRING)
+  .flag("dry", ["--dry"], {
+    help: "Do not write the changed config to disk, only write it to stdout.",
+  })
+  .action(async (args) => {
+    const config = Configuration.load(
+      args.configArgs.configEntryFile,
+      args.configArgs.project,
+    );
+    config.setString(args.set.section, args.set.option, args.set.value);
+    if (args.set.dry) {
+      console.log(
+        config.stringify({
+          excludeDefaults: true,
+        }),
+      );
+    } else {
+      config.write({
+        excludeDefaults: true,
+      });
+    }
+  });
+
 const deploymentCli = testingCli.subcommand("deploymentArgs", "deployment", {
   help: "Subcommands for handling GNU Taler deployments.",
 });
@@ -1147,23 +1194,6 @@ deploymentCli
     console.log(out);
   });
 
-const deploymentConfigCli = deploymentCli.subcommand("configArgs", "config", {
-  help: "Subcommands the Taler configuration.",
-});
-
-deploymentConfigCli
-  .subcommand("show", "show")
-  .flag("diagnostics", ["-d", "--diagnostics"])
-  .maybeArgument("cfgfile", clk.STRING, {})
-  .action(async (args) => {
-    const cfg = Configuration.load(args.show.cfgfile);
-    console.log(
-      cfg.stringify({
-        diagnostics: args.show.diagnostics,
-      }),
-    );
-  });
-
 testingCli.subcommand("logtest", "logtest").action(async (args) => {
   logger.trace("This is a trace message.");
   logger.info("This is an info message.");
diff --git a/packages/taler-util/src/talerconfig.ts b/packages/taler-util/src/talerconfig.ts
index 83c0044be..2bd7b355f 100644
--- a/packages/taler-util/src/talerconfig.ts
+++ b/packages/taler-util/src/talerconfig.ts
@@ -23,13 +23,12 @@
 /**
  * Imports
  */
-import { AmountJson } from "./amounts.js";
-import { Amounts } from "./amounts.js";
+import { AmountJson, Amounts } from "./amounts.js";
 import { Logger } from "./logging.js";
 
-import nodejs_path from "path";
-import nodejs_os from "os";
 import nodejs_fs from "fs";
+import nodejs_os from "os";
+import nodejs_path from "path";
 
 const logger = new Logger("talerconfig.ts");
 
@@ -76,6 +75,54 @@ interface Section {
 
 type SectionMap = { [sectionName: string]: Section };
 
+/**
+ * Different projects use the GNUnet/Taler-Style config.
+ *
+ * The config source determines where to locate the configuration.
+ */
+export interface ConfigSource {
+  projectName: string;
+  componentName: string;
+  installPathBinary: string;
+  baseConfigVarname: string;
+  prefixVarname: string;
+}
+
+export type ConfigSourceDef = { [x: string]: ConfigSource | undefined };
+
+export const ConfigSources = {
+  ["taler"]: {
+    projectName: "taler",
+    componentName: "taler",
+    installPathBinary: "taler-config",
+    baseConfigVarname: "TALER_BASE_CONFIG",
+    prefixVarname: "TALER_PREFIX",
+  } satisfies ConfigSource,
+  ["libeufin-bank"]: {
+    projectName: "libeufin",
+    componentName: "libeufin-bank",
+    installPathBinary: "libeufin-bank",
+    baseConfigVarname: "LIBEUFIN_BASE_CONFIG",
+    prefixVarname: "LIBEUFIN_PREFIX",
+  } satisfies ConfigSource,
+  ["libeufin-nexus"]: {
+    projectName: "libeufin",
+    componentName: "libeufin-nexus",
+    installPathBinary: "libeufin-nexus",
+    baseConfigVarname: "LIBEUFIN_BASE_CONFIG",
+    prefixVarname: "LIBEUFIN_PREFIX",
+  } satisfies ConfigSource,
+  ["gnunet"]: {
+    projectName: "gnunet",
+    componentName: "gnunet",
+    installPathBinary: "gnunet-config",
+    baseConfigVarname: "GNUNET_BASE_CONFIG",
+    prefixVarname: "GNUNET_PREFIX",
+  } satisfies ConfigSource,
+} satisfies ConfigSourceDef;
+
+const defaultConfigSource: ConfigSource = ConfigSources.taler;
+
 export class ConfigValue<T> {
   constructor(
     private sectionName: string,
@@ -215,7 +262,7 @@ export function pathsub(
   return s;
 }
 
-export interface LoadOptions {
+interface LoadOptions {
   filename?: string;
   banDirectives?: boolean;
 }
@@ -310,6 +357,14 @@ export class Configuration {
 
   private nestLevel = 0;
 
+  /**
+   * Does the entrypoint config file contain complex
+   * directives?
+   */
+  private entrypointIsComplex: boolean = false;
+
+  constructor(private configSource: ConfigSource = defaultConfigSource) {}
+
   private loadFromFilename(
     filename: string,
     isDefaultSource: boolean,
@@ -434,6 +489,9 @@ export class Configuration {
             `invalid configuration, directive in ${fn}:${lineNo} forbidden`,
           );
         }
+        if (!isDefaultSource) {
+          this.entrypointIsComplex = true;
+        }
         const directive = directiveMatch[1].toLowerCase();
         switch (directive) {
           case "inline": {
@@ -521,10 +579,6 @@ export class Configuration {
     }
   }
 
-  loadFromString(s: string, opts: LoadOptions = {}): void {
-    return this.internalLoadFromString(s, false, opts);
-  }
-
   private provideSection(section: string): Section {
     const secNorm = section.toUpperCase();
     if (this.sectionMap[secNorm]) {
@@ -653,7 +707,7 @@ export class Configuration {
     );
   }
 
-  loadDefaultsFromDir(dirname: string): void {
+  private loadDefaultsFromDir(dirname: string): void {
     const files = nodejs_fs.readdirSync(dirname);
     for (const f of files) {
       const fn = nodejs_path.join(dirname, f);
@@ -662,26 +716,28 @@ export class Configuration {
   }
 
   private loadDefaults(): void {
-    let baseConfigDir = process.env["TALER_BASE_CONFIG"];
+    const { projectName, prefixVarname, baseConfigVarname, installPathBinary } =
+      this.configSource;
+    let baseConfigDir = process.env[baseConfigVarname];
     if (!baseConfigDir) {
       /* Try to locate the configuration based on the location
        * of the taler-config binary. */
-      const path = which("taler-config");
+      const path = which(installPathBinary);
       if (path) {
         baseConfigDir = nodejs_fs.realpathSync(
-          nodejs_path.dirname(path) + "/../share/taler/config.d",
+          nodejs_path.dirname(path) + `/../share/${projectName}/config.d`,
         );
       }
     }
     if (!baseConfigDir) {
-      baseConfigDir = "/usr/share/taler/config.d";
+      baseConfigDir = `/usr/share/${projectName}/config.d`;
     }
 
-    let installPrefix = process.env["TALER_PREFIX"];
+    let installPrefix = process.env[prefixVarname];
     if (!installPrefix) {
       /* Try to locate install path based on the location
        * of the taler-config binary. */
-      const path = which("taler-config");
+      const path = which(installPathBinary);
       if (path) {
         installPrefix = nodejs_fs.realpathSync(
           nodejs_path.dirname(path) + "/..",
@@ -695,12 +751,12 @@ export class Configuration {
     this.setStringSystemDefault(
       "PATHS",
       "LIBEXECDIR",
-      `${installPrefix}/taler/libexec/`,
+      `${installPrefix}/${projectName}/libexec/`,
     );
     this.setStringSystemDefault(
       "PATHS",
       "DOCDIR",
-      `${installPrefix}/share/doc/taler/`,
+      `${installPrefix}/share/doc/${projectName}/`,
     );
     this.setStringSystemDefault(
       "PATHS",
@@ -717,58 +773,80 @@ export class Configuration {
     this.setStringSystemDefault(
       "PATHS",
       "LIBDIR",
-      `${installPrefix}/lib/taler/`,
+      `${installPrefix}/lib/${projectName}/`,
     );
     this.setStringSystemDefault(
       "PATHS",
       "DATADIR",
-      `${installPrefix}/share/taler/`,
+      `${installPrefix}/share/${projectName}/`,
     );
 
     this.loadDefaultsFromDir(baseConfigDir);
   }
 
-  getDefaultConfigFilename(): string | undefined {
+  private findDefaultConfigFilename(): string | undefined {
     const xdg = process.env["XDG_CONFIG_HOME"];
     const home = process.env["HOME"];
     let fn: string | undefined;
+    const { projectName, componentName } = this.configSource;
     if (xdg) {
-      fn = nodejs_path.join(xdg, "taler.conf");
+      fn = nodejs_path.join(xdg, `${componentName}.conf`);
     } else if (home) {
-      fn = nodejs_path.join(home, ".config/taler.conf");
+      fn = nodejs_path.join(home, `.config/${componentName}.conf`);
     }
     if (fn && nodejs_fs.existsSync(fn)) {
       return fn;
     }
-    const etc1 = "/etc/taler.conf";
+    const etc1 = `/etc/${componentName}.conf`;
     if (nodejs_fs.existsSync(etc1)) {
       return etc1;
     }
-    const etc2 = "/etc/taler/taler.conf";
+    const etc2 = `/etc/${projectName}/${componentName}.conf`;
     if (nodejs_fs.existsSync(etc2)) {
       return etc2;
     }
     return undefined;
   }
 
-  static load(filename?: string): Configuration {
-    const cfg = new Configuration();
+  static load(
+    filename?: string,
+    configSource?: ConfigSource | string,
+  ): Configuration {
+    let cs: ConfigSource;
+    if (configSource == null) {
+      cs = defaultConfigSource;
+    } else if (typeof configSource === "string") {
+      if (configSource in ConfigSources) {
+        cs = ConfigSources[configSource as keyof typeof ConfigSources];
+      } else {
+        throw Error("invalid config source");
+      }
+    } else {
+      cs = configSource;
+    }
+    const cfg = new Configuration(cs);
     cfg.loadDefaults();
     if (filename) {
       cfg.loadFromFilename(filename, false);
+      cfg.hintEntrypoint = filename;
     } else {
-      const fn = cfg.getDefaultConfigFilename();
+      const fn = cfg.findDefaultConfigFilename();
       if (fn) {
         // It's the default filename for the main config file,
         // but we don't consider the values default values.
         cfg.loadFromFilename(fn, false);
+        cfg.hintEntrypoint = fn;
       }
     }
-    cfg.hintEntrypoint = filename;
     return cfg;
   }
 
   stringify(opts: StringifyOptions = {}): string {
+    if (opts.excludeDefaults && this.entrypointIsComplex) {
+      throw Error(
+        "unable to do diff serialization of config file, as entry point contains complex directives",
+      );
+    }
     let s = "";
     if (opts.diagnostics) {
       s += "# Configuration file diagnostics\n";
@@ -824,7 +902,20 @@ export class Configuration {
     return s;
   }
 
-  write(filename: string, opts: { excludeDefaults?: boolean } = {}): void {
+  write(opts: { excludeDefaults?: boolean } = {}): void {
+    const filename = this.hintEntrypoint;
+    if (!filename) {
+      throw Error(
+        "unknown configuration entrypoing, unable to write back config file",
+      );
+    }
+    nodejs_fs.writeFileSync(
+      filename,
+      this.stringify({ excludeDefaults: opts.excludeDefaults }),
+    );
+  }
+
+  writeTo(filename: string, opts: { excludeDefaults?: boolean } = {}): void {
     nodejs_fs.writeFileSync(
       filename,
       this.stringify({ excludeDefaults: opts.excludeDefaults }),