implement infrastructure for future DB migrations via backup

11 files changed, 189 insertions, 93 deletions

diff --git a/packages/taler-wallet-cli/src/integrationtests/harness.ts b/packages/taler-wallet-cli/src/integrationtests/harness.ts
index 108b78540..4985c5fc1 100644
--- a/packages/taler-wallet-cli/src/integrationtests/harness.ts
+++ b/packages/taler-wallet-cli/src/integrationtests/harness.ts
@@ -78,6 +78,7 @@ import {
   AcceptTipRequest,
   AbortPayWithRefundRequest,
   handleWorkerError,
+  openPromise,
 } from "taler-wallet-core";
 import { URL } from "url";
 import axios, { AxiosError } from "axios";
@@ -94,7 +95,6 @@ import {
 import { ApplyRefundResponse } from "taler-wallet-core";
 import { PendingOperationsResponse } from "taler-wallet-core";
 import { CoinConfig } from "./denomStructures";
-import { after } from "taler-wallet-core/src/util/timer";
 
 const exec = util.promisify(require("child_process").exec);
 
@@ -1114,11 +1114,14 @@ export class ExchangeService implements ExchangeServiceInterface {
       `exchange-httpd-${this.name}`,
     );
 
+    await this.pingUntilAvailable();
     await this.keyup();
   }
 
   async pingUntilAvailable(): Promise<void> {
-    const url = `http://localhost:${this.exchangeConfig.httpPort}/keys`;
+    // We request /management/keys, since /keys can block
+    // when we didn't do the key setup yet.
+    const url = `http://localhost:${this.exchangeConfig.httpPort}/management/keys`;
     await pingProc(this.exchangeHttpProc, url, `exchange (${this.name})`);
   }
 }
@@ -1449,10 +1452,14 @@ export async function runTestWithState(
 ): Promise<TestRunResult> {
   const startMs = new Date().getTime();
 
-  const handleSignal = () => {
+  const p = openPromise();
+  let status: TestStatus;
+
+  const handleSignal = (s: string) => {
     gc.shutdownSync();
-    console.warn("**** received fatal signal, shutting down test harness");
-    process.exit(1);
+    console.warn("**** received fatal proces event, shutting down test harness");
+    status = "fail";
+    p.reject(Error("caught signal"));
   };
 
   process.on("SIGINT", handleSignal);
@@ -1460,10 +1467,9 @@ export async function runTestWithState(
   process.on("unhandledRejection", handleSignal);
   process.on("uncaughtException", handleSignal);
 
-  let status: TestStatus;
   try {
     console.log("running test in directory", gc.testDir);
-    await testMain(gc);
+    await Promise.race([testMain(gc), p.promise]);
     status = "pass";
   } catch (e) {
     console.error("FATAL: test failed with exception", e);
diff --git a/packages/taler-wallet-cli/src/integrationtests/testrunner.ts b/packages/taler-wallet-cli/src/integrationtests/testrunner.ts
index d9804562e..578e9488c 100644
--- a/packages/taler-wallet-cli/src/integrationtests/testrunner.ts
+++ b/packages/taler-wallet-cli/src/integrationtests/testrunner.ts
@@ -66,12 +66,13 @@ const allTests: TestMainFunction[] = [
   runMerchantLongpollingTest,
   runMerchantRefundApiTest,
   runPayAbortTest,
-  runPayPaidTest,
   runPaymentClaimTest,
   runPaymentFaultTest,
   runPaymentIdempotencyTest,
   runPaymentMultipleTest,
+  runPaymentTest,
   runPaymentTransientTest,
+  runPayPaidTest,
   runPaywallFlowTest,
   runRefundAutoTest,
   runRefundGoneTest,
@@ -82,10 +83,9 @@ const allTests: TestMainFunction[] = [
   runTimetravelWithdrawTest,
   runTippingTest,
   runWallettestingTest,
+  runTestWithdrawalManualTest,
   runWithdrawalAbortBankTest,
   runWithdrawalBankIntegratedTest,
-  runWallettestingTest,
-  runPaymentTest,
 ];
 
 export interface TestRunSpec {
@@ -166,7 +166,12 @@ export async function runTests(spec: TestRunSpec) {
     JSON.stringify({ testResults }, undefined, 2),
   );
   console.log(`See ${resultsFile} for details`);
+  console.log(`Skipped: ${numSkip}/${numTotal}`);
+  console.log(`Failed: ${numFail}/${numTotal}`);
   console.log(`Passed: ${numPass}/${numTotal}`);
+  if (numPass < numTotal - numSkip) {
+    process.exit(1);
+  }
 }
 
 export function getTestInfo(): TestInfo[] {
diff --git a/packages/taler-wallet-core/src/db.ts b/packages/taler-wallet-core/src/db.ts
index b13abac57..aed2ce5cb 100644
--- a/packages/taler-wallet-core/src/db.ts
+++ b/packages/taler-wallet-core/src/db.ts
@@ -1,5 +1,11 @@
-import { Stores } from "./types/dbTypes";
-import { openDatabase, Database, Store, Index } from "./util/query";
+import { MetaStores, Stores } from "./types/dbTypes";
+import {
+  openDatabase,
+  Database,
+  Store,
+  Index,
+  AnyStoreMap,
+} from "./util/query";
 import {
   IDBFactory,
   IDBDatabase,
@@ -14,7 +20,11 @@ import { Logger } from "./util/logging";
  * for all previous versions must be written, which should be
  * avoided.
  */
-const TALER_DB_NAME = "taler-wallet-prod-v1";
+const TALER_DB_NAME = "taler-wallet-main-v2";
+
+const TALER_META_DB_NAME = "taler-wallet-meta";
+
+const CURRENT_DB_CONFIG_KEY = "currentMainDbName";
 
 /**
  * Current database minor version, should be incremented
@@ -23,78 +33,134 @@ const TALER_DB_NAME = "taler-wallet-prod-v1";
  * backwards-compatible way or object stores and indices
  * are added.
  */
-export const WALLET_DB_MINOR_VERSION = 3;
+export const WALLET_DB_MINOR_VERSION = 1;
 
 const logger = new Logger("db.ts");
 
-/**
- * Return a promise that resolves
- * to the taler wallet db.
- */
-export function openTalerDatabase(
-  idbFactory: IDBFactory,
-  onVersionChange: () => void,
-): Promise<IDBDatabase> {
-  const onUpgradeNeeded = (
-    db: IDBDatabase,
-    oldVersion: number,
-    newVersion: number,
-    upgradeTransaction: IDBTransaction,
-  ): void => {
-    if (oldVersion === 0) {
-      for (const n in Stores) {
-        if ((Stores as any)[n] instanceof Store) {
-          const si: Store<string, any> = (Stores as any)[n];
-          const s = db.createObjectStore(si.name, si.storeParams);
-          for (const indexName in si as any) {
-            if ((si as any)[indexName] instanceof Index) {
-              const ii: Index<string, string, any, any> = (si as any)[
-                indexName
-              ];
-              s.createIndex(ii.indexName, ii.keyPath, ii.options);
-            }
+function upgradeFromStoreMap(
+  storeMap: AnyStoreMap,
+  db: IDBDatabase,
+  oldVersion: number,
+  newVersion: number,
+  upgradeTransaction: IDBTransaction,
+): void {
+  if (oldVersion === 0) {
+    for (const n in storeMap) {
+      if ((storeMap as any)[n] instanceof Store) {
+        const si: Store<string, any> = (storeMap as any)[n];
+        const s = db.createObjectStore(si.name, si.storeParams);
+        for (const indexName in si as any) {
+          if ((si as any)[indexName] instanceof Index) {
+            const ii: Index<string, string, any, any> = (si as any)[indexName];
+            s.createIndex(ii.indexName, ii.keyPath, ii.options);
           }
         }
       }
-      return;
-    }
-    if (oldVersion === newVersion) {
-      return;
     }
-    logger.info(`upgrading database from ${oldVersion} to ${newVersion}`);
-    for (const n in Stores) {
-      if ((Stores as any)[n] instanceof Store) {
-        const si: Store<string, any> = (Stores as any)[n];
-        let s: IDBObjectStore;
-        const storeVersionAdded = si.storeParams?.versionAdded ?? 1;
-        if (storeVersionAdded > oldVersion) {
-          s = db.createObjectStore(si.name, si.storeParams);
-        } else {
-          s = upgradeTransaction.objectStore(si.name);
-        }
-        for (const indexName in si as any) {
-          if ((si as any)[indexName] instanceof Index) {
-            const ii: Index<string, string, any, any> = (si as any)[indexName];
-            const indexVersionAdded = ii.options?.versionAdded ?? 0;
-            if (
-              indexVersionAdded > oldVersion ||
-              storeVersionAdded > oldVersion
-            ) {
-              s.createIndex(ii.indexName, ii.keyPath, ii.options);
-            }
+    return;
+  }
+  if (oldVersion === newVersion) {
+    return;
+  }
+  logger.info(`upgrading database from ${oldVersion} to ${newVersion}`);
+  for (const n in Stores) {
+    if ((Stores as any)[n] instanceof Store) {
+      const si: Store<string, any> = (Stores as any)[n];
+      let s: IDBObjectStore;
+      const storeVersionAdded = si.storeParams?.versionAdded ?? 1;
+      if (storeVersionAdded > oldVersion) {
+        s = db.createObjectStore(si.name, si.storeParams);
+      } else {
+        s = upgradeTransaction.objectStore(si.name);
+      }
+      for (const indexName in si as any) {
+        if ((si as any)[indexName] instanceof Index) {
+          const ii: Index<string, string, any, any> = (si as any)[indexName];
+          const indexVersionAdded = ii.options?.versionAdded ?? 0;
+          if (
+            indexVersionAdded > oldVersion ||
+            storeVersionAdded > oldVersion
+          ) {
+            s.createIndex(ii.indexName, ii.keyPath, ii.options);
           }
         }
       }
     }
-  };
+  }
+}
 
-  return openDatabase(
+function onTalerDbUpgradeNeeded(
+  db: IDBDatabase,
+  oldVersion: number,
+  newVersion: number,
+  upgradeTransaction: IDBTransaction,
+) {
+  upgradeFromStoreMap(Stores, db, oldVersion, newVersion, upgradeTransaction);
+}
+
+function onMetaDbUpgradeNeeded(
+  db: IDBDatabase,
+  oldVersion: number,
+  newVersion: number,
+  upgradeTransaction: IDBTransaction,
+) {
+  upgradeFromStoreMap(
+    MetaStores,
+    db,
+    oldVersion,
+    newVersion,
+    upgradeTransaction,
+  );
+}
+
+/**
+ * Return a promise that resolves
+ * to the taler wallet db.
+ */
+export async function openTalerDatabase(
+  idbFactory: IDBFactory,
+  onVersionChange: () => void,
+): Promise<Database<typeof Stores>> {
+  const metaDbHandle = await openDatabase(
+    idbFactory,
+    TALER_META_DB_NAME,
+    1,
+    () => {},
+    onMetaDbUpgradeNeeded,
+  );
+
+  const metaDb = new Database(metaDbHandle, MetaStores);
+  let currentMainVersion: string | undefined;
+  await metaDb.runWithWriteTransaction([MetaStores.metaConfig], async (tx) => {
+    const dbVersionRecord = await tx.get(
+      MetaStores.metaConfig,
+      CURRENT_DB_CONFIG_KEY,
+    );
+    if (!dbVersionRecord) {
+      currentMainVersion = TALER_DB_NAME;
+      await tx.put(MetaStores.metaConfig, {
+        key: CURRENT_DB_CONFIG_KEY,
+        value: TALER_DB_NAME,
+      });
+    } else {
+      currentMainVersion = dbVersionRecord.key;
+    }
+  });
+
+  if (currentMainVersion !== TALER_DB_NAME) {
+    // In the future, the migration logic will be implemented here.
+    throw Error(`migration from database ${currentMainVersion} not supported`);
+  }
+
+  const mainDbHandle = await openDatabase(
     idbFactory,
     TALER_DB_NAME,
     WALLET_DB_MINOR_VERSION,
     onVersionChange,
-    onUpgradeNeeded,
+    onTalerDbUpgradeNeeded,
   );
+
+  return new Database(mainDbHandle, Stores);
 }
 
 export function deleteTalerDatabase(idbFactory: IDBFactory): void {
diff --git a/packages/taler-wallet-core/src/headless/helpers.ts b/packages/taler-wallet-core/src/headless/helpers.ts
index 30b670032..3d380ad49 100644
--- a/packages/taler-wallet-core/src/headless/helpers.ts
+++ b/packages/taler-wallet-core/src/headless/helpers.ts
@@ -34,6 +34,7 @@ import { NodeHttpLib } from "./NodeHttpLib";
 import { Logger } from "../util/logging";
 import { SynchronousCryptoWorkerFactory } from "../crypto/workers/synchronousWorker";
 import type { IDBFactory } from "idb-bridge/lib/idbtypes";
+import { Stores } from "../types/dbTypes";
 
 const logger = new Logger("headless/helpers.ts");
 
@@ -149,9 +150,7 @@ export async function getDefaultNodeWallet(
     workerFactory = new SynchronousCryptoWorkerFactory();
   }
 
-  const dbWrap = new Database(myDb);
-
-  const w = new Wallet(dbWrap, myHttpLib, workerFactory);
+  const w = new Wallet(myDb, myHttpLib, workerFactory);
   if (args.notifyHandler) {
     w.addNotificationListener(args.notifyHandler);
   }
diff --git a/packages/taler-wallet-core/src/index.ts b/packages/taler-wallet-core/src/index.ts
index 3d52ed762..c446a0ffa 100644
--- a/packages/taler-wallet-core/src/index.ts
+++ b/packages/taler-wallet-core/src/index.ts
@@ -34,6 +34,7 @@ export {
 export * from "./operations/versions";
 
 export * from "./db";
+export * from "./types/dbTypes";
 
 // Internationalization
 export * from "./i18n";
diff --git a/packages/taler-wallet-core/src/operations/exchanges.ts b/packages/taler-wallet-core/src/operations/exchanges.ts
index 52da6be62..7cc4fe101 100644
--- a/packages/taler-wallet-core/src/operations/exchanges.ts
+++ b/packages/taler-wallet-core/src/operations/exchanges.ts
@@ -29,6 +29,7 @@ import {
   DenominationStatus,
   WireFee,
   ExchangeUpdateReason,
+  MetaStores,
 } from "../types/dbTypes";
 import { canonicalizeBaseUrl } from "../util/helpers";
 import * as Amounts from "../util/amounts";
diff --git a/packages/taler-wallet-core/src/operations/state.ts b/packages/taler-wallet-core/src/operations/state.ts
index 1733f13bb..60aee4c3f 100644
--- a/packages/taler-wallet-core/src/operations/state.ts
+++ b/packages/taler-wallet-core/src/operations/state.ts
@@ -23,6 +23,7 @@ import { PendingOperationsResponse } from "../types/pendingTypes";
 import { WalletNotification } from "../types/notifications";
 import { Database } from "../util/query";
 import { openPromise, OpenedPromise } from "../util/promiseUtils";
+import { Stores } from "../types/dbTypes";
 
 type NotificationListener = (n: WalletNotification) => void;
 
@@ -59,7 +60,7 @@ export class InternalWalletState {
     // the actual value nullable. 
     // Check if we are in a DB migration / garbage collection
     // and throw an error in that case.
-    public db: Database,
+    public db: Database<typeof Stores>,
     public http: HttpRequestLibrary,
     cryptoWorkerFactory: CryptoWorkerFactory,
   ) {
diff --git a/packages/taler-wallet-core/src/types/dbTypes.ts b/packages/taler-wallet-core/src/types/dbTypes.ts
index f55dcb2f6..0cfc8801b 100644
--- a/packages/taler-wallet-core/src/types/dbTypes.ts
+++ b/packages/taler-wallet-core/src/types/dbTypes.ts
@@ -1591,9 +1591,6 @@ class TipsStore extends Store<"tips", TipRecord> {
     this,
     "tipsByMerchantTipIdAndOriginIndex",
     ["merchantTipId", "merchantBaseUrl"],
-    {
-      versionAdded: 2,
-    },
   );
 }
 
@@ -1657,7 +1654,7 @@ class BackupProvidersStore extends Store<
   BackupProviderRecord
 > {
   constructor() {
-    super("backupProviders", { keyPath: "baseUrl", versionAdded: 3 });
+    super("backupProviders", { keyPath: "baseUrl" });
   }
 }
 
@@ -1688,3 +1685,13 @@ export const Stores = {
   bankWithdrawUris: new BankWithdrawUrisStore(),
   backupProviders: new BackupProvidersStore(),
 };
+
+export class MetaConfigStore extends Store<"metaConfig", ConfigRecord<any>> {
+  constructor() {
+    super("metaConfig", { keyPath: "key" });
+  }
+}
+
+export const MetaStores = {
+  metaConfig: new MetaConfigStore(),
+};
diff --git a/packages/taler-wallet-core/src/util/query.ts b/packages/taler-wallet-core/src/util/query.ts
index 35aab81e9..fdcab4fa1 100644
--- a/packages/taler-wallet-core/src/util/query.ts
+++ b/packages/taler-wallet-core/src/util/query.ts
@@ -269,6 +269,8 @@ class ResultStream<T> {
   }
 }
 
+export type AnyStoreMap = { [s: string]: Store<any, any> };
+
 type StoreName<S> = S extends Store<infer N, any> ? N : never;
 type StoreContent<S> = S extends Store<any, infer R> ? R : never;
 type IndexRecord<Ind> = Ind extends Index<any, any, any, infer R> ? R : never;
@@ -462,8 +464,7 @@ export class Index<
 }
 
 /**
- * Return a promise that resolves
- * to the taler wallet db.
+ * Return a promise that resolves to the opened IndexedDB database.
  */
 export function openDatabase(
   idbFactory: IDBFactory,
@@ -480,7 +481,7 @@ export function openDatabase(
   return new Promise<IDBDatabase>((resolve, reject) => {
     const req = idbFactory.open(databaseName, databaseVersion);
     req.onerror = (e) => {
-      logger.error("taler database error", e);
+      logger.error("database error", e);
       reject(new Error("database error"));
     };
     req.onsuccess = (e) => {
@@ -508,8 +509,8 @@ export function openDatabase(
   });
 }
 
-export class Database {
-  constructor(private db: IDBDatabase) {}
+export class Database<StoreMap extends AnyStoreMap> {
+  constructor(private db: IDBDatabase, stores: StoreMap) {}
 
   static deleteDatabase(idbFactory: IDBFactory, dbName: string): void {
     idbFactory.deleteDatabase(dbName);
@@ -571,10 +572,10 @@ export class Database {
     });
   }
 
-  async get<N extends string, T>(
-    store: Store<N, T>,
+  async get<N extends keyof StoreMap, S extends StoreMap[N]>(
+    store: S,
     key: IDBValidKey,
-  ): Promise<T | undefined> {
+  ): Promise<StoreContent<S> | undefined> {
     const tx = this.db.transaction([store.name], "readonly");
     const req = tx.objectStore(store.name).get(key);
     const v = await requestToPromise(req);
@@ -634,14 +635,22 @@ export class Database {
     return new ResultStream<IndexRecord<Ind>>(req);
   }
 
-  async runWithReadTransaction<T, StoreTypes extends Store<string, any>>(
+  async runWithReadTransaction<
+    T,
+    N extends keyof StoreMap,
+    StoreTypes extends StoreMap[N]
+  >(
     stores: StoreTypes[],
     f: (t: TransactionHandle<StoreTypes>) => Promise<T>,
   ): Promise<T> {
     return runWithTransaction<T, StoreTypes>(this.db, stores, f, "readonly");
   }
 
-  async runWithWriteTransaction<T, StoreTypes extends Store<string, any>>(
+  async runWithWriteTransaction<
+    T,
+    N extends keyof StoreMap,
+    StoreTypes extends StoreMap[N]
+  >(
     stores: StoreTypes[],
     f: (t: TransactionHandle<StoreTypes>) => Promise<T>,
   ): Promise<T> {
diff --git a/packages/taler-wallet-core/src/wallet.ts b/packages/taler-wallet-core/src/wallet.ts
index 56e3d82d1..631ac9509 100644
--- a/packages/taler-wallet-core/src/wallet.ts
+++ b/packages/taler-wallet-core/src/wallet.ts
@@ -24,7 +24,7 @@
  */
 import { CryptoWorkerFactory } from "./crypto/workers/cryptoApi";
 import { HttpRequestLibrary } from "./util/http";
-import { Database } from "./util/query";
+import { Database, Store } from "./util/query";
 
 import { Amounts, AmountJson } from "./util/amounts";
 
@@ -52,6 +52,7 @@ import {
   ReserveRecordStatus,
   CoinSourceType,
   RefundState,
+  MetaStores,
 } from "./types/dbTypes";
 import { CoinDumpJson, WithdrawUriInfoResponse } from "./types/talerTypes";
 import {
@@ -200,12 +201,12 @@ export class Wallet {
   private stopped = false;
   private memoRunRetryLoop = new AsyncOpMemoSingle<void>();
 
-  get db(): Database {
+  get db(): Database<typeof Stores> {
     return this.ws.db;
   }
 
   constructor(
-    db: Database,
+    db: Database<typeof Stores>,
     http: HttpRequestLibrary,
     cryptoWorkerFactory: CryptoWorkerFactory,
   ) {
diff --git a/packages/taler-wallet-webextension/src/wxBackend.ts b/packages/taler-wallet-webextension/src/wxBackend.ts
index e1dcdde49..95cd5f021 100644
--- a/packages/taler-wallet-webextension/src/wxBackend.ts
+++ b/packages/taler-wallet-webextension/src/wxBackend.ts
@@ -24,7 +24,6 @@
  * Imports.
  */
 import { isFirefox, getPermissionsApi } from "./compat";
-import MessageSender = chrome.runtime.MessageSender;
 import { extendedPermissions } from "./permissions";
 
 import {
@@ -40,6 +39,7 @@ import {
   CoreApiResponse,
   WalletDiagnostics,
   CoreApiResponseSuccess,
+  Stores,
 } from "taler-wallet-core";
 import { BrowserHttpLib } from "./browserHttpLib";
 import { BrowserCryptoWorkerFactory } from "./browserCryptoWorkerFactory";
@@ -50,7 +50,7 @@ import { BrowserCryptoWorkerFactory } from "./browserCryptoWorkerFactory";
  */
 let currentWallet: Wallet | undefined;
 
-let currentDatabase: IDBDatabase | undefined;
+let currentDatabase: Database<typeof Stores> | undefined;
 
 /**
  * Last version if an outdated DB, if applicable.
@@ -135,7 +135,7 @@ async function dispatch(
         setupHeaderListener();
         r = wrapResponse({ newValue: true });
       } else {
-        await new Promise((resolve, reject) => {
+        await new Promise<void>((resolve, reject) => {
           getPermissionsApi().remove(extendedPermissions, (rem) => {
             console.log("permissions removed:", rem);
             resolve();
@@ -246,7 +246,7 @@ async function reinitWallet(): Promise<void> {
   const http = new BrowserHttpLib();
   console.log("setting wallet");
   const wallet = new Wallet(
-    new Database(currentDatabase),
+    currentDatabase,
     http,
     new BrowserCryptoWorkerFactory(),
   );