wallet-core: implement database fixups

6 files changed, 289 insertions, 251 deletions

diff --git a/packages/taler-wallet-core/src/db-utils.ts b/packages/taler-wallet-core/src/db-utils.ts
deleted file mode 100644
index fe39a0fda..000000000
--- a/packages/taler-wallet-core/src/db-utils.ts
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- This file is part of GNU Taler
- (C) 2021 Taler Systems S.A.
-
- GNU Taler is free software; you can redistribute it and/or modify it under the
- terms of the GNU General Public License as published by the Free Software
- Foundation; either version 3, or (at your option) any later version.
-
- GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along with
- GNU Taler; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
- */
-
-/**
- * Imports.
- */
-import { IDBDatabase, IDBFactory, IDBTransaction } from "@gnu-taler/idb-bridge";
-import { Logger } from "@gnu-taler/taler-util";
-import {
-  CURRENT_DB_CONFIG_KEY,
-  TALER_DB_NAME,
-  TALER_META_DB_NAME,
-  walletMetadataStore,
-  WalletStoresV1,
-  WALLET_DB_MINOR_VERSION,
-} from "./db.js";
-import {
-  DbAccess,
-  IndexDescriptor,
-  openDatabase,
-  StoreDescriptor,
-  StoreWithIndexes,
-} from "./util/query.js";
-
-const logger = new Logger("db-utils.ts");
-
-function upgradeFromStoreMap(
-  storeMap: any,
-  db: IDBDatabase,
-  oldVersion: number,
-  newVersion: number,
-  upgradeTransaction: IDBTransaction,
-): void {
-  if (oldVersion === 0) {
-    for (const n in storeMap) {
-      const swi: StoreWithIndexes<
-        any,
-        StoreDescriptor<unknown>,
-        any
-      > = storeMap[n];
-      const storeDesc: StoreDescriptor<unknown> = swi.store;
-      const s = db.createObjectStore(swi.storeName, {
-        autoIncrement: storeDesc.autoIncrement,
-        keyPath: storeDesc.keyPath,
-      });
-      for (const indexName in swi.indexMap as any) {
-        const indexDesc: IndexDescriptor = swi.indexMap[indexName];
-        s.createIndex(indexDesc.name, indexDesc.keyPath, {
-          multiEntry: indexDesc.multiEntry,
-          unique: indexDesc.unique,
-        });
-      }
-    }
-    return;
-  }
-  if (oldVersion === newVersion) {
-    return;
-  }
-  logger.info(`upgrading database from ${oldVersion} to ${newVersion}`);
-  for (const n in storeMap) {
-    const swi: StoreWithIndexes<any, StoreDescriptor<unknown>, any> = storeMap[
-      n
-    ];
-    const storeDesc: StoreDescriptor<unknown> = swi.store;
-    const storeAddedVersion = storeDesc.versionAdded ?? 0;
-    if (storeAddedVersion <= oldVersion) {
-      continue;
-    }
-    const s = db.createObjectStore(swi.storeName, {
-      autoIncrement: storeDesc.autoIncrement,
-      keyPath: storeDesc.keyPath,
-    });
-    for (const indexName in swi.indexMap as any) {
-      const indexDesc: IndexDescriptor = swi.indexMap[indexName];
-      const indexAddedVersion = indexDesc.versionAdded ?? 0;
-      if (indexAddedVersion <= oldVersion) {
-        continue;
-      }
-      s.createIndex(indexDesc.name, indexDesc.keyPath, {
-        multiEntry: indexDesc.multiEntry,
-        unique: indexDesc.unique,
-      });
-    }
-  }
-}
-
-function promiseFromTransaction(transaction: IDBTransaction): Promise<void> {
-  return new Promise<void>((resolve, reject) => {
-    transaction.oncomplete = () => {
-      resolve();
-    };
-    transaction.onerror = () => {
-      reject();
-    };
-  });
-}
-
-/**
- * Purge all data in the given database.
- */
-export function clearDatabase(db: IDBDatabase): Promise<void> {
-  // db.objectStoreNames is a DOMStringList, so we need to convert
-  let stores: string[] = [];
-  for (let i = 0; i < db.objectStoreNames.length; i++) {
-    stores.push(db.objectStoreNames[i]);
-  }
-  const tx = db.transaction(stores, "readwrite");
-  for (const store of stores) {
-    tx.objectStore(store).clear();
-  }
-  return promiseFromTransaction(tx);
-}
-
-function onTalerDbUpgradeNeeded(
-  db: IDBDatabase,
-  oldVersion: number,
-  newVersion: number,
-  upgradeTransaction: IDBTransaction,
-) {
-  upgradeFromStoreMap(
-    WalletStoresV1,
-    db,
-    oldVersion,
-    newVersion,
-    upgradeTransaction,
-  );
-}
-
-function onMetaDbUpgradeNeeded(
-  db: IDBDatabase,
-  oldVersion: number,
-  newVersion: number,
-  upgradeTransaction: IDBTransaction,
-) {
-  upgradeFromStoreMap(
-    walletMetadataStore,
-    db,
-    oldVersion,
-    newVersion,
-    upgradeTransaction,
-  );
-}
-
-/**
- * Return a promise that resolves
- * to the taler wallet db.
- */
-export async function openTalerDatabase(
-  idbFactory: IDBFactory,
-  onVersionChange: () => void,
-): Promise<DbAccess<typeof WalletStoresV1>> {
-  const metaDbHandle = await openDatabase(
-    idbFactory,
-    TALER_META_DB_NAME,
-    1,
-    () => {},
-    onMetaDbUpgradeNeeded,
-  );
-
-  const metaDb = new DbAccess(metaDbHandle, walletMetadataStore);
-  let currentMainVersion: string | undefined;
-  await metaDb
-    .mktx((stores) => [stores.metaConfig])
-    .runReadWrite(async (tx) => {
-      const dbVersionRecord = await tx.metaConfig.get(CURRENT_DB_CONFIG_KEY);
-      if (!dbVersionRecord) {
-        currentMainVersion = TALER_DB_NAME;
-        await tx.metaConfig.put({
-          key: CURRENT_DB_CONFIG_KEY,
-          value: TALER_DB_NAME,
-        });
-      } else {
-        currentMainVersion = dbVersionRecord.value;
-      }
-    });
-
-  if (currentMainVersion !== TALER_DB_NAME) {
-    switch (currentMainVersion) {
-      case "taler-wallet-main-v2":
-      case "taler-wallet-main-v3":
-      case "taler-wallet-main-v4": // temporary, we might migrate v4 later
-      case "taler-wallet-main-v5":
-      case "taler-wallet-main-v6":
-      case "taler-wallet-main-v7":
-      case "taler-wallet-main-v8":
-        // We consider this a pre-release
-        // development version, no migration is done.
-        await metaDb
-          .mktx((stores) => [stores.metaConfig])
-          .runReadWrite(async (tx) => {
-            await tx.metaConfig.put({
-              key: CURRENT_DB_CONFIG_KEY,
-              value: TALER_DB_NAME,
-            });
-          });
-        break;
-      default:
-        throw Error(
-          `migration from database ${currentMainVersion} not supported`,
-        );
-    }
-  }
-
-  const mainDbHandle = await openDatabase(
-    idbFactory,
-    TALER_DB_NAME,
-    WALLET_DB_MINOR_VERSION,
-    onVersionChange,
-    onTalerDbUpgradeNeeded,
-  );
-
-  return new DbAccess(mainDbHandle, WalletStoresV1);
-}
-
-export async function deleteTalerDatabase(
-  idbFactory: IDBFactory,
-): Promise<void> {
-  return new Promise((resolve, reject) => {
-    const req = idbFactory.deleteDatabase(TALER_DB_NAME);
-    req.onerror = () => reject(req.error);
-    req.onsuccess = () => resolve();
-  });
-}
diff --git a/packages/taler-wallet-core/src/db.ts b/packages/taler-wallet-core/src/db.ts
index c56c3a9b5..ef44adc96 100644
--- a/packages/taler-wallet-core/src/db.ts
+++ b/packages/taler-wallet-core/src/db.ts
@@ -17,7 +17,12 @@
 /**
  * Imports.
  */
-import { Event, IDBDatabase } from "@gnu-taler/idb-bridge";
+import {
+  Event,
+  IDBDatabase,
+  IDBFactory,
+  IDBTransaction,
+} from "@gnu-taler/idb-bridge";
 import {
   AgeCommitmentProof,
   AmountJson,
@@ -51,13 +56,21 @@ import {
   AttentionPriority,
   AttentionInfo,
   AbsoluteTime,
+  Logger,
 } from "@gnu-taler/taler-util";
 import {
+  DbAccess,
   describeContents,
   describeIndex,
   describeStore,
+  GetReadWriteAccess,
+  IndexDescriptor,
+  openDatabase,
+  StoreDescriptor,
+  StoreWithIndexes,
 } from "./util/query.js";
 import { RetryInfo, RetryTags } from "./util/retries.js";
+import { Wallet } from "./wallet.js";
 
 /**
  * This file contains the database schema of the Taler wallet together
@@ -106,7 +119,7 @@ export const CURRENT_DB_CONFIG_KEY = "currentMainDbName";
  * backwards-compatible way or object stores and indices
  * are added.
  */
-export const WALLET_DB_MINOR_VERSION = 1;
+export const WALLET_DB_MINOR_VERSION = 2;
 
 /**
  * Ranges for operation status fields.
@@ -1327,7 +1340,6 @@ export type WgInfo =
   | WgInfoBankPeerPush
   | WgInfoBankRecoup;
 
-
 export interface WithdrawalKycPendingInfo {
   paytoHash: string;
   requirementRow: number;
@@ -2183,12 +2195,28 @@ export const WalletStoresV1 = {
     "userAttention",
     describeContents<UserAttentionRecord>({
       keyPath: ["entityId", "info.type"],
+      versionAdded: 2,
+    }),
+    {},
+  ),
+  fixups: describeStore(
+    "fixups",
+    describeContents<FixupRecord>({
+      keyPath: "fixupName",
+      versionAdded: 2,
     }),
     {},
   ),
 };
 
 /**
+ * An applied migration.
+ */
+export interface FixupRecord {
+  fixupName: string;
+}
+
+/**
  * User accounts
  */
 export interface BankAccountsRecord {
@@ -2320,3 +2348,250 @@ export async function importDb(db: IDBDatabase, object: any): Promise<void> {
   }
   throw Error("could not import database");
 }
+
+export interface FixupDescription {
+  name: string;
+  fn(tx: GetReadWriteAccess<typeof WalletStoresV1>): Promise<void>;
+}
+
+/**
+ * Manual migrations between minor versions of the DB schema.
+ */
+export const walletDbFixups: FixupDescription[] = [
+  {
+    name: "RefreshGroupRecord_currency",
+    async fn(tx): Promise<void> {
+      await tx.refreshGroups.iter().forEachAsync(async (rg) => {
+        if (rg.currency) {
+          return;
+        }
+        // Empty refresh group without input coin, delete it!
+        if (rg.inputPerCoin.length === 0) {
+          await tx.refreshGroups.delete(rg.refreshGroupId);
+          return;
+        }
+        rg.currency = Amounts.parseOrThrow(rg.inputPerCoin[0]).currency;
+        await tx.refreshGroups.put(rg);
+      });
+    },
+  },
+];
+
+const logger = new Logger("db.ts");
+
+export async function applyFixups(
+  db: DbAccess<typeof WalletStoresV1>,
+): Promise<void> {
+  await db.mktxAll().runReadWrite(async (tx) => {
+    for (const fixupInstruction of walletDbFixups) {
+      const fixupRecord = await tx.fixups.get(fixupInstruction.name);
+      if (fixupRecord) {
+        return;
+      }
+      logger.info(`applying DB fixup ${fixupInstruction.name}`);
+      await fixupInstruction.fn(tx);
+    }
+  });
+}
+
+function upgradeFromStoreMap(
+  storeMap: any,
+  db: IDBDatabase,
+  oldVersion: number,
+  newVersion: number,
+  upgradeTransaction: IDBTransaction,
+): void {
+  if (oldVersion === 0) {
+    for (const n in storeMap) {
+      const swi: StoreWithIndexes<
+        any,
+        StoreDescriptor<unknown>,
+        any
+      > = storeMap[n];
+      const storeDesc: StoreDescriptor<unknown> = swi.store;
+      const s = db.createObjectStore(swi.storeName, {
+        autoIncrement: storeDesc.autoIncrement,
+        keyPath: storeDesc.keyPath,
+      });
+      for (const indexName in swi.indexMap as any) {
+        const indexDesc: IndexDescriptor = swi.indexMap[indexName];
+        s.createIndex(indexDesc.name, indexDesc.keyPath, {
+          multiEntry: indexDesc.multiEntry,
+          unique: indexDesc.unique,
+        });
+      }
+    }
+    return;
+  }
+  if (oldVersion === newVersion) {
+    return;
+  }
+  logger.info(`upgrading database from ${oldVersion} to ${newVersion}`);
+  for (const n in storeMap) {
+    const swi: StoreWithIndexes<any, StoreDescriptor<unknown>, any> = storeMap[
+      n
+    ];
+    const storeDesc: StoreDescriptor<unknown> = swi.store;
+    const storeAddedVersion = storeDesc.versionAdded ?? 0;
+    if (storeAddedVersion <= oldVersion) {
+      continue;
+    }
+    const s = db.createObjectStore(swi.storeName, {
+      autoIncrement: storeDesc.autoIncrement,
+      keyPath: storeDesc.keyPath,
+    });
+    for (const indexName in swi.indexMap as any) {
+      const indexDesc: IndexDescriptor = swi.indexMap[indexName];
+      const indexAddedVersion = indexDesc.versionAdded ?? 0;
+      if (indexAddedVersion <= oldVersion) {
+        continue;
+      }
+      s.createIndex(indexDesc.name, indexDesc.keyPath, {
+        multiEntry: indexDesc.multiEntry,
+        unique: indexDesc.unique,
+      });
+    }
+  }
+}
+
+function promiseFromTransaction(transaction: IDBTransaction): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    transaction.oncomplete = () => {
+      resolve();
+    };
+    transaction.onerror = () => {
+      reject();
+    };
+  });
+}
+
+/**
+ * Purge all data in the given database.
+ */
+export function clearDatabase(db: IDBDatabase): Promise<void> {
+  // db.objectStoreNames is a DOMStringList, so we need to convert
+  let stores: string[] = [];
+  for (let i = 0; i < db.objectStoreNames.length; i++) {
+    stores.push(db.objectStoreNames[i]);
+  }
+  const tx = db.transaction(stores, "readwrite");
+  for (const store of stores) {
+    tx.objectStore(store).clear();
+  }
+  return promiseFromTransaction(tx);
+}
+
+function onTalerDbUpgradeNeeded(
+  db: IDBDatabase,
+  oldVersion: number,
+  newVersion: number,
+  upgradeTransaction: IDBTransaction,
+) {
+  upgradeFromStoreMap(
+    WalletStoresV1,
+    db,
+    oldVersion,
+    newVersion,
+    upgradeTransaction,
+  );
+}
+
+function onMetaDbUpgradeNeeded(
+  db: IDBDatabase,
+  oldVersion: number,
+  newVersion: number,
+  upgradeTransaction: IDBTransaction,
+) {
+  upgradeFromStoreMap(
+    walletMetadataStore,
+    db,
+    oldVersion,
+    newVersion,
+    upgradeTransaction,
+  );
+}
+
+/**
+ * Return a promise that resolves
+ * to the taler wallet db.
+ */
+export async function openTalerDatabase(
+  idbFactory: IDBFactory,
+  onVersionChange: () => void,
+): Promise<DbAccess<typeof WalletStoresV1>> {
+  const metaDbHandle = await openDatabase(
+    idbFactory,
+    TALER_META_DB_NAME,
+    1,
+    () => {},
+    onMetaDbUpgradeNeeded,
+  );
+
+  const metaDb = new DbAccess(metaDbHandle, walletMetadataStore);
+  let currentMainVersion: string | undefined;
+  await metaDb
+    .mktx((stores) => [stores.metaConfig])
+    .runReadWrite(async (tx) => {
+      const dbVersionRecord = await tx.metaConfig.get(CURRENT_DB_CONFIG_KEY);
+      if (!dbVersionRecord) {
+        currentMainVersion = TALER_DB_NAME;
+        await tx.metaConfig.put({
+          key: CURRENT_DB_CONFIG_KEY,
+          value: TALER_DB_NAME,
+        });
+      } else {
+        currentMainVersion = dbVersionRecord.value;
+      }
+    });
+
+  if (currentMainVersion !== TALER_DB_NAME) {
+    switch (currentMainVersion) {
+      case "taler-wallet-main-v2":
+      case "taler-wallet-main-v3":
+      case "taler-wallet-main-v4": // temporary, we might migrate v4 later
+      case "taler-wallet-main-v5":
+      case "taler-wallet-main-v6":
+      case "taler-wallet-main-v7":
+      case "taler-wallet-main-v8":
+        // We consider this a pre-release
+        // development version, no migration is done.
+        await metaDb
+          .mktx((stores) => [stores.metaConfig])
+          .runReadWrite(async (tx) => {
+            await tx.metaConfig.put({
+              key: CURRENT_DB_CONFIG_KEY,
+              value: TALER_DB_NAME,
+            });
+          });
+        break;
+      default:
+        throw Error(
+          `major migration from database major=${currentMainVersion} not supported`,
+        );
+    }
+  }
+
+  const mainDbHandle = await openDatabase(
+    idbFactory,
+    TALER_DB_NAME,
+    WALLET_DB_MINOR_VERSION,
+    onVersionChange,
+    onTalerDbUpgradeNeeded,
+  );
+
+  const handle = new DbAccess(mainDbHandle, WalletStoresV1);
+
+  await applyFixups(handle);
+
+  return handle;
+}
+
+export async function deleteTalerDatabase(
+  idbFactory: IDBFactory,
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const req = idbFactory.deleteDatabase(TALER_DB_NAME);
+    req.onerror = () => reject(req.error);
+    req.onsuccess = () => resolve();
+  });
+}
diff --git a/packages/taler-wallet-core/src/headless/helpers.ts b/packages/taler-wallet-core/src/headless/helpers.ts
index 64edf8fb0..fbeb84c67 100644
--- a/packages/taler-wallet-core/src/headless/helpers.ts
+++ b/packages/taler-wallet-core/src/headless/helpers.ts
@@ -34,7 +34,7 @@ import { Logger, WalletNotification } from "@gnu-taler/taler-util";
 import * as fs from "fs";
 import { NodeThreadCryptoWorkerFactory } from "../crypto/workers/nodeThreadWorker.js";
 import { SynchronousCryptoWorkerFactoryNode } from "../crypto/workers/synchronousWorkerFactoryNode.js";
-import { openTalerDatabase } from "../db-utils.js";
+import { openTalerDatabase } from "../index.js";
 import { HttpRequestLibrary } from "../util/http.js";
 import { SetTimeoutTimerAPI } from "../util/timer.js";
 import { Wallet } from "../wallet.js";
diff --git a/packages/taler-wallet-core/src/index.ts b/packages/taler-wallet-core/src/index.ts
index e48c9430f..031656a6c 100644
--- a/packages/taler-wallet-core/src/index.ts
+++ b/packages/taler-wallet-core/src/index.ts
@@ -29,7 +29,6 @@ export * from "./util/http.js";
 export * from "./versions.js";
 
 export * from "./db.js";
-export * from "./db-utils.js";
 
 // Crypto and crypto workers
 // export * from "./crypto/workers/nodeThreadWorker.js";
diff --git a/packages/taler-wallet-core/src/util/query.ts b/packages/taler-wallet-core/src/util/query.ts
index 9e960821d..4eb354f3e 100644
--- a/packages/taler-wallet-core/src/util/query.ts
+++ b/packages/taler-wallet-core/src/util/query.ts
@@ -303,7 +303,7 @@ export interface StoreOptions {
   autoIncrement?: boolean;
 
   /**
-   * Database version that this store was added in, or
+   * First minor database version that this store was added in, or
    * undefined if added in the first version.
    */
   versionAdded?: number;
diff --git a/packages/taler-wallet-core/src/wallet.ts b/packages/taler-wallet-core/src/wallet.ts
index e15c6110c..f73cdac70 100644
--- a/packages/taler-wallet-core/src/wallet.ts
+++ b/packages/taler-wallet-core/src/wallet.ts
@@ -55,7 +55,6 @@ import {
   codecForInitiatePeerPushPaymentRequest,
   codecForIntegrationTestArgs,
   codecForListKnownBankAccounts,
-  codecForUserAttentionsRequest,
   codecForPrepareDepositRequest,
   codecForPreparePayRequest,
   codecForPreparePeerPullPaymentRequest,
@@ -70,6 +69,8 @@ import {
   codecForTrackDepositGroupRequest,
   codecForTransactionByIdRequest,
   codecForTransactionsRequest,
+  codecForUserAttentionByIdRequest,
+  codecForUserAttentionsRequest,
   codecForWithdrawFakebankRequest,
   codecForWithdrawTestBalance,
   CoinDumpJson,
@@ -92,6 +93,7 @@ import {
   KnownBankAccounts,
   KnownBankAccountsInfo,
   Logger,
+  ManualWithdrawalDetails,
   NotificationType,
   parsePaytoUri,
   RefreshReason,
@@ -99,17 +101,15 @@ import {
   URL,
   WalletCoreVersion,
   WalletNotification,
-  codecForUserAttentionByIdRequest,
-  ManualWithdrawalDetails,
 } from "@gnu-taler/taler-util";
 import { TalerCryptoInterface } from "./crypto/cryptoImplementation.js";
 import {
   CryptoDispatcher,
   CryptoWorkerFactory,
 } from "./crypto/workers/crypto-dispatcher.js";
-import { clearDatabase } from "./db-utils.js";
 import {
   AuditorTrustRecord,
+  clearDatabase,
   CoinSourceType,
   ConfigRecordKey,
   DenominationRecord,
@@ -134,6 +134,11 @@ import {
   RecoupOperations,
   RefreshOperations,
 } from "./internal-wallet-state.js";
+import {
+  getUserAttentions,
+  getUserAttentionsUnreadCount,
+  markAttentionRequestAsRead,
+} from "./operations/attention.js";
 import { exportBackup } from "./operations/backup/export.js";
 import {
   addBackupProvider,
@@ -151,11 +156,6 @@ import {
 import { setWalletDeviceId } from "./operations/backup/state.js";
 import { getBalances } from "./operations/balance.js";
 import {
-  getUserAttentions,
-  getUserAttentionsUnreadCount,
-  markAttentionRequestAsRead,
-} from "./operations/attention.js";
-import {
   getExchangeTosStatus,
   makeExchangeListItem,
   runOperationWithErrorReporting,