From 668d7a213e21a776958d985b0758495d967d9f73 Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Wed, 11 Jan 2023 14:19:24 +0100 Subject: wallet-core: implement database fixups --- packages/taler-wallet-core/src/db-utils.ts | 236 ----------------- packages/taler-wallet-core/src/db.ts | 281 ++++++++++++++++++++- packages/taler-wallet-core/src/headless/helpers.ts | 2 +- packages/taler-wallet-core/src/index.ts | 1 - packages/taler-wallet-core/src/util/query.ts | 2 +- packages/taler-wallet-core/src/wallet.ts | 18 +- 6 files changed, 289 insertions(+), 251 deletions(-) delete mode 100644 packages/taler-wallet-core/src/db-utils.ts diff --git a/packages/taler-wallet-core/src/db-utils.ts b/packages/taler-wallet-core/src/db-utils.ts deleted file mode 100644 index fe39a0fda..000000000 --- a/packages/taler-wallet-core/src/db-utils.ts +++ /dev/null @@ -1,236 +0,0 @@ -/* - This file is part of GNU Taler - (C) 2021 Taler Systems S.A. - - GNU Taler is free software; you can redistribute it and/or modify it under the - terms of the GNU General Public License as published by the Free Software - Foundation; either version 3, or (at your option) any later version. - - GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR - A PARTICULAR PURPOSE. See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along with - GNU Taler; see the file COPYING. If not, see - */ - -/** - * Imports. - */ -import { IDBDatabase, IDBFactory, IDBTransaction } from "@gnu-taler/idb-bridge"; -import { Logger } from "@gnu-taler/taler-util"; -import { - CURRENT_DB_CONFIG_KEY, - TALER_DB_NAME, - TALER_META_DB_NAME, - walletMetadataStore, - WalletStoresV1, - WALLET_DB_MINOR_VERSION, -} from "./db.js"; -import { - DbAccess, - IndexDescriptor, - openDatabase, - StoreDescriptor, - StoreWithIndexes, -} from "./util/query.js"; - -const logger = new Logger("db-utils.ts"); - -function upgradeFromStoreMap( - storeMap: any, - db: IDBDatabase, - oldVersion: number, - newVersion: number, - upgradeTransaction: IDBTransaction, -): void { - if (oldVersion === 0) { - for (const n in storeMap) { - const swi: StoreWithIndexes< - any, - StoreDescriptor, - any - > = storeMap[n]; - const storeDesc: StoreDescriptor = swi.store; - const s = db.createObjectStore(swi.storeName, { - autoIncrement: storeDesc.autoIncrement, - keyPath: storeDesc.keyPath, - }); - for (const indexName in swi.indexMap as any) { - const indexDesc: IndexDescriptor = swi.indexMap[indexName]; - s.createIndex(indexDesc.name, indexDesc.keyPath, { - multiEntry: indexDesc.multiEntry, - unique: indexDesc.unique, - }); - } - } - return; - } - if (oldVersion === newVersion) { - return; - } - logger.info(`upgrading database from ${oldVersion} to ${newVersion}`); - for (const n in storeMap) { - const swi: StoreWithIndexes, any> = storeMap[ - n - ]; - const storeDesc: StoreDescriptor = swi.store; - const storeAddedVersion = storeDesc.versionAdded ?? 0; - if (storeAddedVersion <= oldVersion) { - continue; - } - const s = db.createObjectStore(swi.storeName, { - autoIncrement: storeDesc.autoIncrement, - keyPath: storeDesc.keyPath, - }); - for (const indexName in swi.indexMap as any) { - const indexDesc: IndexDescriptor = swi.indexMap[indexName]; - const indexAddedVersion = indexDesc.versionAdded ?? 0; - if (indexAddedVersion <= oldVersion) { - continue; - } - s.createIndex(indexDesc.name, indexDesc.keyPath, { - multiEntry: indexDesc.multiEntry, - unique: indexDesc.unique, - }); - } - } -} - -function promiseFromTransaction(transaction: IDBTransaction): Promise { - return new Promise((resolve, reject) => { - transaction.oncomplete = () => { - resolve(); - }; - transaction.onerror = () => { - reject(); - }; - }); -} - -/** - * Purge all data in the given database. - */ -export function clearDatabase(db: IDBDatabase): Promise { - // db.objectStoreNames is a DOMStringList, so we need to convert - let stores: string[] = []; - for (let i = 0; i < db.objectStoreNames.length; i++) { - stores.push(db.objectStoreNames[i]); - } - const tx = db.transaction(stores, "readwrite"); - for (const store of stores) { - tx.objectStore(store).clear(); - } - return promiseFromTransaction(tx); -} - -function onTalerDbUpgradeNeeded( - db: IDBDatabase, - oldVersion: number, - newVersion: number, - upgradeTransaction: IDBTransaction, -) { - upgradeFromStoreMap( - WalletStoresV1, - db, - oldVersion, - newVersion, - upgradeTransaction, - ); -} - -function onMetaDbUpgradeNeeded( - db: IDBDatabase, - oldVersion: number, - newVersion: number, - upgradeTransaction: IDBTransaction, -) { - upgradeFromStoreMap( - walletMetadataStore, - db, - oldVersion, - newVersion, - upgradeTransaction, - ); -} - -/** - * Return a promise that resolves - * to the taler wallet db. - */ -export async function openTalerDatabase( - idbFactory: IDBFactory, - onVersionChange: () => void, -): Promise> { - const metaDbHandle = await openDatabase( - idbFactory, - TALER_META_DB_NAME, - 1, - () => {}, - onMetaDbUpgradeNeeded, - ); - - const metaDb = new DbAccess(metaDbHandle, walletMetadataStore); - let currentMainVersion: string | undefined; - await metaDb - .mktx((stores) => [stores.metaConfig]) - .runReadWrite(async (tx) => { - const dbVersionRecord = await tx.metaConfig.get(CURRENT_DB_CONFIG_KEY); - if (!dbVersionRecord) { - currentMainVersion = TALER_DB_NAME; - await tx.metaConfig.put({ - key: CURRENT_DB_CONFIG_KEY, - value: TALER_DB_NAME, - }); - } else { - currentMainVersion = dbVersionRecord.value; - } - }); - - if (currentMainVersion !== TALER_DB_NAME) { - switch (currentMainVersion) { - case "taler-wallet-main-v2": - case "taler-wallet-main-v3": - case "taler-wallet-main-v4": // temporary, we might migrate v4 later - case "taler-wallet-main-v5": - case "taler-wallet-main-v6": - case "taler-wallet-main-v7": - case "taler-wallet-main-v8": - // We consider this a pre-release - // development version, no migration is done. - await metaDb - .mktx((stores) => [stores.metaConfig]) - .runReadWrite(async (tx) => { - await tx.metaConfig.put({ - key: CURRENT_DB_CONFIG_KEY, - value: TALER_DB_NAME, - }); - }); - break; - default: - throw Error( - `migration from database ${currentMainVersion} not supported`, - ); - } - } - - const mainDbHandle = await openDatabase( - idbFactory, - TALER_DB_NAME, - WALLET_DB_MINOR_VERSION, - onVersionChange, - onTalerDbUpgradeNeeded, - ); - - return new DbAccess(mainDbHandle, WalletStoresV1); -} - -export async function deleteTalerDatabase( - idbFactory: IDBFactory, -): Promise { - return new Promise((resolve, reject) => { - const req = idbFactory.deleteDatabase(TALER_DB_NAME); - req.onerror = () => reject(req.error); - req.onsuccess = () => resolve(); - }); -} diff --git a/packages/taler-wallet-core/src/db.ts b/packages/taler-wallet-core/src/db.ts index c56c3a9b5..ef44adc96 100644 --- a/packages/taler-wallet-core/src/db.ts +++ b/packages/taler-wallet-core/src/db.ts @@ -17,7 +17,12 @@ /** * Imports. */ -import { Event, IDBDatabase } from "@gnu-taler/idb-bridge"; +import { + Event, + IDBDatabase, + IDBFactory, + IDBTransaction, +} from "@gnu-taler/idb-bridge"; import { AgeCommitmentProof, AmountJson, @@ -51,13 +56,21 @@ import { AttentionPriority, AttentionInfo, AbsoluteTime, + Logger, } from "@gnu-taler/taler-util"; import { + DbAccess, describeContents, describeIndex, describeStore, + GetReadWriteAccess, + IndexDescriptor, + openDatabase, + StoreDescriptor, + StoreWithIndexes, } from "./util/query.js"; import { RetryInfo, RetryTags } from "./util/retries.js"; +import { Wallet } from "./wallet.js"; /** * This file contains the database schema of the Taler wallet together @@ -106,7 +119,7 @@ export const CURRENT_DB_CONFIG_KEY = "currentMainDbName"; * backwards-compatible way or object stores and indices * are added. */ -export const WALLET_DB_MINOR_VERSION = 1; +export const WALLET_DB_MINOR_VERSION = 2; /** * Ranges for operation status fields. @@ -1327,7 +1340,6 @@ export type WgInfo = | WgInfoBankPeerPush | WgInfoBankRecoup; - export interface WithdrawalKycPendingInfo { paytoHash: string; requirementRow: number; @@ -2183,11 +2195,27 @@ export const WalletStoresV1 = { "userAttention", describeContents({ keyPath: ["entityId", "info.type"], + versionAdded: 2, + }), + {}, + ), + fixups: describeStore( + "fixups", + describeContents({ + keyPath: "fixupName", + versionAdded: 2, }), {}, ), }; +/** + * An applied migration. + */ +export interface FixupRecord { + fixupName: string; +} + /** * User accounts */ @@ -2320,3 +2348,250 @@ export async function importDb(db: IDBDatabase, object: any): Promise { } throw Error("could not import database"); } + +export interface FixupDescription { + name: string; + fn(tx: GetReadWriteAccess): Promise; +} + +/** + * Manual migrations between minor versions of the DB schema. + */ +export const walletDbFixups: FixupDescription[] = [ + { + name: "RefreshGroupRecord_currency", + async fn(tx): Promise { + await tx.refreshGroups.iter().forEachAsync(async (rg) => { + if (rg.currency) { + return; + } + // Empty refresh group without input coin, delete it! + if (rg.inputPerCoin.length === 0) { + await tx.refreshGroups.delete(rg.refreshGroupId); + return; + } + rg.currency = Amounts.parseOrThrow(rg.inputPerCoin[0]).currency; + await tx.refreshGroups.put(rg); + }); + }, + }, +]; + +const logger = new Logger("db.ts"); + +export async function applyFixups( + db: DbAccess, +): Promise { + await db.mktxAll().runReadWrite(async (tx) => { + for (const fixupInstruction of walletDbFixups) { + const fixupRecord = await tx.fixups.get(fixupInstruction.name); + if (fixupRecord) { + return; + } + logger.info(`applying DB fixup ${fixupInstruction.name}`); + await fixupInstruction.fn(tx); + } + }); +} + +function upgradeFromStoreMap( + storeMap: any, + db: IDBDatabase, + oldVersion: number, + newVersion: number, + upgradeTransaction: IDBTransaction, +): void { + if (oldVersion === 0) { + for (const n in storeMap) { + const swi: StoreWithIndexes< + any, + StoreDescriptor, + any + > = storeMap[n]; + const storeDesc: StoreDescriptor = swi.store; + const s = db.createObjectStore(swi.storeName, { + autoIncrement: storeDesc.autoIncrement, + keyPath: storeDesc.keyPath, + }); + for (const indexName in swi.indexMap as any) { + const indexDesc: IndexDescriptor = swi.indexMap[indexName]; + s.createIndex(indexDesc.name, indexDesc.keyPath, { + multiEntry: indexDesc.multiEntry, + unique: indexDesc.unique, + }); + } + } + return; + } + if (oldVersion === newVersion) { + return; + } + logger.info(`upgrading database from ${oldVersion} to ${newVersion}`); + for (const n in storeMap) { + const swi: StoreWithIndexes, any> = storeMap[ + n + ]; + const storeDesc: StoreDescriptor = swi.store; + const storeAddedVersion = storeDesc.versionAdded ?? 0; + if (storeAddedVersion <= oldVersion) { + continue; + } + const s = db.createObjectStore(swi.storeName, { + autoIncrement: storeDesc.autoIncrement, + keyPath: storeDesc.keyPath, + }); + for (const indexName in swi.indexMap as any) { + const indexDesc: IndexDescriptor = swi.indexMap[indexName]; + const indexAddedVersion = indexDesc.versionAdded ?? 0; + if (indexAddedVersion <= oldVersion) { + continue; + } + s.createIndex(indexDesc.name, indexDesc.keyPath, { + multiEntry: indexDesc.multiEntry, + unique: indexDesc.unique, + }); + } + } +} + +function promiseFromTransaction(transaction: IDBTransaction): Promise { + return new Promise((resolve, reject) => { + transaction.oncomplete = () => { + resolve(); + }; + transaction.onerror = () => { + reject(); + }; + }); +} + +/** + * Purge all data in the given database. + */ +export function clearDatabase(db: IDBDatabase): Promise { + // db.objectStoreNames is a DOMStringList, so we need to convert + let stores: string[] = []; + for (let i = 0; i < db.objectStoreNames.length; i++) { + stores.push(db.objectStoreNames[i]); + } + const tx = db.transaction(stores, "readwrite"); + for (const store of stores) { + tx.objectStore(store).clear(); + } + return promiseFromTransaction(tx); +} + +function onTalerDbUpgradeNeeded( + db: IDBDatabase, + oldVersion: number, + newVersion: number, + upgradeTransaction: IDBTransaction, +) { + upgradeFromStoreMap( + WalletStoresV1, + db, + oldVersion, + newVersion, + upgradeTransaction, + ); +} + +function onMetaDbUpgradeNeeded( + db: IDBDatabase, + oldVersion: number, + newVersion: number, + upgradeTransaction: IDBTransaction, +) { + upgradeFromStoreMap( + walletMetadataStore, + db, + oldVersion, + newVersion, + upgradeTransaction, + ); +} + +/** + * Return a promise that resolves + * to the taler wallet db. + */ +export async function openTalerDatabase( + idbFactory: IDBFactory, + onVersionChange: () => void, +): Promise> { + const metaDbHandle = await openDatabase( + idbFactory, + TALER_META_DB_NAME, + 1, + () => {}, + onMetaDbUpgradeNeeded, + ); + + const metaDb = new DbAccess(metaDbHandle, walletMetadataStore); + let currentMainVersion: string | undefined; + await metaDb + .mktx((stores) => [stores.metaConfig]) + .runReadWrite(async (tx) => { + const dbVersionRecord = await tx.metaConfig.get(CURRENT_DB_CONFIG_KEY); + if (!dbVersionRecord) { + currentMainVersion = TALER_DB_NAME; + await tx.metaConfig.put({ + key: CURRENT_DB_CONFIG_KEY, + value: TALER_DB_NAME, + }); + } else { + currentMainVersion = dbVersionRecord.value; + } + }); + + if (currentMainVersion !== TALER_DB_NAME) { + switch (currentMainVersion) { + case "taler-wallet-main-v2": + case "taler-wallet-main-v3": + case "taler-wallet-main-v4": // temporary, we might migrate v4 later + case "taler-wallet-main-v5": + case "taler-wallet-main-v6": + case "taler-wallet-main-v7": + case "taler-wallet-main-v8": + // We consider this a pre-release + // development version, no migration is done. + await metaDb + .mktx((stores) => [stores.metaConfig]) + .runReadWrite(async (tx) => { + await tx.metaConfig.put({ + key: CURRENT_DB_CONFIG_KEY, + value: TALER_DB_NAME, + }); + }); + break; + default: + throw Error( + `major migration from database major=${currentMainVersion} not supported`, + ); + } + } + + const mainDbHandle = await openDatabase( + idbFactory, + TALER_DB_NAME, + WALLET_DB_MINOR_VERSION, + onVersionChange, + onTalerDbUpgradeNeeded, + ); + + const handle = new DbAccess(mainDbHandle, WalletStoresV1); + + await applyFixups(handle); + + return handle; +} + +export async function deleteTalerDatabase( + idbFactory: IDBFactory, +): Promise { + return new Promise((resolve, reject) => { + const req = idbFactory.deleteDatabase(TALER_DB_NAME); + req.onerror = () => reject(req.error); + req.onsuccess = () => resolve(); + }); +} diff --git a/packages/taler-wallet-core/src/headless/helpers.ts b/packages/taler-wallet-core/src/headless/helpers.ts index 64edf8fb0..fbeb84c67 100644 --- a/packages/taler-wallet-core/src/headless/helpers.ts +++ b/packages/taler-wallet-core/src/headless/helpers.ts @@ -34,7 +34,7 @@ import { Logger, WalletNotification } from "@gnu-taler/taler-util"; import * as fs from "fs"; import { NodeThreadCryptoWorkerFactory } from "../crypto/workers/nodeThreadWorker.js"; import { SynchronousCryptoWorkerFactoryNode } from "../crypto/workers/synchronousWorkerFactoryNode.js"; -import { openTalerDatabase } from "../db-utils.js"; +import { openTalerDatabase } from "../index.js"; import { HttpRequestLibrary } from "../util/http.js"; import { SetTimeoutTimerAPI } from "../util/timer.js"; import { Wallet } from "../wallet.js"; diff --git a/packages/taler-wallet-core/src/index.ts b/packages/taler-wallet-core/src/index.ts index e48c9430f..031656a6c 100644 --- a/packages/taler-wallet-core/src/index.ts +++ b/packages/taler-wallet-core/src/index.ts @@ -29,7 +29,6 @@ export * from "./util/http.js"; export * from "./versions.js"; export * from "./db.js"; -export * from "./db-utils.js"; // Crypto and crypto workers // export * from "./crypto/workers/nodeThreadWorker.js"; diff --git a/packages/taler-wallet-core/src/util/query.ts b/packages/taler-wallet-core/src/util/query.ts index 9e960821d..4eb354f3e 100644 --- a/packages/taler-wallet-core/src/util/query.ts +++ b/packages/taler-wallet-core/src/util/query.ts @@ -303,7 +303,7 @@ export interface StoreOptions { autoIncrement?: boolean; /** - * Database version that this store was added in, or + * First minor database version that this store was added in, or * undefined if added in the first version. */ versionAdded?: number; diff --git a/packages/taler-wallet-core/src/wallet.ts b/packages/taler-wallet-core/src/wallet.ts index e15c6110c..f73cdac70 100644 --- a/packages/taler-wallet-core/src/wallet.ts +++ b/packages/taler-wallet-core/src/wallet.ts @@ -55,7 +55,6 @@ import { codecForInitiatePeerPushPaymentRequest, codecForIntegrationTestArgs, codecForListKnownBankAccounts, - codecForUserAttentionsRequest, codecForPrepareDepositRequest, codecForPreparePayRequest, codecForPreparePeerPullPaymentRequest, @@ -70,6 +69,8 @@ import { codecForTrackDepositGroupRequest, codecForTransactionByIdRequest, codecForTransactionsRequest, + codecForUserAttentionByIdRequest, + codecForUserAttentionsRequest, codecForWithdrawFakebankRequest, codecForWithdrawTestBalance, CoinDumpJson, @@ -92,6 +93,7 @@ import { KnownBankAccounts, KnownBankAccountsInfo, Logger, + ManualWithdrawalDetails, NotificationType, parsePaytoUri, RefreshReason, @@ -99,17 +101,15 @@ import { URL, WalletCoreVersion, WalletNotification, - codecForUserAttentionByIdRequest, - ManualWithdrawalDetails, } from "@gnu-taler/taler-util"; import { TalerCryptoInterface } from "./crypto/cryptoImplementation.js"; import { CryptoDispatcher, CryptoWorkerFactory, } from "./crypto/workers/crypto-dispatcher.js"; -import { clearDatabase } from "./db-utils.js"; import { AuditorTrustRecord, + clearDatabase, CoinSourceType, ConfigRecordKey, DenominationRecord, @@ -134,6 +134,11 @@ import { RecoupOperations, RefreshOperations, } from "./internal-wallet-state.js"; +import { + getUserAttentions, + getUserAttentionsUnreadCount, + markAttentionRequestAsRead, +} from "./operations/attention.js"; import { exportBackup } from "./operations/backup/export.js"; import { addBackupProvider, @@ -150,11 +155,6 @@ import { } from "./operations/backup/index.js"; import { setWalletDeviceId } from "./operations/backup/state.js"; import { getBalances } from "./operations/balance.js"; -import { - getUserAttentions, - getUserAttentionsUnreadCount, - markAttentionRequestAsRead, -} from "./operations/attention.js"; import { getExchangeTosStatus, makeExchangeListItem, -- cgit v1.2.3