/*
This file is part of TALER
(C) 2016 GNUnet e.V.
TALER is free software; you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
Foundation; either version 3, or (at your option) any later version.
TALER is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
TALER; see the file COPYING. If not, see
*/
/**
* Database query abstractions.
* @module Query
* @author Florian Dold
*/
/**
* Imports.
*/
import { openPromise } from "./promiseUtils.js";
import {
IDBRequest,
IDBTransaction,
IDBValidKey,
IDBDatabase,
IDBFactory,
IDBVersionChangeEvent,
IDBCursor,
IDBKeyPath,
IDBKeyRange,
} from "@gnu-taler/idb-bridge";
import { Codec, Logger, j2s } from "@gnu-taler/taler-util";
const logger = new Logger("query.ts");
/**
* Exception that should be thrown by client code to abort a transaction.
*/
export const TransactionAbort = Symbol("transaction_abort");
/**
* Options for an index.
*/
export interface IndexOptions {
/**
* If true and the path resolves to an array, create an index entry for
* each member of the array (instead of one index entry containing the full array).
*
* Defaults to false.
*/
multiEntry?: boolean;
/**
* Database version that this store was added in, or
* undefined if added in the first version.
*/
versionAdded?: number;
/**
* Does this index enforce unique keys?
*
* Defaults to false.
*/
unique?: boolean;
}
function requestToPromise(req: IDBRequest): Promise {
const stack = Error("Failed request was started here.");
return new Promise((resolve, reject) => {
req.onsuccess = () => {
resolve(req.result);
};
req.onerror = () => {
console.error("error in DB request", req.error);
reject(req.error);
console.error("Request failed:", stack);
};
});
}
type CursorResult = CursorEmptyResult | CursorValueResult;
interface CursorEmptyResult {
hasValue: false;
}
interface CursorValueResult {
hasValue: true;
value: T;
}
class TransactionAbortedError extends Error {
constructor(m: string) {
super(m);
// Set the prototype explicitly.
Object.setPrototypeOf(this, TransactionAbortedError.prototype);
}
}
class ResultStream {
private currentPromise: Promise;
private gotCursorEnd = false;
private awaitingResult = false;
constructor(private req: IDBRequest) {
this.awaitingResult = true;
let p = openPromise();
this.currentPromise = p.promise;
req.onsuccess = () => {
if (!this.awaitingResult) {
throw Error("BUG: invariant violated");
}
const cursor = req.result;
if (cursor) {
this.awaitingResult = false;
p.resolve();
p = openPromise();
this.currentPromise = p.promise;
} else {
this.gotCursorEnd = true;
p.resolve();
}
};
req.onerror = () => {
p.reject(req.error);
};
}
async toArray(): Promise {
const arr: T[] = [];
while (true) {
const x = await this.next();
if (x.hasValue) {
arr.push(x.value);
} else {
break;
}
}
return arr;
}
async map(f: (x: T) => R): Promise {
const arr: R[] = [];
while (true) {
const x = await this.next();
if (x.hasValue) {
arr.push(f(x.value));
} else {
break;
}
}
return arr;
}
async mapAsync(f: (x: T) => Promise): Promise {
const arr: R[] = [];
while (true) {
const x = await this.next();
if (x.hasValue) {
arr.push(await f(x.value));
} else {
break;
}
}
return arr;
}
async forEachAsync(f: (x: T) => Promise): Promise {
while (true) {
const x = await this.next();
if (x.hasValue) {
await f(x.value);
} else {
break;
}
}
}
async forEach(f: (x: T) => void): Promise {
while (true) {
const x = await this.next();
if (x.hasValue) {
f(x.value);
} else {
break;
}
}
}
async filter(f: (x: T) => boolean): Promise {
const arr: T[] = [];
while (true) {
const x = await this.next();
if (x.hasValue) {
if (f(x.value)) {
arr.push(x.value);
}
} else {
break;
}
}
return arr;
}
async next(): Promise> {
if (this.gotCursorEnd) {
return { hasValue: false };
}
if (!this.awaitingResult) {
const cursor: IDBCursor | undefined = this.req.result;
if (!cursor) {
throw Error("assertion failed");
}
this.awaitingResult = true;
cursor.continue();
}
await this.currentPromise;
if (this.gotCursorEnd) {
return { hasValue: false };
}
const cursor = this.req.result;
if (!cursor) {
throw Error("assertion failed");
}
return { hasValue: true, value: cursor.value };
}
}
/**
* Return a promise that resolves to the opened IndexedDB database.
*/
export function openDatabase(
idbFactory: IDBFactory,
databaseName: string,
databaseVersion: number | undefined,
onVersionChange: () => void,
onUpgradeNeeded: (
db: IDBDatabase,
oldVersion: number,
newVersion: number,
upgradeTransaction: IDBTransaction,
) => void,
): Promise {
return new Promise((resolve, reject) => {
const req = idbFactory.open(databaseName, databaseVersion);
req.onerror = (e) => {
logger.error("database error", e);
reject(new Error("database error"));
};
req.onsuccess = (e) => {
req.result.onversionchange = (evt: IDBVersionChangeEvent) => {
logger.info(
`handling versionchange on ${databaseName} from ${evt.oldVersion} to ${evt.newVersion}`,
);
req.result.close();
onVersionChange();
};
resolve(req.result);
};
req.onupgradeneeded = (e) => {
const db = req.result;
const newVersion = e.newVersion;
if (!newVersion) {
throw Error("upgrade needed, but new version unknown");
}
const transaction = req.transaction;
if (!transaction) {
throw Error("no transaction handle available in upgrade handler");
}
logger.info(
`handling upgradeneeded event on ${databaseName} from ${e.oldVersion} to ${e.newVersion}`,
);
onUpgradeNeeded(db, e.oldVersion, newVersion, transaction);
};
});
}
export interface IndexDescriptor {
name: string;
keyPath: IDBKeyPath | IDBKeyPath[];
multiEntry?: boolean;
unique?: boolean;
versionAdded?: number;
}
export interface StoreDescriptor {
_dummy: undefined & RecordType;
keyPath?: IDBKeyPath | IDBKeyPath[];
autoIncrement?: boolean;
/**
* Database version that this store was added in, or
* undefined if added in the first version.
*/
versionAdded?: number;
}
export interface StoreOptions {
keyPath?: IDBKeyPath | IDBKeyPath[];
autoIncrement?: boolean;
/**
* First minor database version that this store was added in, or
* undefined if added in the first version.
*/
versionAdded?: number;
}
export function describeContents(
options: StoreOptions,
): StoreDescriptor {
return {
keyPath: options.keyPath,
_dummy: undefined as any,
autoIncrement: options.autoIncrement,
versionAdded: options.versionAdded,
};
}
export function describeIndex(
name: string,
keyPath: IDBKeyPath | IDBKeyPath[],
options: IndexOptions = {},
): IndexDescriptor {
return {
keyPath,
name,
multiEntry: options.multiEntry,
unique: options.unique,
versionAdded: options.versionAdded,
};
}
interface IndexReadOnlyAccessor {
iter(query?: IDBKeyRange | IDBValidKey): ResultStream;
get(query: IDBValidKey): Promise;
getAll(
query?: IDBKeyRange | IDBValidKey,
count?: number,
): Promise;
count(query?: IDBValidKey): Promise;
}
type GetIndexReadOnlyAccess = {
[P in keyof IndexMap]: IndexReadOnlyAccessor;
};
interface IndexReadWriteAccessor {
iter(query: IDBKeyRange | IDBValidKey): ResultStream;
get(query: IDBValidKey): Promise;
getAll(
query?: IDBKeyRange | IDBValidKey,
count?: number,
): Promise;
count(query?: IDBValidKey): Promise;
}
type GetIndexReadWriteAccess = {
[P in keyof IndexMap]: IndexReadWriteAccessor;
};
export interface StoreReadOnlyAccessor {
get(key: IDBValidKey): Promise;
iter(query?: IDBValidKey): ResultStream;
indexes: GetIndexReadOnlyAccess;
}
export interface InsertResponse {
/**
* Key of the newly inserted (via put/add) record.
*/
key: IDBValidKey;
}
export interface StoreReadWriteAccessor {
get(key: IDBValidKey): Promise;
iter(query?: IDBValidKey): ResultStream;
put(r: RecordType, key?: IDBValidKey): Promise;
add(r: RecordType, key?: IDBValidKey): Promise;
delete(key: IDBValidKey): Promise;
indexes: GetIndexReadWriteAccess;
}
export interface StoreWithIndexes<
StoreName extends string,
RecordType,
IndexMap,
> {
storeName: StoreName;
store: StoreDescriptor;
indexMap: IndexMap;
/**
* Type marker symbol, to check that the descriptor
* has been created through the right function.
*/
mark: Symbol;
}
const storeWithIndexesSymbol = Symbol("StoreWithIndexesMark");
export function describeStore(
name: StoreName,
s: StoreDescriptor,
m: IndexMap,
): StoreWithIndexes {
return {
storeName: name,
store: s,
indexMap: m,
mark: storeWithIndexesSymbol,
};
}
export function describeStoreV2<
StoreName extends string,
RecordType,
IndexMap extends { [x: string]: IndexDescriptor } = {},
>(args: {
storeName: StoreName;
recordCodec: Codec;
keyPath?: IDBKeyPath | IDBKeyPath[];
autoIncrement?: boolean;
/**
* Database version that this store was added in, or
* undefined if added in the first version.
*/
versionAdded?: number;
indexes?: IndexMap;
}): StoreWithIndexes {
return {
storeName: args.storeName,
store: {
_dummy: undefined as any,
autoIncrement: args.autoIncrement,
keyPath: args.keyPath,
versionAdded: args.versionAdded,
},
indexMap: args.indexes ?? ({} as IndexMap),
mark: storeWithIndexesSymbol,
};
}
type KeyPathComponents = string | number;
/**
* Follow a key path (dot-separated) in an object.
*/
type DerefKeyPath = P extends `${infer PX extends keyof T &
KeyPathComponents}`
? T[PX]
: P extends `${infer P0 extends keyof T & KeyPathComponents}.${infer Rest}`
? DerefKeyPath
: unknown;
/**
* Return a path if it is a valid dot-separate path to an object.
* Otherwise, return "never".
*/
type ValidateKeyPath = P extends `${infer PX extends keyof T &
KeyPathComponents}`
? PX
: P extends `${infer P0 extends keyof T & KeyPathComponents}.${infer Rest}`
? `${P0}.${ValidateKeyPath}`
: never;
// function foo(
// x: T,
// p: P extends ValidateKeyPath ? P : never,
// ): void {}
// foo({x: [0,1,2]}, "x.0");
export type GetReadOnlyAccess = {
[P in keyof BoundStores]: BoundStores[P] extends StoreWithIndexes<
infer StoreName,
infer RecordType,
infer IndexMap
>
? StoreReadOnlyAccessor
: unknown;
};
export type StoreNames = StoreMap extends {
[P in keyof StoreMap]: StoreWithIndexes;
}
? keyof StoreMap
: unknown;
export type DbReadOnlyTransaction<
StoreMap,
Stores extends StoreNames & string,
> = StoreMap extends {
[P in Stores]: StoreWithIndexes;
}
? {
[P in Stores]: StoreMap[P] extends StoreWithIndexes<
infer StoreName,
infer RecordType,
infer IndexMap
>
? StoreReadOnlyAccessor
: unknown;
}
: unknown;
export type DbReadWriteTransaction<
StoreMap,
Stores extends StoreNames & string,
> = StoreMap extends {
[P in Stores]: StoreWithIndexes;
}
? {
[P in Stores]: StoreMap[P] extends StoreWithIndexes<
infer StoreName,
infer RecordType,
infer IndexMap
>
? StoreReadWriteAccessor
: unknown;
}
: unknown;
export type GetReadWriteAccess = {
[P in keyof BoundStores]: BoundStores[P] extends StoreWithIndexes<
infer StoreName,
infer RecordType,
infer IndexMap
>
? StoreReadWriteAccessor
: unknown;
};
type ReadOnlyTransactionFunction = (
t: GetReadOnlyAccess,
rawTx: IDBTransaction,
) => Promise;
type ReadWriteTransactionFunction = (
t: GetReadWriteAccess,
rawTx: IDBTransaction,
) => Promise;
export type DbReadWriteTransactionArr<
StoreMap,
StoresArr extends Array>,
> = StoreMap extends {
[P in string]: StoreWithIndexes;
}
? {
[X in StoresArr[number] &
keyof StoreMap]: StoreMap[X] extends StoreWithIndexes<
infer _StoreName,
infer RecordType,
infer IndexMap
>
? StoreReadWriteAccessor
: unknown;
}
: never;
export type DbReadOnlyTransactionArr<
StoreMap,
StoresArr extends Array>,
> = StoreMap extends {
[P in string]: StoreWithIndexes;
}
? {
[X in StoresArr[number] &
keyof StoreMap]: StoreMap[X] extends StoreWithIndexes<
infer _StoreName,
infer RecordType,
infer IndexMap
>
? StoreReadOnlyAccessor
: unknown;
}
: never;
export interface DbTransactionContext {
runReadWrite(f: ReadWriteTransactionFunction): Promise;
runReadOnly(f: ReadOnlyTransactionFunction): Promise;
}
/**
* Convert the type of an array to a union of the contents.
*
* Example:
* Input ["foo", "bar"]
* Output "foo" | "bar"
*/
export type UnionFromArray = Arr extends {
[X in keyof Arr]: Arr[X] & string;
}
? Arr[keyof Arr & number]
: unknown;
function runTx(
tx: IDBTransaction,
arg: Arg,
f: (t: Arg, t2: IDBTransaction) => Promise,
): Promise {
const stack = Error("Failed transaction was started here.");
return new Promise((resolve, reject) => {
let funResult: any = undefined;
let gotFunResult = false;
let transactionException: any = undefined;
tx.oncomplete = () => {
// This is a fatal error: The transaction completed *before*
// the transaction function returned. Likely, the transaction
// function waited on a promise that is *not* resolved in the
// microtask queue, thus triggering the auto-commit behavior.
// Unfortunately, the auto-commit behavior of IDB can't be switched
// of. There are some proposals to add this functionality in the future.
if (!gotFunResult) {
const msg =
"BUG: transaction closed before transaction function returned";
logger.error(msg);
logger.error(`${stack.stack}`);
reject(Error(msg));
}
resolve(funResult);
};
tx.onerror = () => {
logger.error("error in transaction");
logger.error(`${stack.stack}`);
};
tx.onabort = () => {
let msg: string;
if (tx.error) {
msg = `Transaction aborted (transaction error): ${tx.error}`;
} else if (transactionException !== undefined) {
msg = `Transaction aborted (exception thrown): ${transactionException}`;
} else {
msg = "Transaction aborted (no DB error)";
}
logger.error(msg);
logger.error(`${stack.stack}`);
reject(new TransactionAbortedError(msg));
};
const resP = Promise.resolve().then(() => f(arg, tx));
resP
.then((result) => {
gotFunResult = true;
funResult = result;
})
.catch((e) => {
if (e == TransactionAbort) {
logger.trace("aborting transaction");
} else {
transactionException = e;
console.error("Transaction failed:", e);
console.error(stack);
tx.abort();
}
})
.catch((e) => {
console.error("fatal: aborting transaction failed", e);
});
});
}
function makeReadContext(
tx: IDBTransaction,
storePick: { [n: string]: StoreWithIndexes },
): any {
const ctx: { [s: string]: StoreReadOnlyAccessor } = {};
for (const storeAlias in storePick) {
const indexes: { [s: string]: IndexReadOnlyAccessor } = {};
const swi = storePick[storeAlias];
const storeName = swi.storeName;
for (const indexAlias in storePick[storeAlias].indexMap) {
const indexDescriptor: IndexDescriptor =
storePick[storeAlias].indexMap[indexAlias];
const indexName = indexDescriptor.name;
indexes[indexAlias] = {
get(key) {
const req = tx.objectStore(storeName).index(indexName).get(key);
return requestToPromise(req);
},
iter(query) {
const req = tx
.objectStore(storeName)
.index(indexName)
.openCursor(query);
return new ResultStream(req);
},
getAll(query, count) {
const req = tx
.objectStore(storeName)
.index(indexName)
.getAll(query, count);
return requestToPromise(req);
},
count(query) {
const req = tx.objectStore(storeName).index(indexName).count(query);
return requestToPromise(req);
},
};
}
ctx[storeAlias] = {
indexes,
get(key) {
const req = tx.objectStore(storeName).get(key);
return requestToPromise(req);
},
iter(query) {
const req = tx.objectStore(storeName).openCursor(query);
return new ResultStream(req);
},
};
}
return ctx;
}
function makeWriteContext(
tx: IDBTransaction,
storePick: { [n: string]: StoreWithIndexes },
): any {
const ctx: { [s: string]: StoreReadWriteAccessor } = {};
for (const storeAlias in storePick) {
const indexes: { [s: string]: IndexReadWriteAccessor } = {};
const swi = storePick[storeAlias];
const storeName = swi.storeName;
for (const indexAlias in storePick[storeAlias].indexMap) {
const indexDescriptor: IndexDescriptor =
storePick[storeAlias].indexMap[indexAlias];
const indexName = indexDescriptor.name;
indexes[indexAlias] = {
get(key) {
const req = tx.objectStore(storeName).index(indexName).get(key);
return requestToPromise(req);
},
iter(query) {
const req = tx
.objectStore(storeName)
.index(indexName)
.openCursor(query);
return new ResultStream(req);
},
getAll(query, count) {
const req = tx
.objectStore(storeName)
.index(indexName)
.getAll(query, count);
return requestToPromise(req);
},
count(query) {
const req = tx.objectStore(storeName).index(indexName).count(query);
return requestToPromise(req);
},
};
}
ctx[storeAlias] = {
indexes,
get(key) {
const req = tx.objectStore(storeName).get(key);
return requestToPromise(req);
},
iter(query) {
const req = tx.objectStore(storeName).openCursor(query);
return new ResultStream(req);
},
async add(r, k) {
const req = tx.objectStore(storeName).add(r, k);
const key = await requestToPromise(req);
return {
key: key,
};
},
async put(r, k) {
const req = tx.objectStore(storeName).put(r, k);
const key = await requestToPromise(req);
return {
key: key,
};
},
delete(k) {
const req = tx.objectStore(storeName).delete(k);
return requestToPromise(req);
},
};
}
return ctx;
}
type StoreNamesOf = X extends { [x: number]: infer F }
? F extends { storeName: infer I }
? I
: never
: never;
/**
* Type-safe access to a database with a particular store map.
*
* A store map is the metadata that describes the store.
*/
export class DbAccess {
constructor(
private db: IDBDatabase,
private stores: StoreMap,
) {}
idbHandle(): IDBDatabase {
return this.db;
}
/**
* Run a transaction with all object stores.
*/
mktxAll(): DbTransactionContext {
const storeNames: string[] = [];
const accessibleStores: { [x: string]: StoreWithIndexes } =
{};
for (let i = 0; i < this.db.objectStoreNames.length; i++) {
const sn = this.db.objectStoreNames[i];
const swi = (this.stores as any)[sn] as StoreWithIndexes;
if (!swi) {
throw Error(`store metadata not available (${sn})`);
}
storeNames.push(sn);
accessibleStores[sn] = swi;
}
const storeMapKeys = Object.keys(this.stores as any);
for (const storeMapKey of storeMapKeys) {
const swi = (this.stores as any)[storeMapKey] as StoreWithIndexes<
any,
any,
any
>;
if (!accessibleStores[swi.storeName]) {
const version = this.db.version;
throw Error(
`store '${swi.storeName}' required by schema but not in database (minver=${version})`,
);
}
}
const runReadOnly = (
txf: ReadOnlyTransactionFunction,
): Promise => {
const tx = this.db.transaction(storeNames, "readonly");
const readContext = makeReadContext(tx, accessibleStores);
return runTx(tx, readContext, txf);
};
const runReadWrite = (
txf: ReadWriteTransactionFunction,
): Promise => {
const tx = this.db.transaction(storeNames, "readwrite");
const writeContext = makeWriteContext(tx, accessibleStores);
return runTx(tx, writeContext, txf);
};
return {
runReadOnly,
runReadWrite,
};
}
runReadWriteTx>>(
storeNames: StoreNameArray,
txf: (
tx: DbReadWriteTransactionArr,
) => Promise,
): Promise {
const accessibleStores: { [x: string]: StoreWithIndexes } =
{};
const strStoreNames: string[] = [];
for (const sn of storeNames) {
const swi = (this.stores as any)[sn] as StoreWithIndexes;
strStoreNames.push(swi.storeName);
accessibleStores[swi.storeName] = swi;
}
const tx = this.db.transaction(strStoreNames, "readwrite");
const writeContext = makeWriteContext(tx, accessibleStores);
return runTx(tx, writeContext, txf);
}
runReadOnlyTx>>(
storeNames: StoreNameArray,
txf: (tx: DbReadOnlyTransactionArr) => Promise,
): Promise {
const accessibleStores: { [x: string]: StoreWithIndexes } =
{};
const strStoreNames: string[] = [];
for (const sn of storeNames) {
const swi = (this.stores as any)[sn] as StoreWithIndexes;
strStoreNames.push(swi.storeName);
accessibleStores[swi.storeName] = swi;
}
const tx = this.db.transaction(strStoreNames, "readwrite");
const readContext = makeReadContext(tx, accessibleStores);
return runTx(tx, readContext, txf);
}
/**
* Run a transaction with selected object stores.
*
* The {@link namePicker} must be a function that selects a list of object
* stores from all available object stores.
*/
mktx<
StoreNames extends keyof StoreMap,
Stores extends StoreMap[StoreNames],
StoreList extends Stores[],
BoundStores extends {
[X in StoreNamesOf]: StoreList[number] & { storeName: X };
},
>(namePicker: (x: StoreMap) => StoreList): DbTransactionContext {
const storeNames: string[] = [];
const accessibleStores: { [x: string]: StoreWithIndexes } =
{};
const storePick = namePicker(this.stores) as any;
if (typeof storePick !== "object" || storePick === null) {
throw Error();
}
for (const swiPicked of storePick) {
const swi = swiPicked as StoreWithIndexes;
if (swi.mark !== storeWithIndexesSymbol) {
throw Error("invalid store descriptor returned from selector function");
}
storeNames.push(swi.storeName);
accessibleStores[swi.storeName] = swi;
}
const runReadOnly = (
txf: ReadOnlyTransactionFunction,
): Promise => {
const tx = this.db.transaction(storeNames, "readonly");
const readContext = makeReadContext(tx, accessibleStores);
return runTx(tx, readContext, txf);
};
const runReadWrite = (
txf: ReadWriteTransactionFunction,
): Promise => {
const tx = this.db.transaction(storeNames, "readwrite");
const writeContext = makeWriteContext(tx, accessibleStores);
return runTx(tx, writeContext, txf);
};
return {
runReadOnly,
runReadWrite,
};
}
}