/*
This file is part of TALER
(C) 2016 GNUnet e.V.
TALER is free software; you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
Foundation; either version 3, or (at your option) any later version.
TALER is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
TALER; see the file COPYING. If not, see
*/
/**
* @fileoverview
* Query helpers for IndexedDB databases.
*
* @author Florian Dold
*/
/**
* Imports.
*/
import {
IDBCursor,
IDBDatabase,
IDBFactory,
IDBKeyPath,
IDBKeyRange,
IDBRequest,
IDBTransaction,
IDBTransactionMode,
IDBValidKey,
IDBVersionChangeEvent,
} from "@gnu-taler/idb-bridge";
import {
CancellationToken,
Codec,
Logger,
openPromise,
} from "@gnu-taler/taler-util";
const logger = new Logger("query.ts");
/**
* Exception that should be thrown by client code to abort a transaction.
*/
export const TransactionAbort = Symbol("transaction_abort");
/**
* Options for an index.
*/
export interface IndexOptions {
/**
* If true and the path resolves to an array, create an index entry for
* each member of the array (instead of one index entry containing the full array).
*
* Defaults to false.
*/
multiEntry?: boolean;
/**
* Database version that this store was added in, or
* undefined if added in the first version.
*/
versionAdded?: number;
/**
* Does this index enforce unique keys?
*
* Defaults to false.
*/
unique?: boolean;
}
function requestToPromise(req: IDBRequest): Promise {
const stack = Error("Failed request was started here.");
return new Promise((resolve, reject) => {
req.onsuccess = () => {
resolve(req.result);
};
req.onerror = () => {
console.error("error in DB request", req.error);
reject(req.error);
console.error("Request failed:", stack);
};
});
}
type CursorResult = CursorEmptyResult | CursorValueResult;
interface CursorEmptyResult {
hasValue: false;
}
interface CursorValueResult {
hasValue: true;
value: T;
}
class TransactionAbortedError extends Error {
constructor(m: string) {
super(m);
// Set the prototype explicitly.
Object.setPrototypeOf(this, TransactionAbortedError.prototype);
}
}
class ResultStream {
private currentPromise: Promise;
private gotCursorEnd = false;
private awaitingResult = false;
constructor(private req: IDBRequest) {
this.awaitingResult = true;
let p = openPromise();
this.currentPromise = p.promise;
req.onsuccess = () => {
if (!this.awaitingResult) {
throw Error("BUG: invariant violated");
}
const cursor = req.result;
if (cursor) {
this.awaitingResult = false;
p.resolve();
p = openPromise();
this.currentPromise = p.promise;
} else {
this.gotCursorEnd = true;
p.resolve();
}
};
req.onerror = () => {
p.reject(req.error);
};
}
async toArray(): Promise {
const arr: T[] = [];
while (true) {
const x = await this.next();
if (x.hasValue) {
arr.push(x.value);
} else {
break;
}
}
return arr;
}
async map(f: (x: T) => R): Promise {
const arr: R[] = [];
while (true) {
const x = await this.next();
if (x.hasValue) {
arr.push(f(x.value));
} else {
break;
}
}
return arr;
}
async mapAsync(f: (x: T) => Promise): Promise {
const arr: R[] = [];
while (true) {
const x = await this.next();
if (x.hasValue) {
arr.push(await f(x.value));
} else {
break;
}
}
return arr;
}
async forEachAsync(f: (x: T) => Promise): Promise {
while (true) {
const x = await this.next();
if (x.hasValue) {
await f(x.value);
} else {
break;
}
}
}
async forEach(f: (x: T) => void): Promise {
while (true) {
const x = await this.next();
if (x.hasValue) {
f(x.value);
} else {
break;
}
}
}
async filter(f: (x: T) => boolean): Promise {
const arr: T[] = [];
while (true) {
const x = await this.next();
if (x.hasValue) {
if (f(x.value)) {
arr.push(x.value);
}
} else {
break;
}
}
return arr;
}
async next(): Promise> {
if (this.gotCursorEnd) {
return { hasValue: false };
}
if (!this.awaitingResult) {
const cursor: IDBCursor | undefined = this.req.result;
if (!cursor) {
throw Error("assertion failed");
}
this.awaitingResult = true;
cursor.continue();
}
await this.currentPromise;
if (this.gotCursorEnd) {
return { hasValue: false };
}
const cursor = this.req.result;
if (!cursor) {
throw Error("assertion failed");
}
return { hasValue: true, value: cursor.value };
}
}
/**
* Return a promise that resolves to the opened IndexedDB database.
*/
export function openDatabase(
idbFactory: IDBFactory,
databaseName: string,
databaseVersion: number | undefined,
onVersionChange: () => void,
onUpgradeNeeded: (
db: IDBDatabase,
oldVersion: number,
newVersion: number,
upgradeTransaction: IDBTransaction,
) => void,
): Promise {
return new Promise((resolve, reject) => {
const req = idbFactory.open(databaseName, databaseVersion);
req.onerror = (event) => {
// @ts-expect-error
reject(new Error(`database opening error`, { cause: req.error }));
};
req.onsuccess = (e) => {
req.result.onversionchange = (evt: IDBVersionChangeEvent) => {
logger.info(
`handling versionchange on ${databaseName} from ${evt.oldVersion} to ${evt.newVersion}`,
);
req.result.close();
onVersionChange();
};
resolve(req.result);
};
req.onupgradeneeded = (e) => {
const db = req.result;
const newVersion = e.newVersion;
if (!newVersion) {
// @ts-expect-error
throw Error("upgrade needed, but new version unknown", {
cause: req.error,
});
}
const transaction = req.transaction;
if (!transaction) {
// @ts-expect-error
throw Error("no transaction handle available in upgrade handler", {
cause: req.error,
});
}
logger.info(
`handling upgradeneeded event on ${databaseName} from ${e.oldVersion} to ${e.newVersion}`,
);
onUpgradeNeeded(db, e.oldVersion, newVersion, transaction);
};
});
}
export interface IndexDescriptor {
name: string;
keyPath: IDBKeyPath | IDBKeyPath[];
multiEntry?: boolean;
unique?: boolean;
versionAdded?: number;
}
export interface StoreDescriptor {
_dummy: undefined & RecordType;
keyPath?: IDBKeyPath | IDBKeyPath[];
autoIncrement?: boolean;
/**
* Database version that this store was added in, or
* undefined if added in the first version.
*/
versionAdded?: number;
}
export interface StoreOptions {
keyPath?: IDBKeyPath | IDBKeyPath[];
autoIncrement?: boolean;
/**
* First minor database version that this store was added in, or
* undefined if added in the first version.
*/
versionAdded?: number;
}
export function describeContents(
options: StoreOptions,
): StoreDescriptor {
return {
keyPath: options.keyPath,
_dummy: undefined as any,
autoIncrement: options.autoIncrement,
versionAdded: options.versionAdded,
};
}
export function describeIndex(
name: string,
keyPath: IDBKeyPath | IDBKeyPath[],
options: IndexOptions = {},
): IndexDescriptor {
return {
keyPath,
name,
multiEntry: options.multiEntry,
unique: options.unique,
versionAdded: options.versionAdded,
};
}
interface IndexReadOnlyAccessor {
iter(query?: IDBKeyRange | IDBValidKey): ResultStream;
get(query: IDBValidKey): Promise;
getAll(
query?: IDBKeyRange | IDBValidKey,
count?: number,
): Promise;
getAllKeys(
query?: IDBKeyRange | IDBValidKey,
count?: number,
): Promise;
count(query?: IDBValidKey): Promise;
}
type GetIndexReadOnlyAccess = {
[P in keyof IndexMap]: IndexReadOnlyAccessor;
};
interface IndexReadWriteAccessor {
iter(query: IDBKeyRange | IDBValidKey): ResultStream;
get(query: IDBValidKey): Promise;
getAll(
query?: IDBKeyRange | IDBValidKey,
count?: number,
): Promise;
getAllKeys(
query?: IDBKeyRange | IDBValidKey,
count?: number,
): Promise;
count(query?: IDBValidKey): Promise;
}
type GetIndexReadWriteAccess = {
[P in keyof IndexMap]: IndexReadWriteAccessor;
};
export interface StoreReadOnlyAccessor {
get(key: IDBValidKey): Promise;
getAll(
query?: IDBKeyRange | IDBValidKey,
count?: number,
): Promise;
iter(query?: IDBValidKey): ResultStream;
indexes: GetIndexReadOnlyAccess;
}
export interface InsertResponse {
/**
* Key of the newly inserted (via put/add) record.
*/
key: IDBValidKey;
}
export interface StoreReadWriteAccessor {
get(key: IDBValidKey): Promise;
getAll(
query?: IDBKeyRange | IDBValidKey,
count?: number,
): Promise;
iter(query?: IDBValidKey): ResultStream;
put(r: RecordType, key?: IDBValidKey): Promise;
add(r: RecordType, key?: IDBValidKey): Promise;
delete(key: IDBValidKey): Promise;
indexes: GetIndexReadWriteAccess;
}
export interface StoreWithIndexes<
StoreName extends string,
RecordType,
IndexMap,
> {
storeName: StoreName;
store: StoreDescriptor;
indexMap: IndexMap;
/**
* Type marker symbol, to check that the descriptor
* has been created through the right function.
*/
mark: Symbol;
}
const storeWithIndexesSymbol = Symbol("StoreWithIndexesMark");
export function describeStore(
name: StoreName,
s: StoreDescriptor,
m: IndexMap,
): StoreWithIndexes {
return {
storeName: name,
store: s,
indexMap: m,
mark: storeWithIndexesSymbol,
};
}
export function describeStoreV2<
StoreName extends string,
RecordType,
IndexMap extends { [x: string]: IndexDescriptor } = {},
>(args: {
storeName: StoreName;
recordCodec: Codec;
keyPath?: IDBKeyPath | IDBKeyPath[];
autoIncrement?: boolean;
/**
* Database version that this store was added in, or
* undefined if added in the first version.
*/
versionAdded?: number;
indexes?: IndexMap;
}): StoreWithIndexes {
return {
storeName: args.storeName,
store: {
_dummy: undefined as any,
autoIncrement: args.autoIncrement,
keyPath: args.keyPath,
versionAdded: args.versionAdded,
},
indexMap: args.indexes ?? ({} as IndexMap),
mark: storeWithIndexesSymbol,
};
}
type KeyPathComponents = string | number;
/**
* Follow a key path (dot-separated) in an object.
*/
type DerefKeyPath = P extends `${infer PX extends keyof T &
KeyPathComponents}`
? T[PX]
: P extends `${infer P0 extends keyof T & KeyPathComponents}.${infer Rest}`
? DerefKeyPath
: unknown;
/**
* Return a path if it is a valid dot-separate path to an object.
* Otherwise, return "never".
*/
type ValidateKeyPath = P extends `${infer PX extends keyof T &
KeyPathComponents}`
? PX
: P extends `${infer P0 extends keyof T & KeyPathComponents}.${infer Rest}`
? `${P0}.${ValidateKeyPath}`
: never;
// function foo(
// x: T,
// p: P extends ValidateKeyPath ? P : never,
// ): void {}
// foo({x: [0,1,2]}, "x.0");
export type StoreNames = StoreMap extends {
[P in keyof StoreMap]: StoreWithIndexes;
}
? keyof StoreMap
: unknown;
export type DbReadWriteTransaction<
StoreMap,
StoresArr extends Array>,
> = StoreMap extends {
[P in string]: StoreWithIndexes;
}
? {
[X in StoresArr[number] &
keyof StoreMap]: StoreMap[X] extends StoreWithIndexes<
infer _StoreName,
infer RecordType,
infer IndexMap
>
? StoreReadWriteAccessor
: unknown;
}
: never;
export type DbReadOnlyTransaction<
StoreMap,
StoresArr extends Array>,
> = StoreMap extends {
[P in string]: StoreWithIndexes;
}
? {
[X in StoresArr[number] &
keyof StoreMap]: StoreMap[X] extends StoreWithIndexes<
infer _StoreName,
infer RecordType,
infer IndexMap
>
? StoreReadOnlyAccessor
: unknown;
}
: never;
/**
* Convert the type of an array to a union of the contents.
*
* Example:
* Input ["foo", "bar"]
* Output "foo" | "bar"
*/
export type UnionFromArray = Arr extends {
[X in keyof Arr]: Arr[X] & string;
}
? Arr[keyof Arr & number]
: unknown;
function runTx(
tx: IDBTransaction,
arg: Arg,
f: (t: Arg, t2: IDBTransaction) => Promise,
triggerContext: InternalTriggerContext,
): Promise {
const stack = Error("Failed transaction was started here.");
return new Promise((resolve, reject) => {
let funResult: any = undefined;
let gotFunResult = false;
let transactionException: any = undefined;
tx.oncomplete = () => {
// This is a fatal error: The transaction completed *before*
// the transaction function returned. Likely, the transaction
// function waited on a promise that is *not* resolved in the
// microtask queue, thus triggering the auto-commit behavior.
// Unfortunately, the auto-commit behavior of IDB can't be switched
// of. There are some proposals to add this functionality in the future.
if (!gotFunResult) {
const msg =
"BUG: transaction closed before transaction function returned";
logger.error(msg);
logger.error(`${stack.stack}`);
reject(Error(msg));
}
triggerContext.handleAfterCommit();
resolve(funResult);
};
tx.onerror = () => {
logger.error("error in transaction");
logger.error(`${stack.stack}`);
};
tx.onabort = () => {
let msg: string;
if (tx.error) {
msg = `Transaction aborted (transaction error): ${tx.error}`;
} else if (transactionException !== undefined) {
msg = `Transaction aborted (exception thrown): ${transactionException}`;
} else {
msg = "Transaction aborted (no DB error)";
}
logger.error(msg);
logger.error(`${stack.stack}`);
reject(new TransactionAbortedError(msg));
};
const resP = Promise.resolve().then(() => f(arg, tx));
resP
.then((result) => {
gotFunResult = true;
funResult = result;
})
.catch((e) => {
if (e == TransactionAbort) {
logger.trace("aborting transaction");
} else {
transactionException = e;
console.error("Transaction failed:", e);
console.error(stack);
tx.abort();
}
})
.catch((e) => {
console.error("fatal: aborting transaction failed", e);
});
});
}
function makeReadContext(
tx: IDBTransaction,
storePick: { [n: string]: StoreWithIndexes },
triggerContext: InternalTriggerContext,
): any {
const ctx: { [s: string]: StoreReadOnlyAccessor } = {};
for (const storeAlias in storePick) {
const indexes: { [s: string]: IndexReadOnlyAccessor } = {};
const swi = storePick[storeAlias];
const storeName = swi.storeName;
for (const indexAlias in storePick[storeAlias].indexMap) {
const indexDescriptor: IndexDescriptor =
storePick[storeAlias].indexMap[indexAlias];
const indexName = indexDescriptor.name;
indexes[indexAlias] = {
get(key) {
triggerContext.storesAccessed.add(storeName);
const req = tx.objectStore(storeName).index(indexName).get(key);
return requestToPromise(req);
},
iter(query) {
triggerContext.storesAccessed.add(storeName);
const req = tx
.objectStore(storeName)
.index(indexName)
.openCursor(query);
return new ResultStream(req);
},
getAll(query, count) {
triggerContext.storesAccessed.add(storeName);
const req = tx
.objectStore(storeName)
.index(indexName)
.getAll(query, count);
return requestToPromise(req);
},
getAllKeys(query, count) {
triggerContext.storesAccessed.add(storeName);
const req = tx
.objectStore(storeName)
.index(indexName)
.getAllKeys(query, count);
return requestToPromise(req);
},
count(query) {
triggerContext.storesAccessed.add(storeName);
const req = tx.objectStore(storeName).index(indexName).count(query);
return requestToPromise(req);
},
};
}
ctx[storeAlias] = {
indexes,
get(key) {
triggerContext.storesAccessed.add(storeName);
const req = tx.objectStore(storeName).get(key);
return requestToPromise(req);
},
getAll(query, count) {
triggerContext.storesAccessed.add(storeName);
const req = tx.objectStore(storeName).getAll(query, count);
return requestToPromise(req);
},
iter(query) {
triggerContext.storesAccessed.add(storeName);
const req = tx.objectStore(storeName).openCursor(query);
return new ResultStream(req);
},
};
}
return ctx;
}
function makeWriteContext(
tx: IDBTransaction,
storePick: { [n: string]: StoreWithIndexes },
triggerContext: InternalTriggerContext,
): any {
const ctx: { [s: string]: StoreReadWriteAccessor } = {};
for (const storeAlias in storePick) {
const indexes: { [s: string]: IndexReadWriteAccessor } = {};
const swi = storePick[storeAlias];
const storeName = swi.storeName;
for (const indexAlias in storePick[storeAlias].indexMap) {
const indexDescriptor: IndexDescriptor =
storePick[storeAlias].indexMap[indexAlias];
const indexName = indexDescriptor.name;
indexes[indexAlias] = {
get(key) {
triggerContext.storesAccessed.add(storeName);
const req = tx.objectStore(storeName).index(indexName).get(key);
return requestToPromise(req);
},
iter(query) {
triggerContext.storesAccessed.add(storeName);
const req = tx
.objectStore(storeName)
.index(indexName)
.openCursor(query);
return new ResultStream(req);
},
getAll(query, count) {
triggerContext.storesAccessed.add(storeName);
const req = tx
.objectStore(storeName)
.index(indexName)
.getAll(query, count);
return requestToPromise(req);
},
getAllKeys(query, count) {
triggerContext.storesAccessed.add(storeName);
const req = tx
.objectStore(storeName)
.index(indexName)
.getAllKeys(query, count);
return requestToPromise(req);
},
count(query) {
triggerContext.storesAccessed.add(storeName);
const req = tx.objectStore(storeName).index(indexName).count(query);
return requestToPromise(req);
},
};
}
ctx[storeAlias] = {
indexes,
get(key) {
triggerContext.storesAccessed.add(storeName);
const req = tx.objectStore(storeName).get(key);
return requestToPromise(req);
},
getAll(query, count) {
triggerContext.storesAccessed.add(storeName);
const req = tx.objectStore(storeName).getAll(query, count);
return requestToPromise(req);
},
iter(query) {
triggerContext.storesAccessed.add(storeName);
const req = tx.objectStore(storeName).openCursor(query);
return new ResultStream(req);
},
async add(r, k) {
triggerContext.storesAccessed.add(storeName);
triggerContext.storesModified.add(storeName);
const req = tx.objectStore(storeName).add(r, k);
const key = await requestToPromise(req);
return {
key: key,
};
},
async put(r, k) {
triggerContext.storesAccessed.add(storeName);
triggerContext.storesModified.add(storeName);
const req = tx.objectStore(storeName).put(r, k);
const key = await requestToPromise(req);
return {
key: key,
};
},
delete(k) {
triggerContext.storesAccessed.add(storeName);
triggerContext.storesModified.add(storeName);
const req = tx.objectStore(storeName).delete(k);
return requestToPromise(req);
},
};
}
return ctx;
}
export interface DbAccess {
idbHandle(): IDBDatabase;
runAllStoresReadWriteTx(
options: {
label?: string;
},
txf: (
tx: DbReadWriteTransaction>>,
) => Promise,
): Promise;
runAllStoresReadOnlyTx(
options: {
label?: string;
},
txf: (
tx: DbReadOnlyTransaction>>,
) => Promise,
): Promise;
runReadWriteTx>>(
opts: {
storeNames: StoreNameArray;
label?: string;
},
txf: (tx: DbReadWriteTransaction) => Promise,
): Promise;
runReadOnlyTx>>(
opts: {
storeNames: StoreNameArray;
label?: string;
},
txf: (tx: DbReadOnlyTransaction) => Promise,
): Promise;
}
export interface AfterCommitInfo {
mode: IDBTransactionMode;
scope: Set;
accessedStores: Set;
modifiedStores: Set;
}
export interface TriggerSpec {
/**
* Trigger run after every successful commit, run outside of the transaction.
*/
afterCommit?: (info: AfterCommitInfo) => void;
// onRead(store, value)
// initState () => State
// beforeCommit? (tx: Transaction, s: State | undefined) => Promise;
}
class InternalTriggerContext {
storesScope: Set;
storesAccessed: Set = new Set();
storesModified: Set = new Set();
constructor(
private triggerSpec: TriggerSpec,
private mode: IDBTransactionMode,
scope: string[],
) {
this.storesScope = new Set(scope);
}
handleAfterCommit() {
if (this.triggerSpec.afterCommit) {
this.triggerSpec.afterCommit({
mode: this.mode,
accessedStores: this.storesAccessed,
modifiedStores: this.storesModified,
scope: this.storesScope,
});
}
}
}
/**
* Type-safe access to a database with a particular store map.
*
* A store map is the metadata that describes the store.
*/
export class DbAccessImpl implements DbAccess {
constructor(
private db: IDBDatabase,
private stores: StoreMap,
private triggers: TriggerSpec = {},
private cancellationToken: CancellationToken,
) {}
idbHandle(): IDBDatabase {
return this.db;
}
runAllStoresReadWriteTx(
options: {
label?: string;
},
txf: (
tx: DbReadWriteTransaction>>,
) => Promise,
): Promise {
const accessibleStores: { [x: string]: StoreWithIndexes } =
{};
const strStoreNames: string[] = [];
for (const sn of Object.keys(this.stores as any)) {
const swi = (this.stores as any)[sn] as StoreWithIndexes;
strStoreNames.push(swi.storeName);
accessibleStores[swi.storeName] = swi;
}
const mode = "readwrite";
const triggerContext = new InternalTriggerContext(
this.triggers,
mode,
strStoreNames,
);
const tx = this.db.transaction(strStoreNames, mode);
const writeContext = makeWriteContext(tx, accessibleStores, triggerContext);
return runTx(tx, writeContext, txf, triggerContext);
}
async runAllStoresReadOnlyTx(
options: {
label?: string;
},
txf: (
tx: DbReadOnlyTransaction>>,
) => Promise,
): Promise {
const accessibleStores: { [x: string]: StoreWithIndexes } =
{};
const strStoreNames: string[] = [];
for (const sn of Object.keys(this.stores as any)) {
const swi = (this.stores as any)[sn] as StoreWithIndexes;
strStoreNames.push(swi.storeName);
accessibleStores[swi.storeName] = swi;
}
const mode = "readonly";
const triggerContext = new InternalTriggerContext(
this.triggers,
mode,
strStoreNames,
);
const tx = this.db.transaction(strStoreNames, mode);
const writeContext = makeReadContext(tx, accessibleStores, triggerContext);
const res = await runTx(tx, writeContext, txf, triggerContext);
return res;
}
async runReadWriteTx>>(
opts: {
storeNames: StoreNameArray;
},
txf: (tx: DbReadWriteTransaction) => Promise,
): Promise {
const accessibleStores: { [x: string]: StoreWithIndexes } =
{};
const strStoreNames: string[] = [];
for (const sn of opts.storeNames) {
const swi = (this.stores as any)[sn] as StoreWithIndexes;
strStoreNames.push(swi.storeName);
accessibleStores[swi.storeName] = swi;
}
const mode = "readwrite";
const triggerContext = new InternalTriggerContext(
this.triggers,
mode,
strStoreNames,
);
const tx = this.db.transaction(strStoreNames, mode);
const writeContext = makeWriteContext(tx, accessibleStores, triggerContext);
const res = await runTx(tx, writeContext, txf, triggerContext);
return res;
}
runReadOnlyTx>>(
opts: {
storeNames: StoreNameArray;
},
txf: (tx: DbReadOnlyTransaction) => Promise,
): Promise {
const accessibleStores: { [x: string]: StoreWithIndexes } =
{};
const strStoreNames: string[] = [];
for (const sn of opts.storeNames) {
const swi = (this.stores as any)[sn] as StoreWithIndexes;
strStoreNames.push(swi.storeName);
accessibleStores[swi.storeName] = swi;
}
const mode = "readonly";
const triggerContext = new InternalTriggerContext(
this.triggers,
mode,
strStoreNames,
);
const tx = this.db.transaction(strStoreNames, mode);
const readContext = makeReadContext(tx, accessibleStores, triggerContext);
const res = runTx(tx, readContext, txf, triggerContext);
return res;
}
}