make withdrawal requests sequentially, clean up withdrawal logic a bit

6 files changed, 246 insertions, 110 deletions

diff --git a/packages/taler-integrationtests/src/test-paywall-flow.ts b/packages/taler-integrationtests/src/test-paywall-flow.ts
index a0af0d3b6..0671fecbe 100644
--- a/packages/taler-integrationtests/src/test-paywall-flow.ts
+++ b/packages/taler-integrationtests/src/test-paywall-flow.ts
@@ -121,6 +121,7 @@ runTest(async (t: GlobalTestState) => {
   });
 
   if (publicOrderStatusResp.status != 410) {
+    console.log(publicOrderStatusResp.data);
     throw Error(
       `expected status 410 (after paying), but got ${publicOrderStatusResp.status}`,
     );
diff --git a/packages/taler-wallet-core/src/TalerErrorCode.ts b/packages/taler-wallet-core/src/TalerErrorCode.ts
index d45b1064f..1557007f4 100644
--- a/packages/taler-wallet-core/src/TalerErrorCode.ts
+++ b/packages/taler-wallet-core/src/TalerErrorCode.ts
@@ -22,6 +22,8 @@
  */
 
 export enum TalerErrorCode {
+
+
   /**
    * Special code to indicate no error (or no "code" present).
    * Returned with an HTTP status code of #MHD_HTTP_UNINITIALIZED (0).
@@ -73,21 +75,21 @@ export enum TalerErrorCode {
 
   /**
    * Exchange failed to allocate memory for building JSON reply.
-   * Returned with an HTTP status code of #MHD_HTTP_UNINITIALIZED (0).
+   * Returned with an HTTP status code of #MHD_HTTP_INTERNAL_SERVER_ERROR (500).
    * (A value of 0 indicates that the error is generated client-side).
    */
   JSON_ALLOCATION_FAILURE = 7,
 
   /**
    * HTTP method invalid for this URL.
-   * Returned with an HTTP status code of #MHD_HTTP_UNINITIALIZED (0).
+   * Returned with an HTTP status code of #MHD_HTTP_METHOD_NOT_ALLOWED (405).
    * (A value of 0 indicates that the error is generated client-side).
    */
   METHOD_INVALID = 8,
 
   /**
-   * Operation specified invalid for this URL (resulting in a "NOT FOUND" for the overall response).
-   * Returned with an HTTP status code of #MHD_HTTP_UNINITIALIZED (0).
+   * Operation specified invalid for this endpoint.
+   * Returned with an HTTP status code of #MHD_HTTP_BAD_REQUEST (400).
    * (A value of 0 indicates that the error is generated client-side).
    */
   OPERATION_INVALID = 9,
@@ -149,6 +151,20 @@ export enum TalerErrorCode {
   PAYTO_MALFORMED = 17,
 
   /**
+   * Operation specified unknown for this endpoint.
+   * Returned with an HTTP status code of #MHD_HTTP_NOT_FOUND (404).
+   * (A value of 0 indicates that the error is generated client-side).
+   */
+  OPERATION_UNKNOWN = 18,
+
+  /**
+   * Exchange failed to allocate memory.
+   * Returned with an HTTP status code of #MHD_HTTP_INTERNAL_SERVER_ERROR (500).
+   * (A value of 0 indicates that the error is generated client-side).
+   */
+  ALLOCATION_FAILURE = 19,
+
+  /**
    * The exchange failed to even just initialize its connection to the database.
    * Returned with an HTTP status code of #MHD_HTTP_INTERNAL_SERVER_ERROR (500).
    * (A value of 0 indicates that the error is generated client-side).
@@ -842,11 +858,18 @@ export enum TalerErrorCode {
   REFUND_COIN_NOT_FOUND = 1500,
 
   /**
-   * We could not process the refund request as the coin's transaction history does not permit the requested refund at this time.  The "history" in the response proves this.
+   * We could not process the refund request as the coin's transaction history does not permit the requested refund because then refunds would exceed the deposit amount.  The "history" in the response proves this.
    * Returned with an HTTP status code of #MHD_HTTP_CONFLICT (409).
    * (A value of 0 indicates that the error is generated client-side).
    */
-  REFUND_CONFLICT = 1501,
+  REFUND_CONFLICT_DEPOSIT_INSUFFICIENT = 1501,
+
+  /**
+   * We could not process the refund request as the same refund transaction ID was already used with a different amount. Retrying with a different refund transaction ID may work. The "history" in the response proves this by providing the conflicting entry.
+   * Returned with an HTTP status code of #MHD_HTTP_PRECONDITION_FAILED (412).
+   * (A value of 0 indicates that the error is generated client-side).
+   */
+  REFUND_INCONSITENT_AMOUNT = 1502,
 
   /**
    * The exchange knows about the coin we were asked to refund, but not about the specific /deposit operation.  Hence, we cannot issue a refund (as we do not know if this merchant public key is authorized to do a refund).
@@ -940,6 +963,13 @@ export enum TalerErrorCode {
   REFUND_INVALID_SIGNATURE_BY_EXCHANGE = 1515,
 
   /**
+   * The failure proof returned by the exchange is incorrect. Error code generated client-side.
+   * Returned with an HTTP status code of #MHD_HTTP_UNINITIALIZED (0).
+   * (A value of 0 indicates that the error is generated client-side).
+   */
+  REFUND_INVALID_FAILURE_PROOF_BY_EXCHANGE = 1516,
+
+  /**
    * The wire format specified in the "sender_account_details" is not understood or not supported by this exchange.
    * Returned with an HTTP status code of #MHD_HTTP_NOT_FOUND (404).
    * (A value of 0 indicates that the error is generated client-side).
@@ -1164,6 +1194,20 @@ export enum TalerErrorCode {
   PROPOSAL_INSTANCE_CONFIGURATION_LACKS_WIRE = 2002,
 
   /**
+   * The backend could not locate a required template to generate an HTML reply.
+   * Returned with an HTTP status code of #MHD_HTTP_NOT_ACCEPTABLE (406).
+   * (A value of 0 indicates that the error is generated client-side).
+   */
+  MERCHANT_FAILED_TO_LOAD_TEMPLATE = 2003,
+
+  /**
+   * The backend could not expand the template to generate an HTML reply.
+   * Returned with an HTTP status code of #MHD_HTTP_INTERNAL_SERVER_ERROR (500).
+   * (A value of 0 indicates that the error is generated client-side).
+   */
+  MERCHANT_FAILED_TO_EXPAND_TEMPLATE = 2004,
+
+  /**
    * The merchant failed to provide a meaningful response to a /pay request.  This error is created client-side.
    * Returned with an HTTP status code of #MHD_HTTP_UNINITIALIZED (0).
    * (A value of 0 indicates that the error is generated client-side).
@@ -2886,6 +2930,13 @@ export enum TalerErrorCode {
   BANK_TRANSFER_REQUEST_UID_REUSED = 5500,
 
   /**
+   * The withdrawal operation already has a reserve selected.  The current request conflicts with the existing selection.
+   * Returned with an HTTP status code of #MHD_HTTP_CONFLICT (409).
+   * (A value of 0 indicates that the error is generated client-side).
+   */
+  BANK_WITHDRAWAL_OPERATION_RESERVE_SELECTION_CONFLICT = 5600,
+
+  /**
    * The sync service failed to access its database.
    * Returned with an HTTP status code of #MHD_HTTP_INTERNAL_SERVER_ERROR (500).
    * (A value of 0 indicates that the error is generated client-side).
@@ -3082,6 +3133,13 @@ export enum TalerErrorCode {
   WALLET_INVALID_TALER_PAY_URI = 7008,
 
   /**
+   * The signature on a coin by the exchange's denomination key is invalid after unblinding it.
+   * Returned with an HTTP status code of #MHD_HTTP_UNINITIALIZED (0).
+   * (A value of 0 indicates that the error is generated client-side).
+   */
+  WALLET_EXCHANGE_COIN_SIGNATURE_INVALID = 7009,
+
+  /**
    * The exchange does not know about the reserve (yet), and thus withdrawal can't progress.
    * Returned with an HTTP status code of #MHD_HTTP_NOT_FOUND (404).
    * (A value of 0 indicates that the error is generated client-side).
@@ -3094,4 +3152,5 @@ export enum TalerErrorCode {
    * (A value of 0 indicates that the error is generated client-side).
    */
   END = 9999,
+
 }
diff --git a/packages/taler-wallet-core/src/operations/reserves.ts b/packages/taler-wallet-core/src/operations/reserves.ts
index 58095affd..060226cab 100644
--- a/packages/taler-wallet-core/src/operations/reserves.ts
+++ b/packages/taler-wallet-core/src/operations/reserves.ts
@@ -758,7 +758,6 @@ async function depleteReserve(
         rawWithdrawalAmount: withdrawAmount,
         timestampStart: getTimestampNow(),
         retryInfo: initRetryInfo(),
-        lastErrorPerCoin: {},
         lastError: undefined,
         denomsSel: denomSelectionInfoToState(denomsForWithdraw),
       };
diff --git a/packages/taler-wallet-core/src/operations/tip.ts b/packages/taler-wallet-core/src/operations/tip.ts
index d6768bdb6..84cfa570a 100644
--- a/packages/taler-wallet-core/src/operations/tip.ts
+++ b/packages/taler-wallet-core/src/operations/tip.ts
@@ -281,6 +281,7 @@ async function processTipImpl(
       coinIdx: i,
       withdrawalDone: false,
       withdrawalGroupId: withdrawalGroupId,
+      lastError: undefined,
     };
     planchets.push(planchet);
   }
@@ -294,7 +295,6 @@ async function processTipImpl(
     timestampStart: getTimestampNow(),
     withdrawalGroupId: withdrawalGroupId,
     rawWithdrawalAmount: tipRecord.amount,
-    lastErrorPerCoin: {},
     retryInfo: initRetryInfo(),
     timestampFinish: undefined,
     lastError: undefined,
diff --git a/packages/taler-wallet-core/src/operations/withdraw.ts b/packages/taler-wallet-core/src/operations/withdraw.ts
index 9719772a7..a72a70827 100644
--- a/packages/taler-wallet-core/src/operations/withdraw.ts
+++ b/packages/taler-wallet-core/src/operations/withdraw.ts
@@ -39,6 +39,7 @@ import {
   codecForWithdrawOperationStatusResponse,
   codecForWithdrawResponse,
   WithdrawUriInfoResponse,
+  WithdrawResponse,
 } from "../types/talerTypes";
 import { InternalWalletState } from "./state";
 import { parseWithdrawUri } from "../util/taleruri";
@@ -47,7 +48,7 @@ import { updateExchangeFromUrl, getExchangeTrust } from "./exchanges";
 import { WALLET_EXCHANGE_PROTOCOL_VERSION } from "./versions";
 
 import * as LibtoolVersion from "../util/libtoolVersion";
-import { guardOperationException } from "./errors";
+import { guardOperationException, makeErrorDetails, OperationFailedError } from "./errors";
 import { NotificationType } from "../types/notifications";
 import {
   getTimestampNow,
@@ -57,6 +58,7 @@ import {
 } from "../util/time";
 import { readSuccessResponseJsonOrThrow } from "../util/http";
 import { URL } from "../util/url";
+import { TalerErrorCode } from "../TalerErrorCode";
 
 const logger = new Logger("withdraw.ts");
 
@@ -184,9 +186,13 @@ async function getPossibleDenoms(
 }
 
 /**
- * Given a planchet, withdraw a coin from the exchange.
+ * Generate a planchet for a coin index in a withdrawal group.
+ * Does not actually withdraw the coin yet.
+ *
+ * Split up so that we can parallelize the crypto, but serialize
+ * the exchange requests per reserve.
  */
-async function processPlanchet(
+async function processPlanchetGenerate(
   ws: InternalWalletState,
   withdrawalGroupId: string,
   coinIdx: number,
@@ -259,6 +265,7 @@ async function processPlanchet(
       withdrawalDone: false,
       withdrawSig: r.withdrawSig,
       withdrawalGroupId: withdrawalGroupId,
+      lastError: undefined,
     };
     await ws.db.runWithWriteTransaction([Stores.planchets], async (tx) => {
       const p = await tx.getIndexed(Stores.planchets.byGroupAndIndex, [
@@ -273,8 +280,31 @@ async function processPlanchet(
       planchet = newPlanchet;
     });
   }
+}
+
+/**
+ * Send the withdrawal request for a generated planchet to the exchange.
+ *
+ * The verification of the response is done asynchronously to enable parallelism.
+ */
+async function processPlanchetExchangeRequest(
+  ws: InternalWalletState,
+  withdrawalGroupId: string,
+  coinIdx: number,
+): Promise<WithdrawResponse | undefined> {
+  const withdrawalGroup = await ws.db.get(
+    Stores.withdrawalGroups,
+    withdrawalGroupId,
+  );
+  if (!withdrawalGroup) {
+    return;
+  }
+  let planchet = await ws.db.getIndexed(Stores.planchets.byGroupAndIndex, [
+    withdrawalGroupId,
+    coinIdx,
+  ]);
   if (!planchet) {
-    throw Error("invariant violated");
+    return;
   }
   if (planchet.withdrawalDone) {
     logger.warn("processPlanchet: planchet already withdrawn");
@@ -313,16 +343,62 @@ async function processPlanchet(
     exchange.baseUrl,
   ).href;
 
-  const resp = await ws.http.postJson(reqUrl, wd);
-  const r = await readSuccessResponseJsonOrThrow(
-    resp,
-    codecForWithdrawResponse(),
-  );
+  try {
+    const resp = await ws.http.postJson(reqUrl, wd);
+    const r = await readSuccessResponseJsonOrThrow(
+      resp,
+      codecForWithdrawResponse(),
+    );
+  
+    logger.trace(`got response for /withdraw`);
+    return r;
+  } catch (e) {
+    if (!(e instanceof OperationFailedError)) {
+      throw e;
+    }
+    const errDetails = e.operationError;
+    await ws.db.runWithWriteTransaction([Stores.planchets], async (tx) => {
+      let planchet = await ws.db.getIndexed(Stores.planchets.byGroupAndIndex, [
+        withdrawalGroupId,
+        coinIdx,
+      ]);
+      if (!planchet) {
+        return;
+      }
+      planchet.lastError = errDetails;
+      await tx.put(Stores.planchets, planchet);
+    });
+    return;
+  }
+}
 
-  logger.trace(`got response for /withdraw`);
+async function processPlanchetVerifyAndStoreCoin(
+  ws: InternalWalletState,
+  withdrawalGroupId: string,
+  coinIdx: number,
+  resp: WithdrawResponse,
+): Promise<void> {
+  const withdrawalGroup = await ws.db.get(
+    Stores.withdrawalGroups,
+    withdrawalGroupId,
+  );
+  if (!withdrawalGroup) {
+    return;
+  }
+  let planchet = await ws.db.getIndexed(Stores.planchets.byGroupAndIndex, [
+    withdrawalGroupId,
+    coinIdx,
+  ]);
+  if (!planchet) {
+    return;
+  }
+  if (planchet.withdrawalDone) {
+    logger.warn("processPlanchet: planchet already withdrawn");
+    return;
+  }
 
   const denomSig = await ws.cryptoApi.rsaUnblind(
-    r.ev_sig,
+    resp.ev_sig,
     planchet.blindingKey,
     planchet.denomPub,
   );
@@ -334,11 +410,24 @@ async function processPlanchet(
   );
 
   if (!isValid) {
-    throw Error("invalid RSA signature by the exchange");
+    await ws.db.runWithWriteTransaction([Stores.planchets], async (tx) => {
+      let planchet = await ws.db.getIndexed(Stores.planchets.byGroupAndIndex, [
+        withdrawalGroupId,
+        coinIdx,
+      ]);
+      if (!planchet) {
+        return;
+      }
+      planchet.lastError = makeErrorDetails(
+        TalerErrorCode.WALLET_EXCHANGE_COIN_SIGNATURE_INVALID,
+        "invalid signature from the exchange after unblinding",
+        {},
+      );
+      await tx.put(Stores.planchets, planchet);
+    });
+    return;
   }
 
-  logger.trace(`unblinded and verified`);
-
   const coin: CoinRecord = {
     blindingKey: planchet.blindingKey,
     coinPriv: planchet.coinPriv,
@@ -358,11 +447,9 @@ async function processPlanchet(
     suspended: false,
   };
 
-  let withdrawalGroupFinished = false;
-
   const planchetCoinPub = planchet.coinPub;
 
-  const success = await ws.db.runWithWriteTransaction(
+  const firstSuccess = await ws.db.runWithWriteTransaction(
     [Stores.coins, Stores.withdrawalGroups, Stores.reserves, Stores.planchets],
     async (tx) => {
       const ws = await tx.get(Stores.withdrawalGroups, withdrawalGroupId);
@@ -370,64 +457,21 @@ async function processPlanchet(
         return false;
       }
       const p = await tx.get(Stores.planchets, planchetCoinPub);
-      if (!p) {
-        return false;
-      }
-      if (p.withdrawalDone) {
-        // Already withdrawn
+      if (!p || p.withdrawalDone) {
         return false;
       }
       p.withdrawalDone = true;
       await tx.put(Stores.planchets, p);
-
-      let numTotal = 0;
-
-      for (const ds of ws.denomsSel.selectedDenoms) {
-        numTotal += ds.count;
-      }
-
-      let numDone = 0;
-
-      await tx
-        .iterIndexed(Stores.planchets.byGroup, withdrawalGroupId)
-        .forEach((x) => {
-          if (x.withdrawalDone) {
-            numDone++;
-          }
-        });
-
-      if (numDone > numTotal) {
-        throw Error(
-          "invariant violated (created more planchets than expected)",
-        );
-      }
-
-      if (numDone == numTotal) {
-        ws.timestampFinish = getTimestampNow();
-        ws.lastError = undefined;
-        ws.retryInfo = initRetryInfo(false);
-        withdrawalGroupFinished = true;
-      }
-      await tx.put(Stores.withdrawalGroups, ws);
       await tx.add(Stores.coins, coin);
       return true;
     },
   );
 
-  logger.trace(`withdrawal result stored in DB`);
-
-  if (success) {
+  if (firstSuccess) {
     ws.notify({
       type: NotificationType.CoinWithdrawn,
     });
   }
-
-  if (withdrawalGroupFinished) {
-    ws.notify({
-      type: NotificationType.WithdrawGroupFinished,
-      withdrawalSource: withdrawalGroup.source,
-    });
-  }
 }
 
 export function denomSelectionInfoToState(
@@ -552,27 +596,6 @@ async function resetWithdrawalGroupRetry(
   });
 }
 
-async function processInBatches(
-  workGen: Iterator<Promise<void>>,
-  batchSize: number,
-): Promise<void> {
-  for (;;) {
-    const batch: Promise<void>[] = [];
-    for (let i = 0; i < batchSize; i++) {
-      const wn = workGen.next();
-      if (wn.done) {
-        break;
-      }
-      batch.push(wn.value);
-    }
-    if (batch.length == 0) {
-      break;
-    }
-    logger.trace(`processing withdrawal batch of ${batch.length} elements`);
-    await Promise.all(batch);
-  }
-}
-
 async function processWithdrawGroupImpl(
   ws: InternalWalletState,
   withdrawalGroupId: string,
@@ -591,21 +614,79 @@ async function processWithdrawGroupImpl(
     return;
   }
 
-  const numDenoms = withdrawalGroup.denomsSel.selectedDenoms.length;
-  const genWork = function* (): Iterator<Promise<void>> {
-    let coinIdx = 0;
-    for (let i = 0; i < numDenoms; i++) {
-      const count = withdrawalGroup.denomsSel.selectedDenoms[i].count;
-      for (let j = 0; j < count; j++) {
-        yield processPlanchet(ws, withdrawalGroupId, coinIdx);
-        coinIdx++;
-      }
+  const numTotalCoins = withdrawalGroup.denomsSel.selectedDenoms
+    .map((x) => x.count)
+    .reduce((a, b) => a + b);
+
+  let work: Promise<void>[] = [];
+
+  for (let i = 0; i < numTotalCoins; i++) {
+    work.push(processPlanchetGenerate(ws, withdrawalGroupId, i));
+  }
+
+  // Generate coins concurrently (parallelism only happens in the crypto API workers)
+  await Promise.all(work);
+
+  work = [];
+
+  for (let coinIdx = 0; coinIdx < numTotalCoins; coinIdx++) {
+    const resp = await processPlanchetExchangeRequest(
+      ws,
+      withdrawalGroupId,
+      coinIdx,
+    );
+    if (!resp) {
+      continue;
     }
-  };
+    work.push(
+      processPlanchetVerifyAndStoreCoin(ws, withdrawalGroupId, coinIdx, resp),
+    );
+  }
 
-  // Withdraw coins in batches.
-  // The batch size is relatively large
-  await processInBatches(genWork(), 10);
+  await Promise.all(work);
+
+  let numFinished = 0;
+  let finishedForFirstTime = false;
+
+  await ws.db.runWithWriteTransaction(
+    [Stores.coins, Stores.withdrawalGroups, Stores.reserves, Stores.planchets],
+    async (tx) => {
+      const ws = await tx.get(Stores.withdrawalGroups, withdrawalGroupId);
+      if (!ws) {
+        return;
+      }
+
+      await tx
+        .iterIndexed(Stores.planchets.byGroup, withdrawalGroupId)
+        .forEach((x) => {
+          if (x.withdrawalDone) {
+            numFinished++;
+          }
+        });
+
+      if (ws.timestampFinish === undefined && numFinished == numTotalCoins) {
+        finishedForFirstTime = true;
+        ws.timestampFinish = getTimestampNow();
+        ws.lastError = undefined;
+        ws.retryInfo = initRetryInfo(false);
+      }
+      await tx.put(Stores.withdrawalGroups, ws);
+    },
+  );
+
+  if (numFinished != numTotalCoins) {
+    // FIXME: aggregate individual problems into the big error message here.
+    throw Error(
+      `withdrawal did not finish (${numFinished} / ${numTotalCoins} coins withdrawn)`,
+    );
+  }
+
+  if (finishedForFirstTime) {
+    ws.notify({
+      type: NotificationType.WithdrawGroupFinished,
+      withdrawalSource: withdrawalGroup.source,
+    });
+  }
 }
 
 export async function getExchangeWithdrawalInfo(
diff --git a/packages/taler-wallet-core/src/types/dbTypes.ts b/packages/taler-wallet-core/src/types/dbTypes.ts
index 26cf6915d..3c4c2a250 100644
--- a/packages/taler-wallet-core/src/types/dbTypes.ts
+++ b/packages/taler-wallet-core/src/types/dbTypes.ts
@@ -662,6 +662,8 @@ export interface PlanchetRecord {
 
   withdrawalDone: boolean;
 
+  lastError: OperationErrorDetails | undefined;
+
   /**
    * Public key of the reserve, this might be a reserve not
    * known to the wallet if the planchet is from a tip.
@@ -1504,12 +1506,6 @@ export interface WithdrawalGroupRecord {
    */
   retryInfo: RetryInfo;
 
-  /**
-   * Last error per coin/planchet, or undefined if no error occured for
-   * the coin/planchet.
-   */
-  lastErrorPerCoin: { [coinIndex: number]: OperationErrorDetails };
-
   lastError: OperationErrorDetails | undefined;
 }