aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFlorian Dold <florian.dold@gmail.com>2020-08-20 16:27:20 +0530
committerFlorian Dold <florian.dold@gmail.com>2020-08-20 16:27:20 +0530
commit421e613f92b80c81c856d6b074aa160e80e38e3d (patch)
tree7284e8b856986de3ee4ca1362fa9b52ce1e22de1
parentddf9171c5becb3bb1aebdd3e1a298644f62ed090 (diff)
throttling diagnostics and request timeouts
-rw-r--r--packages/taler-wallet-core/src/TalerErrorCode.ts7
-rw-r--r--packages/taler-wallet-core/src/headless/NodeHttpLib.ts16
-rw-r--r--packages/taler-wallet-core/src/operations/exchanges.ts19
-rw-r--r--packages/taler-wallet-core/src/operations/pay.ts28
-rw-r--r--packages/taler-wallet-core/src/operations/recoup.ts6
-rw-r--r--packages/taler-wallet-core/src/operations/refresh.ts28
-rw-r--r--packages/taler-wallet-core/src/operations/reserves.ts36
-rw-r--r--packages/taler-wallet-core/src/types/dbTypes.ts11
-rw-r--r--packages/taler-wallet-core/src/util/RequestThrottler.ts31
-rw-r--r--packages/taler-wallet-core/src/util/http.ts2
-rw-r--r--packages/taler-wallet-core/src/util/time.ts10
11 files changed, 159 insertions, 35 deletions
diff --git a/packages/taler-wallet-core/src/TalerErrorCode.ts b/packages/taler-wallet-core/src/TalerErrorCode.ts
index 412f3ef8a..ff8511046 100644
--- a/packages/taler-wallet-core/src/TalerErrorCode.ts
+++ b/packages/taler-wallet-core/src/TalerErrorCode.ts
@@ -3203,6 +3203,13 @@ export enum TalerErrorCode {
WALLET_WITHDRAWAL_OPERATION_ABORTED_BY_BANK = 7012,
/**
+ * An HTTP request made by the wallet timed out.
+ * Returned with an HTTP status code of #MHD_HTTP_UNINITIALIZED (0).
+ * (A value of 0 indicates that the error is generated client-side).
+ */
+ WALLET_HTTP_REQUEST_TIMEOUT = 7013,
+
+ /**
* End of error code range.
* Returned with an HTTP status code of #MHD_HTTP_UNINITIALIZED (0).
* (A value of 0 indicates that the error is generated client-side).
diff --git a/packages/taler-wallet-core/src/headless/NodeHttpLib.ts b/packages/taler-wallet-core/src/headless/NodeHttpLib.ts
index 59730ab30..85f37cfa3 100644
--- a/packages/taler-wallet-core/src/headless/NodeHttpLib.ts
+++ b/packages/taler-wallet-core/src/headless/NodeHttpLib.ts
@@ -29,6 +29,7 @@ import { RequestThrottler } from "../util/RequestThrottler";
import Axios from "axios";
import { OperationFailedError, makeErrorDetails } from "../operations/errors";
import { TalerErrorCode } from "../TalerErrorCode";
+import { URL } from "../util/url";
/**
* Implementation of the HTTP request library interface for node.
@@ -50,8 +51,20 @@ export class NodeHttpLib implements HttpRequestLibrary {
body: any,
opt?: HttpRequestOptions,
): Promise<HttpResponse> {
+ const parsedUrl = new URL(url);
if (this.throttlingEnabled && this.throttle.applyThrottle(url)) {
- throw Error("request throttled");
+ throw OperationFailedError.fromCode(
+ TalerErrorCode.WALLET_HTTP_REQUEST_THROTTLED,
+ `request to origin ${parsedUrl.origin} was throttled`,
+ {
+ requestMethod: method,
+ requestUrl: url,
+ throttleStats: this.throttle.getThrottleStats(url),
+ });
+ }
+ let timeout: number | undefined;
+ if (typeof opt?.timeout?.d_ms === "number") {
+ timeout = opt.timeout.d_ms;
}
const resp = await Axios({
method,
@@ -61,6 +74,7 @@ export class NodeHttpLib implements HttpRequestLibrary {
validateStatus: () => true,
transformResponse: (x) => x,
data: body,
+ timeout,
});
const respText = resp.data;
diff --git a/packages/taler-wallet-core/src/operations/exchanges.ts b/packages/taler-wallet-core/src/operations/exchanges.ts
index d40dd7883..3b7f62fe9 100644
--- a/packages/taler-wallet-core/src/operations/exchanges.ts
+++ b/packages/taler-wallet-core/src/operations/exchanges.ts
@@ -43,7 +43,7 @@ import {
WALLET_CACHE_BREAKER_CLIENT_VERSION,
WALLET_EXCHANGE_PROTOCOL_VERSION,
} from "./versions";
-import { getTimestampNow } from "../util/time";
+import { getTimestampNow, Duration } from "../util/time";
import { compare } from "../util/libtoolVersion";
import { createRecoupGroup, processRecoupGroup } from "./recoup";
import { TalerErrorCode } from "../TalerErrorCode";
@@ -96,6 +96,10 @@ async function setExchangeError(
await ws.db.mutate(Stores.exchanges, baseUrl, mut);
}
+function getExchangeRequestTimeout(e: ExchangeRecord): Duration {
+ return { d_ms: 5000 };
+}
+
/**
* Fetch the exchange's /keys and update our database accordingly.
*
@@ -117,7 +121,9 @@ async function updateExchangeWithKeys(
const keysUrl = new URL("keys", baseUrl);
keysUrl.searchParams.set("cacheBreaker", WALLET_CACHE_BREAKER_CLIENT_VERSION);
- const resp = await ws.http.get(keysUrl.href);
+ const resp = await ws.http.get(keysUrl.href, {
+ timeout: getExchangeRequestTimeout(existingExchangeRecord),
+ });
const exchangeKeysJson = await readSuccessResponseJsonOrThrow(
resp,
codecForExchangeKeysJson(),
@@ -303,7 +309,10 @@ async function updateExchangeWithTermsOfService(
Accept: "text/plain",
};
- const resp = await ws.http.get(reqUrl.href, { headers });
+ const resp = await ws.http.get(reqUrl.href, {
+ headers,
+ timeout: getExchangeRequestTimeout(exchange),
+ });
const tosText = await readSuccessResponseTextOrThrow(resp);
const tosEtag = resp.headers.get("etag") || undefined;
@@ -361,7 +370,9 @@ async function updateExchangeWithWireInfo(
const reqUrl = new URL("wire", exchangeBaseUrl);
reqUrl.searchParams.set("cacheBreaker", WALLET_CACHE_BREAKER_CLIENT_VERSION);
- const resp = await ws.http.get(reqUrl.href);
+ const resp = await ws.http.get(reqUrl.href, {
+ timeout: getExchangeRequestTimeout(exchange),
+ });
const wireInfo = await readSuccessResponseJsonOrThrow(
resp,
codecForExchangeWireJson(),
diff --git a/packages/taler-wallet-core/src/operations/pay.ts b/packages/taler-wallet-core/src/operations/pay.ts
index 565fe9c66..f20632344 100644
--- a/packages/taler-wallet-core/src/operations/pay.ts
+++ b/packages/taler-wallet-core/src/operations/pay.ts
@@ -35,6 +35,7 @@ import {
updateRetryInfoTimeout,
PayEventRecord,
WalletContractData,
+ getRetryDuration,
} from "../types/dbTypes";
import { NotificationType } from "../types/notifications";
import {
@@ -58,7 +59,13 @@ import { parsePayUri } from "../util/taleruri";
import { guardOperationException, OperationFailedError } from "./errors";
import { createRefreshGroup, getTotalRefreshCost } from "./refresh";
import { InternalWalletState, EXCHANGE_COINS_LOCK } from "./state";
-import { getTimestampNow, timestampAddDuration } from "../util/time";
+import {
+ getTimestampNow,
+ timestampAddDuration,
+ Duration,
+ durationMax,
+ durationMin,
+} from "../util/time";
import { strcmp, canonicalJson } from "../util/helpers";
import {
readSuccessResponseJsonOrThrow,
@@ -588,6 +595,17 @@ async function resetDownloadProposalRetry(
});
}
+function getProposalRequestTimeout(proposal: ProposalRecord): Duration {
+ return durationMax(
+ { d_ms: 60000 },
+ durationMin({ d_ms: 5000 }, getRetryDuration(proposal.retryInfo)),
+ );
+}
+
+function getPurchaseRequestTimeout(purchase: PurchaseRecord): Duration {
+ return { d_ms: 5000 };
+}
+
async function processDownloadProposalImpl(
ws: InternalWalletState,
proposalId: string,
@@ -620,7 +638,9 @@ async function processDownloadProposalImpl(
requestBody.token = proposal.claimToken;
}
- const resp = await ws.http.postJson(orderClaimUrl, requestBody);
+ const resp = await ws.http.postJson(orderClaimUrl, requestBody, {
+ timeout: getProposalRequestTimeout(proposal),
+ });
const proposalResp = await readSuccessResponseJsonOrThrow(
resp,
codecForProposal(),
@@ -886,7 +906,9 @@ export async function submitPay(
logger.trace("making pay request", JSON.stringify(reqBody, undefined, 2));
const resp = await ws.runSequentialized([EXCHANGE_COINS_LOCK], () =>
- ws.http.postJson(payUrl, reqBody),
+ ws.http.postJson(payUrl, reqBody, {
+ timeout: getPurchaseRequestTimeout(purchase),
+ }),
);
const merchantResp = await readSuccessResponseJsonOrThrow(
diff --git a/packages/taler-wallet-core/src/operations/recoup.ts b/packages/taler-wallet-core/src/operations/recoup.ts
index f855a28cb..ba02f72f8 100644
--- a/packages/taler-wallet-core/src/operations/recoup.ts
+++ b/packages/taler-wallet-core/src/operations/recoup.ts
@@ -40,7 +40,7 @@ import {
import { codecForRecoupConfirmation } from "../types/talerTypes";
import { NotificationType } from "../types/notifications";
-import { forceQueryReserve } from "./reserves";
+import { forceQueryReserve, getReserveRequestTimeout } from "./reserves";
import { Amounts } from "../util/amounts";
import { createRefreshGroup, processRefreshGroup } from "./refresh";
@@ -154,7 +154,9 @@ async function recoupWithdrawCoin(
const recoupRequest = await ws.cryptoApi.createRecoupRequest(coin);
const reqUrl = new URL(`/coins/${coin.coinPub}/recoup`, coin.exchangeBaseUrl);
- const resp = await ws.http.postJson(reqUrl.href, recoupRequest);
+ const resp = await ws.http.postJson(reqUrl.href, recoupRequest, {
+ timeout: getReserveRequestTimeout(reserve),
+ });
const recoupConfirmation = await readSuccessResponseJsonOrThrow(
resp,
codecForRecoupConfirmation(),
diff --git a/packages/taler-wallet-core/src/operations/refresh.ts b/packages/taler-wallet-core/src/operations/refresh.ts
index 430675328..89cc3af43 100644
--- a/packages/taler-wallet-core/src/operations/refresh.ts
+++ b/packages/taler-wallet-core/src/operations/refresh.ts
@@ -42,7 +42,7 @@ import {
import { guardOperationException } from "./errors";
import { NotificationType } from "../types/notifications";
import { getRandomBytes, encodeCrock } from "../crypto/talerCrypto";
-import { getTimestampNow } from "../util/time";
+import { getTimestampNow, Duration } from "../util/time";
import { readSuccessResponseJsonOrThrow, HttpResponse } from "../util/http";
import {
codecForExchangeMeltResponse,
@@ -211,6 +211,10 @@ async function refreshCreateSession(
ws.notify({ type: NotificationType.RefreshStarted });
}
+function getRefreshRequestTimeout(rg: RefreshGroupRecord): Duration {
+ return { d_ms: 5000 };
+}
+
async function refreshMelt(
ws: InternalWalletState,
refreshGroupId: string,
@@ -249,12 +253,11 @@ async function refreshMelt(
};
logger.trace(`melt request for coin:`, meltReq);
- const resp = await ws.runSequentialized(
- [EXCHANGE_COINS_LOCK],
- async () => {
- return await ws.http.postJson(reqUrl.href, meltReq);
- },
- );
+ const resp = await ws.runSequentialized([EXCHANGE_COINS_LOCK], async () => {
+ return await ws.http.postJson(reqUrl.href, meltReq, {
+ timeout: getRefreshRequestTimeout(refreshGroup),
+ });
+ });
const meltResponse = await readSuccessResponseJsonOrThrow(
resp,
@@ -346,12 +349,11 @@ async function refreshReveal(
refreshSession.exchangeBaseUrl,
);
- const resp = await ws.runSequentialized(
- [EXCHANGE_COINS_LOCK],
- async () => {
- return await ws.http.postJson(reqUrl.href, req);
- },
- );
+ const resp = await ws.runSequentialized([EXCHANGE_COINS_LOCK], async () => {
+ return await ws.http.postJson(reqUrl.href, req, {
+ timeout: getRefreshRequestTimeout(refreshGroup),
+ });
+ });
const reveal = await readSuccessResponseJsonOrThrow(
resp,
diff --git a/packages/taler-wallet-core/src/operations/reserves.ts b/packages/taler-wallet-core/src/operations/reserves.ts
index 8adaeea81..fda3c4bcb 100644
--- a/packages/taler-wallet-core/src/operations/reserves.ts
+++ b/packages/taler-wallet-core/src/operations/reserves.ts
@@ -35,6 +35,7 @@ import {
WithdrawalSourceType,
ReserveHistoryRecord,
ReserveBankInfo,
+ getRetryDuration,
} from "../types/dbTypes";
import { Logger } from "../util/logging";
import { Amounts } from "../util/amounts";
@@ -64,7 +65,12 @@ import {
} from "./errors";
import { NotificationType } from "../types/notifications";
import { codecForReserveStatus } from "../types/ReserveStatus";
-import { getTimestampNow } from "../util/time";
+import {
+ getTimestampNow,
+ Duration,
+ durationMin,
+ durationMax,
+} from "../util/time";
import {
reconcileReserveHistory,
summarizeReserveHistory,
@@ -331,10 +337,16 @@ async function registerReserveWithBank(
return;
}
const bankStatusUrl = bankInfo.statusUrl;
- const httpResp = await ws.http.postJson(bankStatusUrl, {
- reserve_pub: reservePub,
- selected_exchange: bankInfo.exchangePaytoUri,
- });
+ const httpResp = await ws.http.postJson(
+ bankStatusUrl,
+ {
+ reserve_pub: reservePub,
+ selected_exchange: bankInfo.exchangePaytoUri,
+ },
+ {
+ timeout: getReserveRequestTimeout(reserve),
+ },
+ );
await readSuccessResponseJsonOrThrow(
httpResp,
codecForBankWithdrawalOperationPostResponse(),
@@ -371,6 +383,13 @@ async function processReserveBankStatus(
);
}
+export function getReserveRequestTimeout(r: ReserveRecord): Duration {
+ return durationMax(
+ { d_ms: 60000 },
+ durationMin({ d_ms: 5000 }, getRetryDuration(r.retryInfo)),
+ );
+}
+
async function processReserveBankStatusImpl(
ws: InternalWalletState,
reservePub: string,
@@ -388,7 +407,9 @@ async function processReserveBankStatusImpl(
return;
}
- const statusResp = await ws.http.get(bankStatusUrl);
+ const statusResp = await ws.http.get(bankStatusUrl, {
+ timeout: getReserveRequestTimeout(reserve),
+ });
const status = await readSuccessResponseJsonOrThrow(
statusResp,
codecForWithdrawOperationStatusResponse(),
@@ -501,6 +522,9 @@ async function updateReserve(
const resp = await ws.http.get(
new URL(`reserves/${reservePub}`, reserve.exchangeBaseUrl).href,
+ {
+ timeout: getReserveRequestTimeout(reserve),
+ },
);
const result = await readSuccessResponseJsonOrErrorCode(
diff --git a/packages/taler-wallet-core/src/types/dbTypes.ts b/packages/taler-wallet-core/src/types/dbTypes.ts
index 82260963b..e36e322d1 100644
--- a/packages/taler-wallet-core/src/types/dbTypes.ts
+++ b/packages/taler-wallet-core/src/types/dbTypes.ts
@@ -117,6 +117,17 @@ export function updateRetryInfoTimeout(
r.nextRetry = { t_ms: t };
}
+export function getRetryDuration(
+ r: RetryInfo,
+ p: RetryPolicy = defaultRetryPolicy,
+): Duration {
+ if (p.backoffDelta.d_ms === "forever") {
+ return { d_ms: "forever" };
+ }
+ const t = p.backoffDelta.d_ms * Math.pow(p.backoffBase, r.retryCounter);
+ return { d_ms: t };
+}
+
export function initRetryInfo(
active = true,
p: RetryPolicy = defaultRetryPolicy,
diff --git a/packages/taler-wallet-core/src/util/RequestThrottler.ts b/packages/taler-wallet-core/src/util/RequestThrottler.ts
index 3b8f22f58..b56f7476a 100644
--- a/packages/taler-wallet-core/src/util/RequestThrottler.ts
+++ b/packages/taler-wallet-core/src/util/RequestThrottler.ts
@@ -30,25 +30,25 @@ const logger = new Logger("RequestThrottler.ts");
/**
* Maximum request per second, per origin.
*/
-const MAX_PER_SECOND = 50;
+const MAX_PER_SECOND = 100;
/**
* Maximum request per minute, per origin.
*/
-const MAX_PER_MINUTE = 100;
+const MAX_PER_MINUTE = 500;
/**
* Maximum request per hour, per origin.
*/
-const MAX_PER_HOUR = 1000;
+const MAX_PER_HOUR = 2000;
/**
* Throttling state for one origin.
*/
class OriginState {
- private tokensSecond: number = MAX_PER_SECOND;
- private tokensMinute: number = MAX_PER_MINUTE;
- private tokensHour: number = MAX_PER_HOUR;
+ tokensSecond: number = MAX_PER_SECOND;
+ tokensMinute: number = MAX_PER_MINUTE;
+ tokensHour: number = MAX_PER_HOUR;
private lastUpdate = getTimestampNow();
private refill(): void {
@@ -57,6 +57,9 @@ class OriginState {
if (d.d_ms === "forever") {
throw Error("assertion failed");
}
+ if (d.d_ms < 0) {
+ return;
+ }
const d_s = d.d_ms / 1000;
this.tokensSecond = Math.min(
MAX_PER_SECOND,
@@ -129,4 +132,20 @@ export class RequestThrottler {
const origin = new URL(requestUrl).origin;
return this.getState(origin).applyThrottle();
}
+
+ /**
+ * Get the throttle statistics for a particular URL.
+ */
+ getThrottleStats(requestUrl: string): Record<string, unknown> {
+ const origin = new URL(requestUrl).origin;
+ const state = this.getState(origin);
+ return {
+ tokensHour: state.tokensHour,
+ tokensMinute: state.tokensMinute,
+ tokensSecond: state.tokensSecond,
+ maxTokensHour: MAX_PER_HOUR,
+ maxTokensMinute: MAX_PER_MINUTE,
+ maxTokensSecond: MAX_PER_SECOND,
+ }
+ }
}
diff --git a/packages/taler-wallet-core/src/util/http.ts b/packages/taler-wallet-core/src/util/http.ts
index 22566daac..44c01a4e5 100644
--- a/packages/taler-wallet-core/src/util/http.ts
+++ b/packages/taler-wallet-core/src/util/http.ts
@@ -26,6 +26,7 @@ import { Codec } from "./codec";
import { OperationFailedError, makeErrorDetails } from "../operations/errors";
import { TalerErrorCode } from "../TalerErrorCode";
import { Logger } from "./logging";
+import { Duration } from "./time";
const logger = new Logger("http.ts");
@@ -43,6 +44,7 @@ export interface HttpResponse {
export interface HttpRequestOptions {
headers?: { [name: string]: string };
+ timeout?: Duration,
}
export enum HttpResponseStatus {
diff --git a/packages/taler-wallet-core/src/util/time.ts b/packages/taler-wallet-core/src/util/time.ts
index 5c2f49d12..ccd75e14b 100644
--- a/packages/taler-wallet-core/src/util/time.ts
+++ b/packages/taler-wallet-core/src/util/time.ts
@@ -95,6 +95,16 @@ export function durationMin(d1: Duration, d2: Duration): Duration {
return { d_ms: Math.min(d1.d_ms, d2.d_ms) };
}
+export function durationMax(d1: Duration, d2: Duration): Duration {
+ if (d1.d_ms === "forever") {
+ return { d_ms: "forever" };
+ }
+ if (d2.d_ms === "forever") {
+ return { d_ms: "forever" };
+ }
+ return { d_ms: Math.max(d1.d_ms, d2.d_ms) };
+}
+
export function timestampCmp(t1: Timestamp, t2: Timestamp): number {
if (t1.t_ms === "never") {
if (t2.t_ms === "never") {