From 1fea75bca3951d39c0a45faf3e903fcec77f9c4f Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Mon, 9 Dec 2019 13:29:11 +0100 Subject: throttling / allow non-json requests --- src/android/index.ts | 20 +++++--- src/headless/helpers.ts | 120 ++++++++++++++++++++++++++----------------- src/util/http.ts | 91 ++++++++++++++++++++------------ src/wallet-impl/exchanges.ts | 14 +++-- src/wallet-impl/pay.ts | 22 ++++++-- src/wallet-impl/payback.ts | 2 +- src/wallet-impl/pending.ts | 2 +- src/wallet-impl/refresh.ts | 13 +++-- src/wallet-impl/reserves.ts | 29 ++++++----- src/wallet-impl/return.ts | 2 +- src/wallet-impl/tip.ts | 15 ++++-- src/wallet-impl/withdraw.ts | 13 +++-- src/wallet.ts | 2 +- src/walletTypes.ts | 4 +- tsconfig.json | 1 + 15 files changed, 225 insertions(+), 125 deletions(-) diff --git a/src/android/index.ts b/src/android/index.ts index 300cffd12..ec5853543 100644 --- a/src/android/index.ts +++ b/src/android/index.ts @@ -26,7 +26,7 @@ import { } from "../headless/helpers"; import { openPromise, OpenedPromise } from "../util/promiseUtils"; import fs = require("fs"); -import { HttpRequestLibrary, HttpResponse } from "../util/http"; +import { HttpRequestLibrary, HttpResponse, HttpRequestOptions } from "../util/http"; // @ts-ignore: special built-in module //import akono = require("akono"); @@ -44,7 +44,7 @@ export class AndroidHttpLib implements HttpRequestLibrary { constructor(private sendMessage: (m: string) => void) {} - get(url: string): Promise { + get(url: string, opt?: HttpRequestOptions): Promise { if (this.useNfcTunnel) { const myId = this.requestId++; const p = openPromise(); @@ -62,11 +62,11 @@ export class AndroidHttpLib implements HttpRequestLibrary { ); return p.promise; } else { - return this.nodeHttpLib.get(url); + return this.nodeHttpLib.get(url, opt); } } - postJson(url: string, body: any): Promise { + postJson(url: string, body: any, opt?: HttpRequestOptions): Promise { if (this.useNfcTunnel) { const myId = this.requestId++; const p = openPromise(); @@ -81,7 +81,7 @@ export class AndroidHttpLib implements HttpRequestLibrary { ); return p.promise; } else { - return this.nodeHttpLib.postJson(url, body); + return this.nodeHttpLib.postJson(url, body, opt); } } @@ -91,8 +91,14 @@ export class AndroidHttpLib implements HttpRequestLibrary { if (!p) { console.error(`no matching request for tunneled HTTP response, id=${myId}`); } - if (msg.status == 200) { - p.resolve({ responseJson: msg.responseJson, status: msg.status }); + if (msg.status != 0) { + const resp: HttpResponse = { + headers: {}, + status: msg.status, + json: async () => JSON.parse(msg.responseText), + text: async () => msg.responseText, + }; + p.resolve(resp); } else { p.reject(new Error(`unexpected HTTP status code ${msg.status}`)); } diff --git a/src/headless/helpers.ts b/src/headless/helpers.ts index c4c84d44b..7a9cd2ca7 100644 --- a/src/headless/helpers.ts +++ b/src/headless/helpers.ts @@ -24,8 +24,11 @@ import { Wallet } from "../wallet"; import { MemoryBackend, BridgeIDBFactory, shimIndexedDB } from "idb-bridge"; import { openTalerDb } from "../db"; -import Axios from "axios"; -import { HttpRequestLibrary } from "../util/http"; +import Axios, { AxiosPromise, AxiosResponse } from "axios"; +import { + HttpRequestLibrary, + HttpRequestOptions, +} from "../util/http"; import * as amounts from "../util/amounts"; import { Bank } from "./bank"; @@ -34,45 +37,73 @@ import { Logger } from "../util/logging"; import { NodeThreadCryptoWorkerFactory } from "../crypto/workers/nodeThreadWorker"; import { NotificationType, WalletNotification } from "../walletTypes"; import { SynchronousCryptoWorkerFactory } from "../crypto/workers/synchronousWorker"; +import { RequestThrottler } from "../util/RequestThrottler"; const logger = new Logger("helpers.ts"); - export class NodeHttpLib implements HttpRequestLibrary { - async get(url: string): Promise { + private throttle = new RequestThrottler(); + + private async req( + method: "post" | "get", + url: string, + body: any, + opt?: HttpRequestOptions, + ) { + if (this.throttle.applyThrottle(url)) { + throw Error("request throttled"); + } + let resp: AxiosResponse; try { - const resp = await Axios({ - method: "get", + resp = await Axios({ + method, url: url, - responseType: "json", + responseType: "text", + headers: opt?.headers, + validateStatus: () => true, + transformResponse: (x) => x, + data: body, }); - return { - responseJson: resp.data, - status: resp.status, - }; } catch (e) { throw e; } + const respText = resp.data; + if (typeof respText !== "string") { + throw Error("unexpected response type"); + } + const makeJson = async () => { + let responseJson; + try { + responseJson = JSON.parse(respText); + } catch (e) { + throw Error("Invalid JSON from HTTP response"); + } + if (responseJson === null || typeof responseJson !== "object") { + throw Error("Invalid JSON from HTTP response"); + } + return responseJson; + }; + return { + headers: resp.headers, + status: resp.status, + text: async () => resp.data, + json: makeJson, + }; + } + + async get( + url: string, + opt?: HttpRequestOptions, + ): Promise { + return this.req("get", url, undefined, opt); } async postJson( url: string, body: any, + opt?: HttpRequestOptions, ): Promise { - try { - const resp = await Axios({ - method: "post", - url: url, - responseType: "json", - data: body, - }); - return { - responseJson: resp.data, - status: resp.status, - }; - } catch (e) { - throw e; - } + return this.req("post", url, body, opt); } } @@ -103,8 +134,6 @@ export interface DefaultNodeWalletArgs { export async function getDefaultNodeWallet( args: DefaultNodeWalletArgs = {}, ): Promise { - - BridgeIDBFactory.enableTracing = false; const myBackend = new MemoryBackend(); myBackend.enableTracing = false; @@ -112,7 +141,9 @@ export async function getDefaultNodeWallet( const storagePath = args.persistentStoragePath; if (storagePath) { try { - const dbContentStr: string = fs.readFileSync(storagePath, { encoding: "utf-8" }); + const dbContentStr: string = fs.readFileSync(storagePath, { + encoding: "utf-8", + }); const dbContent = JSON.parse(dbContentStr); myBackend.importDump(dbContent); } catch (e) { @@ -125,7 +156,9 @@ export async function getDefaultNodeWallet( return; } const dbContent = myBackend.exportDump(); - fs.writeFileSync(storagePath, JSON.stringify(dbContent, undefined, 2), { encoding: "utf-8" }); + fs.writeFileSync(storagePath, JSON.stringify(dbContent, undefined, 2), { + encoding: "utf-8", + }); }; } @@ -164,11 +197,7 @@ export async function getDefaultNodeWallet( const worker = new NodeThreadCryptoWorkerFactory(); - const w = new Wallet( - myDb, - myHttpLib, - worker, - ); + const w = new Wallet(myDb, myHttpLib, worker); if (args.notifyHandler) { w.addNotificationListener(args.notifyHandler); } @@ -193,27 +222,24 @@ export async function withdrawTestBalance( const bankUser = await bank.registerRandomUser(); - logger.trace(`Registered bank user ${JSON.stringify(bankUser)}`) + logger.trace(`Registered bank user ${JSON.stringify(bankUser)}`); - const exchangePaytoUri = await myWallet.getExchangePaytoUri( - exchangeBaseUrl, - ["x-taler-bank"], - ); + const exchangePaytoUri = await myWallet.getExchangePaytoUri(exchangeBaseUrl, [ + "x-taler-bank", + ]); const donePromise = new Promise((resolve, reject) => { - myWallet.addNotificationListener((n) => { - if (n.type === NotificationType.ReserveDepleted && n.reservePub === reservePub ) { + myWallet.addNotificationListener(n => { + if ( + n.type === NotificationType.ReserveDepleted && + n.reservePub === reservePub + ) { resolve(); } }); }); - await bank.createReserve( - bankUser, - amount, - reservePub, - exchangePaytoUri, - ); + await bank.createReserve(bankUser, amount, reservePub, exchangePaytoUri); await myWallet.confirmReserve({ reservePub: reserveResponse.reservePub }); await donePromise; diff --git a/src/util/http.ts b/src/util/http.ts index a2bfab279..ab253b232 100644 --- a/src/util/http.ts +++ b/src/util/http.ts @@ -24,16 +24,25 @@ */ export interface HttpResponse { status: number; - responseJson: object & any; + headers: { [name: string]: string }; + json(): Promise; + text(): Promise; +} + +export interface HttpRequestOptions { + headers?: { [name: string]: string }; } /** - * The request library is bundled into an interface to make mocking easy. + * The request library is bundled into an interface to m responseJson: object & any;ake mocking easy. */ export interface HttpRequestLibrary { - get(url: string): Promise; - - postJson(url: string, body: any): Promise; + get(url: string, opt?: HttpRequestOptions): Promise; + postJson( + url: string, + body: any, + opt?: HttpRequestOptions, + ): Promise; } /** @@ -44,13 +53,20 @@ export class BrowserHttpLib implements HttpRequestLibrary { private req( method: string, url: string, - options?: any, + requestBody?: any, + options?: HttpRequestOptions, ): Promise { return new Promise((resolve, reject) => { const myRequest = new XMLHttpRequest(); myRequest.open(method, url); - if (options && options.req) { - myRequest.send(options.req); + if (options?.headers) { + for (const headerName in options.headers) { + myRequest.setRequestHeader(headerName, options.headers[headerName]); + } + } + myRequest.setRequestHeader; + if (requestBody) { + myRequest.send(requestBody); } else { myRequest.send(); } @@ -63,31 +79,42 @@ export class BrowserHttpLib implements HttpRequestLibrary { myRequest.addEventListener("readystatechange", e => { if (myRequest.readyState === XMLHttpRequest.DONE) { if (myRequest.status === 0) { - reject(Error("HTTP Request failed (status code 0, maybe URI scheme is wrong?)")) - return; - } - if (myRequest.status != 200) { reject( Error( - `HTTP Response with unexpected status code ${myRequest.status}: ${myRequest.statusText}`, + "HTTP Request failed (status code 0, maybe URI scheme is wrong?)", ), ); return; } - let responseJson; - try { - responseJson = JSON.parse(myRequest.responseText); - } catch (e) { - reject(Error("Invalid JSON from HTTP response")); - return; - } - if (responseJson === null || typeof responseJson !== "object") { - reject(Error("Invalid JSON from HTTP response")); - return; - } - const resp = { - responseJson: responseJson, + const makeJson = async () => { + let responseJson; + try { + responseJson = JSON.parse(myRequest.responseText); + } catch (e) { + throw Error("Invalid JSON from HTTP response"); + } + if (responseJson === null || typeof responseJson !== "object") { + throw Error("Invalid JSON from HTTP response"); + } + return responseJson; + }; + + const headers = myRequest.getAllResponseHeaders(); + const arr = headers.trim().split(/[\r\n]+/); + + // Create a map of header names to values + const headerMap: { [name: string]: string } = {}; + arr.forEach(function(line) { + const parts = line.split(": "); + const header = parts.shift(); + const value = parts.join(": "); + headerMap[header!] = value; + }); + const resp: HttpResponse = { status: myRequest.status, + headers: headerMap, + json: makeJson, + text: async () => myRequest.responseText, }; resolve(resp); } @@ -95,15 +122,15 @@ export class BrowserHttpLib implements HttpRequestLibrary { }); } - get(url: string) { - return this.req("get", url); + get(url: string, opt?: HttpRequestOptions) { + return this.req("get", url, undefined, opt); } - postJson(url: string, body: any) { - return this.req("post", url, { req: JSON.stringify(body) }); + postJson(url: string, body: any, opt?: HttpRequestOptions) { + return this.req("post", url, JSON.stringify(body), opt); } - postForm(url: string, form: any) { - return this.req("post", url, { req: form }); + stop() { + // Nothing to do } } diff --git a/src/wallet-impl/exchanges.ts b/src/wallet-impl/exchanges.ts index 42d626a71..9810b9b91 100644 --- a/src/wallet-impl/exchanges.ts +++ b/src/wallet-impl/exchanges.ts @@ -112,7 +112,11 @@ async function updateExchangeWithKeys( let keysResp; try { - keysResp = await ws.http.get(keysUrl.href); + const r = await ws.http.get(keysUrl.href); + if (r.status !== 200) { + throw Error(`unexpected status for keys: ${r.status}`); + } + keysResp = await r.json(); } catch (e) { const m = `Fetching keys failed: ${e.message}`; await setExchangeError(ws, baseUrl, { @@ -126,7 +130,7 @@ async function updateExchangeWithKeys( } let exchangeKeysJson: KeysJson; try { - exchangeKeysJson = KeysJson.checked(keysResp.responseJson); + exchangeKeysJson = KeysJson.checked(keysResp); } catch (e) { const m = `Parsing /keys response failed: ${e.message}`; await setExchangeError(ws, baseUrl, { @@ -242,8 +246,10 @@ async function updateExchangeWithWireInfo( reqUrl.searchParams.set("cacheBreaker", WALLET_CACHE_BREAKER_CLIENT_VERSION); const resp = await ws.http.get(reqUrl.href); - - const wiJson = resp.responseJson; + if (resp.status !== 200) { + throw Error(`/wire response has unexpected status code (${resp.status})`); + } + const wiJson = await resp.json(); if (!wiJson) { throw Error("/wire response malformed"); } diff --git a/src/wallet-impl/pay.ts b/src/wallet-impl/pay.ts index d100ad26c..89b124553 100644 --- a/src/wallet-impl/pay.ts +++ b/src/wallet-impl/pay.ts @@ -441,7 +441,11 @@ export async function abortFailedPayment( throw e; } - const refundResponse = MerchantRefundResponse.checked(resp.responseJson); + if (resp.status !== 200) { + throw Error(`unexpected status for /pay (${resp.status})`); + } + + const refundResponse = MerchantRefundResponse.checked(await resp.json()); await acceptRefundResponse(ws, purchase.proposalId, refundResponse); await runWithWriteTransaction(ws.db, [Stores.purchases], async tx => { @@ -597,7 +601,11 @@ async function processDownloadProposalImpl( throw e; } - const proposalResp = Proposal.checked(resp.responseJson); + if (resp.status !== 200) { + throw Error(`contract download failed with status ${resp.status}`); + } + + const proposalResp = Proposal.checked(await resp.json()); const contractTermsHash = await ws.cryptoApi.hashString( canonicalJson(proposalResp.contract_terms), @@ -717,7 +725,10 @@ export async function submitPay( console.log("payment failed", e); throw e; } - const merchantResp = resp.responseJson; + if (resp.status !== 200) { + throw Error(`unexpected status (${resp.status}) for /pay`); + } + const merchantResp = await resp.json(); console.log("got success from pay URL"); const merchantPub = purchase.contractTerms.merchant_pub; @@ -1317,8 +1328,11 @@ async function processPurchaseQueryRefundImpl( console.error("error downloading refund permission", e); throw e; } + if (resp.status !== 200) { + throw Error(`unexpected status code (${resp.status}) for /refund`); + } - const refundResponse = MerchantRefundResponse.checked(resp.responseJson); + const refundResponse = MerchantRefundResponse.checked(await resp.json()); await acceptRefundResponse(ws, proposalId, refundResponse); } diff --git a/src/wallet-impl/payback.ts b/src/wallet-impl/payback.ts index 56696d771..8cdfbf7ed 100644 --- a/src/wallet-impl/payback.ts +++ b/src/wallet-impl/payback.ts @@ -76,7 +76,7 @@ export async function payback( if (resp.status !== 200) { throw Error(); } - const paybackConfirmation = PaybackConfirmation.checked(resp.responseJson); + const paybackConfirmation = PaybackConfirmation.checked(await resp.json()); if (paybackConfirmation.reserve_pub !== coin.reservePub) { throw Error(`Coin's reserve doesn't match reserve on payback`); } diff --git a/src/wallet-impl/pending.ts b/src/wallet-impl/pending.ts index 729dcf125..022895e95 100644 --- a/src/wallet-impl/pending.ts +++ b/src/wallet-impl/pending.ts @@ -238,7 +238,7 @@ async function gatherCoinsPending( // Refreshing dirty coins is always due. await tx.iter(Stores.coins).forEach(coin => { if (coin.status == CoinStatus.Dirty) { - resp.nextRetryDelay.d_ms = 0; + resp.nextRetryDelay = { d_ms: 0 }; resp.pendingOperations.push({ givesLifeness: true, type: "dirty-coin", diff --git a/src/wallet-impl/refresh.ts b/src/wallet-impl/refresh.ts index a23f34324..a33511c34 100644 --- a/src/wallet-impl/refresh.ts +++ b/src/wallet-impl/refresh.ts @@ -118,16 +118,19 @@ async function refreshMelt( }; logger.trace("melt request:", meltReq); const resp = await ws.http.postJson(reqUrl.href, meltReq); + if (resp.status !== 200) { + throw Error(`unexpected status code ${resp.status} for refresh/melt`); + } - logger.trace("melt response:", resp.responseJson); + const respJson = await resp.json(); + + logger.trace("melt response:", respJson); if (resp.status !== 200) { - console.error(resp.responseJson); + console.error(respJson); throw Error("refresh failed"); } - const respJson = resp.responseJson; - const norevealIndex = respJson.noreveal_index; if (typeof norevealIndex !== "number") { @@ -228,7 +231,7 @@ async function refreshReveal( return; } - const respJson = resp.responseJson; + const respJson = await resp.json(); if (!respJson.ev_sigs || !Array.isArray(respJson.ev_sigs)) { console.error("/refresh/reveal did not contain ev_sigs"); diff --git a/src/wallet-impl/reserves.ts b/src/wallet-impl/reserves.ts index d6568bd30..504cf10f0 100644 --- a/src/wallet-impl/reserves.ts +++ b/src/wallet-impl/reserves.ts @@ -282,7 +282,10 @@ async function processReserveBankStatusImpl( let status: WithdrawOperationStatusResponse; try { const statusResp = await ws.http.get(bankStatusUrl); - status = WithdrawOperationStatusResponse.checked(statusResp.responseJson); + if (statusResp.status !== 200) { + throw Error(`unexpected status ${statusResp.status} for bank status query`); + } + status = WithdrawOperationStatusResponse.checked(await statusResp.json()); } catch (e) { throw e; } @@ -378,22 +381,24 @@ async function updateReserve( let resp; try { resp = await ws.http.get(reqUrl.href); - } catch (e) { - if (e.response?.status === 404) { + if (resp.status === 404) { const m = "The exchange does not know about this reserve (yet)."; await incrementReserveRetry(ws, reservePub, undefined); return; - } else { - const m = e.message; - await incrementReserveRetry(ws, reservePub, { - type: "network", - details: {}, - message: m, - }); - throw new OperationFailedAndReportedError(m); } + if (resp.status !== 200) { + throw Error(`unexpected status code ${resp.status} for reserve/status`) + } + } catch (e) { + const m = e.message; + await incrementReserveRetry(ws, reservePub, { + type: "network", + details: {}, + message: m, + }); + throw new OperationFailedAndReportedError(m); } - const reserveInfo = ReserveStatus.checked(resp.responseJson); + const reserveInfo = ReserveStatus.checked(await resp.json()); const balance = Amounts.parseOrThrow(reserveInfo.balance); await oneShotMutate(ws.db, Stores.reserves, reserve.reservePub, r => { if (r.reserveStatus !== ReserveRecordStatus.QUERYING_STATUS) { diff --git a/src/wallet-impl/return.ts b/src/wallet-impl/return.ts index ec19c00ae..0c142f9a6 100644 --- a/src/wallet-impl/return.ts +++ b/src/wallet-impl/return.ts @@ -238,7 +238,7 @@ async function depositReturnedCoins( console.error("deposit failed due to status code", resp); continue; } - const respJson = resp.responseJson; + const respJson = await resp.json(); if (respJson.status !== "DEPOSIT_OK") { console.error("deposit failed", resp); continue; diff --git a/src/wallet-impl/tip.ts b/src/wallet-impl/tip.ts index e11eb3b42..41463ab18 100644 --- a/src/wallet-impl/tip.ts +++ b/src/wallet-impl/tip.ts @@ -41,10 +41,12 @@ export async function getTipStatus( tipStatusUrl.searchParams.set("tip_id", res.merchantTipId); console.log("checking tip status from", tipStatusUrl.href); const merchantResp = await ws.http.get(tipStatusUrl.href); - console.log("resp:", merchantResp.responseJson); - const tipPickupStatus = TipPickupGetResponse.checked( - merchantResp.responseJson, - ); + if (merchantResp.status !== 200) { + throw Error(`unexpected status ${merchantResp.status} for tip-pickup`); + } + const respJson = await merchantResp.json(); + console.log("resp:", respJson); + const tipPickupStatus = TipPickupGetResponse.checked(respJson); console.log("status", tipPickupStatus); @@ -208,13 +210,16 @@ async function processTipImpl( try { const req = { planchets: planchetsDetail, tip_id: tipRecord.merchantTipId }; merchantResp = await ws.http.postJson(tipStatusUrl.href, req); + if (merchantResp.status !== 200) { + throw Error(`unexpected status ${merchantResp.status} for tip-pickup`); + } console.log("got merchant resp:", merchantResp); } catch (e) { console.log("tipping failed", e); throw e; } - const response = TipResponse.checked(merchantResp.responseJson); + const response = TipResponse.checked(await merchantResp.json()); if (response.reserve_sigs.length !== tipRecord.planchets.length) { throw Error("number of tip responses does not match requested planchets"); diff --git a/src/wallet-impl/withdraw.ts b/src/wallet-impl/withdraw.ts index 96055c9c5..cd3989972 100644 --- a/src/wallet-impl/withdraw.ts +++ b/src/wallet-impl/withdraw.ts @@ -117,8 +117,12 @@ export async function getWithdrawalInfo( throw Error("can't parse URL"); } const resp = await ws.http.get(uriResult.statusUrl); - console.log("resp:", resp.responseJson); - const status = WithdrawOperationStatusResponse.checked(resp.responseJson); + if (resp.status !== 200) { + throw Error(`unexpected status (${resp.status}) from bank for ${uriResult.statusUrl}`); + } + const respJson = await resp.json(); + console.log("resp:", respJson); + const status = WithdrawOperationStatusResponse.checked(respJson); return { amount: Amounts.parseOrThrow(status.amount), confirmTransferUrl: status.confirm_transfer_url, @@ -228,8 +232,11 @@ async function processPlanchet( wd.coin_ev = planchet.coinEv; const reqUrl = new URL("reserve/withdraw", exchange.baseUrl).href; const resp = await ws.http.postJson(reqUrl, wd); + if (resp.status !== 200) { + throw Error(`unexpected status ${resp.status} for withdraw`); + } - const r = resp.responseJson; + const r = await resp.json(); const denomSig = await ws.cryptoApi.rsaUnblind( r.ev_sig, diff --git a/src/wallet.ts b/src/wallet.ts index 328baf722..bf1b11fbd 100644 --- a/src/wallet.ts +++ b/src/wallet.ts @@ -22,7 +22,7 @@ /** * Imports. */ -import { CryptoApi, CryptoWorkerFactory } from "./crypto/workers/cryptoApi"; +import { CryptoWorkerFactory } from "./crypto/workers/cryptoApi"; import { HttpRequestLibrary } from "./util/http"; import { oneShotPut, diff --git a/src/walletTypes.ts b/src/walletTypes.ts index e2be26b03..32a5b0192 100644 --- a/src/walletTypes.ts +++ b/src/walletTypes.ts @@ -829,7 +829,7 @@ export class Timestamp { * Timestamp in milliseconds. */ @Checkable.Number() - t_ms: number; + readonly t_ms: number; static checked: (obj: any) => Timestamp; } @@ -838,7 +838,7 @@ export interface Duration { /** * Duration in milliseconds. */ - d_ms: number; + readonly d_ms: number; } export function getTimestampNow(): Timestamp { diff --git a/tsconfig.json b/tsconfig.json index 1650171d5..2af0ca65b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -48,6 +48,7 @@ "src/index.ts", "src/talerTypes.ts", "src/types-test.ts", + "src/util/RequestThrottler.ts", "src/util/amounts.ts", "src/util/assertUnreachable.ts", "src/util/asyncMemo.ts", -- cgit v1.2.3