Merge branch 'master' into age-withdraw

3 files changed, 148 insertions, 62 deletions

diff --git a/src/benchmark/benchmark-common.conf b/src/benchmark/benchmark-common.conf
index 1cd28d622..5ac4a3974 100644
--- a/src/benchmark/benchmark-common.conf
+++ b/src/benchmark/benchmark-common.conf
@@ -26,9 +26,27 @@ LOOKAHEAD_SIGN="1 d"
 DURATION="2 d"
 LOOKAHEAD_SIGN="1 d"
 
+# Trust local exchange for "EUR" currency
+[merchant-exchange-benchmark]
+EXCHANGE_BASE_URL = http://localhost:8081/
+MASTER_KEY=98NJW3CQHZQGQXTY3K85K531XKPAPAVV4Q5V8PYYRR00NJGZWNVG
+# If currency does not match [TALER] section, the exchange
+# will be ignored!
+CURRENCY = EUR
+
+
 [exchangedb-postgres]
 CONFIG="postgres:///talercheck"
 
+[merchantdb-postgres]
+CONFIG="postgres:///talercheck"
+
+[auditordb-postgres]
+CONFIG="postgres:///talercheck"
+
+[syncdb-postgres]
+CONFIG="postgres:///talercheck"
+
 [exchange-offline]
 MASTER_PRIV_FILE=${TALER_DATA_HOME}/exchange/offline-keys/master.priv
 
@@ -40,12 +58,12 @@ MAX_DEBT_BANK=EUR:1000000000000000.0
 DATABASE=bank-db.sqlite3
 
 [libeufin-nexus]
-DB_CONNECTION="jdbc:postgresql://localhost/talercheck?socketFactory=org.newsclub.net.unix."
-#DB_CONNECTION="jdbc:sqlite:libeufin-nexus.sqlite3"
+#DB_CONNECTION="jdbc:postgresql://localhost/talercheck?socketFactory=org.newsclub.net.unix."
+DB_CONNECTION="jdbc:sqlite:libeufin-nexus.sqlite3"
 
 [libeufin-sandbox]
-DB_CONNECTION="jdbc:postgresql://localhost/talercheck?socketFactory=org.newsclub.net.unix."
-#DB_CONNECTION="jdbc:sqlite:libeufin-sandbox.sqlite3"
+#DB_CONNECTION="jdbc:postgresql://localhost/talercheck?socketFactory=org.newsclub.net.unix."
+DB_CONNECTION="jdbc:sqlite:libeufin-sandbox.sqlite3"
 
 [auditor]
 BASE_URL="http://localhost:8083/"
diff --git a/src/benchmark/taler-benchmark-setup.sh b/src/benchmark/taler-benchmark-setup.sh
index 52e0e08bc..57f2e0cde 100755
--- a/src/benchmark/taler-benchmark-setup.sh
+++ b/src/benchmark/taler-benchmark-setup.sh
@@ -17,7 +17,14 @@
 # License along with TALER; see the file COPYING.  If not, see
 # <http://www.gnu.org/licenses/>
 #
-
+# Author: Christian Grothoff
+#
+# This script configures and launches various GNU Taler services.
+# Which ones depend on command-line options. Use "-h" to find out.
+# Prints "<<READY>>" on a separate line once all requested services
+# are running. Close STDIN (or input 'NEWLINE') to stop all started
+# services again.
+#
 set -eu
 
 # Exit, with status code "skip" (no 'real' failure)
@@ -40,26 +47,33 @@ function cleanup()
         kill $n 2> /dev/null || true
     done
     wait
+    rm -f libeufin-nexus.pid libeufin-sandbox.pid
 }
 
 # Install cleanup handler (except for kill -9)
 trap cleanup EXIT
 
 START_AUDITOR=0
+START_BACKUP=0
 START_EXCHANGE=0
 START_FAKEBANK=0
 START_MERCHANT=0
 START_NEXUS=0
 START_SANDBOX=0
+USE_VALGRIND=""
 CONF_ORIG="~/.config/taler.conf"
 LOGLEVEL="DEBUG"
+DEFAULT_SLEEP="0.2"
 
 # Parse command-line options
-while getopts ':abc:efhl:ms' OPTION; do
+while getopts ':abc:efhl:mnsv' OPTION; do
     case "$OPTION" in
         a)
             START_AUDITOR="1"
             ;;
+        b)
+            START_BACKUP="1"
+            ;;
         c)
             CONF_ORIG="$OPTARG"
             ;;
@@ -72,6 +86,7 @@ while getopts ':abc:efhl:ms' OPTION; do
         h)
             echo 'Supported options:'
             echo '  -a           -- start auditor'
+            echo '  -b           -- start backup/sync'
             echo '  -c $CONF     -- set configuration'
             echo '  -e           -- start exchange'
             echo '  -f           -- start fakebank'
@@ -80,6 +95,7 @@ while getopts ':abc:efhl:ms' OPTION; do
             echo '  -m           -- start merchant'
             echo '  -n           -- start nexus'
             echo '  -s           -- start sandbox'
+            echo '  -v           -- use valgrind'
             exit 0
             ;;
         l)
@@ -94,6 +110,10 @@ while getopts ':abc:efhl:ms' OPTION; do
         s)
             START_SANDBOX="1"
             ;;
+        v)
+            USE_VALGRIND="valgrind --leak-check=yes"
+            DEFAULT_SLEEP="2"
+            ;;
         ?)
         exit_fail "Unrecognized command line option"
         ;;
@@ -122,6 +142,13 @@ then
     echo " FOUND"
 fi
 
+if [ "1" = "$START_BACKUP" ]
+then
+    echo -n "Testing for sync-httpd"
+    sync-httpd -h > /dev/null || exit_skip " sync-httpd required"
+    echo " FOUND"
+fi
+
 if [ "1" = "$START_NEXUS" ]
 then
     echo -n "Testing for libeufin-cli"
@@ -135,6 +162,11 @@ CURRENCY=$(taler-config -c "$CONF" -s "TALER" -o "CURRENCY")
 register_sandbox_account() {
     export LIBEUFIN_SANDBOX_USERNAME="$1"
     export LIBEUFIN_SANDBOX_PASSWORD="$2"
+    # FIXME-MS: delete should be removed after we make 'register' idempotent!
+    libeufin-cli sandbox \
+      demobank \
+      delete \
+      --bank-account "$1" &> /dev/null || true
     libeufin-cli sandbox \
       demobank \
       register --name "$3"
@@ -158,37 +190,37 @@ then
     export LIBEUFIN_SANDBOX_DB_CONNECTION=$(taler-config -c "$CONF" -s "libeufin-sandbox" -o "DB_CONNECTION")
 
     # Create the default demobank.
-    libeufin-sandbox config --currency "$CURRENCY" default
+    echo -n "Configuring sandbox "
+    libeufin-sandbox config --currency "$CURRENCY" default &> libeufin-sandbox-config.log
+    echo "DONE"
+    echo -n "Launching sandbox "
     export LIBEUFIN_SANDBOX_ADMIN_PASSWORD="secret"
     libeufin-sandbox serve \
       --port "$SANDBOX_PORT" \
       > libeufin-sandbox-stdout.log \
       2> libeufin-sandbox-stderr.log &
     echo $! > libeufin-sandbox.pid
+    echo "DONE"
     export LIBEUFIN_SANDBOX_URL="http://localhost:$SANDBOX_PORT/"
-    set +e
     OK="0"
     echo -n "Waiting for Sandbox ..."
     for n in $(seq 1 100); do
         echo -n "."
-        sleep 0.2
-        if wget --timeout=1 \
-                --tries=3 \
-                --waitretry=0 \
-                -o /dev/null \
-                -O /dev/null \
-                "$LIBEUFIN_SANDBOX_URL";
-        then
-            OK="1"
-            break
-        fi
+        sleep "$DEFAULT_SLEEP"
+        wget --timeout=1 \
+             --tries=3 \
+             --waitretry=0 \
+             -o /dev/null \
+             -O /dev/null \
+             "$LIBEUFIN_SANDBOX_URL" || continue
+        OK="1"
+        break
     done
     if [ "1" != "$OK" ]
     then
         exit_skip "Failed to launch services (sandbox)"
     fi
     echo "OK"
-    set -e
     echo -n "Register Sandbox users ..."
     register_sandbox_account fortytwo x "Forty Two"
     register_sandbox_account fortythree x "Forty Three"
@@ -215,15 +247,18 @@ then
     export LIBEUFIN_SANDBOX_USERNAME="admin"
     export LIBEUFIN_SANDBOX_PASSWORD="secret"
     echo -n "Create EBICS host at Sandbox.."
+    # FIXME-MS: || true should be removed after we make 'create' idempotent!
     libeufin-cli sandbox \
        --sandbox-url "$LIBEUFIN_SANDBOX_URL" \
-       ebicshost create --host-id talerebics
+       ebicshost create --host-id talerebics &> libeufin-sandbox-ebicshost-create.log || true
     echo "OK"
     echo -n "Create exchange EBICS subscriber at Sandbox.."
+    # FIXME-MS: || true should be removed after we make 'new-ebicssubscriber' idempotent!
     libeufin-cli sandbox \
        demobank new-ebicssubscriber --host-id talerebics \
        --user-id exchangeebics --partner-id talerpartner \
-       --bank-account exchange # that's a username _and_ a bank account name
+       --bank-account exchange &> libeufin-sandbox-ebicsscubscriber.log || true
+    # that's a username _and_ a bank account name
     echo "OK"
     unset LIBEUFIN_SANDBOX_USERNAME
     unset LIBEUFIN_SANDBOX_PASSWORD
@@ -235,7 +270,7 @@ then
 
     # Prepare Nexus, which is the side actually talking
     # to the exchange.
-    export LIBEUFIN_SANDBOX_DB_CONNECTION=$(taler-config -c "$CONF" -s "libeufin-nexus" -o "DB_CONNECTION")
+    export LIBEUFIN_NEXUS_DB_CONNECTION=$(taler-config -c "$CONF" -s "libeufin-nexus" -o "DB_CONNECTION")
 
     # For convenience, username and password are
     # identical to those used at the Sandbox.
@@ -248,27 +283,23 @@ then
     echo $! > libeufin-nexus.pid
     export LIBEUFIN_NEXUS_URL="http://localhost:$NEXUS_PORT"
     echo -n "Waiting for Nexus ..."
-    set +e
     OK="0"
     for n in $(seq 1 100); do
         echo -n "."
-        sleep 0.2
-        if wget --timeout=1 \
-                --tries=3 \
-                --waitretry=0 \
-                -o /dev/null \
-                -O /dev/null \
-                "$LIBEUFIN_NEXUS_URL";
-        then
-            OK="1"
-            break
-        fi
+        sleep "$DEFAULT_SLEEP"
+        wget --timeout=1 \
+             --tries=3 \
+             --waitretry=0 \
+             -o /dev/null \
+             -O /dev/null \
+             "$LIBEUFIN_NEXUS_URL" || continue
+        OK="1"
+        break
     done
     if [ "1" != "$OK" ]
     then
         exit_skip "Failed to launch services (bank)"
     fi
-    set -e
     echo " OK"
 
     export LIBEUFIN_NEXUS_USERNAME=exchange
@@ -326,7 +357,7 @@ fi
 if [ "1" = "$START_FAKEBANK" ]
 then
     echo "Setting up fakebank ..."
-    taler-fakebank-run -c "$CONF" -L "$LOGLEVEL" 2> taler-fakebank-run.log &
+    $USE_VALGRIND taler-fakebank-run -c "$CONF" -L "$LOGLEVEL" 2> taler-fakebank-run.log &
 fi
 
 
@@ -334,6 +365,8 @@ if [ "1" = "$START_EXCHANGE" ]
 then
     echo -n "Starting exchange ..."
 
+    EXCHANGE_PORT=$(taler-config -c "$CONF" -s EXCHANGE -o PORT)
+    EXCHANGE_URL="http://localhost:${EXCHANGE_PORT}/"
     MASTER_PRIV_FILE=$(taler-config -f -c "${CONF}" -s "EXCHANGE-OFFLINE" -o "MASTER_PRIV_FILE")
     MASTER_PRIV_DIR=$(dirname "$MASTER_PRIV_FILE")
     mkdir -p "${MASTER_PRIV_DIR}"
@@ -343,30 +376,54 @@ then
     if [ "$MPUB" != "$MASTER_PUB" ]
     then
         echo -n " patching master_pub ($MASTER_PUB)..."
-        taler-config -c $CONF -s exchange -o MASTER_PUBLIC_KEY -V "$MASTER_PUB"
+        taler-config -c "$CONF" -s exchange -o MASTER_PUBLIC_KEY -V "$MASTER_PUB"
     fi
     taler-exchange-dbinit -c "$CONF"
-    taler-exchange-secmod-eddsa -c "$CONF" -L "$LOGLEVEL" 2> taler-exchange-secmod-eddsa.log &
-    taler-exchange-secmod-rsa -c "$CONF" -L "$LOGLEVEL" 2> taler-exchange-secmod-rsa.log &
-    taler-exchange-secmod-cs -c "$CONF" -L "$LOGLEVEL" 2> taler-exchange-secmod-cs.log &
-    taler-exchange-httpd -c "$CONF" -L "$LOGLEVEL" 2> taler-exchange-httpd.log &
+    $USE_VALGRIND taler-exchange-secmod-eddsa -c "$CONF" -L "$LOGLEVEL" 2> taler-exchange-secmod-eddsa.log &
+    $USE_VALGRIND taler-exchange-secmod-rsa -c "$CONF" -L "$LOGLEVEL" 2> taler-exchange-secmod-rsa.log &
+    $USE_VALGRIND taler-exchange-secmod-cs -c "$CONF" -L "$LOGLEVEL" 2> taler-exchange-secmod-cs.log &
+    $USE_VALGRIND taler-exchange-httpd -c "$CONF" -L "$LOGLEVEL" 2> taler-exchange-httpd.log &
     EXCHANGE_HTTPD_PID=$!
-    taler-exchange-wirewatch -c "$CONF" 2> taler-exchange-wirewatch.log &
+    $USE_VALGRIND taler-exchange-wirewatch -c "$CONF" 2> taler-exchange-wirewatch.log &
     WIREWATCH_PID=$!
+    $USE_VALGRIND taler-exchange-aggregator -c "$CONF" 2> taler-exchange-aggregator.log &
+    AGGREGATOR_PID=$!
+    $USE_VALGRIND taler-exchange-transfer -c "$CONF" 2> taler-exchange-transfer.log &
+    TRANSFER_PID=$!
     echo " DONE"
 fi
 
 if [ "1" = "$START_MERCHANT" ]
 then
     echo -n "Starting merchant ..."
+    MEPUB=$(taler-config -c "$CONF" -s merchant-exchange-benchmark -o MASTER_KEY)
+    MXPUB=${MASTER_PUB:-$(taler-config -c "$CONF" -s exchange -o MASTER_PUBLIC_KEY)}
+    if [ "$MEPUB" != "$MXPUB" ]
+    then
+        echo -n " patching master_pub ($MXPUB)..."
+        taler-config -c "$CONF" -s merchant-exchange-benchmark -o MASTER_KEY -V "$MXPUB"
+    fi
     MERCHANT_PORT=$(taler-config -c "$CONF" -s MERCHANT -o PORT)
     MERCHANT_URL="http://localhost:${MERCHANT_PORT}/"
     taler-merchant-dbinit -c "$CONF"
-    taler-merchant-httpd -c "$CONF" -L "$LOGLEVEL" 2> taler-merchant-httpd.log &
+    $USE_VALGRIND taler-merchant-httpd -c "$CONF" -L "$LOGLEVEL" 2> taler-merchant-httpd.log &
     MERCHANT_HTTPD_PID=$!
+    $USE_VALGRIND taler-merchant-webhook -c "$CONF" -L "$LOGLEVEL" 2> taler-merchant-webhook.log &
+    MERCHANT_WEBHOOK_PID=$!
+    echo " DONE"
+fi
+
+if [ "1" = "$START_BACKUP" ]
+then
+    echo -n "Starting sync ..."
+    SYNC_PORT=$(taler-config -c "$CONF" -s SYNC -o PORT)
+    SYNC_URL="http://localhost:${SYNC_PORT}/"
+    sync-dbinit -c "$CONF"
+    $USE_VALGRIND sync-httpd -c "$CONF" -L "$LOGLEVEL" 2> sync-httpd.log &
     echo " DONE"
 fi
 
+
 if [ "1" = "$START_AUDITOR" ]
 then
     echo -n "Starting auditor ..."
@@ -376,9 +433,10 @@ then
     mkdir -p "$AUDITOR_PRIV_DIR"
     gnunet-ecc -g1 "$AUDITOR_PRIV_FILE" > /dev/null 2> /dev/null
     AUDITOR_PUB=$(gnunet-ecc -p "${AUDITOR_PRIV_FILE}")
+    MAPUB=${MASTER_PUB:-$(taler-config -c "$CONF" -s exchange -o MASTER_PUBLIC_KEY)}
     taler-auditor-dbinit -c "$CONF"
-    taler-auditor-exchange -c "$CONF" -m "$MASTER_PUB" -u "$EXCHANGE_URL"
-    taler-auditor-httpd -L "$LOGLEVEL" -c "$CONF" 2> taler-auditor-httpd.log &
+    taler-auditor-exchange -c "$CONF" -m "$MAPUB" -u "$EXCHANGE_URL"
+    $USE_VALGRIND taler-auditor-httpd -L "$LOGLEVEL" -c "$CONF" 2> taler-auditor-httpd.log &
     echo " DONE"
 fi
 
@@ -390,7 +448,7 @@ then
     for n in $(seq 1 300)
     do
         echo -n "."
-        sleep 0.1
+        sleep "$DEFAULT_SLEEP"
         # bank
         wget --tries=1 \
              --waitretry=0 \
@@ -415,14 +473,14 @@ echo -n "Waiting for Taler services ..."
 for n in $(seq 1 20)
 do
     echo -n "."
-    sleep 0.1
+    sleep "$DEFAULT_SLEEP"
     OK="0"
     if [ "1" = "$START_EXCHANGE" ]
     then
         wget \
             --tries=1 \
             --timeout=1 \
-            "http://localhost:8081/seed" \
+            "http://localhost:8081/config" \
             -o /dev/null \
             -O /dev/null >/dev/null || continue
     fi
@@ -431,7 +489,16 @@ do
         wget \
             --tries=1 \
             --timeout=1 \
-            "http://localhost:9966/" \
+            "${MERCHANT_URL}config" \
+            -o /dev/null \
+            -O /dev/null >/dev/null || continue
+    fi
+    if [ "1" = "$START_BACKUP" ]
+    then
+        wget \
+            --tries=1 \
+            --timeout=1 \
+            "${SYNC_URL}config" \
             -o /dev/null \
             -O /dev/null >/dev/null || continue
     fi
@@ -440,7 +507,7 @@ do
         wget \
             --tries=1 \
             --timeout=1 \
-            "http://localhost:8083/" \
+            "${AUDITOR_URL}config" \
             -o /dev/null \
             -O /dev/null >/dev/null || continue
     fi
@@ -455,14 +522,13 @@ echo " OK"
 
 if [ "1" = "$START_EXCHANGE" ]
 then
-    set +e
     echo -n "Wait for exchange /management/keys to be ready "
     OK="0"
     LAST_RESPONSE=$(mktemp tmp-last-response.XXXXXXXX)
     for n in $(seq 1 50)
     do
         echo -n "."
-        sleep 0.1
+        sleep "$DEFAULT_SLEEP"
         # exchange
         wget \
             --tries=3 \
@@ -471,16 +537,13 @@ then
             "http://localhost:8081/management/keys"\
             -o /dev/null \
             -O "$LAST_RESPONSE" \
-            >/dev/null
-        DENOMS_COUNT=$(jq '.future_denoms|length' < $LAST_RESPONSE)
-        SIGNKEYS_COUNT=$(jq '.future_signkeys|length' < $LAST_RESPONSE)
-        [[ -z "$SIGNKEYS_COUNT" || "$SIGNKEYS_COUNT" == "0" || -z "$DENOMS_COUNT" || "$DENOMS_COUNT" == "0" ]] && continue
+            >/dev/null || continue
         OK="1"
         break;
     done
-    set -e
     if [ "1" != "$OK" ]
     then
+        cat "$LAST_RESPONSE"
         exit_skip "Failed to setup exchange keys, check secmod logs"
     fi
     rm "$LAST_RESPONSE"
@@ -504,7 +567,7 @@ then
             taler-exchange-offline -c "$CONF" \
               enable-account "$EXCHANGE_PAYTO_URI" \
               upload &> "taler-exchange-offline-account-$ASEC.log"
-            echo "OK"
+            echo " OK"
         fi
     done
     if [ "1" = "$START_AUDITOR" ]
@@ -518,29 +581,34 @@ then
 
     echo -n "Checking /keys "
     OK="0"
-    for n in $(seq 1 3)
+    LAST_RESPONSE=$(mktemp tmp-last-response.XXXXXXXX)
+    for n in $(seq 1 10)
     do
         echo -n "."
+        sleep "$DEFAULT_SLEEP"
         wget \
             --tries=1 \
             --timeout=1 \
             "http://localhost:8081/keys" \
             -o /dev/null \
-            -O /dev/null >/dev/null || continue
+            -O "$LAST_RESPONSE" \
+             >/dev/null || continue
         OK="1"
         break
     done
     if [ "1" != "$OK" ]
     then
+        cat "$LAST_RESPONSE"
         exit_skip " Failed to setup keys"
     fi
+    rm "$LAST_RESPONSE"
     echo " OK"
 fi
 
 if [ "1" = "$START_AUDITOR" ]
 then
     echo -n "Setting up auditor signatures ..."
-    taler-auditor-offline -c "$CONF" \
+    timeout 15 taler-auditor-offline -c "$CONF" \
       download \
       sign \
       upload &> taler-auditor-offline.log
diff --git a/src/kyclogic/taler-exchange-kyc-kycaid-converter.sh b/src/kyclogic/taler-exchange-kyc-kycaid-converter.sh
index 96aca2b80..96aca2b80 100644..100755
--- a/src/kyclogic/taler-exchange-kyc-kycaid-converter.sh
+++ b/src/kyclogic/taler-exchange-kyc-kycaid-converter.sh