implement #9153 API (alas, not yet in libtalermerchant, under-tested, and in urgent need for #9176)

2 files changed, 427 insertions, 81 deletions

diff --git a/src/backend/taler-merchant-httpd_config.c b/src/backend/taler-merchant-httpd_config.c
index 96394198..cc70f9f9 100644
--- a/src/backend/taler-merchant-httpd_config.c
+++ b/src/backend/taler-merchant-httpd_config.c
@@ -43,7 +43,7 @@
  * #MERCHANT_PROTOCOL_CURRENT and #MERCHANT_PROTOCOL_AGE in
  * merchant_api_config.c!
  */
-#define MERCHANT_PROTOCOL_VERSION "16:0:12"
+#define MERCHANT_PROTOCOL_VERSION "17:0:13"
 
 
 /**
diff --git a/src/backend/taler-merchant-httpd_private-get-instances-ID-kyc.c b/src/backend/taler-merchant-httpd_private-get-instances-ID-kyc.c
index a3d3bbe7..f8bad978 100644
--- a/src/backend/taler-merchant-httpd_private-get-instances-ID-kyc.c
+++ b/src/backend/taler-merchant-httpd_private-get-instances-ID-kyc.c
@@ -27,7 +27,7 @@
 #include "taler-merchant-httpd_helper.h"
 #include "taler-merchant-httpd_exchanges.h"
 #include <taler/taler_json_lib.h>
-
+#include <regex.h>
 
 /**
  * We do not re-check an acceptable KYC status for
@@ -76,6 +76,17 @@ struct ExchangeKycRequest
   struct TMH_EXCHANGES_KeysOperation *fo;
 
   /**
+   * JSON array of payto-URIs with KYC auth wire transfer
+   * instructions.
+   */
+  json_t *pkaa;
+
+  /**
+   * The keys of the exchange.
+   */
+  struct TALER_EXCHANGE_Keys *keys;
+
+  /**
    * KYC request this exchange request is made for.
    */
   struct KycContext *kc;
@@ -161,6 +172,12 @@ struct KycContext
 
   /**
    * JSON array where we are building up the array with
+   * pending KYC operations.
+   */
+  json_t *voluntary_kycs;
+
+  /**
+   * JSON array where we are building up the array with
    * troubled KYC operations.
    */
   json_t *timeout_kycs;
@@ -281,6 +298,9 @@ kyc_context_cleanup (void *cls)
       TMH_EXCHANGES_keys4exchange_cancel (ekr->fo);
       ekr->fo = NULL;
     }
+    json_decref (ekr->pkaa);
+    if (NULL != ekr->keys)
+      TALER_EXCHANGE_keys_decref (ekr->keys);
     GNUNET_free (ekr->exchange_url);
     GNUNET_free (ekr->payto_uri);
     GNUNET_free (ekr);
@@ -298,6 +318,7 @@ kyc_context_cleanup (void *cls)
   GNUNET_CONTAINER_DLL_remove (kc_head,
                                kc_tail,
                                kc);
+  json_decref (kc->voluntary_kycs);
   json_decref (kc->pending_kycs);
   json_decref (kc->timeout_kycs);
   GNUNET_free (kc);
@@ -441,6 +462,9 @@ ekr_finished (struct ExchangeKycRequest *ekr)
   GNUNET_CONTAINER_DLL_remove (kc->exchange_pending_head,
                                kc->exchange_pending_tail,
                                ekr);
+  json_decref (ekr->pkaa);
+  if (NULL != ekr->keys)
+    TALER_EXCHANGE_keys_decref (ekr->keys);
   GNUNET_free (ekr->exchange_url);
   GNUNET_free (ekr->payto_uri);
   GNUNET_free (ekr);
@@ -464,7 +488,7 @@ ekr_finished (struct ExchangeKycRequest *ekr)
   }
   resume_kyc_with_response (
     kc,
-    kc->response_code, /* MHD_HTTP_OK or MHD_HTTP_BAD_GATEWAY */
+    kc->response_code, /* MHD_HTTP_OK or MHD_HTTP_ACCEPTED */
     TALER_MHD_MAKE_JSON_PACK (
       GNUNET_JSON_pack_array_incref ("pending_kycs",
                                      kc->pending_kycs),
@@ -546,6 +570,233 @@ store_kyc_status (
 
 
 /**
+ * Pack the given @a limit into the JSON @a limits array.
+ *
+ * @param limit account limit to pack
+ * @param[in,out] limits JSON array to extend
+ */
+static void
+pack_limit (const struct TALER_EXCHANGE_AccountLimit *limit,
+            json_t *limits)
+{
+  json_t *jl;
+
+  jl = GNUNET_JSON_PACK (
+    TALER_JSON_pack_kycte ("operation_type",
+                           limit->operation_type),
+    GNUNET_JSON_pack_time_rel ("timeframe",
+                               limit->timeframe),
+    TALER_JSON_pack_amount ("threshold",
+                            &limit->threshold),
+    GNUNET_JSON_pack_bool ("soft_limit",
+                           limit->soft_limit)
+    );
+  GNUNET_assert (0 ==
+                 json_array_append_new (limits,
+                                        jl));
+}
+
+
+/**
+ * Return JSON array with AccountLimit objects giving
+ * the current limits for this exchange.
+ *
+ * @param[in,out] ekr overall request context
+ * @param account_kyc_status KYC status of the specific exchange, NULL if generic
+ */
+static json_t *
+get_exchange_limits (
+  struct ExchangeKycRequest *ekr,
+  const struct TALER_EXCHANGE_AccountKycStatus *account_kyc_status)
+{
+  json_t *limits;
+
+  limits = json_array ();
+  GNUNET_assert (NULL != limits);
+  if (NULL == account_kyc_status)
+  {
+    const struct TALER_EXCHANGE_Keys *keys = ekr->keys;
+
+    GNUNET_assert (NULL != keys);
+    for (unsigned int i = 0; i<keys->hard_limits_length; i++)
+    {
+      const struct TALER_EXCHANGE_AccountLimit *limit
+        = &keys->hard_limits[i];
+
+      pack_limit (limit,
+                  limits);
+    }
+    for (unsigned int i = 0; i<keys->zero_limits_length; i++)
+    {
+      const struct TALER_EXCHANGE_ZeroLimitedOperation *zlimit
+        = &keys->zero_limits[i];
+      json_t *jl;
+      struct TALER_Amount zero;
+
+      GNUNET_assert (GNUNET_OK ==
+                     TALER_amount_set_zero (keys->currency,
+                                            &zero));
+      jl = GNUNET_JSON_PACK (
+        TALER_JSON_pack_kycte ("operation_type",
+                               zlimit->operation_type),
+        GNUNET_JSON_pack_time_rel ("timeframe",
+                                   GNUNET_TIME_UNIT_ZERO),
+        TALER_JSON_pack_amount ("threshold",
+                                &zero),
+        GNUNET_JSON_pack_bool ("soft_limit",
+                               true)
+        );
+      GNUNET_assert (0 ==
+                     json_array_append_new (limits,
+                                            jl));
+    }
+  }
+  else
+  {
+    for (unsigned int i = 0; i<account_kyc_status->limits_length; i++)
+    {
+      const struct TALER_EXCHANGE_AccountLimit *limit
+        = &account_kyc_status->limits[i];
+
+      pack_limit (limit,
+                  limits);
+    }
+  }
+  return limits;
+}
+
+
+/**
+ * Update exchange KYC account status, storing it
+ * in the database and returning it in the response.
+ *
+ * @param[in,out] ekr our request context
+ * @param ks overall HTTP response
+ * @param account_kyc_status specific KYC status
+ */
+static void
+update_account_status (
+  struct ExchangeKycRequest *ekr,
+  const struct TALER_EXCHANGE_KycStatus *ks,
+  const struct TALER_EXCHANGE_AccountKycStatus *account_kyc_status)
+{
+  char *kyc_url;
+
+  GNUNET_break (store_kyc_status (ekr,
+                                  ks,
+                                  account_kyc_status));
+  {
+    char *ats;
+
+    ats = GNUNET_STRINGS_data_to_string_alloc (
+      &account_kyc_status->access_token,
+      sizeof (account_kyc_status->access_token));
+    GNUNET_asprintf (&kyc_url,
+                     "%s/kyc-spa/%s",
+                     ekr->exchange_url,
+                     ats);
+    GNUNET_free (ats);
+  }
+
+  GNUNET_assert (
+    0 ==
+    json_array_append_new (
+      ekr->kc->pending_kycs,
+      GNUNET_JSON_PACK (
+        GNUNET_JSON_pack_allow_null (
+          GNUNET_JSON_pack_array_steal (
+            "limits",
+            get_exchange_limits (ekr,
+                                 account_kyc_status))),
+        (TALER_EC_NONE == ks->hr.ec)
+        ? GNUNET_JSON_pack_allow_null (
+          GNUNET_JSON_pack_string (
+            "dummy",
+            NULL))
+        : GNUNET_JSON_pack_uint64 ("exchange_code",
+                                   ks->hr.ec),
+        GNUNET_JSON_pack_uint64 ("exchange_http_status",
+                                 ks->hr.http_status),
+        GNUNET_JSON_pack_data_auto (
+          "access_token",
+          &account_kyc_status->access_token),
+        GNUNET_JSON_pack_string (
+          "kyc_url",
+          kyc_url),
+        GNUNET_JSON_pack_string (
+          "exchange_url",
+          ekr->exchange_url),
+        GNUNET_JSON_pack_string (
+          "payto_uri",
+          ekr->payto_uri))));
+  GNUNET_free (kyc_url);
+}
+
+
+/**
+ * Return exchange KYC account status when KYC auth
+ * is required for authorization to the KYC state.
+ *
+ * @param[in,out] ekr our request context
+ * @param ks overall HTTP response
+ */
+static void
+return_auth_required (
+  struct ExchangeKycRequest *ekr,
+  const struct TALER_EXCHANGE_KycStatus *ks)
+{
+  struct KycContext *kc = ekr->kc;
+  enum GNUNET_DB_QueryStatus qs;
+
+  qs = TMH_db->account_kyc_set_status (
+    TMH_db->cls,
+    kc->mi->settings.id,
+    &ekr->h_wire,
+    ekr->exchange_url,
+    ekr->exchange_kyc_serial,
+    GNUNET_TIME_timestamp_get (),
+    ks->hr.http_status,
+    ks->hr.ec,
+    NULL,
+    0,
+    NULL,
+    false,
+    true);
+  if (qs < 0)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "Failed to store KYC status in database!\n");
+  }
+
+  GNUNET_assert (
+    0 ==
+    json_array_append_new (
+      kc->pending_kycs,
+      GNUNET_JSON_PACK (
+        GNUNET_JSON_pack_allow_null (
+          GNUNET_JSON_pack_array_steal (
+            "limits",
+            get_exchange_limits (ekr,
+                                 NULL))),
+        (TALER_EC_NONE == ks->hr.ec)
+        ? GNUNET_JSON_pack_allow_null (
+          GNUNET_JSON_pack_string (
+            "dummy",
+            NULL))
+        : GNUNET_JSON_pack_uint64 ("exchange_code",
+                                   ks->hr.ec),
+        GNUNET_JSON_pack_uint64 ("exchange_http_status",
+                                 ks->hr.http_status),
+        GNUNET_JSON_pack_array_incref ("payto_kycauths",
+                                       ekr->pkaa),
+        GNUNET_JSON_pack_string ("exchange_url",
+                                 ekr->exchange_url),
+        GNUNET_JSON_pack_string ("payto_uri",
+                                 ekr->payto_uri))));
+}
+
+
+/**
  * Function called with the result of a KYC check.
  *
  * @param cls a `struct ExchangeKycRequest *`
@@ -560,50 +811,24 @@ exchange_check_cb (
   struct KycContext *kc = ekr->kc;
 
   ekr->kyc = NULL;
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Checking KYC status of `%s' at `%s' is %u\n",
+              ekr->payto_uri,
+              ekr->exchange_url,
+              ks->hr.http_status);
   switch (ks->hr.http_status)
   {
   case MHD_HTTP_OK:
-    GNUNET_break (store_kyc_status (ekr,
-                                    ks,
-                                    &ks->details.ok));
+    update_account_status (ekr,
+                           ks,
+                           &ks->details.ok);
     break;
   case MHD_HTTP_ACCEPTED:
-    {
-      const struct TALER_EXCHANGE_AccountKycStatus *account_kyc_status
-        = &ks->details.accepted;
-      char *kyc_url;
-
-      GNUNET_break (store_kyc_status (ekr,
-                                      ks,
-                                      account_kyc_status));
-      {
-        char *ats;
-
-        ats = GNUNET_STRINGS_data_to_string_alloc (
-          &account_kyc_status->access_token,
-          sizeof (account_kyc_status->access_token));
-        GNUNET_asprintf (&kyc_url,
-                         "%s/kyc-spa/%s",
-                         ekr->exchange_url,
-                         ats);
-        GNUNET_free (ats);
-      }
-      GNUNET_assert (
-        0 ==
-        json_array_append_new (
-          kc->pending_kycs,
-          GNUNET_JSON_PACK (
-            GNUNET_JSON_pack_data_auto ("access_token",
-                                        &account_kyc_status->access_token),
-            GNUNET_JSON_pack_string ("kyc_url",
-                                     kyc_url),
-            GNUNET_JSON_pack_string ("exchange_url",
-                                     ekr->exchange_url),
-            GNUNET_JSON_pack_string ("payto_uri",
-                                     ekr->payto_uri))));
-      GNUNET_free (kyc_url);
-      break;
-    }
+    kc->response_code = MHD_HTTP_ACCEPTED;
+    update_account_status (ekr,
+                           ks,
+                           &ks->details.accepted);
+    break;
   case MHD_HTTP_NO_CONTENT:
     {
       enum GNUNET_DB_QueryStatus qs;
@@ -629,44 +854,11 @@ exchange_check_cb (
       }
     }
     break;
-  case MHD_HTTP_FORBIDDEN:
-  case MHD_HTTP_NOT_FOUND:
-  case MHD_HTTP_CONFLICT:
-  case MHD_HTTP_UNAVAILABLE_FOR_LEGAL_REASONS: // FIXME: real?
-    {
-      enum GNUNET_DB_QueryStatus qs;
-
-      GNUNET_assert (
-        0 ==
-        json_array_append_new (
-          kc->timeout_kycs,
-          GNUNET_JSON_PACK (
-            GNUNET_JSON_pack_string ("exchange_url",
-                                     ekr->exchange_url),
-            GNUNET_JSON_pack_uint64 ("exchange_code",
-                                     ks->hr.ec),
-            GNUNET_JSON_pack_uint64 ("exchange_http_status",
-                                     ks->hr.http_status))));
-      qs = TMH_db->account_kyc_set_status (
-        TMH_db->cls,
-        kc->mi->settings.id,
-        &ekr->h_wire,
-        ekr->exchange_url,
-        ekr->exchange_kyc_serial,
-        GNUNET_TIME_timestamp_get (),
-        ks->hr.http_status,
-        ks->hr.ec,
-        NULL,
-        0,
-        NULL,
-        false,
-        true);
-      if (qs < 0)
-      {
-        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                    "Failed to store KYC status in database!\n");
-      }
-    }
+  case MHD_HTTP_FORBIDDEN: /* bad signature */
+  case MHD_HTTP_NOT_FOUND: /* account unknown */
+  case MHD_HTTP_CONFLICT: /* no account_pub known */
+    return_auth_required (ekr,
+                          ks);
     break;
   default:
     {
@@ -715,6 +907,138 @@ exchange_check_cb (
 
 
 /**
+ * Figure out which exchange accounts from @a keys could
+ * be used for a KYC auth wire transfer from the account
+ * that @a ekr is checking. Will set the "pkaa" array
+ * in @a ekr.
+ *
+ * @param[in,out] request we are processing
+ * @param keys exchange keys
+ */
+static void
+determine_eligible_accounts (
+  struct ExchangeKycRequest *ekr,
+  const struct TALER_EXCHANGE_Keys *keys)
+{
+  struct KycContext *kc = ekr->kc;
+  struct TALER_Amount kyc_amount;
+  char *merchant_pub_str;
+
+  ekr->pkaa = json_array ();
+  GNUNET_assert (NULL != ekr->pkaa);
+  {
+    const struct TALER_EXCHANGE_GlobalFee *gf;
+
+    gf = TALER_EXCHANGE_get_global_fee (keys,
+                                        GNUNET_TIME_timestamp_get ());
+    if (NULL == gf)
+    {
+      GNUNET_assert (GNUNET_OK ==
+                     TALER_amount_set_zero (keys->currency,
+                                            &kyc_amount));
+    }
+    else
+    {
+      /* FIXME: history fee should be globally renamed to KYC fee... */
+      kyc_amount = gf->fees.history;
+    }
+  }
+
+  merchant_pub_str
+    = GNUNET_STRINGS_data_to_string_alloc (
+        &kc->mi->merchant_pub,
+        sizeof (kc->mi->merchant_pub));
+  /* For all accounts of the exchange */
+  for (unsigned int i = 0; i<keys->accounts_len; i++)
+  {
+    struct TALER_EXCHANGE_WireAccount *account
+      = &keys->accounts[i];
+    bool account_restricted = false;
+    const char *exchange_account_payto
+      = account->payto_uri;
+
+    /* KYC auth transfers are never supported with conversion */
+    if (NULL != account->conversion_url)
+      continue;
+
+    /* filter by source account by credit_restrictions */
+    for (unsigned int j = 0; j<account->credit_restrictions_length; j++)
+    {
+      const struct TALER_EXCHANGE_AccountRestriction *ar
+        = &account->credit_restrictions[j];
+
+      switch (ar->type)
+      {
+      case TALER_EXCHANGE_AR_INVALID:
+        continue;
+      case TALER_EXCHANGE_AR_DENY:
+        account_restricted = true;
+        break;
+      case TALER_EXCHANGE_AR_REGEX:
+        {
+          regex_t ex;
+          bool allowed = false;
+
+          if (0 != regcomp (&ex,
+                            ar->details.regex.posix_egrep,
+                            REG_NOSUB | REG_EXTENDED))
+          {
+            GNUNET_break_op (0);
+            continue;
+          }
+          if (regexec (&ex,
+                       ekr->payto_uri,
+                       0, NULL,
+                       REG_STARTEND))
+          {
+            GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                        "Merchant account `%s' allowed by regex\n",
+                        ekr->payto_uri);
+            allowed = true;
+          }
+          regfree (&ex);
+          if (! allowed)
+            account_restricted = true;
+          break;
+        }
+      }   /* end switch */
+    }   /* end loop over credit restrictions */
+    if (account_restricted)
+      continue;
+    /* exchange account is allowed, add it */
+    {
+      char *payto_kycauth;
+
+      if (TALER_amount_is_zero (&kyc_amount))
+        GNUNET_asprintf (&payto_kycauth,
+                         "%s%csubject=%s",
+                         exchange_account_payto,
+                         (NULL == strchr (exchange_account_payto,
+                                          '?'))
+                       ? '?'
+                       : '&',
+                         merchant_pub_str);
+      else
+        GNUNET_asprintf (&payto_kycauth,
+                         "%s%camount=%s&subject=%s",
+                         exchange_account_payto,
+                         (NULL == strchr (exchange_account_payto,
+                                          '?'))
+                       ? '?'
+                       : '&',
+                         TALER_amount2s (&kyc_amount),
+                         merchant_pub_str);
+      GNUNET_assert (0 ==
+                     json_array_append_new (ekr->pkaa,
+                                            json_string (payto_kycauth)));
+      GNUNET_free (payto_kycauth);
+    }
+  }
+  GNUNET_free (merchant_pub_str);
+}
+
+
+/**
  * Function called with the result of a #TMH_EXCHANGES_keys4exchange()
  * operation.  Runs the KYC check against the exchange.
  *
@@ -736,6 +1060,9 @@ kyc_with_exchange (void *cls,
   ekr->fo = NULL;
   if (NULL == keys)
   {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "Failed to download `%s/keys`\n",
+                ekr->exchange_url);
     kc->response_code = MHD_HTTP_BAD_GATEWAY;
     GNUNET_assert (
       0 ==
@@ -747,9 +1074,28 @@ kyc_with_exchange (void *cls,
     ekr_finished (ekr);
     return;
   }
+  determine_eligible_accounts (ekr,
+                               keys);
+  if (0 == json_array_size (ekr->pkaa))
+  {
+    /* No KYC auth wire transfers are possible to this exchange from
+       our merchant bank account, so we cannot use this account with
+       this exchange if it has any KYC requirements! */
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "KYC auth to `%s' impossible for merchant account `%s'\n",
+                ekr->exchange_url,
+                ekr->payto_uri);
+    ekr_finished (ekr);
+    return;
+  }
+  ekr->keys = TALER_EXCHANGE_keys_incref (keys);
   TALER_payto_hash (ekr->payto_uri,
                     &h_payto);
   ap.merchant_priv = kc->mi->merchant_priv;
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Checking KYC status of `%s' at `%s'\n",
+              ekr->payto_uri,
+              ekr->exchange_url);
   ekr->kyc = TALER_EXCHANGE_kyc_check (
     TMH_curl_ctx,
     ekr->exchange_url,
@@ -797,7 +1143,7 @@ kyc_status_cb (
     kc->kyc_serial_pending = true;
     return;
   }
-  kc->response_code = MHD_HTTP_ACCEPTED;
+  kc->response_code = MHD_HTTP_OK;
   ekr = GNUNET_new (struct ExchangeKycRequest);
   GNUNET_CONTAINER_DLL_insert (kc->exchange_pending_head,
                                kc->exchange_pending_tail,