towards new /kyc API: test_kyc_api still fails, but getting close

2 files changed, 404 insertions, 610 deletions

diff --git a/src/backend/taler-merchant-httpd_private-get-instances-ID-kyc.c b/src/backend/taler-merchant-httpd_private-get-instances-ID-kyc.c
index 69e962d0..7fb229b2 100644
--- a/src/backend/taler-merchant-httpd_private-get-instances-ID-kyc.c
+++ b/src/backend/taler-merchant-httpd_private-get-instances-ID-kyc.c
@@ -27,6 +27,7 @@
 #include "taler-merchant-httpd_helper.h"
 #include "taler-merchant-httpd_exchanges.h"
 #include <taler/taler_json_lib.h>
+#include <taler/taler_dbevents.h>
 #include <regex.h>
 
 /**
@@ -77,7 +78,8 @@ struct ExchangeKycRequest
 
   /**
    * JSON array of payto-URIs with KYC auth wire transfer
-   * instructions.
+   * instructions.  Provided if @e auth_ok is false and
+   * @e kyc_auth_conflict is false.
    */
   json_t *pkaa;
 
@@ -92,14 +94,10 @@ struct ExchangeKycRequest
   struct KycContext *kc;
 
   /**
-   * Hash of the wire account (with salt) we are checking.
-   */
-  struct TALER_MerchantWireHashP h_wire;
-
-  /**
-   * Handle for the actual HTTP request to the exchange.
+   * JSON array of AccountLimits that apply, NULL if
+   * unknown (and likely defaults apply).
    */
-  struct TALER_EXCHANGE_KycCheckHandle *kyc;
+  json_t *jlimits;
 
   /**
    * Our account's payto URI.
@@ -112,10 +110,62 @@ struct ExchangeKycRequest
   char *exchange_url;
 
   /**
+   * Hash of the wire account (with salt) we are checking.
+   */
+  struct TALER_MerchantWireHashP h_wire;
+
+  /**
+   * Current access token for the KYC SPA. Only set
+   * if @e auth_ok is true.
+   */
+  struct TALER_AccountAccessTokenP access_token;
+
+  /**
    * Timestamp when we last got a reply from the exchange.
    */
   struct GNUNET_TIME_Timestamp last_check;
 
+  /**
+   * Last HTTP status code obtained via /kyc-check from
+   * the exchange.
+   */
+  unsigned int last_http_status;
+
+  /**
+   * Last Taler error code returned from /kyc-check.
+   */
+  enum TALER_ErrorCode last_ec;
+
+  /**
+   * True if this account
+   * cannot work at this exchange because KYC auth is
+   * impossible.
+   */
+  bool kyc_auth_conflict;
+
+  /**
+   * We could not get /keys from the exchange.
+   */
+  bool no_keys;
+
+  /**
+   * True if @e access_token is available.
+   */
+  bool auth_ok;
+
+  /**
+   * True if we believe no KYC is currently required
+   * for this account at this exchange.
+   */
+  bool kyc_ok;
+
+  /**
+   * True if the exchange exposed to us that the account
+   * is currently under AML review.
+   */
+  bool in_aml_review;
+
+
 };
 
 
@@ -150,11 +200,6 @@ struct KycContext
   struct TMH_HandlerContext *hc;
 
   /**
-   * Task to trigger on request timeout, or NULL.
-   */
-  struct GNUNET_SCHEDULER_Task *timeout_task;
-
-  /**
    * Response to return, NULL if we don't have one yet.
    */
   struct MHD_Response *response;
@@ -163,19 +208,7 @@ struct KycContext
    * JSON array where we are building up the array with
    * pending KYC operations.
    */
-  json_t *pending_kycs;
-
-  /**
-   * JSON array where we are building up the array with
-   * pending KYC operations.
-   */
-  json_t *voluntary_kycs;
-
-  /**
-   * JSON array where we are building up the array with
-   * troubled KYC operations.
-   */
-  json_t *timeout_kycs;
+  json_t *kycs_data;
 
   /**
    * Head of DLL of requests we are making to an
@@ -196,6 +229,12 @@ struct KycContext
   const char *exchange_url;
 
   /**
+   * Notification handler from database on changes
+   * to the KYC status.
+   */
+  struct GNUNET_DB_EventHandler *eh;
+
+  /**
    * Set to the h_wire of the merchant account if
    * @a have_h_wire is true, used to filter by account.
    */
@@ -222,6 +261,11 @@ struct KycContext
   enum GNUNET_GenericReturnValue suspended;
 
   /**
+   * What state are we long-polling for?
+   */
+  enum TALER_EXCHANGE_KycLongPollTarget lpt;
+
+  /**
    * True if @e h_wire was given.
    */
   bool have_h_wire;
@@ -230,7 +274,7 @@ struct KycContext
    * We're still waiting on the exchange to determine
    * the KYC status of our deposit(s).
    */
-  bool kyc_serial_pending;
+  bool return_immediately;
 
 };
 
@@ -253,11 +297,6 @@ TMH_force_kyc_resume ()
        NULL != kc;
        kc = kc->next)
   {
-    if (NULL != kc->timeout_task)
-    {
-      GNUNET_SCHEDULER_cancel (kc->timeout_task);
-      kc->timeout_task = NULL;
-    }
     if (GNUNET_YES == kc->suspended)
     {
       kc->suspended = GNUNET_SYSERR;
@@ -283,27 +322,23 @@ kyc_context_cleanup (void *cls)
     GNUNET_CONTAINER_DLL_remove (kc->exchange_pending_head,
                                  kc->exchange_pending_tail,
                                  ekr);
-    if (NULL != ekr->kyc)
-    {
-      TALER_EXCHANGE_kyc_check_cancel (ekr->kyc);
-      ekr->kyc = NULL;
-    }
     if (NULL != ekr->fo)
     {
       TMH_EXCHANGES_keys4exchange_cancel (ekr->fo);
       ekr->fo = NULL;
     }
     json_decref (ekr->pkaa);
+    json_decref (ekr->jlimits);
     if (NULL != ekr->keys)
       TALER_EXCHANGE_keys_decref (ekr->keys);
     GNUNET_free (ekr->exchange_url);
     GNUNET_free (ekr->payto_uri);
     GNUNET_free (ekr);
   }
-  if (NULL != kc->timeout_task)
+  if (NULL != kc->eh)
   {
-    GNUNET_SCHEDULER_cancel (kc->timeout_task);
-    kc->timeout_task = NULL;
+    TMH_db->event_listen_cancel (kc->eh);
+    kc->eh = NULL;
   }
   if (NULL != kc->response)
   {
@@ -313,73 +348,42 @@ kyc_context_cleanup (void *cls)
   GNUNET_CONTAINER_DLL_remove (kc_head,
                                kc_tail,
                                kc);
-  json_decref (kc->voluntary_kycs);
-  json_decref (kc->pending_kycs);
-  json_decref (kc->timeout_kycs);
+  json_decref (kc->kycs_data);
   GNUNET_free (kc);
 }
 
 
 /**
- * Resume the given KYC context and send the given response.  Stores the
+ * Resume the given KYC context and send the final response.  Stores the
  * response in the @a kc and signals MHD to resume the connection.  Also
  * ensures MHD runs immediately.
  *
  * @param kc KYC context
- * @param response_code response code to use
- * @param response response data to send back
  */
 static void
-resume_kyc_with_response (struct KycContext *kc,
-                          unsigned int response_code,
-                          struct MHD_Response *response)
+resume_kyc_with_response (struct KycContext *kc)
 {
   char dat[128];
 
-  kc->response_code = response_code;
-  kc->response = response;
-  switch (response_code)
-  {
-  case MHD_HTTP_OK:
-    /* KYC failed, cache briefly */
-    TALER_MHD_get_date_string (GNUNET_TIME_relative_to_absolute (
-                                 EXPIRATION_KYC_FAILURE),
-                               dat);
-    GNUNET_break (MHD_YES ==
-                  MHD_add_response_header (response,
-                                           MHD_HTTP_HEADER_EXPIRES,
-                                           dat));
-    GNUNET_break (MHD_YES ==
-                  MHD_add_response_header (response,
-                                           MHD_HTTP_HEADER_CACHE_CONTROL,
-                                           "max-age=300"));
-    break;
-  case MHD_HTTP_NO_CONTENT:
-    /* KYC passed, cache for a long time! */
-    TALER_MHD_get_date_string (GNUNET_TIME_relative_to_absolute (
-                                 EXPIRATION_KYC_SUCCESS),
-                               dat);
-    GNUNET_break (MHD_YES ==
-                  MHD_add_response_header (response,
-                                           MHD_HTTP_HEADER_EXPIRES,
-                                           dat));
-    GNUNET_break (MHD_YES ==
-                  MHD_add_response_header (response,
-                                           MHD_HTTP_HEADER_CACHE_CONTROL,
-                                           "max-age=3600"));
-    break;
-  case MHD_HTTP_BAD_GATEWAY:
-  case MHD_HTTP_GATEWAY_TIMEOUT:
-    break; /* no caching */
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+  kc->response_code = MHD_HTTP_OK;
+  kc->response = TALER_MHD_MAKE_JSON_PACK (
+    GNUNET_JSON_pack_array_incref ("kyc_data",
+                                   kc->kycs_data));
+  /* KYC failed, cache briefly */
+  TALER_MHD_get_date_string (GNUNET_TIME_relative_to_absolute (
+                               EXPIRATION_KYC_FAILURE),
+                             dat);
+  GNUNET_break (MHD_YES ==
+                MHD_add_response_header (kc->response,
+                                         MHD_HTTP_HEADER_EXPIRES,
+                                         dat));
+  GNUNET_break (MHD_YES ==
+                MHD_add_response_header (kc->response,
+                                         MHD_HTTP_HEADER_CACHE_CONTROL,
+                                         "max-age=300"));
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Resuming /kyc handling as exchange interaction is done (%u)\n",
-              response_code);
-  if (NULL != kc->timeout_task)
-  {
-    GNUNET_SCHEDULER_cancel (kc->timeout_task);
-    kc->timeout_task = NULL;
-  }
+              MHD_HTTP_OK);
   GNUNET_assert (GNUNET_YES == kc->suspended);
   kc->suspended = GNUNET_NO;
   MHD_resume_connection (kc->connection);
@@ -388,58 +392,26 @@ resume_kyc_with_response (struct KycContext *kc,
 
 
 /**
- * Handle a timeout for the processing of the kyc request.
+ * Handle a DB event about an update relevant
+ * for the processing of the kyc request.
  *
  * @param cls our `struct KycContext`
+ * @param extra additional event data provided
+ * @param extra_size number of bytes in @a extra
  */
 static void
-handle_kyc_timeout (void *cls)
+kyc_change_cb (void *cls,
+               const void *extra,
+               size_t extra_size)
 {
   struct KycContext *kc = cls;
-  struct ExchangeKycRequest *ekr;
 
-  kc->timeout_task = NULL;
-  while (NULL != (ekr = kc->exchange_pending_head))
-  {
-    GNUNET_CONTAINER_DLL_remove (kc->exchange_pending_head,
-                                 kc->exchange_pending_tail,
-                                 ekr);
-    if (NULL != ekr->kyc)
-    {
-      TALER_EXCHANGE_kyc_check_cancel (ekr->kyc);
-      ekr->kyc = NULL;
-    }
-    if (NULL != ekr->fo)
-    {
-      TMH_EXCHANGES_keys4exchange_cancel (ekr->fo);
-      ekr->fo = NULL;
-    }
-    GNUNET_assert (
-      0 ==
-      json_array_append_new (
-        kc->timeout_kycs,
-        GNUNET_JSON_PACK (
-          GNUNET_JSON_pack_string ("exchange_url",
-                                   ekr->exchange_url),
-          GNUNET_JSON_pack_uint64 ("exchange_code",
-                                   TALER_EC_MERCHANT_GENERIC_EXCHANGE_TIMEOUT),
-          GNUNET_JSON_pack_uint64 ("exchange_http_status",
-                                   0))));
-    GNUNET_free (ekr->exchange_url);
-    GNUNET_free (ekr->payto_uri);
-    GNUNET_free (ekr);
-  }
   GNUNET_assert (GNUNET_YES == kc->suspended);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Resuming KYC with gateway timeout\n");
-  resume_kyc_with_response (
-    kc,
-    MHD_HTTP_GATEWAY_TIMEOUT,
-    TALER_MHD_MAKE_JSON_PACK (
-      GNUNET_JSON_pack_array_incref ("pending_kycs",
-                                     kc->pending_kycs),
-      GNUNET_JSON_pack_array_incref ("timeout_kycs",
-                                     kc->timeout_kycs)));
+  kc->suspended = GNUNET_NO;
+  MHD_resume_connection (kc->connection);
+  TALER_MHD_daemon_trigger (); /* we resumed, kick MHD */
 }
 
 
@@ -472,107 +444,6 @@ pack_limit (const struct TALER_EXCHANGE_AccountLimit *limit,
 
 
 /**
- * We are done with the KYC request @a ekr.  Remove it from the work list and
- * check if we are done overall.
- *
- * @param[in] ekr key request that is done (and will be freed)
- */
-static void
-ekr_finished (struct ExchangeKycRequest *ekr)
-{
-  struct KycContext *kc = ekr->kc;
-
-  GNUNET_CONTAINER_DLL_remove (kc->exchange_pending_head,
-                               kc->exchange_pending_tail,
-                               ekr);
-  json_decref (ekr->pkaa);
-  if (NULL != ekr->keys)
-    TALER_EXCHANGE_keys_decref (ekr->keys);
-  GNUNET_free (ekr->exchange_url);
-  GNUNET_free (ekr->payto_uri);
-  GNUNET_free (ekr);
-  if (NULL != kc->exchange_pending_head)
-    return; /* wait for more */
-  /* All exchange requests done, create final
-     big response from cumulated replies */
-  if ( (0 == json_array_size (kc->pending_kycs)) &&
-       (0 == json_array_size (kc->timeout_kycs)) )
-  {
-    /* special case: all KYC operations did succeed
-       after we asked at the exchanges => 204 */
-    struct MHD_Response *response;
-
-    response = MHD_create_response_from_buffer_static (0,
-                                                       "");
-    resume_kyc_with_response (kc,
-                              MHD_HTTP_NO_CONTENT,
-                              response);
-    return;
-  }
-  resume_kyc_with_response (
-    kc,
-    kc->response_code, /* MHD_HTTP_OK or MHD_HTTP_ACCEPTED */
-    TALER_MHD_MAKE_JSON_PACK (
-      GNUNET_JSON_pack_array_incref ("pending_kycs",
-                                     kc->pending_kycs),
-      GNUNET_JSON_pack_array_incref ("timeout_kycs",
-                                     kc->timeout_kycs)));
-}
-
-
-/**
- * Store KYC response from the exchange in the
- * local database.
- *
- * @param ekr request context
- * @param ks HTTP response details
- * @param account_kyc_status account KYC status details
- * @return true if the operation was successful
- */
-static bool
-store_kyc_status (
-  const struct ExchangeKycRequest *ekr,
-  const struct TALER_EXCHANGE_KycStatus *ks,
-  const struct TALER_EXCHANGE_AccountKycStatus *account_kyc_status)
-{
-  json_t *jlimits;
-  enum GNUNET_DB_QueryStatus qs;
-
-  jlimits = json_array ();
-  GNUNET_assert (NULL != jlimits);
-  for (unsigned int i = 0; i<account_kyc_status->limits_length; i++)
-  {
-    const struct TALER_EXCHANGE_AccountLimit *limit
-      = &account_kyc_status->limits[i];
-
-    pack_limit (limit,
-                jlimits);
-  }
-
-  qs = TMH_db->account_kyc_set_status (
-    TMH_db->cls,
-    ekr->kc->mi->settings.id,
-    &ekr->h_wire,
-    ekr->exchange_url,
-    GNUNET_TIME_timestamp_get (),
-    ks->hr.http_status,
-    ks->hr.ec,
-    &account_kyc_status->access_token,
-    jlimits,
-    account_kyc_status->aml_review,
-    false);
-  json_decref (jlimits);
-  if (qs < 0)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "Failed to store KYC status in database!\n");
-    return false;
-  }
-  return true;
-}
-
-
-/**
  * Return JSON array with AccountLimit objects giving
  * the current limits for this exchange.
  *
@@ -581,297 +452,141 @@ store_kyc_status (
  */
 static json_t *
 get_exchange_limits (
-  struct ExchangeKycRequest *ekr,
-  const struct TALER_EXCHANGE_AccountKycStatus *account_kyc_status)
+  struct ExchangeKycRequest *ekr)
 {
+  const struct TALER_EXCHANGE_Keys *keys = ekr->keys;
   json_t *limits;
 
+  if (NULL != ekr->jlimits)
+    return json_incref (ekr->jlimits);
+  if (NULL == keys)
+    return NULL;
   limits = json_array ();
   GNUNET_assert (NULL != limits);
-  if (NULL == account_kyc_status)
+  for (unsigned int i = 0; i<keys->hard_limits_length; i++)
   {
-    const struct TALER_EXCHANGE_Keys *keys = ekr->keys;
-
-    GNUNET_assert (NULL != keys);
-    for (unsigned int i = 0; i<keys->hard_limits_length; i++)
-    {
-      const struct TALER_EXCHANGE_AccountLimit *limit
-        = &keys->hard_limits[i];
-
-      pack_limit (limit,
-                  limits);
-    }
-    for (unsigned int i = 0; i<keys->zero_limits_length; i++)
-    {
-      const struct TALER_EXCHANGE_ZeroLimitedOperation *zlimit
-        = &keys->zero_limits[i];
-      json_t *jl;
-      struct TALER_Amount zero;
+    const struct TALER_EXCHANGE_AccountLimit *limit
+      = &keys->hard_limits[i];
 
-      GNUNET_assert (GNUNET_OK ==
-                     TALER_amount_set_zero (keys->currency,
-                                            &zero));
-      jl = GNUNET_JSON_PACK (
-        TALER_JSON_pack_kycte ("operation_type",
-                               zlimit->operation_type),
-        GNUNET_JSON_pack_time_rel ("timeframe",
-                                   GNUNET_TIME_UNIT_ZERO),
-        TALER_JSON_pack_amount ("threshold",
-                                &zero),
-        GNUNET_JSON_pack_bool ("soft_limit",
-                               true)
-        );
-      GNUNET_assert (0 ==
-                     json_array_append_new (limits,
-                                            jl));
-    }
+    pack_limit (limit,
+                limits);
   }
-  else
+  for (unsigned int i = 0; i<keys->zero_limits_length; i++)
   {
-    for (unsigned int i = 0; i<account_kyc_status->limits_length; i++)
-    {
-      const struct TALER_EXCHANGE_AccountLimit *limit
-        = &account_kyc_status->limits[i];
-
-      pack_limit (limit,
-                  limits);
-    }
+    const struct TALER_EXCHANGE_ZeroLimitedOperation *zlimit
+      = &keys->zero_limits[i];
+    json_t *jl;
+    struct TALER_Amount zero;
+
+    GNUNET_assert (GNUNET_OK ==
+                   TALER_amount_set_zero (keys->currency,
+                                          &zero));
+    jl = GNUNET_JSON_PACK (
+      TALER_JSON_pack_kycte ("operation_type",
+                             zlimit->operation_type),
+      GNUNET_JSON_pack_time_rel ("timeframe",
+                                 GNUNET_TIME_UNIT_ZERO),
+      TALER_JSON_pack_amount ("threshold",
+                              &zero),
+      GNUNET_JSON_pack_bool ("soft_limit",
+                             true)
+      );
+    GNUNET_assert (0 ==
+                   json_array_append_new (limits,
+                                          jl));
   }
   return limits;
 }
 
 
 /**
- * Update exchange KYC account status, storing it
- * in the database and returning it in the response.
+ * Take data from @a ekr to expand our response.
  *
- * @param[in,out] ekr our request context
- * @param ks overall HTTP response
- * @param account_kyc_status specific KYC status
+ * @param ekr exchange we are done inspecting
  */
 static void
-update_account_status (
-  struct ExchangeKycRequest *ekr,
-  const struct TALER_EXCHANGE_KycStatus *ks,
-  const struct TALER_EXCHANGE_AccountKycStatus *account_kyc_status)
+ekr_expand_response (struct ExchangeKycRequest *ekr)
 {
-  char *kyc_url;
-
-  GNUNET_break (store_kyc_status (ekr,
-                                  ks,
-                                  account_kyc_status));
-  {
-    char *ats;
-
-    ats = GNUNET_STRINGS_data_to_string_alloc (
-      &account_kyc_status->access_token,
-      sizeof (account_kyc_status->access_token));
-    GNUNET_asprintf (&kyc_url,
-                     "%s/kyc-spa/%s",
-                     ekr->exchange_url,
-                     ats);
-    GNUNET_free (ats);
-  }
-
   GNUNET_assert (
     0 ==
     json_array_append_new (
-      ekr->kc->pending_kycs,
+      ekr->kc->kycs_data,
       GNUNET_JSON_PACK (
-        GNUNET_JSON_pack_allow_null (
-          GNUNET_JSON_pack_array_steal (
-            "limits",
-            get_exchange_limits (ekr,
-                                 account_kyc_status))),
-        (TALER_EC_NONE == ks->hr.ec)
-        ? GNUNET_JSON_pack_allow_null (
-          GNUNET_JSON_pack_string (
-            "dummy",
-            NULL))
-        : GNUNET_JSON_pack_uint64 ("exchange_code",
-                                   ks->hr.ec),
-        GNUNET_JSON_pack_uint64 ("exchange_http_status",
-                                 ks->hr.http_status),
-        GNUNET_JSON_pack_data_auto (
-          "access_token",
-          &account_kyc_status->access_token),
         GNUNET_JSON_pack_string (
-          "kyc_url",
-          kyc_url),
+          "payto_uri",
+          ekr->payto_uri),
         GNUNET_JSON_pack_string (
           "exchange_url",
           ekr->exchange_url),
-        GNUNET_JSON_pack_string (
-          "payto_uri",
-          ekr->payto_uri))));
-  GNUNET_free (kyc_url);
-}
-
-
-/**
- * Return exchange KYC account status when KYC auth
- * is required for authorization to the KYC state.
- *
- * @param[in,out] ekr our request context
- * @param ks overall HTTP response
- */
-static void
-return_auth_required (
-  struct ExchangeKycRequest *ekr,
-  const struct TALER_EXCHANGE_KycStatus *ks)
-{
-  struct KycContext *kc = ekr->kc;
-  enum GNUNET_DB_QueryStatus qs;
-
-  qs = TMH_db->account_kyc_set_status (
-    TMH_db->cls,
-    kc->mi->settings.id,
-    &ekr->h_wire,
-    ekr->exchange_url,
-    GNUNET_TIME_timestamp_get (),
-    ks->hr.http_status,
-    ks->hr.ec,
-    NULL,
-    NULL,
-    false,
-    true);
-  if (qs < 0)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "Failed to store KYC status in database!\n");
-  }
-
-  GNUNET_assert (
-    0 ==
-    json_array_append_new (
-      kc->pending_kycs,
-      GNUNET_JSON_PACK (
-        GNUNET_JSON_pack_allow_null (
-          GNUNET_JSON_pack_array_steal (
-            "limits",
-            get_exchange_limits (ekr,
-                                 NULL))),
-        (TALER_EC_NONE == ks->hr.ec)
+        GNUNET_JSON_pack_bool ("no_keys",
+                               ekr->no_keys),
+        GNUNET_JSON_pack_bool ("auth_conflict",
+                               ekr->kyc_auth_conflict),
+        GNUNET_JSON_pack_uint64 ("exchange_http_status",
+                                 ekr->last_http_status),
+        (TALER_EC_NONE == ekr->last_ec)
         ? GNUNET_JSON_pack_allow_null (
           GNUNET_JSON_pack_string (
             "dummy",
             NULL))
         : GNUNET_JSON_pack_uint64 ("exchange_code",
-                                   ks->hr.ec),
-        GNUNET_JSON_pack_uint64 ("exchange_http_status",
-                                 ks->hr.http_status),
-        GNUNET_JSON_pack_array_incref ("payto_kycauths",
-                                       ekr->pkaa),
-        GNUNET_JSON_pack_string ("exchange_url",
-                                 ekr->exchange_url),
-        GNUNET_JSON_pack_string ("payto_uri",
-                                 ekr->payto_uri))));
+                                   ekr->last_ec),
+        ekr->auth_ok
+        ? GNUNET_JSON_pack_data_auto (
+          "access_token",
+          &ekr->access_token)
+        : GNUNET_JSON_pack_allow_null (
+          GNUNET_JSON_pack_string (
+            "dummy",
+            NULL)),
+        GNUNET_JSON_pack_allow_null (
+          GNUNET_JSON_pack_array_steal (
+            "limits",
+            get_exchange_limits (ekr))),
+        GNUNET_JSON_pack_allow_null (
+          GNUNET_JSON_pack_array_incref ("payto_kycauths",
+                                         ekr->pkaa))
+        )));
 }
 
 
 /**
- * Function called with the result of a KYC check.
+ * We are done with the KYC request @a ekr.  Remove it from the work list and
+ * check if we are done overall.
  *
- * @param cls a `struct ExchangeKycRequest *`
- * @param ks the account's KYC status details
+ * @param[in] ekr key request that is done (and will be freed)
  */
 static void
-exchange_check_cb (
-  void *cls,
-  const struct TALER_EXCHANGE_KycStatus *ks)
+ekr_finished (struct ExchangeKycRequest *ekr)
 {
-  struct ExchangeKycRequest *ekr = cls;
   struct KycContext *kc = ekr->kc;
 
-  ekr->kyc = NULL;
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Checking KYC status of `%s' at `%s' is %u\n",
-              ekr->payto_uri,
-              ekr->exchange_url,
-              ks->hr.http_status);
-  switch (ks->hr.http_status)
+  ekr_expand_response (ekr);
+  GNUNET_CONTAINER_DLL_remove (kc->exchange_pending_head,
+                               kc->exchange_pending_tail,
+                               ekr);
+  json_decref (ekr->jlimits);
+  json_decref (ekr->pkaa);
+  if (NULL != ekr->keys)
+    TALER_EXCHANGE_keys_decref (ekr->keys);
+  GNUNET_free (ekr->exchange_url);
+  GNUNET_free (ekr->payto_uri);
+  GNUNET_free (ekr);
+
+  if (NULL != kc->exchange_pending_head)
+    return; /* wait for more */
+
+  if ( (! kc->return_immediately) &&
+       (! GNUNET_TIME_absolute_is_past (kc->timeout)) )
   {
-  case MHD_HTTP_OK:
-    update_account_status (ekr,
-                           ks,
-                           &ks->details.ok);
-    break;
-  case MHD_HTTP_ACCEPTED:
-    kc->response_code = MHD_HTTP_ACCEPTED;
-    update_account_status (ekr,
-                           ks,
-                           &ks->details.accepted);
-    break;
-  case MHD_HTTP_NO_CONTENT:
-    {
-      enum GNUNET_DB_QueryStatus qs;
-
-      qs = TMH_db->account_kyc_set_status (
-        TMH_db->cls,
-        kc->mi->settings.id,
-        &ekr->h_wire,
-        ekr->exchange_url,
-        GNUNET_TIME_timestamp_get (),
-        MHD_HTTP_NO_CONTENT,
-        TALER_EC_NONE,
-        NULL,
-        NULL,
-        false,
-        true);
-      if (qs < 0)
-      {
-        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                    "Failed to store KYC status in database!\n");
-      }
-    }
-    break;
-  case MHD_HTTP_FORBIDDEN: /* bad signature */
-  case MHD_HTTP_NOT_FOUND: /* account unknown */
-  case MHD_HTTP_CONFLICT: /* no account_pub known */
-    return_auth_required (ekr,
-                          ks);
-    break;
-  default:
-    {
-      enum GNUNET_DB_QueryStatus qs;
-
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                  "Exchange responded with HTTP status %u (%d) to /kyc-check request!\n",
-                  ks->hr.http_status,
-                  ks->hr.ec);
-      kc->response_code = MHD_HTTP_BAD_GATEWAY;
-      GNUNET_assert (
-        0 ==
-        json_array_append_new (
-          kc->timeout_kycs,
-          GNUNET_JSON_PACK (
-            GNUNET_JSON_pack_string ("exchange_url",
-                                     ekr->exchange_url),
-            GNUNET_JSON_pack_uint64 ("exchange_code",
-                                     ks->hr.ec),
-            GNUNET_JSON_pack_uint64 ("exchange_http_status",
-                                     ks->hr.http_status))));
-      qs = TMH_db->account_kyc_set_status (
-        TMH_db->cls,
-        kc->mi->settings.id,
-        &ekr->h_wire,
-        ekr->exchange_url,
-        GNUNET_TIME_timestamp_get (),
-        ks->hr.http_status,
-        ks->hr.ec,
-        NULL,
-        NULL,
-        false,
-        true);
-      if (qs < 0)
-      {
-        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                    "Failed to store KYC status in database!\n");
-      }
-      break;
-    }
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Remaining suspended: long poll target %d not reached\n",
+                kc->lpt);
+    return;
   }
-  ekr_finished (ekr);
+  /* All exchange requests done, create final
+     big response from cumulated replies */
+  resume_kyc_with_response (kc);
 }
 
 
@@ -882,14 +597,13 @@ exchange_check_cb (
  * in @a ekr.
  *
  * @param[in,out] request we are processing
- * @param keys exchange keys
  */
 static void
 determine_eligible_accounts (
-  struct ExchangeKycRequest *ekr,
-  const struct TALER_EXCHANGE_Keys *keys)
+  struct ExchangeKycRequest *ekr)
 {
   struct KycContext *kc = ekr->kc;
+  const struct TALER_EXCHANGE_Keys *keys = ekr->keys;
   struct TALER_Amount kyc_amount;
   char *merchant_pub_str;
 
@@ -1022,58 +736,35 @@ kyc_with_exchange (void *cls,
                    struct TMH_Exchange *exchange)
 {
   struct ExchangeKycRequest *ekr = cls;
-  struct KycContext *kc = ekr->kc;
-  struct TALER_PaytoHashP h_payto;
-  union TALER_AccountPrivateKeyP ap;
 
   (void) exchange;
   ekr->fo = NULL;
   if (NULL == keys)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "Failed to download `%s/keys`\n",
+                "Failed to download `%skeys`\n",
                 ekr->exchange_url);
-    kc->response_code = MHD_HTTP_BAD_GATEWAY;
-    GNUNET_assert (
-      0 ==
-      json_array_append_new (
-        kc->timeout_kycs,
-        GNUNET_JSON_PACK (
-          TALER_JSON_pack_ec (
-            TALER_EC_MERCHANT_GENERIC_EXCHANGE_KEYS_FAILURE))));
+    ekr->no_keys = true;
     ekr_finished (ekr);
     return;
   }
-  determine_eligible_accounts (ekr,
-                               keys);
-  if (0 == json_array_size (ekr->pkaa))
+  ekr->keys = TALER_EXCHANGE_keys_incref (keys);
+  if (! ekr->auth_ok)
   {
-    /* No KYC auth wire transfers are possible to this exchange from
-       our merchant bank account, so we cannot use this account with
-       this exchange if it has any KYC requirements! */
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                "KYC auth to `%s' impossible for merchant account `%s'\n",
-                ekr->exchange_url,
-                ekr->payto_uri);
-    ekr_finished (ekr);
-    return;
+    determine_eligible_accounts (ekr);
+    if (0 == json_array_size (ekr->pkaa))
+    {
+      /* No KYC auth wire transfers are possible to this exchange from
+         our merchant bank account, so we cannot use this account with
+         this exchange if it has any KYC requirements! */
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                  "KYC auth to `%s' impossible for merchant account `%s'\n",
+                  ekr->exchange_url,
+                  ekr->payto_uri);
+      ekr->kyc_auth_conflict = true;
+    }
   }
-  ekr->keys = TALER_EXCHANGE_keys_incref (keys);
-  TALER_payto_hash (ekr->payto_uri,
-                    &h_payto);
-  ap.merchant_priv = kc->mi->merchant_priv;
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Checking KYC status of `%s' at `%s'\n",
-              ekr->payto_uri,
-              ekr->exchange_url);
-  ekr->kyc = TALER_EXCHANGE_kyc_check (
-    TMH_curl_ctx,
-    ekr->exchange_url,
-    &h_payto,
-    &ap,
-    GNUNET_TIME_absolute_get_remaining (kc->timeout),
-    &exchange_check_cb,
-    ekr);
+  ekr_finished (ekr);
 }
 
 
@@ -1083,46 +774,84 @@ kyc_with_exchange (void *cls,
  *
  * @param cls our `struct KycContext *`
  * @param h_wire hash of the wire account
- * @param exchange_kyc_serial serial number for the KYC process at the exchange, 0 if unknown
  * @param payto_uri payto:// URI of the merchant's bank account
  * @param exchange_url base URL of the exchange for which this is a status
  * @param last_check when did we last get an update on our KYC status from the exchange
  * @param kyc_ok true if we satisfied the KYC requirements
+ * @param access_token access token for the KYC SPA, NULL if we cannot access it yet (need KYC auth wire transfer)
+ * @param last_http_status last HTTP status from /kyc-check
+ * @param last_ec last Taler error code from /kyc-check
+ * @param in_aml_review true if the account is pending review
+ * @param jlimits JSON array of applicable AccountLimits, or NULL if unknown (like defaults apply)
  */
 static void
 kyc_status_cb (
   void *cls,
   const struct TALER_MerchantWireHashP *h_wire,
-  uint64_t exchange_kyc_serial,
   const char *payto_uri,
   const char *exchange_url,
   struct GNUNET_TIME_Timestamp last_check,
-  bool kyc_ok)
+  bool kyc_ok,
+  const struct TALER_AccountAccessTokenP *access_token,
+  unsigned int last_http_status,
+  enum TALER_ErrorCode last_ec,
+  bool in_aml_review,
+  const json_t *jlimits)
 {
   struct KycContext *kc = cls;
   struct ExchangeKycRequest *ekr;
 
-  if (kyc_ok &&
-      (GNUNET_TIME_relative_cmp (
-         GNUNET_TIME_absolute_get_duration (last_check.abs_time),
-         <,
-         STALE_KYC_TIMEOUT)) )
-    return; /* KYC ok, ignore! */
-  kc->response_code = MHD_HTTP_OK;
+  switch (kc->lpt)
+  {
+  case TALER_EXCHANGE_KLPT_NONE:
+    break;
+  case TALER_EXCHANGE_KLPT_KYC_AUTH_TRANSFER:
+    if (NULL != access_token)
+      kc->return_immediately = true;
+    break;
+  case TALER_EXCHANGE_KLPT_INVESTIGATION_DONE:
+    if (! in_aml_review)
+      kc->return_immediately = true;
+    break;
+  case TALER_EXCHANGE_KLPT_KYC_OK:
+    if (kyc_ok)
+      kc->return_immediately = true;
+    break;
+  }
   ekr = GNUNET_new (struct ExchangeKycRequest);
   GNUNET_CONTAINER_DLL_insert (kc->exchange_pending_head,
                                kc->exchange_pending_tail,
                                ekr);
+  ekr->last_http_status = last_http_status;
+  ekr->last_ec = last_ec;
+  if (NULL != jlimits)
+    ekr->jlimits = json_incref ((json_t *) jlimits);
   ekr->h_wire = *h_wire;
   ekr->exchange_url = GNUNET_strdup (exchange_url);
   ekr->payto_uri = GNUNET_strdup (payto_uri);
   ekr->last_check = last_check;
+  ekr->kyc_ok = kyc_ok;
   ekr->kc = kc;
-  ekr->fo = TMH_EXCHANGES_keys4exchange (
-    exchange_url,
-    false,
-    &kyc_with_exchange,
-    ekr);
+  ekr->in_aml_review = in_aml_review;
+  ekr->auth_ok = (NULL != access_token);
+  if ( (! ekr->auth_ok) ||
+       (NULL == ekr->jlimits) )
+  {
+    /* Figure out wire transfer instructions */
+    if (GNUNET_NO == kc->suspended)
+    {
+      MHD_suspend_connection (kc->connection);
+      kc->suspended = GNUNET_YES;
+    }
+    ekr->fo = TMH_EXCHANGES_keys4exchange (
+      exchange_url,
+      false,
+      &kyc_with_exchange,
+      ekr);
+    return;
+  }
+  ekr->access_token = *access_token;
+  ekr_finished (ekr);
 }
 
 
@@ -1153,32 +882,38 @@ get_instances_ID_kyc (
                                  kc);
     kc->connection = connection;
     kc->hc = hc;
-    kc->pending_kycs = json_array ();
-    GNUNET_assert (NULL != kc->pending_kycs);
-    kc->timeout_kycs = json_array ();
-    GNUNET_assert (NULL != kc->timeout_kycs);
-
+    kc->kycs_data = json_array ();
+    GNUNET_assert (NULL != kc->kycs_data);
     TALER_MHD_parse_request_timeout (connection,
                                      &kc->timeout);
-    if (! GNUNET_TIME_absolute_is_past (kc->timeout))
-      kc->timeout_task
-        = GNUNET_SCHEDULER_add_at (kc->timeout,
-                                   &handle_kyc_timeout,
-                                   kc);
-
+    {
+      uint64_t num = 0;
+      int val;
+
+      TALER_MHD_parse_request_number (connection,
+                                      "lpt",
+                                      &num);
+      val = (int) num;
+      if ( (val < 0) ||
+           (val > TALER_EXCHANGE_KLPT_MAX) )
+      {
+        /* Protocol violation, but we can be graceful and
+           just ignore the long polling! */
+        GNUNET_break_op (0);
+        val = TALER_EXCHANGE_KLPT_NONE;
+      }
+      kc->lpt = (enum TALER_EXCHANGE_KycLongPollTarget) val;
+    }
+    kc->return_immediately
+      = (TALER_EXCHANGE_KLPT_NONE == kc->lpt);
     /* process 'exchange_url' argument */
     kc->exchange_url = MHD_lookup_connection_value (
       connection,
       MHD_GET_ARGUMENT_KIND,
       "exchange_url");
     if ( (NULL != kc->exchange_url) &&
-         (! TALER_url_valid_charset (kc->exchange_url) ||
-          ( (0 != strncasecmp (kc->exchange_url,
-                               "http://",
-                               strlen ("http://"))) &&
-            (0 != strncasecmp (kc->exchange_url,
-                               "https://",
-                               strlen ("https://"))) ) ) )
+         ( (! TALER_url_valid_charset (kc->exchange_url)) ||
+           (! TALER_is_web_url (kc->exchange_url)) ) )
     {
       GNUNET_break_op (0);
       return TALER_MHD_reply_with_error (
@@ -1192,68 +927,101 @@ get_instances_ID_kyc (
                                       "h_wire",
                                       &kc->h_wire,
                                       kc->have_h_wire);
-    /* Check our database */
+
+    if ( (TALER_EXCHANGE_KLPT_NONE != kc->lpt) &&
+         (! GNUNET_TIME_absolute_is_past (kc->timeout)) )
     {
-      enum GNUNET_DB_QueryStatus qs;
-
-      qs = TMH_db->account_kyc_get_status (
-        TMH_db->cls,
-        mi->settings.id,
-        kc->have_h_wire
-        ? &kc->h_wire
-        : NULL,
-        kc->exchange_url,
-        &kyc_status_cb,
-        kc);
-      if (qs < 0)
+      if (kc->have_h_wire)
       {
-        GNUNET_break (0);
-        return TALER_MHD_reply_with_ec (
-          connection,
-          TALER_EC_GENERIC_DB_FETCH_FAILED,
-          "account_kyc_get_status");
+        struct TALER_MERCHANTDB_MerchantKycStatusChangeEventP ev = {
+          .header.size = htons (sizeof (ev)),
+          .header.type = htons (
+            TALER_DBEVENT_MERCHANT_EXCHANGE_KYC_STATUS_CHANGED
+            ),
+          .h_wire = kc->h_wire
+        };
+
+        kc->eh = TMH_db->event_listen (
+          TMH_db->cls,
+          &ev.header,
+          GNUNET_TIME_absolute_get_remaining (kc->timeout),
+          &kyc_change_cb,
+          kc);
       }
-    }
-    if (kc->kyc_serial_pending)
+      else
+      {
+        struct GNUNET_DB_EventHeaderP hdr = {
+          .size = htons (sizeof (hdr)),
+          .type = htons (TALER_DBEVENT_MERCHANT_KYC_STATUS_CHANGED)
+        };
+
+        kc->eh = TMH_db->event_listen (
+          TMH_db->cls,
+          &hdr,
+          GNUNET_TIME_absolute_get_remaining (kc->timeout),
+          &kyc_change_cb,
+          kc);
+      }
+    } /* end register LISTEN hooks */
+  } /* end 1st time initialization */
+
+  if (GNUNET_SYSERR == kc->suspended)
+    return MHD_NO; /* during shutdown, we don't generate any more replies */
+  GNUNET_assert (GNUNET_NO == kc->suspended);
+
+  if (NULL != kc->response)
+    return MHD_queue_response (connection,
+                               kc->response_code,
+                               kc->response);
+
+  /* Check our database */
+  {
+    enum GNUNET_DB_QueryStatus qs;
+
+    GNUNET_break (0 ==
+                  json_array_clear (kc->kycs_data));
+    qs = TMH_db->account_kyc_get_status (
+      TMH_db->cls,
+      mi->settings.id,
+      kc->have_h_wire
+      ? &kc->h_wire
+      : NULL,
+      kc->exchange_url,
+      &kyc_status_cb,
+      kc);
+    if (qs < 0)
     {
-      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                  "Exchange legitimization UUID unknown, assuming KYC pending\n");
-      return TALER_MHD_REPLY_JSON_PACK (
+      /* Database error */
+      GNUNET_break (0);
+      if (GNUNET_YES == kc->suspended)
+      {
+        /* must have suspended before DB error, resume! */
+        MHD_resume_connection (connection);
+        kc->suspended = GNUNET_NO;
+      }
+      return TALER_MHD_reply_with_ec (
         connection,
-        MHD_HTTP_SERVICE_UNAVAILABLE,
-        GNUNET_JSON_pack_string ("hint",
-                                 "awaiting legitimization UUID"));
+        TALER_EC_GENERIC_DB_FETCH_FAILED,
+        "account_kyc_get_status");
     }
-    if (NULL == kc->exchange_pending_head)
+    if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
+    {
+      /* no matching accounts, could not have suspended */
+      GNUNET_assert (GNUNET_NO == kc->suspended);
       return TALER_MHD_reply_static (connection,
                                      MHD_HTTP_NO_CONTENT,
                                      NULL,
                                      NULL,
                                      0);
-    MHD_suspend_connection (connection);
-    kc->suspended = GNUNET_YES;
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Suspending KYC request handling while checking with the exchange(s)\n");
-    return MHD_YES;
-  }
-  if (GNUNET_SYSERR == kc->suspended)
-    return MHD_NO; /* during shutdown, we don't generate any more replies */
-  GNUNET_assert (GNUNET_NO == kc->suspended);
-  if (0 != kc->response_code)
-  {
-    /* We are *done* processing the request, just queue the response (!) */
-    if (UINT_MAX == kc->response_code)
-    {
-      GNUNET_break (0);
-      return MHD_NO; /* hard error */
     }
-    return MHD_queue_response (connection,
-                               kc->response_code,
-                               kc->response);
   }
-  /* we should never get here */
-  GNUNET_break (0);
-  return MHD_NO;
+  if (GNUNET_YES == kc->suspended)
+    return MHD_YES;
+  /* Should have generated a response */
+  GNUNET_break (NULL != kc->response);
+  return MHD_queue_response (connection,
+                             kc->response_code,
+                             kc->response);
 }
 
 
diff --git a/src/backend/taler-merchant-kyccheck.c b/src/backend/taler-merchant-kyccheck.c
index 279be891..f001285c 100644
--- a/src/backend/taler-merchant-kyccheck.c
+++ b/src/backend/taler-merchant-kyccheck.c
@@ -39,6 +39,14 @@
           30)
 
 /**
+ * How long do we wait between requests if all we wait
+ * for is a change in the AML investigation status?
+ */
+#define AML_FREQ GNUNET_TIME_relative_multiply ( \
+          GNUNET_TIME_UNIT_HOURS, \
+          6)
+
+/**
  * How many inquiries do we process concurrently at most.
  */
 #define OPEN_INQUIRY_LIMIT 1024
@@ -467,8 +475,15 @@ exchange_check_cb (
     store_kyc_status (i,
                       &ks->details.ok);
     i->backoff = GNUNET_TIME_UNIT_ZERO;
-    /* KYC is OK, only check again if triggered */
-    i->due = GNUNET_TIME_UNIT_FOREVER_ABS;
+    if (i->aml_review)
+    {
+      i->due = GNUNET_TIME_relative_to_absolute (AML_FREQ);
+    }
+    else
+    {
+      /* KYC is OK, only check again if triggered */
+      i->due = GNUNET_TIME_UNIT_FOREVER_ABS;
+    }
     break;
   case MHD_HTTP_ACCEPTED:
     i->last_kyc_check = GNUNET_TIME_timestamp_get ();
@@ -573,6 +588,7 @@ static void
 inquiry_work (void *cls)
 {
   struct Inquiry *i = cls;
+  enum TALER_EXCHANGE_KycLongPollTarget lpt;
 
   i->task = NULL;
   if (! GNUNET_TIME_absolute_is_past (i->due))
@@ -600,11 +616,19 @@ inquiry_work (void *cls)
               i->e->keys->exchange_url);
   i->timeout
     = GNUNET_TIME_relative_to_absolute (EXCHANGE_TIMEOUT);
+  lpt = TALER_EXCHANGE_KLPT_NONE;
+  if (! i->auth_ok)
+    lpt = TALER_EXCHANGE_KLPT_NONE;
+  else if (! i->kyc_ok)
+    lpt = TALER_EXCHANGE_KLPT_KYC_OK;
+  else if (i->aml_review)
+    lpt = TALER_EXCHANGE_KLPT_INVESTIGATION_DONE;
   i->kyc = TALER_EXCHANGE_kyc_check (
     ctx,
     i->e->keys->exchange_url,
     &i->a->h_payto,
     &i->a->ap,
+    lpt,
     EXCHANGE_TIMEOUT,
     &exchange_check_cb,
     i);
@@ -997,7 +1021,9 @@ find_keys (const char *exchange_url)
   }
   if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
   {
-    GNUNET_break (0);
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "No %s/keys yet!\n",
+                exchange_url);
     return;
   }
   for (e = e_head; NULL != e; e = e->next)