skeleton for kyccheck helper

2 files changed, 490 insertions, 164 deletions

diff --git a/src/backend/taler-merchant-kyccheck.c b/src/backend/taler-merchant-kyccheck.c
index 36970886..297106de 100644
--- a/src/backend/taler-merchant-kyccheck.c
+++ b/src/backend/taler-merchant-kyccheck.c
@@ -40,17 +40,6 @@
  */
 #define OPEN_INQUIRY_LIMIT 1024
 
-/**
- * How many inquiries do we process concurrently per exchange at most.
- */
-#define EXCHANGE_INQUIRY_LIMIT 16
-
-
-/**
- * Information about an inquiry job.
- */
-struct Inquiry;
-
 
 /**
  * Information about an exchange.
@@ -68,53 +57,48 @@ struct Exchange
   struct Exchange *prev;
 
   /**
-   * Head of active inquiries.
+   * The keys of this exchange
    */
-  struct Inquiry *w_head;
+  struct TALER_EXCHANGE_Keys *keys;
 
-  /**
-   * Tail of active inquiries.
-   */
-  struct Inquiry *w_tail;
+};
 
-  /**
-   * Which exchange are we tracking here.
-   */
-  char *exchange_url;
 
+/**
+ * Information about an Account.
+ */
+struct Account
+{
   /**
-   * The keys of this exchange
+   * Kept in a DLL.
    */
-  struct TALER_EXCHANGE_Keys *keys;
+  struct Account *next;
 
   /**
-   * How many active inquiries do we have right now with this exchange.
+   * Kept in a DLL.
    */
-  unsigned int exchange_inquiries;
+  struct Account *prev;
 
   /**
-   * How soon can may we, at the earliest, re-download /keys?
+   * Head of inquiries for this account.
    */
-  struct GNUNET_TIME_Absolute first_retry;
+  struct Inquiry *i_head;
 
   /**
-   * How long should we wait between the next retry?
+   * Tail of inquiries for this account.
    */
-  struct GNUNET_TIME_Relative retry_delay;
+  struct Inquiry *i_tail;
 
   /**
-   * How long should we wait between requests
-   * for transfer details?
+   * The payto-URI of this account.
    */
-  struct GNUNET_TIME_Relative transfer_delay;
+  char *merchant_account_uri;
 
   /**
-   * False to indicate that there is an ongoing
-   * /keys transfer we are waiting for;
-   * true to indicate that /keys data is up-to-date.
+   * Database generation when this account
+   * was last active.
    */
-  bool ready;
-
+  uint64_t account_gen;
 };
 
 
@@ -134,9 +118,34 @@ struct Inquiry
   struct Inquiry *prev;
 
   /**
-   * Kept in a DLL.
+   * Main task for this inquiry.
+   */
+  struct GNUNET_SCHEDULER_Task *task;
+
+  /**
+   * Which exchange is this inquiry about.
+   */
+  struct Exchange *e;
+
+  /**
+   * Which account is this inquiry about.
+   */
+  struct Account *a;
+
+  /**
+   * Did we not run this inquiry due to limits?
+   */
+  bool limited;
+
+  /**
+   * Do we believe this account's KYC is in good shape?
+   */
+  bool kyc_ok;
+
+  /**
+   * True if we did this account's KYC AUTH transfer.
    */
-  struct Exchange *exchange;
+  bool auth_ok;
 
 };
 
@@ -152,6 +161,16 @@ static struct Exchange *e_head;
 static struct Exchange *e_tail;
 
 /**
+ * Head of accounts.
+ */
+static struct Account *a_head;
+
+/**
+ * Tail of accounts.
+ */
+static struct Account *a_tail;
+
+/**
  * The merchant's configuration.
  */
 static const struct GNUNET_CONFIGURATION_Handle *cfg;
@@ -172,25 +191,32 @@ static struct GNUNET_CURL_Context *ctx;
 static struct GNUNET_CURL_RescheduleContext *rc;
 
 /**
- * Main task for #find_work().
+ * Event handler to learn that there may be new bank
+ * accounts to check.
  */
-static struct GNUNET_SCHEDULER_Task *task;
+static struct GNUNET_DB_EventHandler *eh_accounts;
 
 /**
- * Event handler to learn that there are new transfers
- * to check.
+ * Event handler to learn that there may be new exchange
+ * keys to check.
  */
-static struct GNUNET_DB_EventHandler *eh;
+static struct GNUNET_DB_EventHandler *eh_keys;
 
 /**
- * How many active inquiries do we have right now.
+ * Main task to discover (new) accounts.
  */
-static unsigned int active_inquiries;
+static struct GNUNET_SCHEDULER_Task *account_task;
+
+/**
+ * Counter determining how often we have called
+ * "select_accounts" on the database.
+ */
+static uint64_t database_gen;
 
 /**
- * Set to true if we ever encountered any problem.
+ * How many active inquiries do we have right now.
  */
-static bool found_problem;
+static unsigned int active_inquiries;
 
 /**
  * Value to return from main(). 0 on success, non-zero on errors.
@@ -212,41 +238,49 @@ static bool at_limit;
 
 
 /**
- * Finds new transfers that require work in the merchant database.
+ * Do KYC check work for the give inquiry in @a i.
  *
- * @param cls NULL
+ * @param i inquiry to work on
  */
 static void
-find_work (void *cls);
+inquiry_work (struct Inquiry *i);
 
 
 /**
- * Free resources of @a w.
- *
- * @param[in] w inquiry job to terminate
+ * An inquiry finished, check if we should resume
+ * others.
  */
 static void
-end_inquiry (struct Inquiry *w)
+end_inquiry (void)
 {
-  struct Exchange *e = w->exchange;
-
   GNUNET_assert (active_inquiries > 0);
   active_inquiries--;
-  GNUNET_CONTAINER_DLL_remove (e->w_head,
-                               e->w_tail,
-                               w);
-  GNUNET_free (w);
   if ( (active_inquiries < OPEN_INQUIRY_LIMIT / 2) &&
-       (NULL == task) &&
        (at_limit) )
   {
     at_limit = false;
-    GNUNET_assert (NULL == task);
-    task = GNUNET_SCHEDULER_add_now (&find_work,
-                                     NULL);
+    for (struct Account *a = a_head;
+         NULL != a;
+         a = a->next)
+    {
+      for (struct Inquiry *i = a->i_head;
+           NULL != i;
+           i = i->next)
+      {
+        if (! i->limited)
+          continue;
+        GNUNET_assert (NULL == i->task);
+        /* done synchronously so that the active_inquiries
+           is updated immediately */
+        inquiry_work (i);
+        if (at_limit)
+          break;
+      }
+      if (at_limit)
+        break;
+    }
   }
-  if ( (NULL == task) &&
-       (! at_limit) &&
+  if ( (! at_limit) &&
        (0 == active_inquiries) &&
        (test_mode) )
   {
@@ -258,83 +292,24 @@ end_inquiry (struct Inquiry *w)
 }
 
 
-/**
- * We're being aborted with CTRL-C (or SIGTERM). Shut down.
- *
- * @param cls closure (NULL)
- */
-static void
-shutdown_task (void *cls)
-{
-  (void) cls;
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Running shutdown\n");
-  while (NULL != e_head)
-  {
-    struct Exchange *e = e_head;
-
-    while (NULL != e->w_head)
-    {
-      struct Inquiry *w = e->w_head;
-
-      end_inquiry (w);
-    }
-    GNUNET_free (e->exchange_url);
-    if (NULL != e->keys)
-    {
-      TALER_EXCHANGE_keys_decref (e->keys);
-      e->keys = NULL;
-    }
-    GNUNET_CONTAINER_DLL_remove (e_head,
-                                 e_tail,
-                                 e);
-    GNUNET_free (e);
-  }
-  if (NULL != eh)
-  {
-    db_plugin->event_listen_cancel (eh);
-    eh = NULL;
-  }
-  if (NULL != task)
-  {
-    GNUNET_SCHEDULER_cancel (task);
-    task = NULL;
-  }
-  TALER_MERCHANTDB_plugin_unload (db_plugin);
-  db_plugin = NULL;
-  cfg = NULL;
-  if (NULL != ctx)
-  {
-    GNUNET_CURL_fini (ctx);
-    ctx = NULL;
-  }
-  if (NULL != rc)
-  {
-    GNUNET_CURL_gnunet_rc_destroy (rc);
-    rc = NULL;
-  }
-}
-
-
 static void
-find_work (void *cls)
+inquiry_work (struct Inquiry *i)
 {
   //  enum GNUNET_DB_QueryStatus qs;
-  int limit;
 
-  (void) cls;
-  task = NULL;
   GNUNET_assert (OPEN_INQUIRY_LIMIT >= active_inquiries);
-  limit = OPEN_INQUIRY_LIMIT - active_inquiries;
-  if (0 == limit)
+  if (OPEN_INQUIRY_LIMIT <= active_inquiries)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 "Not looking for work: at limit\n");
+    i->limited = true;
     at_limit = true;
     return;
   }
   at_limit = false;
+
 #if 0
+  active_inquiries++;
   // FIXME: do actual work!
   qs = db_plugin->select_open_transfers (db_plugin->cls,
                                          limit,
@@ -347,27 +322,163 @@ find_work (void *cls)
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
-  if (qs == limit)
+#endif
+  if ( (0 == active_inquiries) &&
+       (test_mode) )
   {
-    /* DB limited response, re-trigger DB interaction
-       the moment we significantly fall below the
-       limit */
-    at_limit = true;
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "No more open inquiries and in test mode. Existing.\n");
+    GNUNET_SCHEDULER_shutdown ();
+    return;
   }
-  if (0 == active_inquiries)
+}
+
+
+/**
+ * Start the KYC checking for account @a at exchange @a e.
+ *
+ * @param e an exchange
+ * @param a an account
+ */
+static void
+start_inquiry (struct Exchange *e,
+               struct Account *a)
+{
+  struct Inquiry *i;
+
+  i = GNUNET_new (struct Inquiry);
+  i->e = e;
+  i->a = a;
+  GNUNET_CONTAINER_DLL_insert (a->i_head,
+                               a->i_tail,
+                               i);
+  // FIXME: initial import from DB here!?
+  inquiry_work (i);
+}
+
+
+/**
+ * Stop KYC inquiry @a i.
+ *
+ * @param[in] i the inquiry to stop
+ */
+static void
+stop_inquiry (struct Inquiry *i)
+{
+  struct Account *a = i->a;
+
+  GNUNET_CONTAINER_DLL_remove (a->i_head,
+                               a->i_tail,
+                               i);
+  if (NULL != i->task)
   {
-    if (test_mode)
+    GNUNET_SCHEDULER_cancel (i->task);
+    i->task = NULL;
+  }
+  // FIXME: other clean-ups related to i here!
+  GNUNET_free (i);
+}
+
+
+/**
+ * Start inquries for all exchanges on account @a a.
+ *
+ * @param a an account
+ */
+static void
+start_inquiries (struct Account *a)
+{
+  for (struct Exchange *e = e_head;
+       NULL != e;
+       e = e->next)
+    start_inquiry (e,
+                   a);
+}
+
+
+/**
+ * Stop all inquries involving account @a a.
+ *
+ * @param a an account
+ */
+static void
+stop_inquiries (struct Account *a)
+{
+  struct Inquiry *i;
+
+  while (NULL != (i = a->i_head))
+    stop_inquiry (i);
+}
+
+
+/**
+ * Callback invoked with information about a bank account.
+ *
+ * @param cls closure
+ * @param ad details about the account
+ */
+static void
+account_cb (
+  void *cls,
+  const struct TALER_MERCHANTDB_AccountDetails *ad)
+{
+  const char *payto_uri = ad->payto_uri;
+
+  if (! ad->active)
+    return;
+  for (struct Account *a = a_head;
+       NULL != a;
+       a = a->next)
+  {
+    if (0 ==
+        strcmp (payto_uri,
+                a->merchant_account_uri))
     {
-      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                  "No more open inquiries and in test mode. Existing.\n");
-      GNUNET_SCHEDULER_shutdown ();
+      a->account_gen = database_gen;
       return;
     }
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                "No open inquiries found, waiting for notification to resume\n")
-    ;
   }
-#endif
+  {
+    struct Account *a = GNUNET_new (struct Account);
+
+    a->account_gen = database_gen;
+    a->merchant_account_uri = GNUNET_strdup (payto_uri);
+    GNUNET_CONTAINER_DLL_insert (a_head,
+                                 a_tail,
+                                 a);
+    start_inquiries (a);
+  }
+}
+
+
+/**
+ * The set of bank accounts has changed, update our
+ * list of active inquiries.
+ */
+static void
+find_accounts (void *cls)
+{
+  enum GNUNET_DB_QueryStatus qs;
+
+  (void) cls;
+  account_task = NULL;
+  database_gen++;
+  qs = db_plugin->select_accounts (db_plugin->cls,
+                                   NULL, /* all instances */
+                                   &account_cb,
+                                   NULL);
+  if (qs < 0)
+  {
+    GNUNET_break (0);
+    return;
+  }
+  for (struct Account *a = a_head;
+       NULL != a;
+       a = a->next)
+  {
+    if (a->account_gen < database_gen)
+      stop_inquiries (a);
+  }
 }
 
 
@@ -387,10 +498,206 @@ account_changed (void *cls,
   (void) cls;
   (void) extra;
   (void) extra_size;
-  if (NULL != task)
+  if (NULL != account_task)
     return;
-  task = GNUNET_SCHEDULER_add_now (&find_work,
-                                   NULL);
+  account_task
+    = GNUNET_SCHEDULER_add_now (&find_accounts,
+                                NULL);
+}
+
+
+/**
+ * Interact with the database to get the current set
+ * of exchange keys known to us.
+ *
+ * @param exchange_url the exchange URL to check
+ */
+static void
+find_keys (const char *exchange_url)
+{
+  enum GNUNET_DB_QueryStatus qs;
+  struct TALER_EXCHANGE_Keys *keys;
+  struct Exchange *e;
+
+  qs = db_plugin->select_exchange_keys (db_plugin->cls,
+                                        exchange_url,
+                                        &keys);
+  if (qs < 0)
+  {
+    GNUNET_break (0);
+    return;
+  }
+  if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
+  {
+    GNUNET_break (0);
+    return;
+  }
+  for (e = e_head; NULL != e; e = e->next)
+  {
+    if (0 == strcmp (e->keys->exchange_url,
+                     keys->exchange_url))
+    {
+      TALER_EXCHANGE_keys_decref (e->keys);
+      e->keys = keys;
+      return;
+    }
+  }
+  e = GNUNET_new (struct Exchange);
+  e->keys = keys;
+  GNUNET_CONTAINER_DLL_insert (e_head,
+                               e_tail,
+                               e);
+  for (struct Account *a = a_head;
+       NULL != a;
+       a = a->next)
+  {
+    if (a->account_gen < database_gen)
+      start_inquiry (e,
+                     a);
+  }
+}
+
+
+/**
+ * Function called when transfers are added to the merchant database.  We look
+ * for more work.
+ *
+ * @param cls closure (NULL)
+ * @param extra additional event data provided
+ * @param extra_size number of bytes in @a extra
+ */
+static void
+keys_changed (void *cls,
+              const void *extra,
+              size_t extra_size)
+{
+  const char *url = extra;
+
+  (void) cls;
+  if ( (NULL == extra) ||
+       (0 == extra_size) )
+  {
+    GNUNET_break (0);
+    return;
+  }
+  if ('\0' != url[extra_size - 1])
+  {
+    GNUNET_break (0);
+    return;
+  }
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Received keys change notification: reload `%s'\n",
+              url);
+  find_keys (url);
+}
+
+
+/**
+ * Function called on each configuration section. Finds sections
+ * about exchanges, parses the entries.
+ *
+ * @param cls NULL
+ * @param section name of the section
+ */
+static void
+accept_exchanges (void *cls,
+                  const char *section)
+{
+  char *url;
+
+  (void) cls;
+  if (0 !=
+      strncasecmp (section,
+                   "merchant-exchange-",
+                   strlen ("merchant-exchange-")))
+    return;
+  if (GNUNET_YES ==
+      GNUNET_CONFIGURATION_get_value_yesno (cfg,
+                                            section,
+                                            "DISABLED"))
+    return;
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_string (cfg,
+                                             section,
+                                             "EXCHANGE_BASE_URL",
+                                             &url))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+                               section,
+                               "EXCHANGE_BASE_URL");
+    global_ret = EXIT_NOTCONFIGURED;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  find_keys (url);
+  GNUNET_free (url);
+}
+
+
+/**
+ * We're being aborted with CTRL-C (or SIGTERM). Shut down.
+ *
+ * @param cls closure (NULL)
+ */
+static void
+shutdown_task (void *cls)
+{
+  (void) cls;
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Running shutdown\n");
+  while (NULL != e_head)
+  {
+    struct Exchange *e = e_head;
+
+    if (NULL != e->keys)
+    {
+      TALER_EXCHANGE_keys_decref (e->keys);
+      e->keys = NULL;
+    }
+    GNUNET_CONTAINER_DLL_remove (e_head,
+                                 e_tail,
+                                 e);
+    GNUNET_free (e);
+  }
+  while (NULL != a_head)
+  {
+    struct Account *a = a_head;
+
+    stop_inquiries (a);
+    GNUNET_CONTAINER_DLL_remove (a_head,
+                                 a_tail,
+                                 a);
+    GNUNET_free (a->merchant_account_uri);
+    GNUNET_free (a);
+  }
+  if (NULL != eh_accounts)
+  {
+    db_plugin->event_listen_cancel (eh_accounts);
+    eh_accounts = NULL;
+  }
+  if (NULL != account_task)
+  {
+    GNUNET_SCHEDULER_cancel (account_task);
+    account_task = NULL;
+  }
+  if (NULL != eh_keys)
+  {
+    db_plugin->event_listen_cancel (eh_keys);
+    eh_keys = NULL;
+  }
+  TALER_MERCHANTDB_plugin_unload (db_plugin);
+  db_plugin = NULL;
+  cfg = NULL;
+  if (NULL != ctx)
+  {
+    GNUNET_CURL_fini (ctx);
+    ctx = NULL;
+  }
+  if (NULL != rc)
+  {
+    GNUNET_CURL_gnunet_rc_destroy (rc);
+    rc = NULL;
+  }
 }
 
 
@@ -445,18 +752,36 @@ run (void *cls,
   {
     struct GNUNET_DB_EventHeaderP es = {
       .size = htons (sizeof (es)),
+      .type = htons (TALER_DBEVENT_MERCHANT_EXCHANGE_KEYS)
+    };
+
+    eh_keys
+      = db_plugin->event_listen (db_plugin->cls,
+                                 &es,
+                                 GNUNET_TIME_UNIT_FOREVER_REL,
+                                 &keys_changed,
+                                 NULL);
+  }
+  GNUNET_CONFIGURATION_iterate_sections (cfg,
+                                         &accept_exchanges,
+                                         NULL);
+  {
+    struct GNUNET_DB_EventHeaderP es = {
+      .size = htons (sizeof (es)),
       .type = htons (TALER_DBEVENT_MERCHANT_ACCOUNTS_CHANGED)
     };
 
-    eh = db_plugin->event_listen (db_plugin->cls,
-                                  &es,
-                                  GNUNET_TIME_UNIT_FOREVER_REL,
-                                  &account_changed,
-                                  NULL);
+    eh_accounts
+      = db_plugin->event_listen (db_plugin->cls,
+                                 &es,
+                                 GNUNET_TIME_UNIT_FOREVER_REL,
+                                 &account_changed,
+                                 NULL);
   }
-  GNUNET_assert (NULL == task);
-  task = GNUNET_SCHEDULER_add_now (&find_work,
-                                   NULL);
+  GNUNET_assert (NULL == account_task);
+  account_task
+    = GNUNET_SCHEDULER_add_now (&find_accounts,
+                                NULL);
 }
 
 
@@ -500,9 +825,6 @@ main (int argc,
     return EXIT_INVALIDARGUMENT;
   if (GNUNET_NO == ret)
     return EXIT_SUCCESS;
-  if ( (found_problem) &&
-       (0 == global_ret) )
-    global_ret = 7;
   return global_ret;
 }
 
diff --git a/src/backenddb/pg_select_accounts.c b/src/backenddb/pg_select_accounts.c
index e88fbb57..eaf26015 100644
--- a/src/backenddb/pg_select_accounts.c
+++ b/src/backenddb/pg_select_accounts.c
@@ -127,7 +127,9 @@ TMH_PG_select_accounts (void *cls,
     .pg = pg
   };
   struct GNUNET_PQ_QueryParam params[] = {
-    GNUNET_PQ_query_param_string (id),
+    NULL == id
+    ? GNUNET_PQ_query_param_null ()
+    : GNUNET_PQ_query_param_string (id),
     GNUNET_PQ_query_param_end
   };
   enum GNUNET_DB_QueryStatus qs;
@@ -143,10 +145,12 @@ TMH_PG_select_accounts (void *cls,
            ",credit_facade_credentials"
            ",active"
            " FROM merchant_accounts"
-           " WHERE merchant_serial="
-           "   (SELECT merchant_serial "
-           "      FROM merchant_instances"
-           "     WHERE merchant_id=$1);");
+           " WHERE"
+           "  ($1 IS NULL) OR"
+           "  (merchant_serial="
+           "    (SELECT merchant_serial "
+           "       FROM merchant_instances"
+           "      WHERE merchant_id=$1));");
   qs = GNUNET_PQ_eval_prepared_multi_select (pg->conn,
                                              "select_accounts",
                                              params,