diff options
author | Christian Grothoff <christian@grothoff.org> | 2024-09-06 15:52:15 +0200 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2024-09-06 15:52:15 +0200 |
commit | c3a9329d6e841e72b16273fff64743cba73f70e2 (patch) | |
tree | 4038d5646119c4689dd37008b92aa6a26de31da2 | |
parent | 7732fee20314a6deb2559632593ddd27f2c21b4c (diff) |
renaming taler-merchant-exchange to taler-merchant-reconciliation, preparing ground for new taler-merchant-kyccheck and taler-merchant-exchangekeyupdate helpers
-rw-r--r-- | debian/taler-merchant.taler-merchant-exchangekeyupdate.service | 17 | ||||
-rw-r--r-- | debian/taler-merchant.taler-merchant-kyccheck.service (renamed from debian/taler-merchant.taler-merchant-exchange.service) | 4 | ||||
-rw-r--r-- | debian/taler-merchant.taler-merchant-reconciliation.service | 17 | ||||
-rw-r--r-- | debian/taler-merchant.taler-merchant.target | 8 | ||||
-rw-r--r-- | doc/Makefile.am | 12 | ||||
m--------- | doc/prebuilt | 0 | ||||
-rw-r--r-- | src/backend/.gitignore | 6 | ||||
-rw-r--r-- | src/backend/Makefile.am | 75 | ||||
-rw-r--r-- | src/backend/taler-merchant-exchangekeyupdate.c | 547 | ||||
-rw-r--r-- | src/backend/taler-merchant-kyccheck.c | 510 | ||||
-rw-r--r-- | src/backend/taler-merchant-reconciliation.c (renamed from src/backend/taler-merchant-exchange.c) | 14 | ||||
-rw-r--r-- | src/testing/test_kyc_api.c | 4 | ||||
-rw-r--r-- | src/testing/test_merchant_api.c | 2 | ||||
-rwxr-xr-x | src/testing/test_merchant_order_creation.sh | 4 | ||||
-rwxr-xr-x | src/testing/test_merchant_transfer_tracking.sh | 16 | ||||
-rwxr-xr-x | src/testing/test_merchant_wirewatch.sh | 4 | ||||
-rw-r--r-- | src/testing/testing_api_cmd_tme.c | 30 |
17 files changed, 1203 insertions, 67 deletions
diff --git a/debian/taler-merchant.taler-merchant-exchangekeyupdate.service b/debian/taler-merchant.taler-merchant-exchangekeyupdate.service new file mode 100644 index 00000000..b2e597f2 --- /dev/null +++ b/debian/taler-merchant.taler-merchant-exchangekeyupdate.service @@ -0,0 +1,17 @@ +[Unit] +Description=GNU Taler merchant exchange configuration data download service +After=postgres.service + +[Service] +User=taler-merchant-httpd +Type=simple +Restart=always +RestartMode=direct +RestartSec=1s +RestartPreventExitStatus=9 +ExecStart=/usr/bin/taler-merchant-exchangekeyupdate -c /etc/taler/taler.conf -L INFO +PrivateTmp=yes +PrivateDevices=yes +ProtectSystem=full +RuntimeMaxSec=3600s +Slice=taler-merchant.slice diff --git a/debian/taler-merchant.taler-merchant-exchange.service b/debian/taler-merchant.taler-merchant-kyccheck.service index f61dc546..8417ce1d 100644 --- a/debian/taler-merchant.taler-merchant-exchange.service +++ b/debian/taler-merchant.taler-merchant-kyccheck.service @@ -1,5 +1,5 @@ [Unit] -Description=GNU Taler merchant-exchange transaction reconciliation service +Description=GNU Taler merchant KYC status check service After=postgres.service [Service] @@ -9,7 +9,7 @@ Restart=always RestartMode=direct RestartSec=1s RestartPreventExitStatus=9 -ExecStart=/usr/bin/taler-merchant-exchange -c /etc/taler/taler.conf -L INFO +ExecStart=/usr/bin/taler-merchant-kyccheck -c /etc/taler/taler.conf -L INFO PrivateTmp=yes PrivateDevices=yes ProtectSystem=full diff --git a/debian/taler-merchant.taler-merchant-reconciliation.service b/debian/taler-merchant.taler-merchant-reconciliation.service new file mode 100644 index 00000000..eef8f0f5 --- /dev/null +++ b/debian/taler-merchant.taler-merchant-reconciliation.service @@ -0,0 +1,17 @@ +[Unit] +Description=GNU Taler merchant transaction reconciliation service +After=postgres.service + +[Service] +User=taler-merchant-httpd +Type=simple +Restart=always +RestartMode=direct +RestartSec=1s +RestartPreventExitStatus=9 +ExecStart=/usr/bin/taler-merchant-reconciliation -c /etc/taler/taler.conf -L INFO +PrivateTmp=yes +PrivateDevices=yes +ProtectSystem=full +RuntimeMaxSec=3600s +Slice=taler-merchant.slice diff --git a/debian/taler-merchant.taler-merchant.target b/debian/taler-merchant.taler-merchant.target index bfab54f5..81e78f6e 100644 --- a/debian/taler-merchant.taler-merchant.target +++ b/debian/taler-merchant.taler-merchant.target @@ -2,11 +2,13 @@ Description=GNU Taler merchant After=postgres.service network.target +Wants=taler-merchant-depositcheck.service +Wants=taler-merchant-exchangekeyupdate.service Wants=taler-merchant-httpd.service -Wants=taler-merchant-wirewatch.service -Wants=taler-merchant-exchange.service +Wants=taler-merchant-kyccheck.service +Wants=taler-merchant-reconciliation.service Wants=taler-merchant-webhook.service -Wants=taler-merchant-depositcheck.service +Wants=taler-merchant-wirewatch.service [Install] WantedBy=multi-user.target diff --git a/doc/Makefile.am b/doc/Makefile.am index 9efaf49d..979f320f 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -2,12 +2,14 @@ SUBDIRS = . doxygen man_MANS = \ prebuilt/man/taler-merchant-benchmark.1 \ - prebuilt/man/taler-merchant-dbconfig.1\ - prebuilt/man/taler-merchant-dbinit.1\ - prebuilt/man/taler-merchant-depositcheck.1\ - prebuilt/man/taler-merchant-exchange.1\ + prebuilt/man/taler-merchant-dbconfig.1 \ + prebuilt/man/taler-merchant-dbinit.1 \ + prebuilt/man/taler-merchant-depositcheck.1 \ + prebuilt/man/taler-merchant-exchangekeyupdate.1 \ prebuilt/man/taler-merchant-httpd.1 \ - prebuilt/man/taler-merchant-passwd.1\ + prebuilt/man/taler-merchant-kyccheck.1 \ + prebuilt/man/taler-merchant-passwd.1 \ + prebuilt/man/taler-merchant-reconciliation.1 \ prebuilt/man/taler-merchant-webhook.1 \ prebuilt/man/taler-merchant-wirewatch.1 diff --git a/doc/prebuilt b/doc/prebuilt -Subproject ab2e09b5a3711ab04f1f77f79158cc006cab319 +Subproject b420ea339295885530ecadba0ed849d42415456 diff --git a/src/backend/.gitignore b/src/backend/.gitignore index 69cbfcee..932c5282 100644 --- a/src/backend/.gitignore +++ b/src/backend/.gitignore @@ -1,4 +1,6 @@ +taler-merchant-depositcheck +taler-merchant-exchangekeyupdate +taler-merchant-kyccheck +taler-merchant-reconciliation taler-merchant-webhook taler-merchant-wirewatch -taler-merchant-exchange -taler-merchant-depositcheck diff --git a/src/backend/Makefile.am b/src/backend/Makefile.am index c9c408de..cf429777 100644 --- a/src/backend/Makefile.am +++ b/src/backend/Makefile.am @@ -17,11 +17,48 @@ EXTRA_DIST = \ bin_PROGRAMS = \ taler-merchant-depositcheck \ - taler-merchant-exchange \ + taler-merchant-exchangekeyupdate \ taler-merchant-httpd \ + taler-merchant-kyccheck \ + taler-merchant-reconciliation \ taler-merchant-webhook \ taler-merchant-wirewatch +taler_merchant_depositcheck_SOURCES = \ + taler-merchant-depositcheck.c +taler_merchant_depositcheck_LDADD = \ + $(top_builddir)/src/backenddb/libtalermerchantdb.la \ + -ltalerexchange \ + -ltalerjson \ + -ltalerutil \ + -ltalerpq \ + -ljansson \ + -lgnunetcurl \ + -lgnunetjson \ + -lgnunetutil \ + -lcurl \ + $(XLIB) +taler_merchant_depositcheck_CFLAGS = \ + $(AM_CFLAGS) + + +taler_merchant_exchangekeyupdate_SOURCES = \ + taler-merchant-exchangekeyupdate.c +taler_merchant_exchangekeyupdate_LDADD = \ + $(top_builddir)/src/backenddb/libtalermerchantdb.la \ + -ltalerexchange \ + -ltalerjson \ + -ltalerutil \ + -ltalerpq \ + -lgnunetjson \ + -lgnunetcurl \ + -lgnunetutil \ + -lcurl \ + $(XLIB) +taler_merchant_exchangekeyupdate_CFLAGS = \ + $(AM_CFLAGS) + + taler_merchant_httpd_SOURCES = \ taler-merchant-httpd.c taler-merchant-httpd.h \ taler-merchant-httpd_config.c taler-merchant-httpd_config.h \ @@ -187,9 +224,9 @@ taler_merchant_httpd_CFLAGS = \ $(AM_CFLAGS) -taler_merchant_exchange_SOURCES = \ - taler-merchant-exchange.c -taler_merchant_exchange_LDADD = \ +taler_merchant_kyccheck_SOURCES = \ + taler-merchant-kyccheck.c +taler_merchant_kyccheck_LDADD = \ $(top_builddir)/src/backenddb/libtalermerchantdb.la \ -ltalerexchange \ -ltalerjson \ @@ -200,33 +237,34 @@ taler_merchant_exchange_LDADD = \ -lgnunetutil \ -lcurl \ $(XLIB) -taler_merchant_exchange_CFLAGS = \ +taler_merchant_kyccheck_CFLAGS = \ $(AM_CFLAGS) -taler_merchant_webhook_SOURCES = \ - taler-merchant-webhook.c -taler_merchant_webhook_LDADD = \ +taler_merchant_reconciliation_SOURCES = \ + taler-merchant-reconciliation.c +taler_merchant_reconciliation_LDADD = \ $(top_builddir)/src/backenddb/libtalermerchantdb.la \ - -ltalertemplating \ - -ltalermhd \ + -ltalerexchange \ -ltalerjson \ -ltalerutil \ -ltalerpq \ - -ljansson \ - -lgnunetcurl \ -lgnunetjson \ + -lgnunetcurl \ -lgnunetutil \ -lcurl \ $(XLIB) -taler_merchant_webhook_CFLAGS = \ +taler_merchant_reconciliation_CFLAGS = \ $(AM_CFLAGS) -taler_merchant_depositcheck_SOURCES = \ - taler-merchant-depositcheck.c -taler_merchant_depositcheck_LDADD = \ + + +taler_merchant_webhook_SOURCES = \ + taler-merchant-webhook.c +taler_merchant_webhook_LDADD = \ $(top_builddir)/src/backenddb/libtalermerchantdb.la \ - -ltalerexchange \ + -ltalertemplating \ + -ltalermhd \ -ltalerjson \ -ltalerutil \ -ltalerpq \ @@ -236,9 +274,10 @@ taler_merchant_depositcheck_LDADD = \ -lgnunetutil \ -lcurl \ $(XLIB) -taler_merchant_depositcheck_CFLAGS = \ +taler_merchant_webhook_CFLAGS = \ $(AM_CFLAGS) + taler_merchant_wirewatch_SOURCES = \ taler-merchant-wirewatch.c taler_merchant_wirewatch_LDADD = \ diff --git a/src/backend/taler-merchant-exchangekeyupdate.c b/src/backend/taler-merchant-exchangekeyupdate.c new file mode 100644 index 00000000..c3ded9d1 --- /dev/null +++ b/src/backend/taler-merchant-exchangekeyupdate.c @@ -0,0 +1,547 @@ +/* + This file is part of TALER + Copyright (C) 2023 Taler Systems SA + + TALER is free software; you can redistribute it and/or modify it under the + terms of the GNU Affero General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License along with + TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/> +*/ +/** + * @file taler-merchant-exchangekeyupdate.c + * @brief Process that ensures our /keys data for all exchanges is current + * @author Christian Grothoff + */ +#include "platform.h" +#include <gnunet/gnunet_util_lib.h> +#include <jansson.h> +#include <pthread.h> +#include <taler/taler_dbevents.h> +#include "taler_merchant_bank_lib.h" +#include "taler_merchantdb_lib.h" +#include "taler_merchantdb_plugin.h" + +/** + * Timeout for the exchange interaction. Rather long as we should do + * long-polling and do not want to wake up too often. + */ +#define EXCHANGE_TIMEOUT GNUNET_TIME_relative_multiply ( \ + GNUNET_TIME_UNIT_MINUTES, \ + 30) + +/** + * How many inquiries do we process concurrently at most. + */ +#define OPEN_INQUIRY_LIMIT 1024 + + +/** + * Information about an inquiry job. + */ +struct Inquiry; + + +/** + * Information about an exchange. + */ +struct Exchange +{ + /** + * Kept in a DLL. + */ + struct Exchange *next; + + /** + * Kept in a DLL. + */ + struct Exchange *prev; + + /** + * Which exchange are we tracking here. + */ + char *exchange_url; + + /** + * A connection to this exchange + */ + struct TALER_EXCHANGE_GetKeysHandle *conn; + + /** + * The keys of this exchange + */ + struct TALER_EXCHANGE_Keys *keys; + + /** + * Task where we retry fetching /keys from the exchange. + */ + struct GNUNET_SCHEDULER_Task *retry_task; + + /** + * How soon can may we, at the earliest, re-download /keys? + */ + struct GNUNET_TIME_Absolute first_retry; + + /** + * How long should we wait between the next retry? + */ + struct GNUNET_TIME_Relative retry_delay; + + /** + * How long should we wait between requests + * for transfer details? + */ + struct GNUNET_TIME_Relative transfer_delay; + + /** + * False to indicate that there is an ongoing + * /keys transfer we are waiting for; + * true to indicate that /keys data is up-to-date. + */ + bool ready; + +}; + + +/** + * Head of known exchanges. + */ +static struct Exchange *e_head; + +/** + * Tail of known exchanges. + */ +static struct Exchange *e_tail; + +/** + * The merchant's configuration. + */ +static const struct GNUNET_CONFIGURATION_Handle *cfg; + +/** + * Our database plugin. + */ +static struct TALER_MERCHANTDB_Plugin *db_plugin; + +/** + * Handle to the context for interacting with the bank. + */ +static struct GNUNET_CURL_Context *ctx; + +/** + * Scheduler context for running the @e ctx. + */ +static struct GNUNET_CURL_RescheduleContext *rc; + +/** + * Main task for #find_work(). + */ +static struct GNUNET_SCHEDULER_Task *task; + +/** + * How many active inquiries do we have right now. + */ +static unsigned int active_inquiries; + +/** + * Set to true if we ever encountered any problem. + */ +static bool found_problem; + +/** + * Value to return from main(). 0 on success, non-zero on errors. + */ +static int global_ret; + +/** + * #GNUNET_YES if we are in test mode and should exit when idle. + */ +static int test_mode; + +/** + * True if the last DB query was limited by the + * #OPEN_INQUIRY_LIMIT and we thus should check again + * as soon as we are substantially below that limit, + * and not only when we get a DB notification. + */ +static bool at_limit; + + +/** + * Function that initiates a /keys download. + * + * @param cls a `struct Exchange *` + */ +static void +download_keys (void *cls); + + +/** + * Finds new transfers that require work in the merchant database. + * + * @param cls NULL + */ +static void +find_work (void *cls); + + +/** + * Free resources of @a e. + * + * @param[in] e inquiry job to terminate + */ +static void +end_inquiry (struct Exchange *e) +{ + GNUNET_assert (active_inquiries > 0); + + active_inquiries--; + if ( (active_inquiries < OPEN_INQUIRY_LIMIT / 2) && + (NULL == task) && + (at_limit) ) + { + at_limit = false; + GNUNET_assert (NULL == task); + task = GNUNET_SCHEDULER_add_now (&find_work, + NULL); + } + if ( (NULL == task) && + (! at_limit) && + (0 == active_inquiries) && + (test_mode) ) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "No more open inquiries and in test mode. Existing.\n"); + GNUNET_SCHEDULER_shutdown (); + return; + } +} + + +/** + * Function called with information about who is auditing + * a particular exchange and what keys the exchange is using. + * + * @param cls closure with a `struct Exchange *` + * @param kr response data + * @param[in] keys the keys of the exchange + */ +static void +cert_cb ( + void *cls, + const struct TALER_EXCHANGE_KeysResponse *kr, + struct TALER_EXCHANGE_Keys *keys) +{ + struct Exchange *e = cls; + struct GNUNET_TIME_Absolute n; + + e->conn = NULL; + switch (kr->hr.http_status) + { + case MHD_HTTP_OK: + e->ready = true; + TALER_EXCHANGE_keys_decref (e->keys); + e->keys = keys; + end_inquiry (e); + /* Reset back-off */ + e->retry_delay = GNUNET_TIME_UNIT_ZERO; + /* Success: rate limit at once per minute */ + e->first_retry = GNUNET_TIME_relative_to_absolute ( + GNUNET_TIME_UNIT_MINUTES); + n = GNUNET_TIME_absolute_max (e->first_retry, + keys->key_data_expiration.abs_time); + if (NULL != e->retry_task) + GNUNET_SCHEDULER_cancel (e->retry_task); + e->retry_task = GNUNET_SCHEDULER_add_at (n, + &download_keys, + e); + break; + default: + e->retry_delay + = GNUNET_TIME_STD_BACKOFF (e->retry_delay); + e->first_retry + = GNUNET_TIME_relative_to_absolute (e->retry_delay); + if (NULL != e->retry_task) + GNUNET_SCHEDULER_cancel (e->retry_task); + e->retry_task = GNUNET_SCHEDULER_add_delayed (e->retry_delay, + &download_keys, + e); + break; + } +} + + +static void +download_keys (void *cls) +{ + struct Exchange *e = cls; + struct GNUNET_TIME_Relative n; + + /* If we do not hear back again soon, try again automatically */ + n = GNUNET_TIME_STD_BACKOFF (e->retry_delay); + n = GNUNET_TIME_relative_max (n, + GNUNET_TIME_UNIT_MINUTES); + e->retry_task = GNUNET_SCHEDULER_add_delayed (n, + &download_keys, + e); + + if ( (NULL == e->keys) || + (GNUNET_TIME_absolute_is_past (e->keys->key_data_expiration.abs_time)) ) + { + e->conn = TALER_EXCHANGE_get_keys (ctx, + e->exchange_url, + e->keys, + &cert_cb, + e); + if (NULL != e->conn) + active_inquiries++; + } +} + + +/** + * Lookup our internal data structure for the given + * @a exchange_url or create one if we do not yet have + * one. + * + * @param exchange_url base URL of the exchange + * @return our state for this exchange + */ +static struct Exchange * +find_exchange (const char *exchange_url) +{ + struct Exchange *e; + + for (e = e_head; NULL != e; e = e->next) + if (0 == strcmp (exchange_url, + e->exchange_url)) + return e; + e = GNUNET_new (struct Exchange); + e->exchange_url = GNUNET_strdup (exchange_url); + GNUNET_CONTAINER_DLL_insert (e_head, + e_tail, + e); + e->retry_task = GNUNET_SCHEDULER_add_now (&download_keys, + e); + return e; +} + + +/** + * We're being aborted with CTRL-C (or SIGTERM). Shut down. + * + * @param cls closure (NULL) + */ +static void +shutdown_task (void *cls) +{ + (void) cls; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Running shutdown\n"); + while (NULL != e_head) + { + struct Exchange *e = e_head; + + GNUNET_free (e->exchange_url); + if (NULL != e->conn) + { + TALER_EXCHANGE_get_keys_cancel (e->conn); + e->conn = NULL; + } + if (NULL != e->keys) + { + TALER_EXCHANGE_keys_decref (e->keys); + e->keys = NULL; + } + if (NULL != e->retry_task) + { + GNUNET_SCHEDULER_cancel (e->retry_task); + e->retry_task = NULL; + } + GNUNET_CONTAINER_DLL_remove (e_head, + e_tail, + e); + GNUNET_free (e); + } + if (NULL != task) + { + GNUNET_SCHEDULER_cancel (task); + task = NULL; + } + TALER_MERCHANTDB_plugin_unload (db_plugin); + db_plugin = NULL; + cfg = NULL; + if (NULL != ctx) + { + GNUNET_CURL_fini (ctx); + ctx = NULL; + } + if (NULL != rc) + { + GNUNET_CURL_gnunet_rc_destroy (rc); + rc = NULL; + } +} + + +static void +find_work (void *cls) +{ + // enum GNUNET_DB_QueryStatus qs; + int limit; + + (void) cls; + task = NULL; + GNUNET_assert (OPEN_INQUIRY_LIMIT >= active_inquiries); + limit = OPEN_INQUIRY_LIMIT - active_inquiries; + if (0 == limit) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Not looking for work: at limit\n"); + at_limit = true; + return; + } + at_limit = false; +#if FIXME + qs = db_plugin->select_open_transfers (db_plugin->cls, + limit, + &start_inquiry, + NULL); + if (qs < 0) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Failed to obtain open transfers from database\n"); + GNUNET_SCHEDULER_shutdown (); + return; + } + if (qs == limit) + { + /* DB limited response, re-trigger DB interaction + the moment we significantly fall below the + limit */ + at_limit = true; + } +#endif + if (0 == active_inquiries) + { + if (test_mode) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "No more open inquiries and in test mode. Existing.\n"); + GNUNET_SCHEDULER_shutdown (); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "No open inquiries found, waiting for notification to resume\n") + ; + } +} + + +/** + * First task. + * + * @param cls closure, NULL + * @param args remaining command-line arguments + * @param cfgfile name of the configuration file used (for saving, can be NULL!) + * @param c configuration + */ +static void +run (void *cls, + char *const *args, + const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + (void) args; + (void) cfgfile; + + cfg = c; + GNUNET_SCHEDULER_add_shutdown (&shutdown_task, + NULL); + ctx = GNUNET_CURL_init (&GNUNET_CURL_gnunet_scheduler_reschedule, + &rc); + rc = GNUNET_CURL_gnunet_rc_create (ctx); + if (NULL == ctx) + { + GNUNET_break (0); + GNUNET_SCHEDULER_shutdown (); + global_ret = EXIT_FAILURE; + return; + } + if (NULL == + (db_plugin = TALER_MERCHANTDB_plugin_load (cfg))) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Failed to initialize DB subsystem\n"); + GNUNET_SCHEDULER_shutdown (); + global_ret = EXIT_NOTCONFIGURED; + return; + } + if (GNUNET_OK != + db_plugin->connect (db_plugin->cls)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Failed to connect to database\n"); + GNUNET_SCHEDULER_shutdown (); + global_ret = EXIT_FAILURE; + return; + } + GNUNET_assert (NULL == task); + task = GNUNET_SCHEDULER_add_now (&find_work, + NULL); +} + + +/** + * The main function of taler-merchant-exchangekeyupdate + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, + char *const *argv) +{ + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_option_timetravel ('T', + "timetravel"), + GNUNET_GETOPT_option_flag ('t', + "test", + "run in test mode and exit when idle", + &test_mode), + GNUNET_GETOPT_option_version (VERSION "-" VCS_VERSION), + GNUNET_GETOPT_OPTION_END + }; + enum GNUNET_GenericReturnValue ret; + + if (GNUNET_OK != + GNUNET_STRINGS_get_utf8_args (argc, argv, + &argc, &argv)) + return EXIT_INVALIDARGUMENT; + TALER_OS_init (); + ret = GNUNET_PROGRAM_run ( + argc, argv, + "taler-merchant-exchangekeyupdate", + gettext_noop ( + "background process that ensures our key and configuration data on exchanges is up-to-date"), + options, + &run, NULL); + GNUNET_free_nz ((void *) argv); + if (GNUNET_SYSERR == ret) + return EXIT_INVALIDARGUMENT; + if (GNUNET_NO == ret) + return EXIT_SUCCESS; + if ( (found_problem) && + (0 == global_ret) ) + global_ret = 7; + return global_ret; +} + + +/* end of taler-merchant-exchangekeyupdate.c */ diff --git a/src/backend/taler-merchant-kyccheck.c b/src/backend/taler-merchant-kyccheck.c new file mode 100644 index 00000000..36970886 --- /dev/null +++ b/src/backend/taler-merchant-kyccheck.c @@ -0,0 +1,510 @@ +/* + This file is part of TALER + Copyright (C) 2024 Taler Systems SA + + TALER is free software; you can redistribute it and/or modify it under the + terms of the GNU Affero General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License along with + TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/> +*/ +/** + * @file taler-merchant-kyccheck.c + * @brief Process that check the KYC status of our bank accounts at all exchanges + * @author Christian Grothoff + */ +#include "platform.h" +#include <gnunet/gnunet_util_lib.h> +#include <jansson.h> +#include <pthread.h> +#include <taler/taler_dbevents.h> +#include "taler_merchant_bank_lib.h" +#include "taler_merchantdb_lib.h" +#include "taler_merchantdb_plugin.h" + +/** + * Timeout for the exchange interaction. Rather long as we should do + * long-polling and do not want to wake up too often. + */ +#define EXCHANGE_TIMEOUT GNUNET_TIME_relative_multiply ( \ + GNUNET_TIME_UNIT_MINUTES, \ + 30) + +/** + * How many inquiries do we process concurrently at most. + */ +#define OPEN_INQUIRY_LIMIT 1024 + +/** + * How many inquiries do we process concurrently per exchange at most. + */ +#define EXCHANGE_INQUIRY_LIMIT 16 + + +/** + * Information about an inquiry job. + */ +struct Inquiry; + + +/** + * Information about an exchange. + */ +struct Exchange +{ + /** + * Kept in a DLL. + */ + struct Exchange *next; + + /** + * Kept in a DLL. + */ + struct Exchange *prev; + + /** + * Head of active inquiries. + */ + struct Inquiry *w_head; + + /** + * Tail of active inquiries. + */ + struct Inquiry *w_tail; + + /** + * Which exchange are we tracking here. + */ + char *exchange_url; + + /** + * The keys of this exchange + */ + struct TALER_EXCHANGE_Keys *keys; + + /** + * How many active inquiries do we have right now with this exchange. + */ + unsigned int exchange_inquiries; + + /** + * How soon can may we, at the earliest, re-download /keys? + */ + struct GNUNET_TIME_Absolute first_retry; + + /** + * How long should we wait between the next retry? + */ + struct GNUNET_TIME_Relative retry_delay; + + /** + * How long should we wait between requests + * for transfer details? + */ + struct GNUNET_TIME_Relative transfer_delay; + + /** + * False to indicate that there is an ongoing + * /keys transfer we are waiting for; + * true to indicate that /keys data is up-to-date. + */ + bool ready; + +}; + + +/** + * Information about an inquiry job. + */ +struct Inquiry +{ + /** + * Kept in a DLL. + */ + struct Inquiry *next; + + /** + * Kept in a DLL. + */ + struct Inquiry *prev; + + /** + * Kept in a DLL. + */ + struct Exchange *exchange; + +}; + + +/** + * Head of known exchanges. + */ +static struct Exchange *e_head; + +/** + * Tail of known exchanges. + */ +static struct Exchange *e_tail; + +/** + * The merchant's configuration. + */ +static const struct GNUNET_CONFIGURATION_Handle *cfg; + +/** + * Our database plugin. + */ +static struct TALER_MERCHANTDB_Plugin *db_plugin; + +/** + * Handle to the context for interacting with the bank. + */ +static struct GNUNET_CURL_Context *ctx; + +/** + * Scheduler context for running the @e ctx. + */ +static struct GNUNET_CURL_RescheduleContext *rc; + +/** + * Main task for #find_work(). + */ +static struct GNUNET_SCHEDULER_Task *task; + +/** + * Event handler to learn that there are new transfers + * to check. + */ +static struct GNUNET_DB_EventHandler *eh; + +/** + * How many active inquiries do we have right now. + */ +static unsigned int active_inquiries; + +/** + * Set to true if we ever encountered any problem. + */ +static bool found_problem; + +/** + * Value to return from main(). 0 on success, non-zero on errors. + */ +static int global_ret; + +/** + * #GNUNET_YES if we are in test mode and should exit when idle. + */ +static int test_mode; + +/** + * True if the last DB query was limited by the + * #OPEN_INQUIRY_LIMIT and we thus should check again + * as soon as we are substantially below that limit, + * and not only when we get a DB notification. + */ +static bool at_limit; + + +/** + * Finds new transfers that require work in the merchant database. + * + * @param cls NULL + */ +static void +find_work (void *cls); + + +/** + * Free resources of @a w. + * + * @param[in] w inquiry job to terminate + */ +static void +end_inquiry (struct Inquiry *w) +{ + struct Exchange *e = w->exchange; + + GNUNET_assert (active_inquiries > 0); + active_inquiries--; + GNUNET_CONTAINER_DLL_remove (e->w_head, + e->w_tail, + w); + GNUNET_free (w); + if ( (active_inquiries < OPEN_INQUIRY_LIMIT / 2) && + (NULL == task) && + (at_limit) ) + { + at_limit = false; + GNUNET_assert (NULL == task); + task = GNUNET_SCHEDULER_add_now (&find_work, + NULL); + } + if ( (NULL == task) && + (! at_limit) && + (0 == active_inquiries) && + (test_mode) ) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "No more open inquiries and in test mode. Existing.\n"); + GNUNET_SCHEDULER_shutdown (); + return; + } +} + + +/** + * We're being aborted with CTRL-C (or SIGTERM). Shut down. + * + * @param cls closure (NULL) + */ +static void +shutdown_task (void *cls) +{ + (void) cls; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Running shutdown\n"); + while (NULL != e_head) + { + struct Exchange *e = e_head; + + while (NULL != e->w_head) + { + struct Inquiry *w = e->w_head; + + end_inquiry (w); + } + GNUNET_free (e->exchange_url); + if (NULL != e->keys) + { + TALER_EXCHANGE_keys_decref (e->keys); + e->keys = NULL; + } + GNUNET_CONTAINER_DLL_remove (e_head, + e_tail, + e); + GNUNET_free (e); + } + if (NULL != eh) + { + db_plugin->event_listen_cancel (eh); + eh = NULL; + } + if (NULL != task) + { + GNUNET_SCHEDULER_cancel (task); + task = NULL; + } + TALER_MERCHANTDB_plugin_unload (db_plugin); + db_plugin = NULL; + cfg = NULL; + if (NULL != ctx) + { + GNUNET_CURL_fini (ctx); + ctx = NULL; + } + if (NULL != rc) + { + GNUNET_CURL_gnunet_rc_destroy (rc); + rc = NULL; + } +} + + +static void +find_work (void *cls) +{ + // enum GNUNET_DB_QueryStatus qs; + int limit; + + (void) cls; + task = NULL; + GNUNET_assert (OPEN_INQUIRY_LIMIT >= active_inquiries); + limit = OPEN_INQUIRY_LIMIT - active_inquiries; + if (0 == limit) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Not looking for work: at limit\n"); + at_limit = true; + return; + } + at_limit = false; +#if 0 + // FIXME: do actual work! + qs = db_plugin->select_open_transfers (db_plugin->cls, + limit, + &start_inquiry, + NULL); + if (qs < 0) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Failed to obtain open transfers from database\n"); + GNUNET_SCHEDULER_shutdown (); + return; + } + if (qs == limit) + { + /* DB limited response, re-trigger DB interaction + the moment we significantly fall below the + limit */ + at_limit = true; + } + if (0 == active_inquiries) + { + if (test_mode) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "No more open inquiries and in test mode. Existing.\n"); + GNUNET_SCHEDULER_shutdown (); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "No open inquiries found, waiting for notification to resume\n") + ; + } +#endif +} + + +/** + * Function called when transfers are added to the merchant database. We look + * for more work. + * + * @param cls closure (NULL) + * @param extra additional event data provided + * @param extra_size number of bytes in @a extra + */ +static void +account_changed (void *cls, + const void *extra, + size_t extra_size) +{ + (void) cls; + (void) extra; + (void) extra_size; + if (NULL != task) + return; + task = GNUNET_SCHEDULER_add_now (&find_work, + NULL); +} + + +/** + * First task. + * + * @param cls closure, NULL + * @param args remaining command-line arguments + * @param cfgfile name of the configuration file used (for saving, can be NULL!) + * @param c configuration + */ +static void +run (void *cls, + char *const *args, + const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + (void) args; + (void) cfgfile; + + cfg = c; + GNUNET_SCHEDULER_add_shutdown (&shutdown_task, + NULL); + ctx = GNUNET_CURL_init (&GNUNET_CURL_gnunet_scheduler_reschedule, + &rc); + rc = GNUNET_CURL_gnunet_rc_create (ctx); + if (NULL == ctx) + { + GNUNET_break (0); + GNUNET_SCHEDULER_shutdown (); + global_ret = EXIT_FAILURE; + return; + } + if (NULL == + (db_plugin = TALER_MERCHANTDB_plugin_load (cfg))) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Failed to initialize DB subsystem\n"); + GNUNET_SCHEDULER_shutdown (); + global_ret = EXIT_NOTCONFIGURED; + return; + } + if (GNUNET_OK != + db_plugin->connect (db_plugin->cls)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Failed to connect to database\n"); + GNUNET_SCHEDULER_shutdown (); + global_ret = EXIT_FAILURE; + return; + } + { + struct GNUNET_DB_EventHeaderP es = { + .size = htons (sizeof (es)), + .type = htons (TALER_DBEVENT_MERCHANT_ACCOUNTS_CHANGED) + }; + + eh = db_plugin->event_listen (db_plugin->cls, + &es, + GNUNET_TIME_UNIT_FOREVER_REL, + &account_changed, + NULL); + } + GNUNET_assert (NULL == task); + task = GNUNET_SCHEDULER_add_now (&find_work, + NULL); +} + + +/** + * The main function of taler-merchant-kyccheck + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, + char *const *argv) +{ + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_option_timetravel ('T', + "timetravel"), + GNUNET_GETOPT_option_flag ('t', + "test", + "run in test mode and exit when idle", + &test_mode), + GNUNET_GETOPT_option_version (VERSION "-" VCS_VERSION), + GNUNET_GETOPT_OPTION_END + }; + enum GNUNET_GenericReturnValue ret; + + if (GNUNET_OK != + GNUNET_STRINGS_get_utf8_args (argc, argv, + &argc, &argv)) + return EXIT_INVALIDARGUMENT; + TALER_OS_init (); + ret = GNUNET_PROGRAM_run ( + argc, argv, + "taler-merchant-kyccheck", + gettext_noop ( + "background process that checks the KYC state of our bank accounts at various exchanges"), + options, + &run, NULL); + GNUNET_free_nz ((void *) argv); + if (GNUNET_SYSERR == ret) + return EXIT_INVALIDARGUMENT; + if (GNUNET_NO == ret) + return EXIT_SUCCESS; + if ( (found_problem) && + (0 == global_ret) ) + global_ret = 7; + return global_ret; +} + + +/* end of taler-merchant-kyccheck.c */ diff --git a/src/backend/taler-merchant-exchange.c b/src/backend/taler-merchant-reconciliation.c index d26786d5..23c9a926 100644 --- a/src/backend/taler-merchant-exchange.c +++ b/src/backend/taler-merchant-reconciliation.c @@ -14,7 +14,7 @@ TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/> */ /** - * @file taler-merchant-exchange.c + * @file taler-merchant-reconciliation.c * @brief Process that reconciles information about incoming bank transfers with orders by asking the exchange * @author Christian Grothoff */ @@ -1157,9 +1157,9 @@ find_work (void *cls) GNUNET_SCHEDULER_shutdown (); return; } - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "No open inquiries found, waiting for notification to resume\n") - ; + GNUNET_log ( + GNUNET_ERROR_TYPE_INFO, + "No open inquiries found, waiting for notification to resume\n"); } } @@ -1260,7 +1260,7 @@ run (void *cls, /** - * The main function of taler-merchant-exchange + * The main function of taler-merchant-reconciliation * * @param argc number of arguments from the command line * @param argv command line arguments @@ -1289,7 +1289,7 @@ main (int argc, TALER_OS_init (); ret = GNUNET_PROGRAM_run ( argc, argv, - "taler-merchant-exchange", + "taler-merchant-reconciliation", gettext_noop ( "background process that reconciles bank transfers with orders by asking the exchange"), options, @@ -1306,4 +1306,4 @@ main (int argc, } -/* end of taler-merchant-exchange.c */ +/* end of taler-merchant-reconciliation.c */ diff --git a/src/testing/test_kyc_api.c b/src/testing/test_kyc_api.c index 267ea578..5f34995a 100644 --- a/src/testing/test_kyc_api.c +++ b/src/testing/test_kyc_api.c @@ -303,7 +303,7 @@ run (void *cls, "deposit-simple", NULL), TALER_TESTING_cmd_run_tme ( - "run taler-merchant-exchange-1", + "run taler-merchant-reconciliation-1", CONFIG_FILE), TALER_TESTING_cmd_merchant_get_transfers ( "get-transfers-1", @@ -473,7 +473,7 @@ run (void *cls, "deposit-simple", NULL), TALER_TESTING_cmd_run_tme ( - "run taler-merchant-exchange-2-aml", + "run taler-merchant-reconciliation-2-aml", CONFIG_FILE), TALER_TESTING_cmd_merchant_get_transfers ( "get-transfers-aml", diff --git a/src/testing/test_merchant_api.c b/src/testing/test_merchant_api.c index 5cc2cc67..f5cd61ab 100644 --- a/src/testing/test_merchant_api.c +++ b/src/testing/test_merchant_api.c @@ -495,7 +495,7 @@ run (void *cls, MHD_HTTP_NO_CONTENT, "deposit-simple", NULL), - TALER_TESTING_cmd_run_tme ("run taler-merchant-exchange-1", + TALER_TESTING_cmd_run_tme ("run taler-merchant-reconciliation-1", config_file), TALER_TESTING_cmd_merchant_post_transfer2 ("post-transfer-bad", merchant_url, diff --git a/src/testing/test_merchant_order_creation.sh b/src/testing/test_merchant_order_creation.sh index 2fe069af..a52d7b41 100755 --- a/src/testing/test_merchant_order_creation.sh +++ b/src/testing/test_merchant_order_creation.sh @@ -601,9 +601,9 @@ then fi echo " OK" -echo -n "Testing taler-merchant-exchange ..." +echo -n "Testing taler-merchant-reconciliation ..." set -e -taler-merchant-exchange -L INFO -c "$CONF" -t &> taler-merchant-exchange.log +taler-merchant-reconciliation -L INFO -c "$CONF" -t &> taler-merchant-reconciliation.log echo " OK" diff --git a/src/testing/test_merchant_transfer_tracking.sh b/src/testing/test_merchant_transfer_tracking.sh index 41a20c11..832489c9 100755 --- a/src/testing/test_merchant_transfer_tracking.sh +++ b/src/testing/test_merchant_transfer_tracking.sh @@ -307,8 +307,8 @@ fi echo "OK" -echo -n "Fetching running taler-merchant-exchange on bogus transfer ..." -taler-merchant-exchange -c "$CONF" -L INFO -t &> taler-merchant-exchange-bad.log +echo -n "Fetching running taler-merchant-reconciliation on bogus transfer ..." +taler-merchant-reconciliation -c "$CONF" -L INFO -t &> taler-merchant-reconciliation-bad.log echo "OK" echo -n "Fetching wire transfers of 'test' instance ..." @@ -367,8 +367,8 @@ then fi echo " OK" -echo -n "Fetching running taler-merchant-exchange on good transfer ..." -taler-merchant-exchange -c $CONF -L INFO -t &> taler-merchant-exchange-bad.log +echo -n "Fetching running taler-merchant-reconciliation on good transfer ..." +taler-merchant-reconciliation -c $CONF -L INFO -t &> taler-merchant-reconciliation-bad.log echo "OK" echo -n "Fetching wire transfers of TEST instance ..." @@ -537,8 +537,8 @@ then fi echo " OK" -echo -n "Fetching running taler-merchant-exchange on good transfer ..." -taler-merchant-exchange -c $CONF -L INFO -t &> taler-merchant-exchange2.log +echo -n "Fetching running taler-merchant-reconciliation on good transfer ..." +taler-merchant-reconciliation -c $CONF -L INFO -t &> taler-merchant-reconciliation2.log echo "OK" echo -n "Fetching wire transfers of TEST instance ..." @@ -684,8 +684,8 @@ fi echo "OK" -echo -n "Fetching running taler-merchant-exchange on good transfer ..." -taler-merchant-exchange -c $CONF -L INFO -t &> taler-merchant-exchange2.log +echo -n "Fetching running taler-merchant-reconciliation on good transfer ..." +taler-merchant-reconciliation -c $CONF -L INFO -t &> taler-merchant-reconciliation2.log echo "OK" echo -n "Checking order status ..." diff --git a/src/testing/test_merchant_wirewatch.sh b/src/testing/test_merchant_wirewatch.sh index 61bd2049..5b509ce8 100755 --- a/src/testing/test_merchant_wirewatch.sh +++ b/src/testing/test_merchant_wirewatch.sh @@ -347,10 +347,10 @@ fi echo " OK" echo -n "Integrating wire transfer data with exchange..." -taler-merchant-exchange \ +taler-merchant-reconciliation \ -c "$CONF" \ -t \ - -L INFO &> merchant-exchange.log + -L INFO &> merchant-reconciliation.log echo " OK" echo -n "Checking order status ..." diff --git a/src/testing/testing_api_cmd_tme.c b/src/testing/testing_api_cmd_tme.c index b84e1df5..549fe182 100644 --- a/src/testing/testing_api_cmd_tme.c +++ b/src/testing/testing_api_cmd_tme.c @@ -18,7 +18,7 @@ */ /** * @file testing/testing_api_cmd_tme.c - * @brief run the taler-merchant-exchange command + * @brief run the taler-merchant-reconciliation command * @author Christian Grothoff */ #include "platform.h" @@ -29,15 +29,15 @@ /** - * State for a "taler-merchant-exchange" CMD. + * State for a "taler-merchant-reconciliation" CMD. */ struct MerchantExchangeState { /** - * Process for taler-merchant-exchange + * Process for taler-merchant-reconciliation */ - struct GNUNET_OS_Process *merchant_exchange_proc; + struct GNUNET_OS_Process *merchant_reconciliation_proc; /** * Configuration file used by the program. @@ -47,7 +47,7 @@ struct MerchantExchangeState /** - * Run the command; use the `taler-merchant-exchange' program. + * Run the command; use the `taler-merchant-reconciliation' program. * * @param cls closure. * @param cmd command currently being executed. @@ -61,16 +61,16 @@ tme_run (void *cls, struct MerchantExchangeState *ws = cls; (void) cmd; - ws->merchant_exchange_proc + ws->merchant_reconciliation_proc = GNUNET_OS_start_process (GNUNET_OS_INHERIT_STD_ALL, NULL, NULL, NULL, - "taler-merchant-exchange", - "taler-merchant-exchange", + "taler-merchant-reconciliation", + "taler-merchant-reconciliation", "-c", ws->config_filename, "-t", /* exit when done */ "-L", "DEBUG", NULL); - if (NULL == ws->merchant_exchange_proc) + if (NULL == ws->merchant_reconciliation_proc) { GNUNET_break (0); TALER_TESTING_interpreter_fail (is); @@ -94,14 +94,14 @@ tme_cleanup (void *cls, struct MerchantExchangeState *ws = cls; (void) cmd; - if (NULL != ws->merchant_exchange_proc) + if (NULL != ws->merchant_reconciliation_proc) { GNUNET_break (0 == - GNUNET_OS_process_kill (ws->merchant_exchange_proc, + GNUNET_OS_process_kill (ws->merchant_reconciliation_proc, SIGKILL)); - GNUNET_OS_process_wait (ws->merchant_exchange_proc); - GNUNET_OS_process_destroy (ws->merchant_exchange_proc); - ws->merchant_exchange_proc = NULL; + GNUNET_OS_process_wait (ws->merchant_reconciliation_proc); + GNUNET_OS_process_destroy (ws->merchant_reconciliation_proc); + ws->merchant_reconciliation_proc = NULL; } GNUNET_free (ws); } @@ -124,7 +124,7 @@ tme_traits (void *cls, { struct MerchantExchangeState *ws = cls; struct TALER_TESTING_Trait traits[] = { - TALER_TESTING_make_trait_process (&ws->merchant_exchange_proc), + TALER_TESTING_make_trait_process (&ws->merchant_reconciliation_proc), TALER_TESTING_trait_end () }; |