/* This file is part of TALER Copyright (C) 2023, 2024 Taler Systems SA TALER is free software; you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation; either version 3, or (at your option) any later version. TALER is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with TALER; see the file COPYING. If not, see */ /** * @file taler-merchant-exchangekeyupdate.c * @brief Process that ensures our /keys data for all exchanges is current * @author Christian Grothoff */ #include "platform.h" #include #include #include #include #include "taler_merchant_bank_lib.h" #include "taler_merchantdb_lib.h" #include "taler_merchantdb_plugin.h" /** * Maximum frequency for the exchange interaction. */ #define EXCHANGE_MAXFREQ GNUNET_TIME_relative_multiply ( \ GNUNET_TIME_UNIT_MINUTES, \ 5) /** * How many inquiries do we process concurrently at most. */ #define OPEN_INQUIRY_LIMIT 1024 /** * How often do we retry after DB serialization errors (at most)? */ #define MAX_RETRIES 3 /** * Information about an exchange. */ struct Exchange { /** * Kept in a DLL. */ struct Exchange *next; /** * Kept in a DLL. */ struct Exchange *prev; /** * Base URL of the exchange are we tracking here. */ char *exchange_url; /** * Expected currency of the exchange. */ char *currency; /** * A /keys request to this exchange, NULL if not active. */ struct TALER_EXCHANGE_GetKeysHandle *conn; /** * The keys of this exchange, NULL if not known. */ struct TALER_EXCHANGE_Keys *keys; /** * Task where we retry fetching /keys from the exchange. */ struct GNUNET_SCHEDULER_Task *retry_task; /** * Master public key expected for this exchange. */ struct TALER_MasterPublicKeyP master_pub; /** * How soon can may we, at the earliest, re-download /keys? */ struct GNUNET_TIME_Absolute first_retry; /** * How long should we wait between the next retry? * Used for exponential back-offs. */ struct GNUNET_TIME_Relative retry_delay; /** * Are we waiting for /keys downloads due to our * hard limit? */ bool limited; }; /** * Head of known exchanges. */ static struct Exchange *e_head; /** * Tail of known exchanges. */ static struct Exchange *e_tail; /** * The merchant's configuration. */ static const struct GNUNET_CONFIGURATION_Handle *cfg; /** * Our database plugin. */ static struct TALER_MERCHANTDB_Plugin *db_plugin; /** * Our event handler listening for /keys forced downloads. */ static struct GNUNET_DB_EventHandler *eh; /** * Handle to the context for interacting with the bank. */ static struct GNUNET_CURL_Context *ctx; /** * Scheduler context for running the @e ctx. */ static struct GNUNET_CURL_RescheduleContext *rc; /** * How many active inquiries do we have right now. */ static unsigned int active_inquiries; /** * Value to return from main(). 0 on success, non-zero on errors. */ static int global_ret; /** * #GNUNET_YES if we are in test mode and should exit when idle. */ static int test_mode; /** * True if the last DB query was limited by the * #OPEN_INQUIRY_LIMIT and we thus should check again * as soon as we are substantially below that limit, * and not only when we get a DB notification. */ static bool at_limit; /** * Function that initiates a /keys download. * * @param cls a `struct Exchange *` */ static void download_keys (void *cls); /** * An inquiry finished, check if we need to start more. */ static void end_inquiry (void) { GNUNET_assert (active_inquiries > 0); active_inquiries--; if ( (active_inquiries < OPEN_INQUIRY_LIMIT / 2) && (at_limit) ) { at_limit = false; for (struct Exchange *e = e_head; NULL != e; e = e->next) { if (! e->limited) continue; e->limited = false; /* done synchronously so that the active_inquiries is updated immediately */ download_keys (e); if (at_limit) break; } } if ( (! at_limit) && (0 == active_inquiries) && (test_mode) ) { GNUNET_log (GNUNET_ERROR_TYPE_INFO, "No more open inquiries and in test mode. Existing.\n"); GNUNET_SCHEDULER_shutdown (); return; } } /** * Add account restriction @a a to array of @a restrictions. * * @param[in,out] restrictions JSON array to build * @param r restriction to add to @a restrictions * @return #GNUNET_SYSERR if @a r is malformed */ static enum GNUNET_GenericReturnValue add_restriction (json_t *restrictions, const struct TALER_EXCHANGE_AccountRestriction *r) { json_t *jr; jr = NULL; switch (r->type) { case TALER_EXCHANGE_AR_INVALID: GNUNET_break_op (0); return GNUNET_SYSERR; case TALER_EXCHANGE_AR_DENY: jr = GNUNET_JSON_PACK ( GNUNET_JSON_pack_string ("type", "deny") ); break; case TALER_EXCHANGE_AR_REGEX: jr = GNUNET_JSON_PACK ( GNUNET_JSON_pack_string ( "type", "regex"), GNUNET_JSON_pack_string ( "regex", r->details.regex.posix_egrep), GNUNET_JSON_pack_string ( "human_hint", r->details.regex.human_hint), GNUNET_JSON_pack_object_incref ( "human_hint_i18n", (json_t *) r->details.regex.human_hint_i18n) ); break; } if (NULL == jr) { GNUNET_break_op (0); return GNUNET_SYSERR; } GNUNET_assert (0 == json_array_append_new (restrictions, jr)); return GNUNET_OK; } /** * Update our information in the database about the * /keys of an exchange. Run inside of a database * transaction scope that will re-try and/or commit * depending on the return value. * * @param keys information to persist * @return transaction status */ static enum GNUNET_DB_QueryStatus insert_keys_data (const struct TALER_EXCHANGE_Keys *keys) { enum GNUNET_DB_QueryStatus qs; /* store exchange online signing keys in our DB */ for (unsigned int i = 0; inum_sign_keys; i++) { const struct TALER_EXCHANGE_SigningPublicKey *sign_key = &keys->sign_keys[i]; qs = db_plugin->insert_exchange_signkey ( db_plugin->cls, &keys->master_pub, &sign_key->key, sign_key->valid_from, sign_key->valid_until, sign_key->valid_legal, &sign_key->master_sig); /* 0 is OK, we may already have the key in the DB! */ if (0 > qs) { GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); return qs; } } qs = db_plugin->insert_exchange_keys (db_plugin->cls, keys); if (0 > qs) { GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); return qs; } qs = db_plugin->delete_exchange_accounts (db_plugin->cls, &keys->master_pub); if (0 > qs) { GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); return qs; } for (unsigned int i = 0; iaccounts_len; i++) { const struct TALER_EXCHANGE_WireAccount *account = &keys->accounts[i]; json_t *debit_restrictions; json_t *credit_restrictions; debit_restrictions = json_array (); GNUNET_assert (NULL != debit_restrictions); credit_restrictions = json_array (); GNUNET_assert (NULL != credit_restrictions); for (unsigned int j = 0; jdebit_restrictions_length; j++) { if (GNUNET_OK != add_restriction (debit_restrictions, &account->debit_restrictions[j])) { db_plugin->rollback (db_plugin->cls); GNUNET_break (0); json_decref (debit_restrictions); json_decref (credit_restrictions); return GNUNET_DB_STATUS_HARD_ERROR; } } for (unsigned int j = 0; jcredit_restrictions_length; j++) { if (GNUNET_OK != add_restriction (credit_restrictions, &account->credit_restrictions[j])) { db_plugin->rollback (db_plugin->cls); GNUNET_break (0); json_decref (debit_restrictions); json_decref (credit_restrictions); return GNUNET_DB_STATUS_HARD_ERROR; } } qs = db_plugin->insert_exchange_account ( db_plugin->cls, &keys->master_pub, account->fpayto_uri, account->conversion_url, debit_restrictions, credit_restrictions, &account->master_sig); json_decref (debit_restrictions); json_decref (credit_restrictions); if (qs < 0) { GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); return qs; } } /* end 'for all accounts' */ for (unsigned int i = 0; ifees_len; i++) { const struct TALER_EXCHANGE_WireFeesByMethod *fbm = &keys->fees[i]; const char *wire_method = fbm->method; const struct TALER_EXCHANGE_WireAggregateFees *fees = fbm->fees_head; while (NULL != fees) { struct GNUNET_HashCode h_wire_method; GNUNET_CRYPTO_hash (wire_method, strlen (wire_method) + 1, &h_wire_method); qs = db_plugin->store_wire_fee_by_exchange ( db_plugin->cls, &keys->master_pub, &h_wire_method, &fees->fees, fees->start_date, fees->end_date, &fees->master_sig); if (0 > qs) { GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); return qs; } fees = fees->next; } /* all fees for this method */ } /* for all methods (i) */ { struct GNUNET_DB_EventHeaderP es = { .size = ntohs (sizeof (es)), .type = ntohs (TALER_DBEVENT_MERCHANT_EXCHANGE_KEYS) }; db_plugin->event_notify (db_plugin->cls, &es, keys->exchange_url, strlen (keys->exchange_url) + 1); } return qs; } /** * Run database transaction to store the @a keys in * the merchant database (and notify other processes * that may care about them). * * @param keys the keys to store * @return true on success */ static bool store_keys (struct TALER_EXCHANGE_Keys *keys) { enum GNUNET_DB_QueryStatus qs; db_plugin->preflight (db_plugin->cls); for (unsigned int r = 0; rstart (db_plugin->cls, "update exchange key data")) { db_plugin->rollback (db_plugin->cls); GNUNET_break (0); return false; } qs = insert_keys_data (keys); if (qs < 0) { db_plugin->rollback (db_plugin->cls); if (GNUNET_DB_STATUS_SOFT_ERROR == qs) continue; GNUNET_break (0); return false; } qs = db_plugin->commit (db_plugin->cls); if (qs < 0) { db_plugin->rollback (db_plugin->cls); if (GNUNET_DB_STATUS_SOFT_ERROR == qs) continue; GNUNET_break (0); return false; } } /* end of retry loop */ if (qs < 0) { GNUNET_break (0); return false; } return true; } /** * Function called with information about who is auditing * a particular exchange and what keys the exchange is using. * * @param cls closure with a `struct Exchange *` * @param kr response data * @param[in] keys the keys of the exchange */ static void cert_cb ( void *cls, const struct TALER_EXCHANGE_KeysResponse *kr, struct TALER_EXCHANGE_Keys *keys) { struct Exchange *e = cls; struct GNUNET_TIME_Absolute n; e->conn = NULL; switch (kr->hr.http_status) { case MHD_HTTP_OK: TALER_EXCHANGE_keys_decref (e->keys); e->keys = NULL; if (0 != strcmp (e->currency, keys->currency)) { GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "/keys response from `%s' is for currency `%s', but we expected `%s'. Ignoring response.\n", e->exchange_url, keys->currency, e->currency); break; } if (0 != GNUNET_memcmp (&keys->master_pub, &e->master_pub)) { GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Master public key in %skeys response does not match. Ignoring response.\n", e->exchange_url); break; } if (! store_keys (keys)) break; e->keys = TALER_EXCHANGE_keys_incref (keys); /* Reset back-off */ e->retry_delay = EXCHANGE_MAXFREQ; /* limit retry */ e->first_retry = GNUNET_TIME_relative_to_absolute ( EXCHANGE_MAXFREQ); /* Limit by expiration */ n = GNUNET_TIME_absolute_max (e->first_retry, keys->key_data_expiration.abs_time); if (NULL != e->retry_task) GNUNET_SCHEDULER_cancel (e->retry_task); e->retry_task = GNUNET_SCHEDULER_add_at (n, &download_keys, e); end_inquiry (); return; default: break; } /* Try again (soon-ish) */ e->retry_delay = GNUNET_TIME_STD_BACKOFF (e->retry_delay); n = GNUNET_TIME_absolute_max ( e->first_retry, GNUNET_TIME_relative_to_absolute (e->retry_delay)); GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Will download %skeys in %s\n", e->exchange_url, GNUNET_TIME_relative2s ( GNUNET_TIME_absolute_get_remaining (n), true)); if (NULL != e->retry_task) GNUNET_SCHEDULER_cancel (e->retry_task); e->retry_task = GNUNET_SCHEDULER_add_at (n, &download_keys, e); end_inquiry (); } static void download_keys (void *cls) { struct Exchange *e = cls; e->retry_task = NULL; GNUNET_break (OPEN_INQUIRY_LIMIT >= active_inquiries); if (OPEN_INQUIRY_LIMIT <= active_inquiries) { GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Cannot run job: at limit\n"); e->limited = true; at_limit = true; return; } e->retry_delay = GNUNET_TIME_STD_BACKOFF (e->retry_delay); e->conn = TALER_EXCHANGE_get_keys (ctx, e->exchange_url, e->keys, &cert_cb, e); if (NULL != e->conn) { active_inquiries++; } else { struct GNUNET_TIME_Relative n; n = GNUNET_TIME_relative_max (e->retry_delay, EXCHANGE_MAXFREQ); e->retry_task = GNUNET_SCHEDULER_add_delayed (n, &download_keys, e); } } /** * Lookup exchange by @a exchange_url. Create one * if it does not exist. * * @param exchange_url base URL to match against * @return NULL if not found */ static struct Exchange * lookup_exchange (const char *exchange_url) { for (struct Exchange *e = e_head; NULL != e; e = e->next) if (0 == strcmp (e->exchange_url, exchange_url)) return e; GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Got notification about unknown exchange `%s'\n", exchange_url); return NULL; } /** * Force immediate (re)loading of /keys for an exchange. * * @param cls NULL * @param extra base URL of the exchange that changed * @param extra_len number of bytes in @a extra */ static void force_exchange_keys (void *cls, const void *extra, size_t extra_len) { const char *url = extra; struct Exchange *e; if ( (NULL == extra) || (0 == extra_len) ) { GNUNET_break (0); return; } if ('\0' != url[extra_len - 1]) { GNUNET_break (0); return; } GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Received keys change notification: reload `%s'\n", url); e = lookup_exchange (url); if (NULL == e) { GNUNET_break (0); return; } if (NULL != e->conn) { GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Already downloading %skeys\n", url); return; } GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Will download %skeys in %s\n", url, GNUNET_TIME_relative2s ( GNUNET_TIME_absolute_get_remaining ( e->first_retry), true)); if (NULL != e->retry_task) GNUNET_SCHEDULER_cancel (e->retry_task); e->retry_task = GNUNET_SCHEDULER_add_at (e->first_retry, &download_keys, e); } /** * Function called on each configuration section. Finds sections * about exchanges, parses the entries. * * @param cls NULL * @param section name of the section */ static void accept_exchanges (void *cls, const char *section) { char *url; char *mks; char *currency; (void) cls; if (0 != strncasecmp (section, "merchant-exchange-", strlen ("merchant-exchange-"))) return; if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, section, "DISABLED")) return; if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg, section, "EXCHANGE_BASE_URL", &url)) { GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, section, "EXCHANGE_BASE_URL"); global_ret = EXIT_NOTCONFIGURED; GNUNET_SCHEDULER_shutdown (); return; } for (struct Exchange *e = e_head; NULL != e; e = e->next) { if (0 == strcmp (url, e->exchange_url)) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Exchange `%s' configured in multiple sections, maybe set DISABLED=YES in section `%s'?\n", url, section); GNUNET_free (url); global_ret = EXIT_NOTCONFIGURED; GNUNET_SCHEDULER_shutdown (); return; } } if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg, section, "CURRENCY", ¤cy)) { GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, section, "CURRENCY"); GNUNET_free (url); global_ret = EXIT_NOTCONFIGURED; GNUNET_SCHEDULER_shutdown (); return; } if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg, section, "MASTER_KEY", &mks)) { GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, section, "MASTER_KEY"); global_ret = EXIT_NOTCONFIGURED; GNUNET_SCHEDULER_shutdown (); GNUNET_free (currency); GNUNET_free (url); return; } { struct Exchange *e; e = GNUNET_new (struct Exchange); e->exchange_url = url; e->currency = currency; GNUNET_CONTAINER_DLL_insert (e_head, e_tail, e); if (GNUNET_OK != GNUNET_CRYPTO_eddsa_public_key_from_string ( mks, strlen (mks), &e->master_pub.eddsa_pub)) { GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR, section, "MASTER_KEY", "malformed EdDSA key"); global_ret = EXIT_NOTCONFIGURED; GNUNET_SCHEDULER_shutdown (); GNUNET_free (mks); return; } GNUNET_free (mks); { enum GNUNET_DB_QueryStatus qs; struct TALER_EXCHANGE_Keys *keys = NULL; qs = db_plugin->select_exchange_keys (db_plugin->cls, url, &keys); if (qs < 0) { GNUNET_break (0); global_ret = EXIT_FAILURE; GNUNET_SCHEDULER_shutdown (); return; } if ( (NULL != keys) && (0 != strcmp (keys->currency, e->currency)) ) { GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "/keys cached in our database were for currency `%s', but we expected `%s'. Fetching /keys again.\n", keys->currency, e->currency); TALER_EXCHANGE_keys_decref (keys); keys = NULL; } if ( (NULL != keys) && (0 != GNUNET_memcmp (&e->master_pub, &keys->master_pub)) ) { /* master pub differs => fetch keys again */ GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Master public key of exchange `%s' differs from our configuration. Fetching /keys again.\n", e->exchange_url); TALER_EXCHANGE_keys_decref (keys); keys = NULL; } e->keys = keys; if (NULL == keys) { /* done synchronously so that the active_inquiries is updated immediately */ download_keys (e); } else { e->retry_task = GNUNET_SCHEDULER_add_at (keys->key_data_expiration.abs_time, &download_keys, e); } } GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Exchange `%s' setup\n", e->exchange_url); } } /** * We're being aborted with CTRL-C (or SIGTERM). Shut down. * * @param cls closure (NULL) */ static void shutdown_task (void *cls) { (void) cls; GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Running shutdown\n"); while (NULL != e_head) { struct Exchange *e = e_head; GNUNET_free (e->exchange_url); GNUNET_free (e->currency); if (NULL != e->conn) { TALER_EXCHANGE_get_keys_cancel (e->conn); e->conn = NULL; } if (NULL != e->keys) { TALER_EXCHANGE_keys_decref (e->keys); e->keys = NULL; } if (NULL != e->retry_task) { GNUNET_SCHEDULER_cancel (e->retry_task); e->retry_task = NULL; } GNUNET_CONTAINER_DLL_remove (e_head, e_tail, e); GNUNET_free (e); } if (NULL != eh) { db_plugin->event_listen_cancel (eh); eh = NULL; } TALER_MERCHANTDB_plugin_unload (db_plugin); db_plugin = NULL; cfg = NULL; if (NULL != ctx) { GNUNET_CURL_fini (ctx); ctx = NULL; } if (NULL != rc) { GNUNET_CURL_gnunet_rc_destroy (rc); rc = NULL; } } /** * First task. * * @param cls closure, NULL * @param args remaining command-line arguments * @param cfgfile name of the configuration file used (for saving, can be NULL!) * @param c configuration */ static void run (void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *c) { (void) args; (void) cfgfile; cfg = c; GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL); ctx = GNUNET_CURL_init (&GNUNET_CURL_gnunet_scheduler_reschedule, &rc); rc = GNUNET_CURL_gnunet_rc_create (ctx); if (NULL == ctx) { GNUNET_break (0); GNUNET_SCHEDULER_shutdown (); global_ret = EXIT_FAILURE; return; } if (NULL == (db_plugin = TALER_MERCHANTDB_plugin_load (cfg))) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Failed to initialize DB subsystem\n"); GNUNET_SCHEDULER_shutdown (); global_ret = EXIT_NOTCONFIGURED; return; } if (GNUNET_OK != db_plugin->connect (db_plugin->cls)) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Failed to connect to database. Consider running taler-merchant-dbinit!\n"); GNUNET_SCHEDULER_shutdown (); global_ret = EXIT_FAILURE; return; } { struct GNUNET_DB_EventHeaderP es = { .size = ntohs (sizeof (es)), .type = ntohs (TALER_DBEVENT_MERCHANT_EXCHANGE_FORCE_KEYS) }; eh = db_plugin->event_listen (db_plugin->cls, &es, GNUNET_TIME_UNIT_FOREVER_REL, &force_exchange_keys, NULL); } GNUNET_CONFIGURATION_iterate_sections (cfg, &accept_exchanges, NULL); if ( (0 == active_inquiries) && (test_mode) ) { GNUNET_log (GNUNET_ERROR_TYPE_INFO, "No more open inquiries and in test mode. Existing.\n"); GNUNET_SCHEDULER_shutdown (); return; } } /** * The main function of taler-merchant-exchangekeyupdate * * @param argc number of arguments from the command line * @param argv command line arguments * @return 0 ok, 1 on error */ int main (int argc, char *const *argv) { struct GNUNET_GETOPT_CommandLineOption options[] = { GNUNET_GETOPT_option_timetravel ('T', "timetravel"), GNUNET_GETOPT_option_flag ('t', "test", "run in test mode and exit when idle", &test_mode), GNUNET_GETOPT_option_version (VERSION "-" VCS_VERSION), GNUNET_GETOPT_OPTION_END }; enum GNUNET_GenericReturnValue ret; TALER_OS_init (); ret = GNUNET_PROGRAM_run ( argc, argv, "taler-merchant-exchangekeyupdate", gettext_noop ( "background process that ensures our key and configuration data on exchanges is up-to-date"), options, &run, NULL); if (GNUNET_SYSERR == ret) return EXIT_INVALIDARGUMENT; if (GNUNET_NO == ret) return EXIT_SUCCESS; return global_ret; } /* end of taler-merchant-exchangekeyupdate.c */