From 6d7645a99b523cb36d692de3e8d3bc80ddd8f21f Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 13 Sep 2017 01:14:31 +0200 Subject: theoretically finished implementation of #4840 (/keys cherry picking), but by far not done testing --- src/exchange-lib/exchange_api_handle.c | 175 +++++++++++++--------- src/exchange/taler-exchange-httpd_keystate.c | 216 +++++++++++++++++++++------ src/include/taler_exchange_service.h | 25 ++-- 3 files changed, 295 insertions(+), 121 deletions(-) (limited to 'src') diff --git a/src/exchange-lib/exchange_api_handle.c b/src/exchange-lib/exchange_api_handle.c index c028b9a9c..99255e449 100644 --- a/src/exchange-lib/exchange_api_handle.c +++ b/src/exchange-lib/exchange_api_handle.c @@ -505,6 +505,7 @@ decode_keys_json (const json_t *resp_obj, enum TALER_EXCHANGE_VersionCompatibility *vc) { struct GNUNET_TIME_Absolute list_issue_date; + struct GNUNET_TIME_Absolute last_denom_issue_date; struct TALER_ExchangeSignatureP sig; struct TALER_ExchangeKeySetPS ks; struct GNUNET_HashContext *hash_context; @@ -513,9 +514,6 @@ decode_keys_json (const json_t *resp_obj, unsigned int revision; unsigned int current; - memset (key_data, - 0, - sizeof (struct TALER_EXCHANGE_Keys)); if (JSON_OBJECT != json_typeof (resp_obj)) return GNUNET_SYSERR; @@ -588,41 +586,23 @@ decode_keys_json (const json_t *resp_obj, json_object_get (resp_obj, "signkeys"))); EXITIF (JSON_ARRAY != json_typeof (sign_keys_array)); - + EXITIF (0 == (key_data->num_sign_keys = + json_array_size (sign_keys_array))); + key_data->sign_keys + = GNUNET_new_array (key_data->num_sign_keys, + struct TALER_EXCHANGE_SigningPublicKey); index = 0; json_array_foreach (sign_keys_array, index, sign_key_obj) { - struct TALER_EXCHANGE_SigningPublicKey sk; - bool found = false; - EXITIF (GNUNET_SYSERR == - parse_json_signkey (&sk, + parse_json_signkey (&key_data->sign_keys[index], sign_key_obj, &key_data->master_pub)); - for (unsigned int j=0;jnum_sign_keys;j++) - { - if (0 == memcmp (&sk, - &key_data->sign_keys[j], - sizeof (sk))) - { - found = true; - break; - } - } - if (found) - { - /* 0:0:0 did not support /keys cherry picking */ - GNUNET_break_op (0 == current); - continue; - } - if (key_data->sign_keys_size == key_data->num_sign_keys) - GNUNET_array_grow (key_data->sign_keys, - key_data->sign_keys_size, - key_data->sign_keys_size * 2 + 2); - key_data->sign_keys[key_data->num_sign_keys++] = sk; } } - /* parse the denomination keys */ + /* parse the denomination keys, merging with the + possibly EXISTING array as required (/keys cherry picking) */ + last_denom_issue_date.abs_value_us = 0LLU; { json_t *denom_keys_array; json_t *denom_key_obj; @@ -632,14 +612,11 @@ decode_keys_json (const json_t *resp_obj, json_object_get (resp_obj, "denoms"))); EXITIF (JSON_ARRAY != json_typeof (denom_keys_array)); - key_data->denom_keys = GNUNET_new_array (key_data->num_denom_keys, - struct TALER_EXCHANGE_DenomPublicKey); - index = 0; json_array_foreach (denom_keys_array, index, denom_key_obj) { struct TALER_EXCHANGE_DenomPublicKey dk; bool found = false; - + EXITIF (GNUNET_SYSERR == parse_json_denomkey (&dk, denom_key_obj, @@ -658,44 +635,72 @@ decode_keys_json (const json_t *resp_obj, if (found) { /* 0:0:0 did not support /keys cherry picking */ - GNUNET_break_op (0 == current); + GNUNET_break_op (0 == current); continue; } if (key_data->denom_keys_size == key_data->num_denom_keys) GNUNET_array_grow (key_data->denom_keys, key_data->denom_keys_size, key_data->denom_keys_size * 2 + 2); - key_data->denom_keys[key_data->num_denom_keys++] = dk; - } + key_data->denom_keys[key_data->num_denom_keys++] = dk; + + /* Update "last_denom_issue_date" */ + last_denom_issue_date + = GNUNET_TIME_absolute_max (last_denom_issue_date, + dk.valid_from); + }; } + key_data->last_denom_issue_date = last_denom_issue_date; /* parse the auditor information */ { json_t *auditors_array; json_t *auditor_info; - unsigned int len; unsigned int index; EXITIF (NULL == (auditors_array = json_object_get (resp_obj, "auditors"))); EXITIF (JSON_ARRAY != json_typeof (auditors_array)); - len = json_array_size (auditors_array); - if (0 != len) - { - key_data->auditors = GNUNET_new_array (len, - struct TALER_EXCHANGE_AuditorInformation); - index = 0; - json_array_foreach (auditors_array, index, auditor_info) { - EXITIF (GNUNET_SYSERR == - parse_json_auditor (&key_data->auditors[index], - auditor_info, - key_data)); + + /* Merge with the existing auditor information we have (/keys cherry picking) */ + index = 0; + json_array_foreach (auditors_array, index, auditor_info) { + struct TALER_EXCHANGE_AuditorInformation ai; + bool found = false; + + EXITIF (GNUNET_SYSERR == + parse_json_auditor (&ai, + auditor_info, + key_data)); + for (unsigned int j=0;jnum_auditors;j++) + { + struct TALER_EXCHANGE_AuditorInformation *aix = &key_data->auditors[j]; + if (0 == memcmp (&ai.auditor_pub, + &aix->auditor_pub, + sizeof (struct TALER_AuditorPublicKeyP))) + { + found = true; + /* Merge denomination key signatures of downloaded /keys into existing + auditor information 'aix'. */ + GNUNET_array_grow (aix->denom_keys, + aix->num_denom_keys, + aix->num_denom_keys + ai.num_denom_keys); + memcpy (&aix->denom_keys[aix->num_denom_keys - ai.num_denom_keys], + ai.denom_keys, + ai.num_denom_keys * sizeof (struct TALER_EXCHANGE_DenomPublicKey *)); + break; + } } - key_data->num_auditors = len; - } + if (found) + continue; /* we are done */ + if (key_data->auditors_size == key_data->num_auditors) + GNUNET_array_grow (key_data->auditors, + key_data->auditors_size, + key_data->auditors_size * 2 + 2); + key_data->auditors[key_data->num_auditors++] = ai; + }; } - key_data->last_issue_date = list_issue_date; - + /* Validate signature... */ ks.purpose.size = htonl (sizeof (ks)); ks.purpose.purpose = htonl (TALER_SIGNATURE_EXCHANGE_KEY_SET); @@ -729,7 +734,7 @@ static void free_key_data (struct TALER_EXCHANGE_Keys *key_data) { GNUNET_array_grow (key_data->sign_keys, - key_data->sign_keys_size, + key_data->num_sign_keys, 0); for (unsigned int i=0;inum_denom_keys;i++) GNUNET_CRYPTO_rsa_public_key_free (key_data->denom_keys[i].key.rsa_public_key); @@ -744,7 +749,7 @@ free_key_data (struct TALER_EXCHANGE_Keys *key_data) GNUNET_free (key_data->auditors[i].auditor_url); } GNUNET_array_grow (key_data->auditors, - key_data->num_auditors, + key_data->auditors_size, 0); GNUNET_free_non_null (key_data->version); key_data->version = NULL; @@ -814,6 +819,47 @@ keys_completed_cb (void *cls, response_code = 0; break; } + memset (&kd, + 0, + sizeof (struct TALER_EXCHANGE_Keys)); + + /* We keep the denomination keys and auditor signatures from the + previous iteration (/keys cherry picking) */ + kd.num_denom_keys = kd_old.num_denom_keys; + kd.denom_keys = GNUNET_new_array (kd.num_denom_keys, + struct TALER_EXCHANGE_DenomPublicKey); + /* First make a shallow copy, we then need another pass for the RSA key... */ + memcpy (kd.denom_keys, + kd_old.denom_keys, + kd_old.num_denom_keys * sizeof (struct TALER_EXCHANGE_DenomPublicKey)); + for (unsigned int i=0;iauditor_pub = aold->auditor_pub; + anew->auditor_url = GNUNET_strdup (aold->auditor_url); + GNUNET_array_grow (anew->denom_keys, + anew->num_denom_keys, + aold->num_denom_keys); + for (unsigned int j=0;jnum_denom_keys;j++) + { + /* offsets will map 1:1 */ + unsigned int off = kd_old.denom_keys - aold->denom_keys[j]; + + GNUNET_assert (off < kd_old.num_denom_keys); + anew->denom_keys[j] = &kd.denom_keys[off]; + } + } + if (GNUNET_OK != decode_keys_json (resp_obj, &kd, @@ -953,7 +999,6 @@ parse_date_string (const char *date, time_t t; char day[4]; char mon[4]; - unsigned int i; unsigned int mday; unsigned int year; unsigned int h; @@ -977,11 +1022,11 @@ parse_date_string (const char *date, now.tm_min = m; now.tm_sec = s; now.tm_wday = 7; - for (i=0;i<7;i++) + for (unsigned int i=0;i<7;i++) if (0 == strcasecmp (days[i], day)) now.tm_wday = i; now.tm_mon = 12; - for (i=0;i<12;i++) + for (unsigned int i=0;i<12;i++) if (0 == strcasecmp (mons[i], mon)) now.tm_mon = i; if ( (7 == now.tm_wday) || @@ -1091,7 +1136,7 @@ request_keys (struct TALER_EXCHANGE_Handle *exchange) GNUNET_asprintf (&arg, "/keys?last_issue_date=%llu", - (unsigned long long) exchange->key_data.last_issue_date.abs_value_us); + (unsigned long long) exchange->key_data.last_denom_issue_date.abs_value_us); kr->url = MAH_path_to_url (exchange, arg); GNUNET_free (arg); @@ -1172,11 +1217,10 @@ TALER_EXCHANGE_test_signing_key (const struct TALER_EXCHANGE_Keys *keys, const struct TALER_ExchangePublicKeyP *pub) { struct GNUNET_TIME_Absolute now; - unsigned int i; /* we will check using a tolerance of 1h for the time */ now = GNUNET_TIME_absolute_get (); - for (i=0;inum_sign_keys;i++) + for (unsigned int i=0;inum_sign_keys;i++) if ( (keys->sign_keys[i].valid_from.abs_value_us <= now.abs_value_us + 60 * 60 * 1000LL * 1000LL) && (keys->sign_keys[i].valid_until.abs_value_us > now.abs_value_us - 60 * 60 * 1000LL * 1000LL) && (0 == memcmp (pub, @@ -1199,9 +1243,7 @@ const struct TALER_EXCHANGE_DenomPublicKey * TALER_EXCHANGE_get_denomination_key (const struct TALER_EXCHANGE_Keys *keys, const struct TALER_DenominationPublicKey *pk) { - unsigned int i; - - for (i=0;inum_denom_keys;i++) + for (unsigned int i=0;inum_denom_keys;i++) if (0 == GNUNET_CRYPTO_rsa_public_key_cmp (pk->rsa_public_key, keys->denom_keys[i].key.rsa_public_key)) return &keys->denom_keys[i]; @@ -1220,9 +1262,7 @@ const struct TALER_EXCHANGE_DenomPublicKey * TALER_EXCHANGE_get_denomination_key_by_hash (const struct TALER_EXCHANGE_Keys *keys, const struct GNUNET_HashCode *hc) { - unsigned int i; - - for (i=0;inum_denom_keys;i++) + for (unsigned int i=0;inum_denom_keys;i++) if (0 == memcmp (hc, &keys->denom_keys[i].h_key, sizeof (struct GNUNET_HashCode))) @@ -1260,5 +1300,4 @@ TALER_EXCHANGE_get_keys_raw (struct TALER_EXCHANGE_Handle *exchange) } - /* end of exchange_api_handle.c */ diff --git a/src/exchange/taler-exchange-httpd_keystate.c b/src/exchange/taler-exchange-httpd_keystate.c index ec0a470a0..ddc04bda0 100644 --- a/src/exchange/taler-exchange-httpd_keystate.c +++ b/src/exchange/taler-exchange-httpd_keystate.c @@ -55,28 +55,24 @@ struct TEH_KS_StateHandle { /** - * JSON array with denomination keys. (Currently not really used - * after initialization.) + * JSON array with denomination keys. */ json_t *denom_keys_array; /** - * JSON array with signing keys. (Currently not really used - * after initialization.) + * JSON array with auditor information. */ - json_t *sign_keys_array; + json_t *auditors_array; /** - * JSON array with auditor information. (Currently not really used - * after initialization.) + * JSON array with revoked denomination keys. */ - json_t *auditors_array; + json_t *payback_array; /** - * JSON array with revoked denomination keys. (Currently not really used - * after initialization). + * JSON array with signing keys. */ - json_t *payback_array; + json_t *sign_keys_array; /** * Cached JSON text that the exchange will send for a "/keys" request. @@ -213,6 +209,97 @@ denom_key_issue_to_json (const struct TALER_DenominationPublicKey *pk, } +/** + * Closure for #denom_keys_to_json(). + */ +struct ResponseBuilderContext +{ + /** + * JSON array with denomination keys. + */ + json_t *denom_keys_array; + + /** + * JSON array with auditor signatures. + */ + json_t *auditors_array; + + /** + * Keys after what issue date do we care about? + */ + struct GNUNET_TIME_Absolute last_issue_date; + + /** + * Flag set to #GNUNET_SYSERR on internal errors + */ + int error; +}; + + +/** + * Add denomination keys past the "last_issue_date" to the + * "denom_keys_array". + * + * @param cls a `struct ResponseBuilderContext` + * @param key hash of the denomination key + * @param value a `struct TALER_EXCHANGEDB_DenominationKeyIssueInformation` + * @return #GNUNET_OK (continue to iterate) + */ +static int +denom_keys_to_json (void *cls, + const struct GNUNET_HashCode *key, + void *value) +{ + struct ResponseBuilderContext *rbc = cls; + struct TALER_EXCHANGEDB_DenominationKeyIssueInformation *dki = value; + + if (rbc->last_issue_date.abs_value_us >= + GNUNET_TIME_absolute_ntoh (dki->issue.properties.start).abs_value_us) + { + /* remove matching entry from 'auditors_array' */ + size_t off; + json_t *val; + json_t *kval; + + kval = GNUNET_JSON_from_data_auto (key); + json_array_foreach (rbc->auditors_array, off, val) { + size_t ioff; + json_t *dkv; + json_t *dka = json_object_get (val, + "denomination_keys"); + if (NULL == dka) + { + GNUNET_break (0); + continue; + } + json_array_foreach (dka, ioff, dkv) { + json_t *ival = json_object_get (dkv, + "denom_pub_h"); + + if (NULL == ival) + { + GNUNET_break (0); + continue; + } + if (json_equal (ival, kval)) + { + json_array_remove (dka, + ioff); + break; + } + }; + }; + return GNUNET_OK; /* skip, key known to client */ + } + if (0 != + json_array_append_new (rbc->denom_keys_array, + denom_key_issue_to_json (&dki->denom_pub, + &dki->issue))) + rbc->error = GNUNET_SYSERR; + return GNUNET_OK; +} + + /** * Get the relative time value that describes how * far in the future do we want to provide coin keys. @@ -325,7 +412,7 @@ struct AddRevocationContext /** - * Execute transaction to add revocations. + * Execute transaction to add revocations. * * @param cls closure with the `struct AddRevocationContext *` * @param connection NULL @@ -340,7 +427,7 @@ add_revocations_transaction (void *cls, int *mhd_ret) { struct AddRevocationContext *arc = cls; - + return TEH_plugin->insert_denomination_revocation (TEH_plugin->cls, session, &arc->dki->issue.properties.denom_hash, @@ -349,7 +436,7 @@ add_revocations_transaction (void *cls, /** - * Execute transaction to add a denomination to the DB. + * Execute transaction to add a denomination to the DB. * * @param cls closure with the `const struct TALER_EXCHANGEDB_DenominationKeyIssueInformation *` * @param connection NULL @@ -432,7 +519,7 @@ reload_keys_denom_iter (void *cls, if (NULL != revocation_master_sig) { struct AddRevocationContext arc; - + GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Adding denomination key `%s' to revocation set\n", alias); @@ -454,9 +541,9 @@ reload_keys_denom_iter (void *cls, GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Giving up, this is fatal. Committing suicide via SIGTERM.\n"); handle_signal (SIGTERM); - return GNUNET_SYSERR; + return GNUNET_SYSERR; } - + GNUNET_assert (0 == json_array_append_new (ctx->payback_array, GNUNET_JSON_from_data_auto (&dki->issue.properties.denom_hash))); @@ -478,8 +565,6 @@ reload_keys_denom_iter (void *cls, GNUNET_CRYPTO_hash_context_read (ctx->hash_context, &denom_key_hash, sizeof (struct GNUNET_HashCode)); - - if (GNUNET_OK != TEH_DB_run_transaction (NULL, NULL, @@ -487,9 +572,9 @@ reload_keys_denom_iter (void *cls, (void *) dki)) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Giving up, this is fatal. Committing suicide via SIGTERM.\n"); + "Could not persist denomination key in DB. Committing suicide via SIGTERM.\n"); handle_signal (SIGTERM); - return GNUNET_SYSERR; + return GNUNET_SYSERR; } res = store_in_map (ctx->denomkey_map, @@ -893,10 +978,9 @@ TEH_KS_acquire_ (const char *location) GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "No valid signing key found!\n"); - keys = json_pack ("{s:s, s:o, s:o, s:o, s:o, s:o, s:o, s:o, s:o}", + keys = json_pack ("{s:s, s:o, s:O, s:o, s:O, s:O, s:o, s:o, s:o}", "version", TALER_PROTOCOL_VERSION, - "master_public_key", - GNUNET_JSON_from_data_auto (&TEH_master_public_key), + "master_public_key", GNUNET_JSON_from_data_auto (&TEH_master_public_key), "signkeys", key_state->sign_keys_array, "denoms", key_state->denom_keys_array, "payback", key_state->payback_array, @@ -905,10 +989,7 @@ TEH_KS_acquire_ (const char *location) "eddsa_pub", GNUNET_JSON_from_data_auto (&key_state->current_sign_key_issue.issue.signkey_pub), "eddsa_sig", GNUNET_JSON_from_data_auto (&sig)); GNUNET_assert (NULL != keys); - key_state->auditors_array = NULL; - key_state->sign_keys_array = NULL; key_state->denom_keys_array = NULL; - key_state->payback_array = NULL; key_state->keys_json = json_dumps (keys, JSON_INDENT (2)); GNUNET_assert (NULL != key_state->keys_json); @@ -1282,7 +1363,7 @@ TEH_KS_handler_keys (struct TEH_RequestHandler *rh, if (NULL != have) { unsigned long long haven; - + if (1 != sscanf (have, "%llu", @@ -1299,26 +1380,75 @@ TEH_KS_handler_keys (struct TEH_RequestHandler *rh, { last_issue_date.abs_value_us = 0LLU; } - key_state = TEH_KS_acquire (); - /* FIXME: #4840: compute /keys delta from last_issue_date */ - (void) last_issue_date; - comp = MHD_NO; - if (NULL != key_state->keys_jsonz) - comp = TEH_RESPONSE_can_compress (connection); - if (MHD_YES == comp) + if (0LLU != last_issue_date.abs_value_us) { - json = key_state->keys_jsonz; - json_len = key_state->keys_jsonz_size; + /* Generate incremental response */ + struct ResponseBuilderContext rbc; + + rbc.error = GNUNET_NO; + rbc.denom_keys_array = json_array (); + rbc.auditors_array = json_deep_copy (key_state->auditors_array); + rbc.last_issue_date = last_issue_date; + GNUNET_CONTAINER_multihashmap_iterate (key_state->denomkey_map, + &denom_keys_to_json, + &rbc); + if (GNUNET_NO == rbc.error) + { + json_t *keys; + + keys = json_pack ("{s:s, s:o, s:O, s:o, s:O, s:o, s:o}", + "version", TALER_PROTOCOL_VERSION, + "master_public_key", GNUNET_JSON_from_data_auto (&TEH_master_public_key), + "signkeys", key_state->sign_keys_array, + "denoms", rbc.denom_keys_array, + "payback", key_state->payback_array, + "auditors", rbc.auditors_array, + "list_issue_date", GNUNET_JSON_from_time_abs (key_state->reload_time)); + rbc.denom_keys_array = NULL; + rbc.auditors_array = NULL; + json = json_dumps (keys, + JSON_INDENT (2)); + json_decref (keys); + json_len = strlen (json); + if (TEH_RESPONSE_can_compress (connection)) + comp = TEH_RESPONSE_body_compress ((void **) &json, + &json_len); + response = MHD_create_response_from_buffer (json_len, + json, + MHD_RESPMEM_MUST_FREE); + } + else + { + /* Try to salvage the situation by returning full reply */ + GNUNET_break (0); + last_issue_date.abs_value_us = 0LLU; + } + if (NULL != rbc.denom_keys_array) + json_decref (rbc.denom_keys_array); + if (NULL != rbc.auditors_array) + json_decref (rbc.auditors_array); } - else + if (0LLU == last_issue_date.abs_value_us) { - json = key_state->keys_json; - json_len = strlen (key_state->keys_json); + /* Generate full response */ + comp = MHD_NO; + if (NULL != key_state->keys_jsonz) + comp = TEH_RESPONSE_can_compress (connection); + if (MHD_YES == comp) + { + json = key_state->keys_jsonz; + json_len = key_state->keys_jsonz_size; + } + else + { + json = key_state->keys_json; + json_len = strlen (key_state->keys_json); + } + response = MHD_create_response_from_buffer (json_len, + json, + MHD_RESPMEM_MUST_COPY); } - response = MHD_create_response_from_buffer (json_len, - json, - MHD_RESPMEM_MUST_COPY); TEH_KS_release (key_state); if (NULL == response) { diff --git a/src/include/taler_exchange_service.h b/src/include/taler_exchange_service.h index c4c41c027..d7056b574 100644 --- a/src/include/taler_exchange_service.h +++ b/src/include/taler_exchange_service.h @@ -206,12 +206,17 @@ struct TALER_EXCHANGE_Keys char *version; /** - * Timestamp indicating the /keys generation. Monotonically - * increasing. Used to fetch /keys incrementally. Set from - * the "list_issue_date" timestamp of /keys. + * Timestamp indicating the /keys generation. */ - struct GNUNET_TIME_Absolute last_issue_date; - + struct GNUNET_TIME_Absolute list_issue_date; + + /** + * Timestamp indicating the creation time of the last + * denomination key in /keys. + * Used to fetch /keys incrementally. + */ + struct GNUNET_TIME_Absolute last_denom_issue_date; + /** * Length of the @e sign_keys array (number of valid entries). */ @@ -228,9 +233,9 @@ struct TALER_EXCHANGE_Keys unsigned int num_auditors; /** - * Actual length of the @e sign_keys array (size of allocation). + * Actual length of the @e auditors array (size of allocation). */ - unsigned int sign_keys_size; + unsigned int auditors_size; /** * Actual length of the @e denom_keys array (size of allocation). @@ -240,7 +245,7 @@ struct TALER_EXCHANGE_Keys }; -/** +/** * How compatible are the protocol version of the exchange and this * client? The bits (1,2,4) can be used to test if the exchange's * version is incompatible, older or newer respectively. @@ -280,7 +285,7 @@ enum TALER_EXCHANGE_VersionCompatibility /** * The exchange is too recent for this implementation. */ - TALER_EXCHANGE_VC_INCOMPATIBLE_NEWER + TALER_EXCHANGE_VC_INCOMPATIBLE_NEWER = TALER_EXCHANGE_VC_INCOMPATIBLE | TALER_EXCHANGE_VC_NEWER, @@ -288,7 +293,7 @@ enum TALER_EXCHANGE_VersionCompatibility * We could not even parse the version data. */ TALER_EXCHANGE_VC_PROTOCOL_ERROR = 8 - + }; -- cgit v1.2.3