From b15713f42e9ecbc67e9b8ffd178d97d08b7377bb Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 10 May 2023 22:09:47 +0200 Subject: add helper logic for JSON2JSON conversion --- src/include/taler_util.h | 54 +++++- src/util/Makefile.am | 13 +- src/util/conversion.c | 402 ++++++++++++++++++++++++++++++++++++++++++++ src/util/test_conversion.c | 149 ++++++++++++++++ src/util/test_conversion.sh | 5 + 5 files changed, 621 insertions(+), 2 deletions(-) create mode 100644 src/util/conversion.c create mode 100644 src/util/test_conversion.c create mode 100755 src/util/test_conversion.sh diff --git a/src/include/taler_util.h b/src/include/taler_util.h index 6c2948221..0b15eb8a8 100644 --- a/src/include/taler_util.h +++ b/src/include/taler_util.h @@ -1,6 +1,6 @@ /* This file is part of TALER - Copyright (C) 2014-2021 Taler Systems SA + Copyright (C) 2014-2023 Taler Systems SA TALER is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -569,6 +569,58 @@ enum GNUNET_GenericReturnValue TALER_JSON_parse_age_groups (const json_t *root, struct TALER_AgeMask *mask); + +/** + * Handle to an external process that will assist + * with some JSON-to-JSON conversion. + */ +struct TALER_JSON_ExternalConversion; + +/** + * Type of a callback that receives a JSON @a result. + * + * @param cls closure + * @param status_type how did the process die + * @apram code termination status code from the process + * @param result some JSON result, NULL if we failed to get an JSON output + */ +typedef void +(*TALER_JSON_JsonCallback) (void *cls, + enum GNUNET_OS_ProcessStatusType status_type, + unsigned long code, + const json_t *result); + + +/** + * Launch some external helper @a binary to convert some @a input + * and eventually call @a cb with the result. + * + * @param input JSON to serialize and pass to the helper process + * @param cb function to call on the result + * @param cb_cls closure for @a cb + * @param binary name of the binary to execute + * @param ... NULL-terminated list of arguments for the @a binary, + * usually starting with again the name of the binary + * @return handle to cancel the operation (and kill the helper) + */ +struct TALER_JSON_ExternalConversion * +TALER_JSON_external_conversion_start (const json_t *input, + TALER_JSON_JsonCallback cb, + void *cb_cls, + const char *binary, + ...); + +/** + * Abort external conversion, killing the process and preventing + * the callback from being called. Must not be called after the + * callback was invoked. + * + * @param[in] ec external conversion handle to cancel + */ +void +TALER_JSON_external_conversion_stop ( + struct TALER_JSON_ExternalConversion *ec); + #undef __TALER_UTIL_LIB_H_INSIDE__ #endif diff --git a/src/util/Makefile.am b/src/util/Makefile.am index 115ea5055..9d1ec9d42 100644 --- a/src/util/Makefile.am +++ b/src/util/Makefile.am @@ -20,7 +20,8 @@ EXTRA_DIST = \ taler-config.in \ test_helper_eddsa.conf \ test_helper_rsa.conf \ - test_helper_cs.conf + test_helper_cs.conf \ + test_conversion.sh bin_PROGRAMS = \ taler-exchange-secmod-eddsa \ @@ -80,6 +81,7 @@ libtalerutil_la_SOURCES = \ aml_signatures.c \ auditor_signatures.c \ config.c \ + conversion.c \ crypto.c \ crypto_confirmation.c \ crypto_contract.c \ @@ -125,6 +127,7 @@ AM_TESTS_ENVIRONMENT=export TALER_PREFIX=$${TALER_PREFIX:-@libdir@};export PATH= check_PROGRAMS = \ test_age_restriction \ test_amount \ + test_conversion \ test_crypto \ test_helper_eddsa \ test_helper_rsa \ @@ -141,6 +144,14 @@ test_age_restriction_LDADD = \ -lgnunetutil \ libtalerutil.la +test_conversion_SOURCES = \ + test_conversion.c +test_conversion_LDADD = \ + -lgnunetjson \ + -lgnunetutil \ + -ljansson \ + libtalerutil.la + test_amount_SOURCES = \ test_amount.c test_amount_LDADD = \ diff --git a/src/util/conversion.c b/src/util/conversion.c new file mode 100644 index 000000000..501ca0145 --- /dev/null +++ b/src/util/conversion.c @@ -0,0 +1,402 @@ +/* + This file is part of TALER + Copyright (C) 2023 Taler Systems SA + + TALER is free software; you can redistribute it and/or modify it under the + terms of the GNU General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + TALER; see the file COPYING. If not, see +*/ +/** + * @file conversion.c + * @brief helper routines to run some external JSON-to-JSON converter + * @author Christian Grothoff + */ +#include "platform.h" +#include "taler_util.h" +#include + + +struct TALER_JSON_ExternalConversion +{ + /** + * Callback to call with the result. + */ + TALER_JSON_JsonCallback cb; + + /** + * Closure for @e cb. + */ + void *cb_cls; + + /** + * Handle to the helper process. + */ + struct GNUNET_OS_Process *helper; + + /** + * Pipe for the stdin of the @e helper. + */ + struct GNUNET_DISK_FileHandle *chld_stdin; + + /** + * Pipe for the stdout of the @e helper. + */ + struct GNUNET_DISK_FileHandle *chld_stdout; + + /** + * Handle to wait on the child to terminate. + */ + struct GNUNET_ChildWaitHandle *cwh; + + /** + * Task to read JSON output from the child. + */ + struct GNUNET_SCHEDULER_Task *read_task; + + /** + * Task to send JSON input to the child. + */ + struct GNUNET_SCHEDULER_Task *write_task; + + /** + * Buffer with data we need to send to the helper. + */ + void *write_buf; + + /** + * Buffer for reading data from the helper. + */ + void *read_buf; + + /** + * Total length of @e write_buf. + */ + size_t write_size; + + /** + * Current write position in @e write_buf. + */ + size_t write_pos; + + /** + * Current size of @a read_buf. + */ + size_t read_size; + + /** + * Current offset in @a read_buf. + */ + size_t read_pos; + +}; + + +/** + * Function called when we can read more data from + * the child process. + * + * @param cls our `struct TALER_JSON_ExternalConversion *` + */ +static void +read_cb (void *cls) +{ + struct TALER_JSON_ExternalConversion *ec = cls; + + ec->read_task = NULL; + while (1) + { + ssize_t ret; + + if (ec->read_size == ec->read_pos) + { + /* Grow input buffer */ + size_t ns; + void *tmp; + + ns = GNUNET_MAX (2 * ec->read_size, + 1024); + if (ns > GNUNET_MAX_MALLOC_CHECKED) + ns = GNUNET_MAX_MALLOC_CHECKED; + if (ec->read_size == ns) + { + /* Helper returned more than 40 MB of data! Stop reading! */ + GNUNET_break (0); + GNUNET_break (GNUNET_OK == + GNUNET_DISK_file_close (ec->chld_stdin)); + return; + } + tmp = GNUNET_malloc_large (ns); + if (NULL == tmp) + { + /* out of memory, also stop reading */ + GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, + "malloc"); + GNUNET_break (GNUNET_OK == + GNUNET_DISK_file_close (ec->chld_stdin)); + return; + } + GNUNET_memcpy (tmp, + ec->read_buf, + ec->read_pos); + GNUNET_free (ec->read_buf); + ec->read_buf = tmp; + ec->read_size = ns; + } + ret = GNUNET_DISK_file_read (ec->chld_stdout, + ec->read_buf, + ec->read_size - ec->read_pos); + if (ret < 0) + { + if ( (EAGAIN != errno) && + (EWOULDBLOCK != errno) && + (EINTR != errno) ) + { + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, + "read"); + return; + } + break; + } + if (0 == ret) + { + /* regular end of stream, good! */ + return; + } + GNUNET_assert (ec->read_size >= ec->read_pos + ret); + ec->read_pos += ret; + } + ec->read_task + = GNUNET_SCHEDULER_add_read_file ( + GNUNET_TIME_UNIT_FOREVER_REL, + ec->chld_stdout, + &read_cb, + ec); +} + + +/** + * Function called when we can write more data to + * the child process. + * + * @param cls our `struct TALER_JSON_ExternalConversion *` + */ +static void +write_cb (void *cls) +{ + struct TALER_JSON_ExternalConversion *ec = cls; + ssize_t ret; + + ec->write_task = NULL; + while (ec->write_size > ec->write_pos) + { + ret = GNUNET_DISK_file_write (ec->chld_stdin, + ec->write_buf + ec->write_pos, + ec->write_size - ec->write_pos); + if (ret < 0) + { + if ( (EAGAIN != errno) && + (EINTR != errno) ) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, + "write"); + break; + } + if (0 == ret) + { + GNUNET_break (0); + break; + } + GNUNET_assert (ec->write_size >= ec->write_pos + ret); + ec->write_pos += ret; + } + if ( (ec->write_size > ec->write_pos) && + ( (EAGAIN == errno) || + (EWOULDBLOCK == errno) || + (EINTR == errno) ) ) + { + ec->write_task + = GNUNET_SCHEDULER_add_write_file ( + GNUNET_TIME_UNIT_FOREVER_REL, + ec->chld_stdin, + &write_cb, + ec); + } + else + { + GNUNET_break (GNUNET_OK == + GNUNET_DISK_file_close (ec->chld_stdin)); + ec->chld_stdin = NULL; + } +} + + +/** + * Defines a GNUNET_ChildCompletedCallback which is sent back + * upon death or completion of a child process. + * + * @param cls handle for the callback + * @param type type of the process + * @param exit_code status code of the process + * + */ +static void +child_done_cb (void *cls, + enum GNUNET_OS_ProcessStatusType type, + long unsigned int exit_code) +{ + struct TALER_JSON_ExternalConversion *ec = cls; + json_t *j; + json_error_t err; + + ec->cwh = NULL; + if (NULL != ec->read_task) + { + GNUNET_SCHEDULER_cancel (ec->read_task); + /* We could get the process termination notification before having drained + the read buffer. So drain it now, just in case. */ + read_cb (ec); + } + if (NULL != ec->read_task) + { + GNUNET_SCHEDULER_cancel (ec->read_task); + ec->read_task = NULL; + } + GNUNET_OS_process_destroy (ec->helper); + ec->helper = NULL; + j = json_loadb (ec->read_buf, + ec->read_pos, + JSON_REJECT_DUPLICATES, + &err); + if (NULL == j) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Failed to parse JSON from helper at %d: %s\n", + err.position, + err.text); + } + ec->cb (ec->cb_cls, + type, + exit_code, + j); + json_decref (j); + TALER_JSON_external_conversion_stop (ec); +} + + +struct TALER_JSON_ExternalConversion * +TALER_JSON_external_conversion_start (const json_t *input, + TALER_JSON_JsonCallback cb, + void *cb_cls, + const char *binary, + ...) +{ + struct TALER_JSON_ExternalConversion *ec; + struct GNUNET_DISK_PipeHandle *pipe_stdin; + struct GNUNET_DISK_PipeHandle *pipe_stdout; + va_list ap; + + ec = GNUNET_new (struct TALER_JSON_ExternalConversion); + ec->cb = cb; + ec->cb_cls = cb_cls; + pipe_stdin = GNUNET_DISK_pipe (GNUNET_DISK_PF_BLOCKING_READ); + GNUNET_assert (NULL != pipe_stdin); + pipe_stdout = GNUNET_DISK_pipe (GNUNET_DISK_PF_BLOCKING_WRITE); + GNUNET_assert (NULL != pipe_stdout); + va_start (ap, + binary); + ec->helper = GNUNET_OS_start_process_va (GNUNET_OS_INHERIT_STD_ERR, + pipe_stdin, + pipe_stdout, + NULL, + binary, + ap); + va_end (ap); + if (NULL == ec->helper) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Failed to run conversion helper `%s'\n", + binary); + GNUNET_break (GNUNET_OK == + GNUNET_DISK_pipe_close (pipe_stdin)); + GNUNET_break (GNUNET_OK == + GNUNET_DISK_pipe_close (pipe_stdout)); + GNUNET_free (ec); + return NULL; + } + ec->chld_stdin = + GNUNET_DISK_pipe_detach_end (pipe_stdin, + GNUNET_DISK_PIPE_END_WRITE); + ec->chld_stdout = + GNUNET_DISK_pipe_detach_end (pipe_stdout, + GNUNET_DISK_PIPE_END_READ); + GNUNET_break (GNUNET_OK == + GNUNET_DISK_pipe_close (pipe_stdin)); + GNUNET_break (GNUNET_OK == + GNUNET_DISK_pipe_close (pipe_stdout)); + ec->write_buf = json_dumps (input, JSON_COMPACT); + ec->write_size = strlen (ec->write_buf); + ec->read_task + = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL, + ec->chld_stdout, + &read_cb, + ec); + ec->write_task + = GNUNET_SCHEDULER_add_write_file (GNUNET_TIME_UNIT_FOREVER_REL, + ec->chld_stdin, + &write_cb, + ec); + ec->cwh = GNUNET_wait_child (ec->helper, + &child_done_cb, + ec); + return ec; +} + + +void +TALER_JSON_external_conversion_stop ( + struct TALER_JSON_ExternalConversion *ec) +{ + if (NULL != ec->cwh) + { + GNUNET_wait_child_cancel (ec->cwh); + ec->cwh = NULL; + } + if (NULL != ec->helper) + { + GNUNET_break (0 == + GNUNET_OS_process_kill (ec->helper, + SIGKILL)); + GNUNET_OS_process_destroy (ec->helper); + } + if (NULL != ec->read_task) + { + GNUNET_SCHEDULER_cancel (ec->read_task); + ec->read_task = NULL; + } + if (NULL != ec->write_task) + { + GNUNET_SCHEDULER_cancel (ec->write_task); + ec->write_task = NULL; + } + if (NULL != ec->chld_stdin) + { + GNUNET_break (GNUNET_OK == + GNUNET_DISK_file_close (ec->chld_stdin)); + ec->chld_stdin = NULL; + } + if (NULL != ec->chld_stdout) + { + GNUNET_break (GNUNET_OK == + GNUNET_DISK_file_close (ec->chld_stdout)); + ec->chld_stdout = NULL; + } + GNUNET_free (ec->read_buf); + free (ec->write_buf); + GNUNET_free (ec); +} diff --git a/src/util/test_conversion.c b/src/util/test_conversion.c new file mode 100644 index 000000000..00cb35e72 --- /dev/null +++ b/src/util/test_conversion.c @@ -0,0 +1,149 @@ +/* + This file is part of TALER + (C) 2023 Taler Systems SA + + TALER is free software; you can redistribute it and/or modify it under the + terms of the GNU General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + TALER; see the file COPYING. If not, see +*/ +/** + * @file util/test_conversion.c + * @brief Tests for conversion logic + * @author Christian Grothoff + */ +#include "platform.h" +#include "taler_util.h" +#include + +/** + * Return value from main(). + */ +static int global_ret; + +/** + * Handle to our helper. + */ +static struct TALER_JSON_ExternalConversion *ec; + + +/** + * Type of a callback that receives a JSON @a result. + * + * @param cls closure + * @param status_type how did the process die + * @apram code termination status code from the process + * @param result some JSON result, NULL if we failed to get an JSON output + */ +static void +conv_cb (void *cls, + enum GNUNET_OS_ProcessStatusType status_type, + unsigned long code, + const json_t *result) +{ + json_t *expect; + + (void) cls; + (void) status_type; + ec = NULL; + global_ret = 3; + if (42 != code) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Unexpected return value from helper: %u\n", + (unsigned int) code); + return; + } + expect = GNUNET_JSON_PACK ( + GNUNET_JSON_pack_string ("foo", + "arg") + ); + if (1 == json_equal (expect, + result)) + { + global_ret = 0; + } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Unexpected JSON result\n"); + json_dumpf (result, + stderr, + JSON_INDENT (2)); + global_ret = 4; + } + json_decref (expect); +} + + +/** + * Function called on shutdown/CTRL-C. + * + * @param cls NULL + */ +static void +do_shutdown (void *cls) +{ + (void) cls; + if (NULL != ec) + { + GNUNET_break (0); + global_ret = 2; + TALER_JSON_external_conversion_stop (ec); + ec = NULL; + } +} + + +/** + * Main test function. + * + * @param cls NULL + */ +static void +run (void *cls) +{ + json_t *input; + + (void) cls; + GNUNET_SCHEDULER_add_shutdown (&do_shutdown, + NULL); + input = GNUNET_JSON_PACK ( + GNUNET_JSON_pack_string ("key", + "foo") + ); + ec = TALER_JSON_external_conversion_start (input, + &conv_cb, + NULL, + "./test_conversion.sh", + "test_conversion.sh", + "arg", + NULL); + json_decref (input); + GNUNET_assert (NULL != ec); +} + + +int +main (int argc, + const char *const argv[]) +{ + (void) argc; + (void) argv; + unsetenv ("XDG_DATA_HOME"); + unsetenv ("XDG_CONFIG_HOME"); + GNUNET_log_setup ("test-conversion", + "WARNING", + NULL); + GNUNET_OS_init (TALER_project_data_default ()); + global_ret = 1; + GNUNET_SCHEDULER_run (&run, + NULL); + return global_ret; +} diff --git a/src/util/test_conversion.sh b/src/util/test_conversion.sh new file mode 100755 index 000000000..26e1a36d8 --- /dev/null +++ b/src/util/test_conversion.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +KEY=$(jq -r .key) +echo -n "{\"$KEY\":\"$1\"}" +exit 42 -- cgit v1.2.3