diff options
| -rw-r--r-- | README | 21 | ||||
| -rw-r--r-- | bitcoinrpc/.gitignore | 2 | ||||
| -rw-r--r-- | bitcoinrpc/__init__.py | 0 | ||||
| -rw-r--r-- | bitcoinrpc/authproxy.py | 140 | ||||
| -rw-r--r-- | jsonrpc/__init__.py | 2 | ||||
| -rw-r--r-- | jsonrpc/authproxy.py | 3 | ||||
| -rw-r--r-- | jsonrpc/json.py | 9 | ||||
| -rw-r--r-- | jsonrpc/proxy.py | 1 | ||||
| -rw-r--r-- | qa/rpc-tests/.gitignore | 2 | ||||
| -rw-r--r-- | qa/rpc-tests/README.md | 38 | ||||
| -rwxr-xr-x | qa/rpc-tests/listtransactions.py | 151 | ||||
| -rwxr-xr-x | qa/rpc-tests/skeleton.py | 79 | ||||
| -rw-r--r-- | qa/rpc-tests/util.py | 136 | ||||
| -rw-r--r-- | setup.py | 15 |
14 files changed, 585 insertions, 14 deletions
diff --git a/README b/README new file mode 100644 index 0000000000..9edd5a2121 --- /dev/null +++ b/README @@ -0,0 +1,21 @@ +AuthServiceProxy is an improved version of python-jsonrpc. + +It includes the following generic improvements: + +- HTTP connections persist for the life of the AuthServiceProxy object +- sends protocol 'version', per JSON-RPC 1.1 +- sends proper, incrementing 'id' +- uses standard Python json lib + +It also includes the following bitcoin-specific details: + +- sends Basic HTTP authentication headers +- parses all JSON numbers that look like floats as Decimal + +Installation: + +- change the first line of setup.py to point to the directory of your installation of python 2.* +- run setup.py + +Note: This will only install bitcoinrpc. If you also want to install jsonrpc to preserve +backwards compatibility, you have to replace 'bitcoinrpc' with 'jsonrpc' in setup.py and run it again. diff --git a/bitcoinrpc/.gitignore b/bitcoinrpc/.gitignore new file mode 100644 index 0000000000..2f78cf5b66 --- /dev/null +++ b/bitcoinrpc/.gitignore @@ -0,0 +1,2 @@ +*.pyc + diff --git a/bitcoinrpc/__init__.py b/bitcoinrpc/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/bitcoinrpc/__init__.py diff --git a/bitcoinrpc/authproxy.py b/bitcoinrpc/authproxy.py new file mode 100644 index 0000000000..2914477170 --- /dev/null +++ b/bitcoinrpc/authproxy.py @@ -0,0 +1,140 @@ + +""" + Copyright 2011 Jeff Garzik + + AuthServiceProxy has the following improvements over python-jsonrpc's + ServiceProxy class: + + - HTTP connections persist for the life of the AuthServiceProxy object + (if server supports HTTP/1.1) + - sends protocol 'version', per JSON-RPC 1.1 + - sends proper, incrementing 'id' + - sends Basic HTTP authentication headers + - parses all JSON numbers that look like floats as Decimal + - uses standard Python json lib + + Previous copyright, from python-jsonrpc/jsonrpc/proxy.py: + + Copyright (c) 2007 Jan-Klaas Kollhof + + This file is part of jsonrpc. + + jsonrpc is free software; you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + This software is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with this software; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +""" + +try: + import http.client as httplib +except ImportError: + import httplib +import base64 +import json +import decimal +try: + import urllib.parse as urlparse +except ImportError: + import urlparse + +USER_AGENT = "AuthServiceProxy/0.1" + +HTTP_TIMEOUT = 30 + + +class JSONRPCException(Exception): + def __init__(self, rpc_error): + Exception.__init__(self) + self.error = rpc_error + + +class AuthServiceProxy(object): + def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connection=None): + self.__service_url = service_url + self.__service_name = service_name + self.__url = urlparse.urlparse(service_url) + if self.__url.port is None: + port = 80 + else: + port = self.__url.port + self.__id_count = 0 + (user, passwd) = (self.__url.username, self.__url.password) + try: + user = user.encode('utf8') + except AttributeError: + pass + try: + passwd = passwd.encode('utf8') + except AttributeError: + pass + authpair = user + b':' + passwd + self.__auth_header = b'Basic ' + base64.b64encode(authpair) + + if connection: + # Callables re-use the connection of the original proxy + self.__conn = connection + elif self.__url.scheme == 'https': + self.__conn = httplib.HTTPSConnection(self.__url.hostname, port, + None, None, False, + timeout) + else: + self.__conn = httplib.HTTPConnection(self.__url.hostname, port, + False, timeout) + + def __getattr__(self, name): + if name.startswith('__') and name.endswith('__'): + # Python internal stuff + raise AttributeError + if self.__service_name is not None: + name = "%s.%s" % (self.__service_name, name) + return AuthServiceProxy(self.__service_url, name, connection=self.__conn) + + def __call__(self, *args): + self.__id_count += 1 + + postdata = json.dumps({'version': '1.1', + 'method': self.__service_name, + 'params': args, + 'id': self.__id_count}) + self.__conn.request('POST', self.__url.path, postdata, + {'Host': self.__url.hostname, + 'User-Agent': USER_AGENT, + 'Authorization': self.__auth_header, + 'Content-type': 'application/json'}) + + response = self._get_response() + if response['error'] is not None: + raise JSONRPCException(response['error']) + elif 'result' not in response: + raise JSONRPCException({ + 'code': -343, 'message': 'missing JSON-RPC result'}) + else: + return response['result'] + + def _batch(self, rpc_call_list): + postdata = json.dumps(list(rpc_call_list)) + self.__conn.request('POST', self.__url.path, postdata, + {'Host': self.__url.hostname, + 'User-Agent': USER_AGENT, + 'Authorization': self.__auth_header, + 'Content-type': 'application/json'}) + + return self._get_response() + + def _get_response(self): + http_response = self.__conn.getresponse() + if http_response is None: + raise JSONRPCException({ + 'code': -342, 'message': 'missing HTTP response from server'}) + + return json.loads(http_response.read().decode('utf8'), + parse_float=decimal.Decimal) diff --git a/jsonrpc/__init__.py b/jsonrpc/__init__.py new file mode 100644 index 0000000000..8441fa3120 --- /dev/null +++ b/jsonrpc/__init__.py @@ -0,0 +1,2 @@ +from .json import loads, dumps, JSONEncodeException, JSONDecodeException +from jsonrpc.proxy import ServiceProxy, JSONRPCException diff --git a/jsonrpc/authproxy.py b/jsonrpc/authproxy.py new file mode 100644 index 0000000000..e90ef361d0 --- /dev/null +++ b/jsonrpc/authproxy.py @@ -0,0 +1,3 @@ +from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException + +__all__ = ['AuthServiceProxy', 'JSONRPCException'] diff --git a/jsonrpc/json.py b/jsonrpc/json.py new file mode 100644 index 0000000000..95398630f7 --- /dev/null +++ b/jsonrpc/json.py @@ -0,0 +1,9 @@ +_json = __import__('json') +loads = _json.loads +dumps = _json.dumps +if hasattr(_json, 'JSONEncodeException'): + JSONEncodeException = _json.JSONEncodeException + JSONDecodeException = _json.JSONDecodeException +else: + JSONEncodeException = TypeError + JSONDecodeException = ValueError diff --git a/jsonrpc/proxy.py b/jsonrpc/proxy.py new file mode 100644 index 0000000000..0d2be1e93b --- /dev/null +++ b/jsonrpc/proxy.py @@ -0,0 +1 @@ +from bitcoinrpc.authproxy import AuthServiceProxy as ServiceProxy, JSONRPCException diff --git a/qa/rpc-tests/.gitignore b/qa/rpc-tests/.gitignore new file mode 100644 index 0000000000..cb41d94423 --- /dev/null +++ b/qa/rpc-tests/.gitignore @@ -0,0 +1,2 @@ +*.pyc +cache diff --git a/qa/rpc-tests/README.md b/qa/rpc-tests/README.md index 15aede6c41..835ff11057 100644 --- a/qa/rpc-tests/README.md +++ b/qa/rpc-tests/README.md @@ -1,26 +1,36 @@ Regression tests of RPC interface ================================= -Bash scripts that use the RPC interface and command-line bitcoin-cli to test -full functionality in -regtest mode. +python-bitcoinrpc: git subtree of https://github.com/jgarzik/python-bitcoinrpc +Changes to python-bitcoinrpc should be made upstream, and then +pulled here using git subtree -wallet.sh : Exercise wallet send/receive code. +skeleton.py : Copy this to create new regression tests. -txnmall.sh : Test proper accounting of malleable transactions +listtransactions.py : Tests for the listtransactions RPC call + +util.py : generally useful functions +Bash-based tests, to be ported to Python: +----------------------------------------- +wallet.sh : Exercise wallet send/receive code. +walletbackup.sh : Exercise wallet backup / dump / import +txnmall.sh : Test proper accounting of malleable transactions conflictedbalance.sh : More testing of malleable transaction handling -util.sh : useful re-usable bash functions +Notes +===== +A 200-block -regtest blockchain and wallets for four nodes +is created the first time a regression test is run and +is stored in the cache/ directory. Each node has 25 mature +blocks (25*50=1250 BTC) in their wallet. -Tips for creating new tests -=========================== +After the first run, the cache/ blockchain and wallets are +copied into a temporary directory and used as the initial +test state. -To cleanup after a failed or interrupted test: +If you get into a bad state, you should be able +to recover with: + rm -rf cache killall bitcoind - rm -rf test.* - -The most difficult part of writing reproducible tests is -keeping multiple nodes in sync. See WaitBlocks, -WaitPeers, and WaitMemPools for how other tests -deal with this. diff --git a/qa/rpc-tests/listtransactions.py b/qa/rpc-tests/listtransactions.py new file mode 100755 index 0000000000..fec3acfbb3 --- /dev/null +++ b/qa/rpc-tests/listtransactions.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python + +# Exercise the listtransactions API + +# Add python-bitcoinrpc to module search path: +import os +import sys +sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "python-bitcoinrpc")) + +import json +import shutil +import subprocess +import tempfile +import traceback + +from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException +from util import * + + +def check_array_result(object_array, to_match, expected): + """ + Pass in array of JSON objects, a dictionary with key/value pairs + to match against, and another dictionary with expected key/value + pairs. + """ + num_matched = 0 + for item in object_array: + all_match = True + for key,value in to_match.items(): + if item[key] != value: + all_match = False + if not all_match: + continue + for key,value in expected.items(): + if item[key] != value: + raise AssertionError("%s : expected %s=%s"%(str(item), str(key), str(value))) + num_matched = num_matched+1 + if num_matched == 0: + raise AssertionError("No objects matched %s"%(str(to_match))) + +def run_test(nodes): + # Simple send, 0 to 1: + txid = nodes[0].sendtoaddress(nodes[1].getnewaddress(), 0.1) + sync_mempools(nodes) + check_array_result(nodes[0].listtransactions(), + {"txid":txid}, + {"category":"send","account":"","amount":Decimal("-0.1"),"confirmations":0}) + check_array_result(nodes[1].listtransactions(), + {"txid":txid}, + {"category":"receive","account":"","amount":Decimal("0.1"),"confirmations":0}) + # mine a block, confirmations should change: + nodes[0].setgenerate(True, 1) + sync_blocks(nodes) + check_array_result(nodes[0].listtransactions(), + {"txid":txid}, + {"category":"send","account":"","amount":Decimal("-0.1"),"confirmations":1}) + check_array_result(nodes[1].listtransactions(), + {"txid":txid}, + {"category":"receive","account":"","amount":Decimal("0.1"),"confirmations":1}) + + # send-to-self: + txid = nodes[0].sendtoaddress(nodes[0].getnewaddress(), 0.2) + check_array_result(nodes[0].listtransactions(), + {"txid":txid, "category":"send"}, + {"amount":Decimal("-0.2")}) + check_array_result(nodes[0].listtransactions(), + {"txid":txid, "category":"receive"}, + {"amount":Decimal("0.2")}) + + # sendmany from node1: twice to self, twice to node2: + send_to = { nodes[0].getnewaddress() : 0.11, nodes[1].getnewaddress() : 0.22, + nodes[0].getaccountaddress("from1") : 0.33, nodes[1].getaccountaddress("toself") : 0.44 } + txid = nodes[1].sendmany("", send_to) + sync_mempools(nodes) + check_array_result(nodes[1].listtransactions(), + {"category":"send","amount":Decimal("-0.11")}, + {"txid":txid} ) + check_array_result(nodes[0].listtransactions(), + {"category":"receive","amount":Decimal("0.11")}, + {"txid":txid} ) + check_array_result(nodes[1].listtransactions(), + {"category":"send","amount":Decimal("-0.22")}, + {"txid":txid} ) + check_array_result(nodes[1].listtransactions(), + {"category":"receive","amount":Decimal("0.22")}, + {"txid":txid} ) + check_array_result(nodes[1].listtransactions(), + {"category":"send","amount":Decimal("-0.33")}, + {"txid":txid} ) + check_array_result(nodes[0].listtransactions(), + {"category":"receive","amount":Decimal("0.33")}, + {"txid":txid, "account" : "from1"} ) + check_array_result(nodes[1].listtransactions(), + {"category":"send","amount":Decimal("-0.44")}, + {"txid":txid, "account" : ""} ) + check_array_result(nodes[1].listtransactions(), + {"category":"receive","amount":Decimal("0.44")}, + {"txid":txid, "account" : "toself"} ) + + +def main(): + import optparse + + parser = optparse.OptionParser(usage="%prog [options]") + parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true", + help="Leave bitcoinds and test.* datadir on exit or error") + parser.add_option("--srcdir", dest="srcdir", default="../../src", + help="Source directory containing bitcoind/bitcoin-cli (default: %default%)") + parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"), + help="Root directory for datadirs") + (options, args) = parser.parse_args() + + os.environ['PATH'] = options.srcdir+":"+os.environ['PATH'] + + check_json_precision() + + success = False + try: + print("Initializing test directory "+options.tmpdir) + if not os.path.isdir(options.tmpdir): + os.makedirs(options.tmpdir) + initialize_chain(options.tmpdir) + + nodes = start_nodes(2, options.tmpdir) + connect_nodes(nodes[1], 0) + sync_blocks(nodes) + run_test(nodes) + + success = True + + except AssertionError as e: + print("Assertion failed: "+e.message) + except Exception as e: + print("Unexpected exception caught during testing: "+str(e)) + stack = traceback.extract_tb(sys.exc_info()[2]) + print(stack[-1]) + + if not options.nocleanup: + print("Cleaning up") + stop_nodes() + shutil.rmtree(options.tmpdir) + + if success: + print("Tests successful") + sys.exit(0) + else: + print("Failed") + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/qa/rpc-tests/skeleton.py b/qa/rpc-tests/skeleton.py new file mode 100755 index 0000000000..0bace6f4eb --- /dev/null +++ b/qa/rpc-tests/skeleton.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python + +# Skeleton for python-based regression tests using +# JSON-RPC + + +# Add python-bitcoinrpc to module search path: +import os +import sys +sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "python-bitcoinrpc")) + +import json +import shutil +import subprocess +import tempfile +import traceback + +from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException +from util import * + + +def run_test(nodes): + # Replace this as appropriate + for node in nodes: + assert_equal(node.getblockcount(), 200) + assert_equal(node.getbalance(), 25*50) + +def main(): + import optparse + + parser = optparse.OptionParser(usage="%prog [options]") + parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true", + help="Leave bitcoinds and test.* datadir on exit or error") + parser.add_option("--srcdir", dest="srcdir", default="../../src", + help="Source directory containing bitcoind/bitcoin-cli (default: %default%)") + parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"), + help="Root directory for datadirs") + (options, args) = parser.parse_args() + + os.environ['PATH'] = options.srcdir+":"+os.environ['PATH'] + + check_json_precision() + + success = False + try: + print("Initializing test directory "+options.tmpdir) + if not os.path.isdir(options.tmpdir): + os.makedirs(options.tmpdir) + initialize_chain(options.tmpdir) + + nodes = start_nodes(2, options.tmpdir) + connect_nodes(nodes[1], 0) + sync_blocks(nodes) + + run_test(nodes) + + success = True + + except AssertionError as e: + print("Assertion failed: "+e.message) + except Exception as e: + print("Unexpected exception caught during testing: "+str(e)) + stack = traceback.extract_tb(sys.exc_info()[2]) + print(stack[-1]) + + if not options.nocleanup: + print("Cleaning up") + stop_nodes() + shutil.rmtree(options.tmpdir) + + if success: + print("Tests successful") + sys.exit(0) + else: + print("Failed") + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/qa/rpc-tests/util.py b/qa/rpc-tests/util.py new file mode 100644 index 0000000000..fbb27ae2df --- /dev/null +++ b/qa/rpc-tests/util.py @@ -0,0 +1,136 @@ +# +# Helpful routines for regression testing +# + +# Add python-bitcoinrpc to module search path: +import os +import sys +sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "python-bitcoinrpc")) + +from decimal import Decimal +import json +import shutil +import subprocess +import time + +from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException +from util import * + +START_P2P_PORT=11000 +START_RPC_PORT=11100 + +def check_json_precision(): + """Make sure json library being used does not lose precision converting BTC values""" + n = Decimal("20000000.00000003") + satoshis = int(json.loads(json.dumps(float(n)))*1.0e8) + if satoshis != 2000000000000003: + raise RuntimeError("JSON encode/decode loses precision") + +def sync_blocks(rpc_connections): + """ + Wait until everybody has the same block count + """ + while True: + counts = [ x.getblockcount() for x in rpc_connections ] + if counts == [ counts[0] ]*len(counts): + break + time.sleep(1) + +def sync_mempools(rpc_connections): + """ + Wait until everybody has the same transactions in their memory + pools + """ + while True: + pool = set(rpc_connections[0].getrawmempool()) + num_match = 1 + for i in range(1, len(rpc_connections)): + if set(rpc_connections[i].getrawmempool()) == pool: + num_match = num_match+1 + if num_match == len(rpc_connections): + break + time.sleep(1) + + +def initialize_chain(test_dir): + """ + Create (or copy from cache) a 200-block-long chain and + 4 wallets. + bitcoind and bitcoin-cli must be in search path. + """ + + if not os.path.isdir(os.path.join("cache", "node0")): + # Create cache directories, run bitcoinds: + bitcoinds = [] + for i in range(4): + datadir = os.path.join("cache", "node"+str(i)) + os.makedirs(datadir) + with open(os.path.join(datadir, "bitcoin.conf"), 'w') as f: + f.write("regtest=1\n"); + f.write("rpcuser=rt\n"); + f.write("rpcpassword=rt\n"); + f.write("port="+str(START_P2P_PORT+i)+"\n"); + f.write("rpcport="+str(START_RPC_PORT+i)+"\n"); + args = [ "bitcoind", "-keypool=1", "-datadir="+datadir ] + if i > 0: + args.append("-connect=127.0.0.1:"+str(START_P2P_PORT)) + bitcoinds.append(subprocess.Popen(args)) + subprocess.check_output([ "bitcoin-cli", "-datadir="+datadir, + "-rpcwait", "getblockcount"]) + + rpcs = [] + for i in range(4): + try: + url = "http://rt:rt@127.0.0.1:%d"%(START_RPC_PORT+i,) + rpcs.append(AuthServiceProxy(url)) + except: + sys.stderr.write("Error connecting to "+url+"\n") + sys.exit(1) + + import pdb; pdb.set_trace() + + # Create a 200-block-long chain; each of the 4 nodes + # gets 25 mature blocks and 25 immature. + for i in range(4): + rpcs[i].setgenerate(True, 25) + sync_blocks(rpcs) + for i in range(4): + rpcs[i].setgenerate(True, 25) + sync_blocks(rpcs) + # Shut them down + for i in range(4): + rpcs[i].stop() + + for i in range(4): + from_dir = os.path.join("cache", "node"+str(i)) + to_dir = os.path.join(test_dir, "node"+str(i)) + shutil.copytree(from_dir, to_dir) + +bitcoind_processes = [] + +def start_nodes(num_nodes, dir): + # Start bitcoinds, and wait for RPC interface to be up and running: + for i in range(num_nodes): + datadir = os.path.join(dir, "node"+str(i)) + args = [ "bitcoind", "-datadir="+datadir ] + bitcoind_processes.append(subprocess.Popen(args)) + subprocess.check_output([ "bitcoin-cli", "-datadir="+datadir, + "-rpcwait", "getblockcount"]) + # Create&return JSON-RPC connections + rpc_connections = [] + for i in range(num_nodes): + url = "http://rt:rt@127.0.0.1:%d"%(START_RPC_PORT+i,) + rpc_connections.append(AuthServiceProxy(url)) + return rpc_connections + +def stop_nodes(): + for process in bitcoind_processes: + process.kill() + +def connect_nodes(from_connection, node_num): + ip_port = "127.0.0.1:"+str(START_P2P_PORT+node_num) + from_connection.addnode(ip_port, "onetry") + +def assert_equal(thing1, thing2): + if thing1 != thing2: + raise AssertionError("%s != %s"%(str(thing1),str(thing2))) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000..b5a217bf93 --- /dev/null +++ b/setup.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +from distutils.core import setup + +setup(name='python-bitcoinrpc', + version='0.1', + description='Enhanced version of python-jsonrpc for use with Bitcoin', + long_description=open('README').read(), + author='Jeff Garzik', + author_email='<jgarzik@exmulti.com>', + maintainer='Jeff Garzik', + maintainer_email='<jgarzik@exmulti.com>', + url='http://www.github.com/jgarzik/python-bitcoinrpc', + packages=['bitcoinrpc'], + classifiers=['License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)', 'Operating System :: OS Independent']) |