Merge bitcoin/bitcoin#19888: rpc, test: Improve getblockstats for unspendables

d885bb2f6ea34cd21dacfebe763a07dbb389c1bd test: Test exclusion of OP_RETURN from getblockstats (Fabian Jahr)
ba9d288b2468f047e5a8e4637dd8749e247ff547 test: Fix getblockstats test data generator (Fabian Jahr)
2ca5a496c2f3cbcc63ea15fa05c1658e7f527bbc rpc: Improve getblockstats (Fabian Jahr)
cb94db119f4643f49da63520d64efc99fb0c0795 validation, index: Add unspendable coinbase helper functions (Fabian Jahr)

Pull request description:

  Fixes #19885

  The genesis block does not have undo data saved to disk so the RPC errored because of that.

ACKs for top commit:
  achow101:
    ACK d885bb2f6ea34cd21dacfebe763a07dbb389c1bd
  aureleoules:
    ACK d885bb2f6ea34cd21dacfebe763a07dbb389c1bd
  stickies-v:
    ACK d885bb2f6

Tree-SHA512: f37bda736ed605b7a41a81eeb4bfbb5d2b8518f847819e5d6a090548a61caf1455623e15165d72589ab3f4478252b00e7b624f9313ad6708cac06dd5edb62e9a

2 files changed, 68 insertions, 42 deletions

diff --git a/test/functional/data/rpc_getblockstats.json b/test/functional/data/rpc_getblockstats.json
index 16dbc5fe60..7d7460aacc 100644
--- a/test/functional/data/rpc_getblockstats.json
+++ b/test/functional/data/rpc_getblockstats.json
@@ -102,8 +102,8 @@
     "00000020f44e7a48b9f221af95f3295c8dcefc5358934a68dc79e2933dc0794b350cad0a90fad2cd50b41d4ef45e76c2a456b98c180632bb4b44e0cd18ce90679fe54e552b4ae75affff7f200000000001020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff0401630101ffffffff0200f2052a010000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88ac0000000000000000266a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf90120000000000000000000000000000000000000000000000000000000000000000000000000",
     "0000002087454276cce83f4d19e0120f6e9728ac5905f7adaf6b27e3f5bbe43ab823f85db7d1f44666531483df3d67c15f2c231718ad93b63b851dce5ff4c4a67f524ffa2b4ae75affff7f200100000001020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff0401640101ffffffff0200f2052a010000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88ac0000000000000000266a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf90120000000000000000000000000000000000000000000000000000000000000000000000000",
     "000000202cdc3e99f07a80252dd6097faa0eddf3f2dde5ae390610e0bca94ecc25931551d31fceb8fe0a682f6017ca3dbb582f3a2f06e5d99ec99c42c8a744dd4c9216b82b4ae75affff7f200300000001020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff0401650101ffffffff0200f2052a010000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88ac0000000000000000266a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf90120000000000000000000000000000000000000000000000000000000000000000000000000",
-    "000000209b3ace9bd510918d20e87518c0cf5976cab3e28cc7af41259a89c6dd7668a32922808b8a082be71bcd6152cb8fd223650b5579a41344ba749e4d17b9bf211a9e2b4ae75affff7f200000000002020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff0401660101ffffffff026c03062a010000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88ac0000000000000000266a24aa21a9edb85d8f3c122c43a72f1e0dd122c8f7af040aa0b0a46001621110fb37818021510120000000000000000000000000000000000000000000000000000000000000000000000000020000000128394022bf44bff30d7399cb5a16e3b94fed67dc174c2e1d77df91bad5a51cb3000000006a47304402201c16d06a5c4353168b3881071aea7d1eb4d88eedfea53a9d6af9abb56da9060002205abf3ae535f1f1b5cfe8ba955535c2b20ac003e7d7720c5b7d2640ac2a04d19001210227d85ba011276cf25b51df6a188b75e604b38770a462b2d0e9fb2fc839ef5d3ffeffffff0294b89a3b000000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88ac00286bee0000000017a91452bab4f229415d0dc5c6d30b162f93a1a0cac5958765000000",
-    "000000200fa168b50a79ad24378a6b0f96e4c9f4ccb657a2663320d5fc1efd8ee7caa10ab42a31c444f2153387530a0979d4dc3dcc134b394c821227b8abff930c03c8412b4ae75affff7f200200000004020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff0401670101ffffffff02e015072a010000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88ac0000000000000000266a24aa21a9ed20376d4bc90f9c689850eec3603cda658ba6295241730473ceb0e970b8d594150120000000000000000000000000000000000000000000000000000000000000000000000000020000000191e549a6cc852bbf1d3f11144b1a34079f64305e6971d2e685d2b40cd386e8a6000000006a47304402200bf62021c0a9a47ced8eba1e0998f5c71b2950763198d83ad284bd791241dbb00220446a05b7c35e7458924de88a8dcccab1ec6a106aa005345e55b482d8eb66337301210227d85ba011276cf25b51df6a188b75e604b38770a462b2d0e9fb2fc839ef5d3ffeffffff02acdbf405000000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88ac94d7a4350000000017a914dfa6f0b17d2c64962c94203e744a9d4179ed22c18766000000020000000112d2f07672102dc6f099c4be308f598e4c4da1a7e0cb462ae14f0444525a1332000000006a47304402200a6a2f544f3f9d299608a7c745e2326de176fb1cac03ae3e74943f4250b8896e02205023a5b4faff99865bf91f1263605a502c723628be9240c0b7bec81d2ed106f101210227d85ba011276cf25b51df6a188b75e604b38770a462b2d0e9fb2fc839ef5d3ffeffffff0200ca9a3b000000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88ac94166bee0000000017a914152cc82f7944f5c416de7dbffb052f7081765d7987660000000200000000010191e549a6cc852bbf1d3f11144b1a34079f64305e6971d2e685d2b40cd386e8a601000000171600147cc872ad7350c37fecab9c4c6d9f08aceb53bdb8feffffff02005ed0b20000000017a914aab1c8c53fe62e283a53efa28097709f4f2ed37b87e0bc9a3b000000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88ac0247304402201b4476f238ed5d515bfcd6927d0d008a4993770763eca73e3ee66f69971831d902200f5215a6dfd90391dd63462cfdf69804fe31224c309ec9c38d33a04dce71c0ee0121028c9d2955a95301b699db62e97d54bf0a91feb44e5cd94bbf5b62f1df57fb643966000000"
+    "000000209b3ace9bd510918d20e87518c0cf5976cab3e28cc7af41259a89c6dd7668a329f03ef4716ad5d88bccfd71088bf2ec3eb5b32e0ff60f35f9becd73052bfa8af12b4ae75affff7f200000000002020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff0401660101ffffffff025803062a010000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88ac0000000000000000266a24aa21a9edb11910e4e0ee6ee6d6bad42736999d1eba649243dc781438e5ef845c7227aaad0120000000000000000000000000000000000000000000000000000000000000000000000000020000000128394022bf44bff30d7399cb5a16e3b94fed67dc174c2e1d77df91bad5a51cb3000000006a47304402200650c6c50bd4952ca13b7aa37d458a36628434bbb968701063cdd36d0725e36c02202e059ccf7a4a049de028c4f140e543baa7e69ea3663e3d1fdfbc8ba7247e82f901210227d85ba011276cf25b51df6a188b75e604b38770a462b2d0e9fb2fc839ef5d3ffeffffff02a8b89a3b000000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88ac00286bee0000000016001423838e991caedd69289d9dac88ca423cca683a2265000000",
+    "0000002061597bf3305ee2334b5d7fec136c2064b69d14955b50cd0b81f49ac33d88e506d80731ce60c2f275d29539a4d04c7e8c72aa4ade3c9baec24881a581fc524c6a2b4ae75affff7f200000000005020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff0401670101ffffffff023840072a010000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88ac0000000000000000266a24aa21a9ed89a3c65f40921b14168368cf4775a710ad17b613cedcff001f1f1ca3c863cc11012000000000000000000000000000000000000000000000000000000000000000000000000002000000000101bb475ac72ba1a45a96be584a689d6d0ace820d9587b0db53dc40a15555b9be770100000000feffffff02c0be9a3b000000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88ac005ed0b200000000160014a57f6871c70569e0543322614c5c85438b796a900247304402207893b5466cdbf9cdedcc73fb8e10e01c4dd9aac4345031ef0c35d91e82ff5fd602207f0e1d7e4270a532060fd1e6ad7881d18acb173fd4fd1d61f3c8ff6707bdb972012103bb4c79ca594b19bbec7ee6302af0ef4191345fa7f03a30ed4e042aeed680924b6600000002000000000101e6a996cbac10b2d6b2bb2796a4ebf639ee21b136e06c7d8c6bc62b7cb4a311870100000000feffffff0200ca9a3b000000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88acc088357700000000160014c48ea35298422796eb75ee9a04ebada66780083902473044022062963ff9f1f0d2c885060c083b97ddd67bd6e8a7daaed6578a62c1f905fd31a6022001836efe75bbce64d29700f5568aed78ce68f098ef0b8efdad0679e2f6c0ceb5012102228c4d459ce647b63b2681597a6e8391e4b5afab147f90747184a0893e4934c26600000002000000000101c654a35d86b4c3112ae9ec75e3bc271e0b8f928620386fdaf9517e8a9b511d850100000000feffffff024038f505000000001976a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88acc0a73f7100000000160014cb1049e20cebfe756d657af0d47a3357c1db3c9702473044022024728eb27e884d08b2b95e670fd58c0e10106a18bc9e01f753f9ae474aa2c95e02201d5abe91302b373507dce1fd2f4b95ddc36e21d23eda6c476cc9efea80338ef90121036be02fd68622ce6740f0d028059af96a310c23c675f044732171df6f64f0fcb2660000000200000000010135c4ee1a780d8279ea7c79d3ad6fbb39315a13e6b51dd51f49db6e2f4796bada0100000000feffffff020000000000000000036a0121c8183f71000000001600147ea4d830ca77c20a23155a176a1472613dc4e7840247304402202db3527431e09ca8034d207134d79fc063939bd402286dd8b3872de3b2a746b402207faae5fb8838e6877822a9209fe2e81d4be4a69ce8164215e03f9b92d75e94f90121024d456d37df6f3436ee1685b78d90d2d24c4dd8c602d5f41405fe7f449b43415f00000000"
   ],
   "mocktime": 1525107225,
   "stats": [
@@ -142,13 +142,15 @@
       "totalfee": 0,
       "txs": 1,
       "utxo_increase": 2,
-      "utxo_size_inc": 163
+      "utxo_increase_actual": 1,
+      "utxo_size_inc": 163,
+      "utxo_size_inc_actual": 75
     },
     {
-      "avgfee": 4460,
+      "avgfee": 4440,
       "avgfeerate": 20,
-      "avgtxsize": 223,
-      "blockhash": "0aa1cae78efd1efcd5203366a257b6ccf4c9e4960f6b8a3724ad790ab568a10f",
+      "avgtxsize": 222,
+      "blockhash": "06e5883dc39af4810bcd505b95149db664206c13ec7f5d4b33e25e30f37b5961",
       "feerate_percentiles": [
         20,
         20,
@@ -158,65 +160,69 @@
       ],
       "height": 102,
       "ins": 1,
-      "maxfee": 4460,
+      "maxfee": 4440,
       "maxfeerate": 20,
-      "maxtxsize": 223,
-      "medianfee": 4460,
+      "maxtxsize": 222,
+      "medianfee": 4440,
       "mediantime": 1525107242,
-      "mediantxsize": 223,
-      "minfee": 4460,
+      "mediantxsize": 222,
+      "minfee": 4440,
       "minfeerate": 20,
-      "mintxsize": 223,
+      "mintxsize": 222,
       "outs": 4,
       "subsidy": 5000000000,
       "swtotal_size": 0,
       "swtotal_weight": 0,
       "swtxs": 0,
       "time": 1525107243,
-      "total_out": 4999995540,
-      "total_size": 223,
-      "total_weight": 892,
-      "totalfee": 4460,
+      "total_out": 4999995560,
+      "total_size": 222,
+      "total_weight": 888,
+      "totalfee": 4440,
       "txs": 2,
       "utxo_increase": 3,
-      "utxo_size_inc": 236
+      "utxo_increase_actual": 2,
+      "utxo_size_inc": 235,
+      "utxo_size_inc_actual": 147
     },
     {
-      "avgfee": 24906,
-      "avgfeerate": 121,
-      "avgtxsize": 231,
-      "blockhash": "53e416e2538bc783c42a7aea566e884321afed893e9e58cf356d6429759dfa46",
+      "avgfee": 21390,
+      "avgfeerate": 155,
+      "avgtxsize": 219,
+      "blockhash": "7474991c2ae3c94c4813d75b4c752028304b773dd4dce8d460dfa2d1e7b542a3",
       "feerate_percentiles": [
         20,
         20,
         20,
-        300,
-        300
+        301,
+        301
       ],
       "height": 103,
-      "ins": 3,
-      "maxfee": 66900,
-      "maxfeerate": 300,
-      "maxtxsize": 249,
-      "medianfee": 4460,
+      "ins": 4,
+      "maxfee": 43200,
+      "maxfeerate": 301,
+      "maxtxsize": 225,
+      "medianfee": 19740,
       "mediantime": 1525107243,
-      "mediantxsize": 223,
-      "minfee": 3360,
+      "mediantxsize": 225,
+      "minfee": 2880,
       "minfeerate": 20,
-      "mintxsize": 223,
-      "outs": 8,
+      "mintxsize": 203,
+      "outs": 10,
       "subsidy": 5000000000,
-      "swtotal_size": 249,
-      "swtotal_weight": 669,
-      "swtxs": 1,
+      "swtotal_size": 878,
+      "swtotal_weight": 2204,
+      "swtxs": 4,
       "time": 1525107243,
-      "total_out": 9999920820,
-      "total_size": 695,
-      "total_weight": 2453,
-      "totalfee": 74720,
-      "txs": 4,
-      "utxo_increase": 5,
-      "utxo_size_inc": 384
+      "total_out": 10899908680,
+      "total_size": 878,
+      "total_weight": 2204,
+      "totalfee": 85560,
+      "txs": 5,
+      "utxo_increase": 6,
+      "utxo_increase_actual": 4,
+      "utxo_size_inc": 441,
+      "utxo_size_inc_actual": 300
     }
   ]
 }
\ No newline at end of file
diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py
index 1ea1ee5659..bf261befcc 100755
--- a/test/functional/rpc_getblockstats.py
+++ b/test/functional/rpc_getblockstats.py
@@ -43,6 +43,10 @@ class GetblockstatsTest(BitcoinTestFramework):
     def generate_test_data(self, filename):
         mocktime = 1525107225
         self.nodes[0].setmocktime(mocktime)
+        self.nodes[0].createwallet(wallet_name='test')
+        privkey = self.nodes[0].get_deterministic_priv_key().key
+        self.nodes[0].importprivkey(privkey)
+
         self.generate(self.nodes[0], COINBASE_MATURITY + 1)
 
         address = self.nodes[0].get_deterministic_priv_key().address
@@ -53,6 +57,8 @@ class GetblockstatsTest(BitcoinTestFramework):
         self.nodes[0].sendtoaddress(address=address, amount=10, subtractfeefromamount=False)
         self.nodes[0].settxfee(amount=0.003)
         self.nodes[0].sendtoaddress(address=address, amount=1, subtractfeefromamount=True)
+        # Send to OP_RETURN output to test its exclusion from statistics
+        self.nodes[0].send(outputs={"data": "21"})
         self.sync_all()
         self.generate(self.nodes[0], 1)
 
@@ -161,6 +167,20 @@ class GetblockstatsTest(BitcoinTestFramework):
         assert_raises_rpc_error(-1, 'getblockstats hash_or_height ( stats )', self.nodes[0].getblockstats, '00', 1, 2)
         assert_raises_rpc_error(-1, 'getblockstats hash_or_height ( stats )', self.nodes[0].getblockstats)
 
+        self.log.info('Test block height 0')
+        genesis_stats = self.nodes[0].getblockstats(0)
+        assert_equal(genesis_stats["blockhash"], "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206")
+        assert_equal(genesis_stats["utxo_increase"], 1)
+        assert_equal(genesis_stats["utxo_size_inc"], 117)
+        assert_equal(genesis_stats["utxo_increase_actual"], 0)
+        assert_equal(genesis_stats["utxo_size_inc_actual"], 0)
+
+        self.log.info('Test tip including OP_RETURN')
+        tip_stats = self.nodes[0].getblockstats(tip)
+        assert_equal(tip_stats["utxo_increase"], 6)
+        assert_equal(tip_stats["utxo_size_inc"], 441)
+        assert_equal(tip_stats["utxo_increase_actual"], 4)
+        assert_equal(tip_stats["utxo_size_inc_actual"], 300)
 
 if __name__ == '__main__':
     GetblockstatsTest().main()