Merge bitcoin/bitcoin#29401: test: Remove struct.pack from almost all places

fa52e13ee81fcc7543890dbd6986fcb55168583f test: Remove struct.pack from almost all places (MarcoFalke)
fa826db477a981b48bff53021f9695a5f6682dc0 scripted-diff: test: Use int.to_bytes over struct packing (MarcoFalke)
faf2a975ad46799d075e3a70c699da0d8182aab9 test: Use int.to_bytes over struct packing (MarcoFalke)
faf3cd659a72473a1aa73c4367a145f4ec64f146 test: Normalize struct.pack format (MarcoFalke)

Pull request description:

  `struct.pack` has many issues:

  * The format string consists of characters that may be confusing and may need to be looked up in the documentation, as opposed to using easy to understand self-documenting code.

  This lead to many test bugs, which weren't hit, which is fine, but still confusing. Ref: https://github.com/bitcoin/bitcoin/pull/29400, https://github.com/bitcoin/bitcoin/pull/29399, https://github.com/bitcoin/bitcoin/pull/29363, fa3886b7c69cbbe564478f30bb2c35e9e6b1cffa, ...

  Fix all issues by using the built-in `int` helpers `to_bytes` via a scripted diff.

  Review notes:

  * For `struct.pack` and `int.to_bytes` the error behavior is the same, although the error messages are not identical.
  * Two `struct.pack` remain. One for float serialization in a C++ code comment, and one for native serialization.

ACKs for top commit:
  achow101:
    ACK fa52e13ee81fcc7543890dbd6986fcb55168583f
  rkrux:
    tACK [fa52e13](https://github.com/bitcoin/bitcoin/pull/29401/commits/fa52e13ee81fcc7543890dbd6986fcb55168583f)
  theStack:
    Code-review ACK fa52e13ee81fcc7543890dbd6986fcb55168583f

Tree-SHA512: ee80d935b68ae43d1654b047e84ceb80abbd20306df35cca848b3f7874634b518560ddcbc7e714e2e7a19241e153dee64556dc4701287ae811e26e4f8c57fe3e

2 files changed, 17 insertions, 18 deletions

diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py
index 00bd1e4017..4b846df94a 100755
--- a/test/functional/test_framework/p2p.py
+++ b/test/functional/test_framework/p2p.py
@@ -411,7 +411,7 @@ class P2PConnection(asyncio.Protocol):
             tmsg = self.magic_bytes
             tmsg += msgtype
             tmsg += b"\x00" * (12 - len(msgtype))
-            tmsg += struct.pack("<I", len(data))
+            tmsg += len(data).to_bytes(4, "little")
             th = sha256(data)
             h = sha256(th)
             tmsg += h[:4]
diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py
index 7b19d31e17..ab3dc2ffb1 100644
--- a/test/functional/test_framework/script.py
+++ b/test/functional/test_framework/script.py
@@ -8,7 +8,6 @@ This file is modified from python-bitcoinlib.
 """
 
 from collections import namedtuple
-import struct
 import unittest
 
 from .key import TaggedHash, tweak_add_pubkey, compute_xonly_pubkey
@@ -58,9 +57,9 @@ class CScriptOp(int):
         elif len(d) <= 0xff:
             return b'\x4c' + bytes([len(d)]) + d  # OP_PUSHDATA1
         elif len(d) <= 0xffff:
-            return b'\x4d' + struct.pack(b'<H', len(d)) + d  # OP_PUSHDATA2
+            return b'\x4d' + len(d).to_bytes(2, "little") + d  # OP_PUSHDATA2
         elif len(d) <= 0xffffffff:
-            return b'\x4e' + struct.pack(b'<I', len(d)) + d  # OP_PUSHDATA4
+            return b'\x4e' + len(d).to_bytes(4, "little") + d  # OP_PUSHDATA4
         else:
             raise ValueError("Data too long to encode in a PUSHDATA op")
 
@@ -670,7 +669,7 @@ def LegacySignatureMsg(script, txTo, inIdx, hashtype):
         txtmp.vin.append(tmp)
 
     s = txtmp.serialize_without_witness()
-    s += struct.pack(b"<I", hashtype)
+    s += hashtype.to_bytes(4, "little")
 
     return (s, None)
 
@@ -726,7 +725,7 @@ def SegwitV0SignatureMsg(script, txTo, inIdx, hashtype, amount):
     if (not (hashtype & SIGHASH_ANYONECANPAY) and (hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE):
         serialize_sequence = bytes()
         for i in txTo.vin:
-            serialize_sequence += struct.pack("<I", i.nSequence)
+            serialize_sequence += i.nSequence.to_bytes(4, "little")
         hashSequence = uint256_from_str(hash256(serialize_sequence))
 
     if ((hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE):
@@ -739,16 +738,16 @@ def SegwitV0SignatureMsg(script, txTo, inIdx, hashtype, amount):
         hashOutputs = uint256_from_str(hash256(serialize_outputs))
 
     ss = bytes()
-    ss += struct.pack("<i", txTo.nVersion)
+    ss += txTo.nVersion.to_bytes(4, "little", signed=True)
     ss += ser_uint256(hashPrevouts)
     ss += ser_uint256(hashSequence)
     ss += txTo.vin[inIdx].prevout.serialize()
     ss += ser_string(script)
-    ss += struct.pack("<q", amount)
-    ss += struct.pack("<I", txTo.vin[inIdx].nSequence)
+    ss += amount.to_bytes(8, "little", signed=True)
+    ss += txTo.vin[inIdx].nSequence.to_bytes(4, "little")
     ss += ser_uint256(hashOutputs)
     ss += txTo.nLockTime.to_bytes(4, "little")
-    ss += struct.pack("<I", hashtype)
+    ss += hashtype.to_bytes(4, "little")
     return ss
 
 def SegwitV0SignatureHash(*args, **kwargs):
@@ -800,13 +799,13 @@ def BIP341_sha_prevouts(txTo):
     return sha256(b"".join(i.prevout.serialize() for i in txTo.vin))
 
 def BIP341_sha_amounts(spent_utxos):
-    return sha256(b"".join(struct.pack("<q", u.nValue) for u in spent_utxos))
+    return sha256(b"".join(u.nValue.to_bytes(8, "little", signed=True) for u in spent_utxos))
 
 def BIP341_sha_scriptpubkeys(spent_utxos):
     return sha256(b"".join(ser_string(u.scriptPubKey) for u in spent_utxos))
 
 def BIP341_sha_sequences(txTo):
-    return sha256(b"".join(struct.pack("<I", i.nSequence) for i in txTo.vin))
+    return sha256(b"".join(i.nSequence.to_bytes(4, "little") for i in txTo.vin))
 
 def BIP341_sha_outputs(txTo):
     return sha256(b"".join(o.serialize() for o in txTo.vout))
@@ -818,8 +817,8 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat
     in_type = hash_type & SIGHASH_ANYONECANPAY
     spk = spent_utxos[input_index].scriptPubKey
     ss = bytes([0, hash_type]) # epoch, hash_type
-    ss += struct.pack("<i", txTo.nVersion)
-    ss += struct.pack("<I", txTo.nLockTime)
+    ss += txTo.nVersion.to_bytes(4, "little", signed=True)
+    ss += txTo.nLockTime.to_bytes(4, "little")
     if in_type != SIGHASH_ANYONECANPAY:
         ss += BIP341_sha_prevouts(txTo)
         ss += BIP341_sha_amounts(spent_utxos)
@@ -835,11 +834,11 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat
     ss += bytes([spend_type])
     if in_type == SIGHASH_ANYONECANPAY:
         ss += txTo.vin[input_index].prevout.serialize()
-        ss += struct.pack("<q", spent_utxos[input_index].nValue)
+        ss += spent_utxos[input_index].nValue.to_bytes(8, "little", signed=True)
         ss += ser_string(spk)
-        ss += struct.pack("<I", txTo.vin[input_index].nSequence)
+        ss += txTo.vin[input_index].nSequence.to_bytes(4, "little")
     else:
-        ss += struct.pack("<I", input_index)
+        ss += input_index.to_bytes(4, "little")
     if (spend_type & 1):
         ss += sha256(ser_string(annex))
     if out_type == SIGHASH_SINGLE:
@@ -850,7 +849,7 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat
     if (scriptpath):
         ss += TaggedHash("TapLeaf", bytes([leaf_ver]) + ser_string(script))
         ss += bytes([0])
-        ss += struct.pack("<i", codeseparator_pos)
+        ss += codeseparator_pos.to_bytes(4, "little", signed=True)
     assert len(ss) ==  175 - (in_type == SIGHASH_ANYONECANPAY) * 49 - (out_type != SIGHASH_ALL and out_type != SIGHASH_SINGLE) * 32 + (annex is not None) * 32 + scriptpath * 37
     return ss