diff options
Diffstat (limited to 'contrib')
22 files changed, 331 insertions, 203 deletions
diff --git a/contrib/asmap/asmap-tool.py b/contrib/asmap/asmap-tool.py index 09c28725e4..33a380a2e7 100755 --- a/contrib/asmap/asmap-tool.py +++ b/contrib/asmap/asmap-tool.py @@ -6,7 +6,9 @@ import argparse import sys import ipaddress +import json import math +from collections import defaultdict import asmap @@ -113,6 +115,18 @@ def main(): parser_diff.add_argument('infile2', type=argparse.FileType('rb'), help="second file to compare (text or binary)") + parser_diff_addrs = subparsers.add_parser("diff_addrs", + help="compute difference between two asmap files for a set of addresses") + parser_diff_addrs.add_argument('-s', '--show-addresses', dest="show_addresses", default=False, action="store_true", + help="include reassigned addresses in the output") + parser_diff_addrs.add_argument("infile1", type=argparse.FileType("rb"), + help="first file to compare (text or binary)") + parser_diff_addrs.add_argument("infile2", type=argparse.FileType("rb"), + help="second file to compare (text or binary)") + parser_diff_addrs.add_argument("addrs_file", type=argparse.FileType("r"), + help="address file containing getnodeaddresses output to use in the comparison " + "(make sure to set the count parameter to zero to get all node addresses, " + "e.g. 'bitcoin-cli getnodeaddresses 0 > addrs.json')") args = parser.parse_args() if args.subcommand is None: parser.print_help() @@ -148,6 +162,33 @@ def main(): f"# {ipv4_changed}{ipv4_change_str} IPv4 addresses changed; " f"{ipv6_changed}{ipv6_change_str} IPv6 addresses changed" ) + elif args.subcommand == "diff_addrs": + state1 = load_file(args.infile1) + state2 = load_file(args.infile2) + address_info = json.load(args.addrs_file) + addrs = {a["address"] for a in address_info if a["network"] in ["ipv4", "ipv6"]} + reassignments = defaultdict(list) + for addr in addrs: + net = ipaddress.ip_network(addr) + prefix = asmap.net_to_prefix(net) + old_asn = state1.lookup(prefix) + new_asn = state2.lookup(prefix) + if new_asn != old_asn: + reassignments[(old_asn, new_asn)].append(addr) + reassignments = sorted(reassignments.items(), key=lambda item: len(item[1]), reverse=True) + num_reassignment_type = defaultdict(int) + for (old_asn, new_asn), reassigned_addrs in reassignments: + num_reassigned = len(reassigned_addrs) + num_reassignment_type[(bool(old_asn), bool(new_asn))] += num_reassigned + old_asn_str = f"AS{old_asn}" if old_asn else "unassigned" + new_asn_str = f"AS{new_asn}" if new_asn else "unassigned" + opt = ": " + ", ".join(reassigned_addrs) if args.show_addresses else "" + print(f"{num_reassigned} address(es) reassigned from {old_asn_str} to {new_asn_str}{opt}") + num_reassignments = sum(len(addrs) for _, addrs in reassignments) + share = num_reassignments / len(addrs) if len(addrs) > 0 else 0 + print(f"Summary: {num_reassignments:,} ({share:.2%}) of {len(addrs):,} addresses were reassigned " + f"(migrations={num_reassignment_type[True, True]}, assignments={num_reassignment_type[False, True]}, " + f"unassignments={num_reassignment_type[True, False]})") else: parser.print_help() sys.exit("No command provided.") diff --git a/contrib/completions/bash/bitcoin-cli.bash b/contrib/completions/bash/bitcoin-cli.bash index 89e01bc09a..b04fdbcb0e 100644 --- a/contrib/completions/bash/bitcoin-cli.bash +++ b/contrib/completions/bash/bitcoin-cli.bash @@ -9,7 +9,7 @@ _bitcoin_rpc() { local rpcargs=() for i in ${COMP_LINE}; do case "$i" in - -conf=*|-datadir=*|-regtest|-rpc*|-testnet) + -conf=*|-datadir=*|-regtest|-rpc*|-testnet|-testnet4) rpcargs=( "${rpcargs[@]}" "$i" ) ;; esac diff --git a/contrib/devtools/bitcoin-tidy/CMakeLists.txt b/contrib/devtools/bitcoin-tidy/CMakeLists.txt index 1260c71423..95345b4782 100644 --- a/contrib/devtools/bitcoin-tidy/CMakeLists.txt +++ b/contrib/devtools/bitcoin-tidy/CMakeLists.txt @@ -25,7 +25,7 @@ find_program(CLANG_TIDY_EXE NAMES "clang-tidy-${LLVM_VERSION_MAJOR}" "clang-tidy message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") message(STATUS "Found clang-tidy: ${CLANG_TIDY_EXE}") -add_library(bitcoin-tidy MODULE bitcoin-tidy.cpp logprintf.cpp) +add_library(bitcoin-tidy MODULE bitcoin-tidy.cpp logprintf.cpp nontrivial-threadlocal.cpp) target_include_directories(bitcoin-tidy SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) # Disable RTTI and exceptions as necessary @@ -58,7 +58,7 @@ else() endif() # Create a dummy library that runs clang-tidy tests as a side-effect of building -add_library(bitcoin-tidy-tests OBJECT EXCLUDE_FROM_ALL example_logprintf.cpp) +add_library(bitcoin-tidy-tests OBJECT EXCLUDE_FROM_ALL example_logprintf.cpp example_nontrivial-threadlocal.cpp) add_dependencies(bitcoin-tidy-tests bitcoin-tidy) set_target_properties(bitcoin-tidy-tests PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_COMMAND}") diff --git a/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp b/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp index 0f34d37793..1ef4494973 100644 --- a/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp +++ b/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "logprintf.h" +#include "nontrivial-threadlocal.h" #include <clang-tidy/ClangTidyModule.h> #include <clang-tidy/ClangTidyModuleRegistry.h> @@ -13,6 +14,7 @@ public: void addCheckFactories(clang::tidy::ClangTidyCheckFactories& CheckFactories) override { CheckFactories.registerCheck<bitcoin::LogPrintfCheck>("bitcoin-unterminated-logprintf"); + CheckFactories.registerCheck<bitcoin::NonTrivialThreadLocal>("bitcoin-nontrivial-threadlocal"); } }; diff --git a/contrib/devtools/bitcoin-tidy/example_nontrivial-threadlocal.cpp b/contrib/devtools/bitcoin-tidy/example_nontrivial-threadlocal.cpp new file mode 100644 index 0000000000..2b74df5d0e --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/example_nontrivial-threadlocal.cpp @@ -0,0 +1,2 @@ +#include <string> +thread_local std::string foo; diff --git a/contrib/devtools/bitcoin-tidy/nontrivial-threadlocal.cpp b/contrib/devtools/bitcoin-tidy/nontrivial-threadlocal.cpp new file mode 100644 index 0000000000..d2bc78a31b --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/nontrivial-threadlocal.cpp @@ -0,0 +1,44 @@ +// Copyright (c) 2023 Bitcoin Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "nontrivial-threadlocal.h" + +#include <clang/AST/ASTContext.h> +#include <clang/ASTMatchers/ASTMatchFinder.h> + + +// Copied from clang-tidy's UnusedRaiiCheck +namespace { +AST_MATCHER(clang::CXXRecordDecl, hasNonTrivialDestructor) { + // TODO: If the dtor is there but empty we don't want to warn either. + return Node.hasDefinition() && Node.hasNonTrivialDestructor(); +} +} // namespace + +namespace bitcoin { + +void NonTrivialThreadLocal::registerMatchers(clang::ast_matchers::MatchFinder* finder) +{ + using namespace clang::ast_matchers; + + /* + thread_local std::string foo; + */ + + finder->addMatcher( + varDecl( + hasThreadStorageDuration(), + hasType(hasCanonicalType(recordType(hasDeclaration(cxxRecordDecl(hasNonTrivialDestructor()))))) + ).bind("nontrivial_threadlocal"), + this); +} + +void NonTrivialThreadLocal::check(const clang::ast_matchers::MatchFinder::MatchResult& Result) +{ + if (const clang::VarDecl* var = Result.Nodes.getNodeAs<clang::VarDecl>("nontrivial_threadlocal")) { + const auto user_diag = diag(var->getBeginLoc(), "Variable with non-trivial destructor cannot be thread_local."); + } +} + +} // namespace bitcoin diff --git a/contrib/devtools/bitcoin-tidy/nontrivial-threadlocal.h b/contrib/devtools/bitcoin-tidy/nontrivial-threadlocal.h new file mode 100644 index 0000000000..c853073467 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/nontrivial-threadlocal.h @@ -0,0 +1,29 @@ +// Copyright (c) 2023 Bitcoin Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef NONTRIVIAL_THREADLOCAL_CHECK_H +#define NONTRIVIAL_THREADLOCAL_CHECK_H + +#include <clang-tidy/ClangTidyCheck.h> + +namespace bitcoin { + +// Warn about any thread_local variable with a non-trivial destructor. +class NonTrivialThreadLocal final : public clang::tidy::ClangTidyCheck +{ +public: + NonTrivialThreadLocal(clang::StringRef Name, clang::tidy::ClangTidyContext* Context) + : clang::tidy::ClangTidyCheck(Name, Context) {} + + bool isLanguageVersionSupported(const clang::LangOptions& LangOpts) const override + { + return LangOpts.CPlusPlus; + } + void registerMatchers(clang::ast_matchers::MatchFinder* Finder) override; + void check(const clang::ast_matchers::MatchFinder::MatchResult& Result) override; +}; + +} // namespace bitcoin + +#endif // NONTRIVIAL_THREADLOCAL_CHECK_H diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 3dddffe324..3c98ee7b20 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -19,7 +19,6 @@ EXCLUDE = [ 'src/qt/bitcoinstrings.cpp', 'src/chainparamsseeds.h', # other external copyrights: - 'src/reverse_iterator.h', 'src/test/fuzz/FuzzedDataProvider.h', 'src/tinyformat.h', 'src/bench/nanobench.h', diff --git a/contrib/devtools/headerssync-params.py b/contrib/devtools/headerssync-params.py index 0198f5db99..f8e415532a 100644..100755 --- a/contrib/devtools/headerssync-params.py +++ b/contrib/devtools/headerssync-params.py @@ -12,13 +12,13 @@ import random # Parameters: # Aim for still working fine at some point in the future. [datetime] -TIME = datetime(2026, 10, 5) +TIME = datetime(2027, 4, 1) # Expected block interval. [timedelta] BLOCK_INTERVAL = timedelta(seconds=600) # The number of headers corresponding to the minchainwork parameter. [headers] -MINCHAINWORK_HEADERS = 804000 +MINCHAINWORK_HEADERS = 856760 # Combined processing bandwidth from all attackers to one victim. [bit/s] # 6 Gbit/s is approximately the speed at which a single thread of a Ryzen 5950X CPU thread can hash diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index f57e9abfec..46f9ee915f 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -38,13 +38,13 @@ def check_ELF_RELRO(binary) -> bool: return have_gnu_relro and have_bindnow -def check_ELF_Canary(binary) -> bool: +def check_ELF_CANARY(binary) -> bool: ''' Check for use of stack canary ''' return binary.has_symbol('__stack_chk_fail') -def check_ELF_separate_code(binary): +def check_ELF_SEPARATE_CODE(binary): ''' Check that sections are appropriately separated in virtual memory, based on their permissions. This checks for missing -Wl,-z,separate-code @@ -105,7 +105,7 @@ def check_ELF_separate_code(binary): return False return True -def check_ELF_control_flow(binary) -> bool: +def check_ELF_CONTROL_FLOW(binary) -> bool: ''' Check for control flow instrumentation ''' @@ -130,7 +130,7 @@ def check_PE_RELOC_SECTION(binary) -> bool: '''Check for a reloc section. This is required for functional ASLR.''' return binary.has_relocations -def check_PE_control_flow(binary) -> bool: +def check_PE_CONTROL_FLOW(binary) -> bool: ''' Check for control flow instrumentation ''' @@ -145,7 +145,7 @@ def check_PE_control_flow(binary) -> bool: return True return False -def check_PE_Canary(binary) -> bool: +def check_PE_CANARY(binary) -> bool: ''' Check for use of stack canary ''' @@ -163,7 +163,7 @@ def check_MACHO_FIXUP_CHAINS(binary) -> bool: ''' return binary.has_dyld_chained_fixups -def check_MACHO_Canary(binary) -> bool: +def check_MACHO_CANARY(binary) -> bool: ''' Check for use of stack canary ''' @@ -182,7 +182,7 @@ def check_NX(binary) -> bool: ''' return binary.has_nx -def check_MACHO_control_flow(binary) -> bool: +def check_MACHO_CONTROL_FLOW(binary) -> bool: ''' Check for control flow instrumentation ''' @@ -192,7 +192,7 @@ def check_MACHO_control_flow(binary) -> bool: return True return False -def check_MACHO_branch_protection(binary) -> bool: +def check_MACHO_BRANCH_PROTECTION(binary) -> bool: ''' Check for branch protection instrumentation ''' @@ -206,8 +206,8 @@ BASE_ELF = [ ('PIE', check_PIE), ('NX', check_NX), ('RELRO', check_ELF_RELRO), - ('Canary', check_ELF_Canary), - ('separate_code', check_ELF_separate_code), + ('CANARY', check_ELF_CANARY), + ('SEPARATE_CODE', check_ELF_SEPARATE_CODE), ] BASE_PE = [ @@ -216,19 +216,19 @@ BASE_PE = [ ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), ('NX', check_NX), ('RELOC_SECTION', check_PE_RELOC_SECTION), - ('CONTROL_FLOW', check_PE_control_flow), - ('Canary', check_PE_Canary), + ('CONTROL_FLOW', check_PE_CONTROL_FLOW), + ('CANARY', check_PE_CANARY), ] BASE_MACHO = [ ('NOUNDEFS', check_MACHO_NOUNDEFS), - ('Canary', check_MACHO_Canary), + ('CANARY', check_MACHO_CANARY), ('FIXUP_CHAINS', check_MACHO_FIXUP_CHAINS), ] CHECKS = { lief.EXE_FORMATS.ELF: { - lief.ARCHITECTURES.X86: BASE_ELF + [('CONTROL_FLOW', check_ELF_control_flow)], + lief.ARCHITECTURES.X86: BASE_ELF + [('CONTROL_FLOW', check_ELF_CONTROL_FLOW)], lief.ARCHITECTURES.ARM: BASE_ELF, lief.ARCHITECTURES.ARM64: BASE_ELF, lief.ARCHITECTURES.PPC: BASE_ELF, @@ -240,39 +240,24 @@ CHECKS = { lief.EXE_FORMATS.MACHO: { lief.ARCHITECTURES.X86: BASE_MACHO + [('PIE', check_PIE), ('NX', check_NX), - ('CONTROL_FLOW', check_MACHO_control_flow)], - lief.ARCHITECTURES.ARM64: BASE_MACHO + [('BRANCH_PROTECTION', check_MACHO_branch_protection)], + ('CONTROL_FLOW', check_MACHO_CONTROL_FLOW)], + lief.ARCHITECTURES.ARM64: BASE_MACHO + [('BRANCH_PROTECTION', check_MACHO_BRANCH_PROTECTION)], } } if __name__ == '__main__': retval: int = 0 for filename in sys.argv[1:]: - try: - binary = lief.parse(filename) - etype = binary.format - arch = binary.abstract.header.architecture - binary.concrete - - if etype == lief.EXE_FORMATS.UNKNOWN: - print(f'{filename}: unknown executable format') - retval = 1 - continue - - if arch == lief.ARCHITECTURES.NONE: - print(f'{filename}: unknown architecture') - retval = 1 - continue - - failed: list[str] = [] - for (name, func) in CHECKS[etype][arch]: - if not func(binary): - failed.append(name) - if failed: - print(f'{filename}: failed {" ".join(failed)}') - retval = 1 - except IOError: - print(f'{filename}: cannot open') + binary = lief.parse(filename) + etype = binary.format + arch = binary.abstract.header.architecture + binary.concrete + + failed: list[str] = [] + for (name, func) in CHECKS[etype][arch]: + if not func(binary): + failed.append(name) + if failed: + print(f'{filename}: failed {" ".join(failed)}') retval = 1 sys.exit(retval) - diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index c4e6bc81e1..1722c7d290 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -245,7 +245,7 @@ def check_MACHO_sdk(binary) -> bool: return False def check_MACHO_lld(binary) -> bool: - if binary.build_version.tools[0].version == [18, 1, 6]: + if binary.build_version.tools[0].version == [18, 1, 8]: return True return False @@ -299,22 +299,14 @@ lief.EXE_FORMATS.PE: [ if __name__ == '__main__': retval: int = 0 for filename in sys.argv[1:]: - try: - binary = lief.parse(filename) - etype = binary.format - if etype == lief.EXE_FORMATS.UNKNOWN: - print(f'{filename}: unknown executable format') - retval = 1 - continue - - failed: list[str] = [] - for (name, func) in CHECKS[etype]: - if not func(binary): - failed.append(name) - if failed: - print(f'{filename}: failed {" ".join(failed)}') - retval = 1 - except IOError: - print(f'{filename}: cannot open') + binary = lief.parse(filename) + etype = binary.format + + failed: list[str] = [] + for (name, func) in CHECKS[etype]: + if not func(binary): + failed.append(name) + if failed: + print(f'{filename}: failed {" ".join(failed)}') retval = 1 sys.exit(retval) diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 7bfd4d98da..4bec6bfe7c 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -15,10 +15,10 @@ from utils import determine_wellknown_cmd def write_testcode(filename): with open(filename, 'w', encoding="utf8") as f: f.write(''' - #include <stdio.h> + #include <cstdio> int main() { - printf("the quick brown fox jumps over the lazy god\\n"); + std::printf("the quick brown fox jumps over the lazy god\\n"); return 0; } ''') @@ -34,17 +34,17 @@ def env_flags() -> list[str]: # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for # reference. flags: list[str] = [] - for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: + for var in ['CXXFLAGS', 'CPPFLAGS', 'LDFLAGS']: flags += filter(None, os.environ.get(var, '').split(' ')) return flags -def call_security_check(cc: str, source: str, executable: str, options) -> tuple: - subprocess.run([*cc,source,'-o',executable] + env_flags() + options, check=True) +def call_security_check(cxx: str, source: str, executable: str, options) -> tuple: + subprocess.run([*cxx,source,'-o',executable] + env_flags() + options, check=True) p = subprocess.run([os.path.join(os.path.dirname(__file__), 'security-check.py'), executable], stdout=subprocess.PIPE, text=True) return (p.returncode, p.stdout.rstrip()) -def get_arch(cc, source, executable): - subprocess.run([*cc, source, '-o', executable] + env_flags(), check=True) +def get_arch(cxx, source, executable): + subprocess.run([*cxx, source, '-o', executable] + env_flags(), check=True) binary = lief.parse(executable) arch = binary.abstract.header.architecture os.remove(executable) @@ -52,95 +52,72 @@ def get_arch(cc, source, executable): class TestSecurityChecks(unittest.TestCase): def test_ELF(self): - source = 'test1.c' + source = 'test1.cpp' executable = 'test1' - cc = determine_wellknown_cmd('CC', 'gcc') + cxx = determine_wellknown_cmd('CXX', 'g++') write_testcode(source) - arch = get_arch(cc, source, executable) + arch = get_arch(cxx, source, executable) if arch == lief.ARCHITECTURES.X86: - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE NX RELRO CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), - (1, executable+': failed RELRO CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), - (1, executable+': failed separate_code CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), - (1, executable+': failed CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code', '-fcf-protection=full']), - (0, '')) + pass_flags = ['-Wl,-znoexecstack', '-Wl,-zrelro', '-Wl,-z,now', '-pie', '-fPIE', '-Wl,-z,separate-code', '-fcf-protection=full'] + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-zexecstack']), (1, executable + ': failed NX')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-no-pie','-fno-PIE']), (1, executable + ': failed PIE')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-znorelro']), (1, executable + ': failed RELRO')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-z,noseparate-code']), (1, executable + ': failed SEPARATE_CODE')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-fcf-protection=none']), (1, executable + ': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags), (0, '')) else: - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE NX RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), - (1, executable+': failed RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), - (1, executable+': failed separate_code')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), - (0, '')) + pass_flags = ['-Wl,-znoexecstack', '-Wl,-zrelro', '-Wl,-z,now', '-pie', '-fPIE', '-Wl,-z,separate-code'] + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-zexecstack']), (1, executable + ': failed NX')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-no-pie','-fno-PIE']), (1, executable + ': failed PIE')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-znorelro']), (1, executable + ': failed RELRO')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-z,noseparate-code']), (1, executable + ': failed SEPARATE_CODE')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags), (0, '')) clean_files(source, executable) def test_PE(self): - source = 'test1.c' + source = 'test1.cpp' executable = 'test1.exe' - cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') + cxx = determine_wellknown_cmd('CXX', 'x86_64-w64-mingw32-g++') write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--disable-nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE','-fno-stack-protector']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE','-fstack-protector-all', '-lssp']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE','-fstack-protector-all', '-lssp']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE','-fstack-protector-all', '-lssp']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) # -pie -fPIE does nothing unless --dynamicbase is also supplied - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE','-fstack-protector-all', '-lssp']), - (1, executable+': failed HIGH_ENTROPY_VA CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE','-fstack-protector-all', '-lssp']), - (1, executable+': failed CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full','-fstack-protector-all', '-lssp']), - (0, '')) + pass_flags = ['-Wl,--nxcompat', '-Wl,--enable-reloc-section', '-Wl,--dynamicbase', '-Wl,--high-entropy-va', '-pie', '-fPIE', '-fcf-protection=full', '-fstack-protector-all', '-lssp'] + + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-fno-stack-protector']), (1, executable + ': failed CANARY')) + # https://github.com/lief-project/LIEF/issues/1076 - in future, we could test this individually. + # self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,--disable-reloc-section']), (1, executable + ': failed RELOC_SECTION')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,--disable-nxcompat']), (1, executable + ': failed NX')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,--disable-dynamicbase']), (1, executable + ': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) # -pie -fPIE does nothing without --dynamicbase + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,--disable-high-entropy-va']), (1, executable + ': failed HIGH_ENTROPY_VA')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-fcf-protection=none']), (1, executable + ': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags), (0, '')) clean_files(source, executable) def test_MACHO(self): - source = 'test1.c' + source = 'test1.cpp' executable = 'test1' - cc = determine_wellknown_cmd('CC', 'clang') + cxx = determine_wellknown_cmd('CXX', 'clang++') write_testcode(source) - arch = get_arch(cc, source, executable) + arch = get_arch(cxx, source, executable) if arch == lief.ARCHITECTURES.X86: - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-no_fixup_chains']), - (1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-fixup_chains']), - (1, executable+': failed NOUNDEFS Canary CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains']), - (1, executable+': failed NOUNDEFS CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all', '-Wl,-fixup_chains']), - (1, executable+': failed CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']), - (0, '')) + pass_flags = ['-Wl,-pie', '-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains'] + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-no_pie', '-Wl,-no_fixup_chains']), (1, executable+': failed FIXUP_CHAINS PIE')) # -fixup_chains is incompatible with -no_pie + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-no_fixup_chains']), (1, executable + ': failed FIXUP_CHAINS')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-fno-stack-protector']), (1, executable + ': failed CANARY')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-flat_namespace']), (1, executable + ': failed NOUNDEFS')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-fcf-protection=none']), (1, executable + ': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags), (0, '')) else: - # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-no_fixup_chains']), - (1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS BRANCH_PROTECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-fixup_chains', '-mbranch-protection=bti']), - (1, executable+': failed NOUNDEFS Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains', '-mbranch-protection=bti']), - (1, executable+': failed NOUNDEFS')) - self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all', '-Wl,-fixup_chains', '-mbranch-protection=bti']), - (0, '')) - + # arm64 darwin doesn't support non-PIE binaries or executable stacks + pass_flags = ['-fstack-protector-all', '-Wl,-fixup_chains', '-mbranch-protection=bti'] + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-mbranch-protection=none']), (1, executable + ': failed BRANCH_PROTECTION')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-no_fixup_chains']), (1, executable + ': failed FIXUP_CHAINS')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-fno-stack-protector']), (1, executable + ': failed CANARY')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags + ['-Wl,-flat_namespace']), (1, executable + ': failed NOUNDEFS')) + self.assertEqual(call_security_check(cxx, source, executable, pass_flags), (0, '')) clean_files(source, executable) diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index b00004586c..454dbc6bd2 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -11,17 +11,17 @@ import unittest from utils import determine_wellknown_cmd -def call_symbol_check(cc: list[str], source, executable, options): +def call_symbol_check(cxx: list[str], source, executable, options): # This should behave the same as AC_TRY_LINK, so arrange well-known flags # in the same order as autoconf would. # # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for # reference. env_flags: list[str] = [] - for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: + for var in ['CXXFLAGS', 'CPPFLAGS', 'LDFLAGS']: env_flags += filter(None, os.environ.get(var, '').split(' ')) - subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) + subprocess.run([*cxx,source,'-o',executable] + env_flags + options, check=True) p = subprocess.run([os.path.join(os.path.dirname(__file__), 'symbol-check.py'), executable], stdout=subprocess.PIPE, text=True) os.remove(source) os.remove(executable) @@ -29,13 +29,13 @@ def call_symbol_check(cc: list[str], source, executable, options): class TestSymbolChecks(unittest.TestCase): def test_ELF(self): - source = 'test1.c' + source = 'test1.cpp' executable = 'test1' - cc = determine_wellknown_cmd('CC', 'gcc') + cxx = determine_wellknown_cmd('CXX', 'g++') # -lutil is part of the libc6 package so a safe bet that it's installed # it's also out of context enough that it's unlikely to ever become a real dependency - source = 'test2.c' + source = 'test2.cpp' executable = 'test2' with open(source, 'w', encoding="utf8") as f: f.write(''' @@ -48,31 +48,31 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-lutil']), + self.assertEqual(call_symbol_check(cxx, source, executable, ['-lutil']), (1, executable + ': libutil.so.1 is not in ALLOWED_LIBRARIES!\n' + executable + ': failed LIBRARY_DEPENDENCIES')) # finally, check a simple conforming binary - source = 'test3.c' + source = 'test3.cpp' executable = 'test3' with open(source, 'w', encoding="utf8") as f: f.write(''' - #include <stdio.h> + #include <cstdio> int main() { - printf("42"); + std::printf("42"); return 0; } ''') - self.assertEqual(call_symbol_check(cc, source, executable, []), + self.assertEqual(call_symbol_check(cxx, source, executable, []), (0, '')) def test_MACHO(self): - source = 'test1.c' + source = 'test1.cpp' executable = 'test1' - cc = determine_wellknown_cmd('CC', 'clang') + cxx = determine_wellknown_cmd('CXX', 'clang++') with open(source, 'w', encoding="utf8") as f: f.write(''' @@ -86,11 +86,11 @@ class TestSymbolChecks(unittest.TestCase): ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-lexpat', '-Wl,-platform_version','-Wl,macos', '-Wl,11.4', '-Wl,11.4']), + self.assertEqual(call_symbol_check(cxx, source, executable, ['-lexpat', '-Wl,-platform_version','-Wl,macos', '-Wl,11.4', '-Wl,11.4']), (1, 'libexpat.1.dylib is not in ALLOWED_LIBRARIES!\n' + f'{executable}: failed DYNAMIC_LIBRARIES MIN_OS SDK')) - source = 'test2.c' + source = 'test2.cpp' executable = 'test2' with open(source, 'w', encoding="utf8") as f: f.write(''' @@ -103,10 +103,10 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-framework', 'CoreGraphics', '-Wl,-platform_version','-Wl,macos', '-Wl,11.4', '-Wl,11.4']), + self.assertEqual(call_symbol_check(cxx, source, executable, ['-framework', 'CoreGraphics', '-Wl,-platform_version','-Wl,macos', '-Wl,11.4', '-Wl,11.4']), (1, f'{executable}: failed MIN_OS SDK')) - source = 'test3.c' + source = 'test3.cpp' executable = 'test3' with open(source, 'w', encoding="utf8") as f: f.write(''' @@ -116,13 +116,13 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,11.0', '-Wl,11.4']), + self.assertEqual(call_symbol_check(cxx, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,11.0', '-Wl,11.4']), (1, f'{executable}: failed SDK')) def test_PE(self): - source = 'test1.c' + source = 'test1.cpp' executable = 'test1.exe' - cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') + cxx = determine_wellknown_cmd('CXX', 'x86_64-w64-mingw32-g++') with open(source, 'w', encoding="utf8") as f: f.write(''' @@ -135,11 +135,11 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-lpdh', '-Wl,--major-subsystem-version', '-Wl,6', '-Wl,--minor-subsystem-version', '-Wl,1']), + self.assertEqual(call_symbol_check(cxx, source, executable, ['-lpdh', '-Wl,--major-subsystem-version', '-Wl,6', '-Wl,--minor-subsystem-version', '-Wl,1']), (1, 'pdh.dll is not in ALLOWED_LIBRARIES!\n' + executable + ': failed DYNAMIC_LIBRARIES')) - source = 'test2.c' + source = 'test2.cpp' executable = 'test2.exe' with open(source, 'w', encoding="utf8") as f: @@ -150,10 +150,10 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,--major-subsystem-version', '-Wl,9', '-Wl,--minor-subsystem-version', '-Wl,9']), + self.assertEqual(call_symbol_check(cxx, source, executable, ['-Wl,--major-subsystem-version', '-Wl,9', '-Wl,--minor-subsystem-version', '-Wl,9']), (1, executable + ': failed SUBSYSTEM_VERSION')) - source = 'test3.c' + source = 'test3.cpp' executable = 'test3.exe' with open(source, 'w', encoding="utf8") as f: f.write(''' @@ -166,7 +166,7 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-lole32', '-Wl,--major-subsystem-version', '-Wl,6', '-Wl,--minor-subsystem-version', '-Wl,1']), + self.assertEqual(call_symbol_check(cxx, source, executable, ['-lole32', '-Wl,--major-subsystem-version', '-Wl,6', '-Wl,--minor-subsystem-version', '-Wl,1']), (0, '')) diff --git a/contrib/devtools/test_utxo_snapshots.sh b/contrib/devtools/test_utxo_snapshots.sh index d7c019f4a6..ad948d4a14 100755 --- a/contrib/devtools/test_utxo_snapshots.sh +++ b/contrib/devtools/test_utxo_snapshots.sh @@ -138,7 +138,7 @@ echo echo "-- Now: add the following to CMainParams::m_assumeutxo_data" echo " in src/kernel/chainparams.cpp, and recompile:" echo -echo " {${RPC_BASE_HEIGHT}, AssumeutxoHash{uint256S(\"0x${RPC_AU}\")}, ${RPC_NCHAINTX}, uint256S(\"0x${RPC_BLOCKHASH}\")}," +echo " {.height = ${RPC_BASE_HEIGHT}, .hash_serialized = AssumeutxoHash{uint256{\"${RPC_AU}\"}}, .m_chain_tx_count = ${RPC_NCHAINTX}, .blockhash = consteval_ctor(uint256{\"${RPC_BLOCKHASH}\"})}," echo echo echo "-- IBDing more blocks to the server node (height=$FINAL_HEIGHT) so there is a diff between snapshot and tip..." diff --git a/contrib/guix/INSTALL.md b/contrib/guix/INSTALL.md index 35ea83e585..10f5835bb8 100644 --- a/contrib/guix/INSTALL.md +++ b/contrib/guix/INSTALL.md @@ -671,6 +671,8 @@ More information: https://github.com/python/cpython/issues/81765 OpenSSL includes tests that will fail once some certificate has expired. The workarounds from the GnuTLS section immediately below can be used. +For openssl-1.1.1l use 2022-05-01 as the date. + ### GnuTLS: test-suite FAIL: status-request-revoked *The derivation is likely identified by: `/gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv`* @@ -705,11 +707,12 @@ authorized. This workaround was described [here](https://issues.guix.gnu.org/44559#5). Basically: -2. Turn off NTP -3. Set system time to 2020-10-01 -4. guix build --no-substitutes /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv -5. Set system time back to accurate current time -6. Turn NTP back on + +1. Turn off NTP +2. Set system time to 2020-10-01 +3. guix build --no-substitutes /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +4. Set system time back to accurate current time +5. Turn NTP back on For example, diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 9bc8c0e75d..fa6933b332 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -71,11 +71,9 @@ unset OBJCPLUS_INCLUDE_PATH export C_INCLUDE_PATH="${NATIVE_GCC}/include" export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" -export OBJC_INCLUDE_PATH="${NATIVE_GCC}/include" -export OBJCPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" case "$HOST" in - *darwin*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; + *darwin*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; # Required for qt/qmake *mingw*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; *) NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)" @@ -180,6 +178,14 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ x86_64_linux_STRIP=x86_64-linux-gnu-strip +case "$HOST" in + *darwin*) + # Unset now that Qt is built + unset C_INCLUDE_PATH + unset CPLUS_INCLUDE_PATH + unset LIBRARY_PATH + ;; +esac ########################### # Source Tarball Building # diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 80bfb2875f..9cc151c706 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -51,7 +51,7 @@ fi time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=f0bb724211872cd6158fce6162e0b8c73efed126 \ + --commit=7bf1d7aeaffba15c4f680f93ae88fbef25427252 \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 44fbfa1c0b..1a47e91b49 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -25,6 +25,7 @@ (guix build-system gnu) (guix build-system python) (guix build-system trivial) + (guix download) (guix gexp) (guix git-download) ((guix licenses) #:prefix license:) @@ -91,7 +92,18 @@ chain for " target " development.")) (home-page (package-home-page xgcc)) (license (package-license xgcc))))) -(define base-gcc gcc-12) +(define base-gcc + (package + (inherit gcc-12) ;; 12.3.0 + (version "12.4.0") + (source (origin + (method url-fetch) + (uri (string-append "mirror://gnu/gcc/gcc-" + version "/gcc-" version ".tar.xz")) + (sha256 + (base32 + "0xcida8l2wykvvzvpcrcn649gj0ijn64gwxbplacpg6c0hk6akvh")))))) + (define base-linux-kernel-headers linux-libre-headers-6.1) (define* (make-bitcoin-cross-toolchain target @@ -116,10 +128,17 @@ desirable for building Bitcoin Core release binaries." (package-with-extra-patches binutils (search-our-patches "binutils-unaligned-default.patch"))) +(define (winpthreads-patches mingw-w64-x86_64-winpthreads) + (package-with-extra-patches mingw-w64-x86_64-winpthreads + (search-our-patches "winpthreads-remap-guix-store.patch"))) + (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" (let* ((xbinutils (binutils-mingw-patches (cross-binutils target))) - (pthreads-xlibc mingw-w64-x86_64-winpthreads) + (machine (substring target 0 (string-index target #\-))) + (pthreads-xlibc (winpthreads-patches (make-mingw-w64 machine + #:xgcc (cross-gcc target #:xgcc (gcc-mingw-patches base-gcc)) + #:with-winpthreads? #t))) (pthreads-xgcc (cross-gcc target #:xgcc (gcc-mingw-patches mingw-w64-base-gcc) #:xbinutils xbinutils @@ -500,13 +519,13 @@ inspecting signatures in Mach-O binaries.") gzip xz ;; Build tools + gcc-toolchain-12 cmake-minimal gnu-make libtool autoconf-2.71 automake pkg-config - bison ;; Scripting python-minimal ;; (3.10) ;; Git @@ -515,22 +534,17 @@ inspecting signatures in Mach-O binaries.") python-lief) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) - (list ;; Native GCC 12 toolchain - gcc-toolchain-12 - zip + (list zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") nsis-x86_64 nss-certs osslsigncode)) ((string-contains target "-linux-") - (list ;; Native GCC 12 toolchain - gcc-toolchain-12 + (list bison (list gcc-toolchain-12 "static") (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") - (list ;; Native GCC 11 toolchain - gcc-toolchain-11 - clang-toolchain-18 + (list clang-toolchain-18 lld-18 (make-lld-wrapper lld-18 #:lld-as-ld? #t) python-signapple diff --git a/contrib/guix/patches/gcc-remap-guix-store.patch b/contrib/guix/patches/gcc-remap-guix-store.patch index a47ef7a2df..a8b41d485b 100644 --- a/contrib/guix/patches/gcc-remap-guix-store.patch +++ b/contrib/guix/patches/gcc-remap-guix-store.patch @@ -1,14 +1,9 @@ -From aad25427e74f387412e8bc9a9d7bbc6c496c792f Mon Sep 17 00:00:00 2001 -From: Andrew Chow <achow101-github@achow101.com> -Date: Wed, 6 Jul 2022 16:49:41 -0400 -Subject: [PATCH] guix: remap guix store paths to /usr +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. ---- - libgcc/Makefile.in | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/libgcc/Makefile.in b/libgcc/Makefile.in -index 851e7657d07..476c2becd1c 100644 --- a/libgcc/Makefile.in +++ b/libgcc/Makefile.in @@ -854,7 +854,7 @@ endif diff --git a/contrib/guix/patches/winpthreads-remap-guix-store.patch b/contrib/guix/patches/winpthreads-remap-guix-store.patch new file mode 100644 index 0000000000..e1f1a6eba5 --- /dev/null +++ b/contrib/guix/patches/winpthreads-remap-guix-store.patch @@ -0,0 +1,17 @@ +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +--- a/mingw-w64-libraries/winpthreads/Makefile.in ++++ b/mingw-w64-libraries/winpthreads/Makefile.in +@@ -478,7 +478,7 @@ top_build_prefix = @top_build_prefix@ + top_builddir = @top_builddir@ + top_srcdir = @top_srcdir@ + SUBDIRS = . tests +-AM_CFLAGS = -Wall -DWIN32_LEAN_AND_MEAN $(am__append_1) ++AM_CFLAGS = -Wall -DWIN32_LEAN_AND_MEAN $(am__append_1) $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) + ACLOCAL_AMFLAGS = -I m4 + lib_LTLIBRARIES = libwinpthread.la + include_HEADERS = include/pthread.h include/sched.h include/semaphore.h include/pthread_unistd.h include/pthread_time.h include/pthread_compat.h include/pthread_signal.h diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 24f6b29a26..74d9830705 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -76,6 +76,16 @@ def getFirstBlockFileId(block_dir_path): blkId = int(firstBlkFn[3:8]) return blkId +def read_xor_key(blocks_path): + NUM_XOR_BYTES = 8 # From InitBlocksdirXorKey::xor_key.size() + try: + xor_filename = os.path.join(blocks_path, "xor.dat") + with open(xor_filename, "rb") as xor_file: + return xor_file.read(NUM_XOR_BYTES) + # support also blockdirs created with pre-v28 versions, where no xor key exists yet + except FileNotFoundError: + return bytes([0] * NUM_XOR_BYTES) + # Block header and extent on disk BlockExtent = namedtuple('BlockExtent', ['fn', 'offset', 'inhdr', 'blkhdr', 'size']) @@ -95,6 +105,7 @@ class BlockDataCopier: self.outFname = None self.blkCountIn = 0 self.blkCountOut = 0 + self.xor_key = read_xor_key(self.settings['input']) self.lastDate = datetime.datetime(2000, 1, 1) self.highTS = 1408893517 - 315360000 @@ -113,6 +124,13 @@ class BlockDataCopier: self.outOfOrderData = {} self.outOfOrderSize = 0 # running total size for items in outOfOrderData + def read_xored(self, f, size): + offset = f.tell() + data = bytearray(f.read(size)) + for i in range(len(data)): + data[i] ^= self.xor_key[(i + offset) % len(self.xor_key)] + return bytes(data) + def writeBlock(self, inhdr, blk_hdr, rawblock): blockSizeOnDisk = len(inhdr) + len(blk_hdr) + len(rawblock) if not self.fileOutput and ((self.outsz + blockSizeOnDisk) > self.maxOutSz): @@ -165,7 +183,7 @@ class BlockDataCopier: '''Fetch block contents from disk given extents''' with open(self.inFileName(extent.fn), "rb") as f: f.seek(extent.offset) - return f.read(extent.size) + return self.read_xored(f, extent.size) def copyOneBlock(self): '''Find the next block to be written in the input, and copy it to the output.''' @@ -190,7 +208,7 @@ class BlockDataCopier: print("Premature end of block data") return - inhdr = self.inF.read(8) + inhdr = self.read_xored(self.inF, 8) if (not inhdr or (inhdr[0] == "\0")): self.inF.close() self.inF = None @@ -207,7 +225,7 @@ class BlockDataCopier: inLenLE = inhdr[4:] su = struct.unpack("<I", inLenLE) inLen = su[0] - 80 # length without header - blk_hdr = self.inF.read(80) + blk_hdr = self.read_xored(self.inF, 80) inExtent = BlockExtent(self.inFn, self.inF.tell(), inhdr, blk_hdr, inLen) self.hash_str = calc_hash_str(blk_hdr) @@ -224,7 +242,7 @@ class BlockDataCopier: if self.blkCountOut == blkHeight: # If in-order block, just copy - rawblock = self.inF.read(inLen) + rawblock = self.read_xored(self.inF, inLen) self.writeBlock(inhdr, blk_hdr, rawblock) # See if we can catch up to prior out-of-order blocks @@ -237,7 +255,7 @@ class BlockDataCopier: # If there is space in the cache, read the data # Reading the data in file sequence instead of seeking and fetching it later is preferred, # but we don't want to fill up memory - self.outOfOrderData[blkHeight] = self.inF.read(inLen) + self.outOfOrderData[blkHeight] = self.read_xored(self.inF, inLen) self.outOfOrderSize += inLen else: # If no space in cache, seek forward self.inF.seek(inLen, os.SEEK_CUR) diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py index f67e7b0f4c..72dd5d28cc 100755 --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -5,11 +5,12 @@ ''' Script to generate list of seed nodes for kernel/chainparams.cpp. -This script expects two text files in the directory that is passed as an +This script expects three text files in the directory that is passed as an argument: nodes_main.txt nodes_test.txt + nodes_testnet4.txt These files must consist of lines in the format @@ -171,6 +172,9 @@ def main(): g.write('\n') with open(os.path.join(indir,'nodes_test.txt'), 'r', encoding="utf8") as f: process_nodes(g, f, 'chainparams_seed_test') + g.write('\n') + with open(os.path.join(indir,'nodes_testnet4.txt'), 'r', encoding="utf8") as f: + process_nodes(g, f, 'chainparams_seed_testnet4') g.write('#endif // BITCOIN_CHAINPARAMSSEEDS_H\n') if __name__ == '__main__': |