aboutsummaryrefslogtreecommitdiff
path: root/contrib
diff options
context:
space:
mode:
Diffstat (limited to 'contrib')
-rw-r--r--contrib/devtools/README.md14
-rwxr-xr-xcontrib/devtools/circular-dependencies.py3
-rwxr-xr-xcontrib/devtools/copyright_header.py21
-rwxr-xr-xcontrib/devtools/gen-manpages.sh3
-rwxr-xr-xcontrib/devtools/security-check.py2
-rwxr-xr-xcontrib/devtools/symbol-check.py136
-rwxr-xr-xcontrib/filter-lcov.py3
-rwxr-xr-xcontrib/gitian-build.py3
-rw-r--r--contrib/gitian-descriptors/gitian-linux.yml2
-rw-r--r--contrib/gitian-descriptors/gitian-osx-signer.yml2
-rw-r--r--contrib/gitian-descriptors/gitian-osx.yml3
-rw-r--r--contrib/gitian-descriptors/gitian-win-signer.yml2
-rw-r--r--contrib/gitian-descriptors/gitian-win.yml2
-rwxr-xr-xcontrib/install_db4.sh3
-rwxr-xr-xcontrib/macdeploy/extract-osx-sdk.sh2
-rwxr-xr-xcontrib/seeds/makeseeds.py10
-rw-r--r--contrib/seeds/suspicious_hosts.txt16
-rw-r--r--contrib/valgrind.supp13
-rwxr-xr-xcontrib/verify-commits/pre-push-hook.sh2
19 files changed, 191 insertions, 51 deletions
diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md
index c35affac59..515a0d8fc6 100644
--- a/contrib/devtools/README.md
+++ b/contrib/devtools/README.md
@@ -103,17 +103,21 @@ Perform basic security checks on a series of executables.
symbol-check.py
===============
-A script to check that the (Linux) executables produced by gitian only contain
-allowed gcc, glibc and libstdc++ version symbols. This makes sure they are
-still compatible with the minimum supported Linux distribution versions.
+A script to check that the executables produced by gitian only contain
+certain symbols and are only linked against allowed libraries.
+
+For Linux this means checking for allowed gcc, glibc and libstdc++ version symbols.
+This makes sure they are still compatible with the minimum supported distribution versions.
+
+For macOS we check that the executables are only linked against libraries we allow.
Example usage after a gitian build:
find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py
-If only supported symbols are used the return value will be 0 and the output will be empty.
+If no errors occur the return value will be 0 and the output will be empty.
-If there are 'unsupported' symbols, the return value will be 1 a list like this will be printed:
+If there are any errors the return value will be 1 and output like this will be printed:
.../64/test_bitcoin: symbol memcpy from unsupported version GLIBC_2.14
.../64/test_bitcoin: symbol __fdelt_chk from unsupported version GLIBC_2.15
diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py
index 2e4657f1dd..6afa4351e7 100755
--- a/contrib/devtools/circular-dependencies.py
+++ b/contrib/devtools/circular-dependencies.py
@@ -1,4 +1,7 @@
#!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
import sys
import re
diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py
index 5307b04197..1b71245aab 100755
--- a/contrib/devtools/copyright_header.py
+++ b/contrib/devtools/copyright_header.py
@@ -19,6 +19,8 @@ EXCLUDE = [
'src/qt/bitcoinstrings.cpp',
'src/chainparamsseeds.h',
# other external copyrights:
+ 'src/reverse_iterator.h',
+ 'src/test/fuzz/FuzzedDataProvider.h',
'src/tinyformat.h',
'test/functional/test_framework/bignum.py',
# python init:
@@ -455,14 +457,14 @@ CPP_HEADER = '''
def get_cpp_header_lines_to_insert(start_year, end_year):
return reversed(get_header_lines(CPP_HEADER, start_year, end_year))
-PYTHON_HEADER = '''
+SCRIPT_HEADER = '''
# Copyright (c) %s The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
'''
-def get_python_header_lines_to_insert(start_year, end_year):
- return reversed(get_header_lines(PYTHON_HEADER, start_year, end_year))
+def get_script_header_lines_to_insert(start_year, end_year):
+ return reversed(get_header_lines(SCRIPT_HEADER, start_year, end_year))
################################################################################
# query git for year of last change
@@ -491,17 +493,18 @@ def file_has_hashbang(file_lines):
return False
return file_lines[0][:2] == '#!'
-def insert_python_header(filename, file_lines, start_year, end_year):
+def insert_script_header(filename, file_lines, start_year, end_year):
if file_has_hashbang(file_lines):
insert_idx = 1
else:
insert_idx = 0
- header_lines = get_python_header_lines_to_insert(start_year, end_year)
+ header_lines = get_script_header_lines_to_insert(start_year, end_year)
for line in header_lines:
file_lines.insert(insert_idx, line)
write_file_lines(filename, file_lines)
def insert_cpp_header(filename, file_lines, start_year, end_year):
+ file_lines.insert(0, '\n')
header_lines = get_cpp_header_lines_to_insert(start_year, end_year)
for line in header_lines:
file_lines.insert(0, line)
@@ -513,8 +516,8 @@ def exec_insert_header(filename, style):
sys.exit('*** %s already has a copyright by The Bitcoin Core developers'
% (filename))
start_year, end_year = get_git_change_year_range(filename)
- if style == 'python':
- insert_python_header(filename, file_lines, start_year, end_year)
+ if style in ['python', 'shell']:
+ insert_script_header(filename, file_lines, start_year, end_year)
else:
insert_cpp_header(filename, file_lines, start_year, end_year)
@@ -555,11 +558,13 @@ def insert_cmd(argv):
if not os.path.isfile(filename):
sys.exit("*** bad filename: %s" % filename)
_, extension = os.path.splitext(filename)
- if extension not in ['.h', '.cpp', '.cc', '.c', '.py']:
+ if extension not in ['.h', '.cpp', '.cc', '.c', '.py', '.sh']:
sys.exit("*** cannot insert for file extension %s" % extension)
if extension == '.py':
style = 'python'
+ elif extension == '.sh':
+ style = 'shell'
else:
style = 'cpp'
exec_insert_header(filename, style)
diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh
index dbdb622877..aa65953d83 100755
--- a/contrib/devtools/gen-manpages.sh
+++ b/contrib/devtools/gen-manpages.sh
@@ -1,4 +1,7 @@
#!/usr/bin/env bash
+# Copyright (c) 2016-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
export LC_ALL=C
TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)}
diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py
index 9941c57479..21d64e893d 100755
--- a/contrib/devtools/security-check.py
+++ b/contrib/devtools/security-check.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-# Copyright (c) 2015-2018 The Bitcoin Core developers
+# Copyright (c) 2015-2020 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
'''
diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py
index 0c59ab6239..f92d997621 100755
--- a/contrib/devtools/symbol-check.py
+++ b/contrib/devtools/symbol-check.py
@@ -15,6 +15,7 @@ import subprocess
import re
import sys
import os
+from typing import List, Optional, Tuple
# Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases
#
@@ -52,8 +53,10 @@ IGNORE_EXPORTS = {
}
READELF_CMD = os.getenv('READELF', '/usr/bin/readelf')
CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt')
+OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool')
+
# Allowed NEEDED libraries
-ALLOWED_LIBRARIES = {
+ELF_ALLOWED_LIBRARIES = {
# bitcoind and bitcoin-qt
'libgcc_s.so.1', # GCC base support
'libc.so.6', # C library
@@ -79,6 +82,25 @@ ARCH_MIN_GLIBC_VER = {
'AArch64':(2,17),
'RISC-V': (2,27)
}
+
+MACHO_ALLOWED_LIBRARIES = {
+# bitcoind and bitcoin-qt
+'libc++.1.dylib', # C++ Standard Library
+'libSystem.B.dylib', # libc, libm, libpthread, libinfo
+# bitcoin-qt only
+'AppKit', # user interface
+'ApplicationServices', # common application tasks.
+'Carbon', # deprecated c back-compat API
+'CoreFoundation', # low level func, data types
+'CoreGraphics', # 2D rendering
+'CoreServices', # operating system services
+'CoreText', # interface for laying out text and handling fonts.
+'Foundation', # base layer functionality for apps/frameworks
+'ImageIO', # read and write image file formats.
+'IOKit', # user-space access to hardware devices and drivers.
+'libobjc.A.dylib', # Objective-C runtime library
+}
+
class CPPFilt(object):
'''
Demangle C++ symbol names.
@@ -98,15 +120,15 @@ class CPPFilt(object):
self.proc.stdout.close()
self.proc.wait()
-def read_symbols(executable, imports=True):
+def read_symbols(executable, imports=True) -> List[Tuple[str, str, str]]:
'''
- Parse an ELF executable and return a list of (symbol,version) tuples
+ Parse an ELF executable and return a list of (symbol,version, arch) tuples
for dynamic, imported symbols.
'''
p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', '-h', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
(stdout, stderr) = p.communicate()
if p.returncode:
- raise IOError('Could not read symbols for %s: %s' % (executable, stderr.strip()))
+ raise IOError('Could not read symbols for {}: {}'.format(executable, stderr.strip()))
syms = []
for line in stdout.splitlines():
line = line.split()
@@ -121,7 +143,7 @@ def read_symbols(executable, imports=True):
syms.append((sym, version, arch))
return syms
-def check_version(max_versions, version, arch):
+def check_version(max_versions, version, arch) -> bool:
if '_' in version:
(lib, _, ver) = version.rpartition('_')
else:
@@ -132,7 +154,7 @@ def check_version(max_versions, version, arch):
return False
return ver <= max_versions[lib] or lib == 'GLIBC' and ver <= ARCH_MIN_GLIBC_VER[arch]
-def read_libraries(filename):
+def elf_read_libraries(filename) -> List[str]:
p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
(stdout, stderr) = p.communicate()
if p.returncode:
@@ -148,26 +170,94 @@ def read_libraries(filename):
raise ValueError('Unparseable (NEEDED) specification')
return libraries
-if __name__ == '__main__':
+def check_imported_symbols(filename) -> bool:
cppfilt = CPPFilt()
+ ok = True
+ for sym, version, arch in read_symbols(filename, True):
+ if version and not check_version(MAX_VERSIONS, version, arch):
+ print('{}: symbol {} from unsupported version {}'.format(filename, cppfilt(sym), version))
+ ok = False
+ return ok
+
+def check_exported_symbols(filename) -> bool:
+ cppfilt = CPPFilt()
+ ok = True
+ for sym,version,arch in read_symbols(filename, False):
+ if arch == 'RISC-V' or sym in IGNORE_EXPORTS:
+ continue
+ print('{}: export of symbol {} not allowed'.format(filename, cppfilt(sym)))
+ ok = False
+ return ok
+
+def check_ELF_libraries(filename) -> bool:
+ ok = True
+ for library_name in elf_read_libraries(filename):
+ if library_name not in ELF_ALLOWED_LIBRARIES:
+ print('{}: NEEDED library {} is not allowed'.format(filename, library_name))
+ ok = False
+ return ok
+
+def macho_read_libraries(filename) -> List[str]:
+ p = subprocess.Popen([OTOOL_CMD, '-L', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
+ (stdout, stderr) = p.communicate()
+ if p.returncode:
+ raise IOError('Error opening file')
+ libraries = []
+ for line in stdout.splitlines():
+ tokens = line.split()
+ if len(tokens) == 1: # skip executable name
+ continue
+ libraries.append(tokens[0].split('/')[-1])
+ return libraries
+
+def check_MACHO_libraries(filename) -> bool:
+ ok = True
+ for dylib in macho_read_libraries(filename):
+ if dylib not in MACHO_ALLOWED_LIBRARIES:
+ print('{} is not in ALLOWED_LIBRARIES!'.format(dylib))
+ ok = False
+ return ok
+
+CHECKS = {
+'ELF': [
+ ('IMPORTED_SYMBOLS', check_imported_symbols),
+ ('EXPORTED_SYMBOLS', check_exported_symbols),
+ ('LIBRARY_DEPENDENCIES', check_ELF_libraries)
+],
+'MACHO': [
+ ('DYNAMIC_LIBRARIES', check_MACHO_libraries)
+]
+}
+
+def identify_executable(executable) -> Optional[str]:
+ with open(filename, 'rb') as f:
+ magic = f.read(4)
+ if magic.startswith(b'MZ'):
+ return 'PE'
+ elif magic.startswith(b'\x7fELF'):
+ return 'ELF'
+ elif magic.startswith(b'\xcf\xfa'):
+ return 'MACHO'
+ return None
+
+if __name__ == '__main__':
retval = 0
for filename in sys.argv[1:]:
- # Check imported symbols
- for sym,version,arch in read_symbols(filename, True):
- if version and not check_version(MAX_VERSIONS, version, arch):
- print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym), version))
- retval = 1
- # Check exported symbols
- if arch != 'RISC-V':
- for sym,version,arch in read_symbols(filename, False):
- if sym in IGNORE_EXPORTS:
- continue
- print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym)))
- retval = 1
- # Check dependency libraries
- for library_name in read_libraries(filename):
- if library_name not in ALLOWED_LIBRARIES:
- print('%s: NEEDED library %s is not allowed' % (filename, library_name))
+ try:
+ etype = identify_executable(filename)
+ if etype is None:
+ print('{}: unknown format'.format(filename))
retval = 1
+ continue
+ failed = []
+ for (name, func) in CHECKS[etype]:
+ if not func(filename):
+ failed.append(name)
+ if failed:
+ print('{}: failed {}'.format(filename, ' '.join(failed)))
+ retval = 1
+ except IOError:
+ print('{}: cannot open'.format(filename))
+ retval = 1
sys.exit(retval)
diff --git a/contrib/filter-lcov.py b/contrib/filter-lcov.py
index df1db76e92..75034616f7 100755
--- a/contrib/filter-lcov.py
+++ b/contrib/filter-lcov.py
@@ -1,4 +1,7 @@
#!/usr/bin/env python3
+# Copyright (c) 2017-2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
import argparse
diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py
index e38fa6fcb0..ade9e00d1b 100755
--- a/contrib/gitian-build.py
+++ b/contrib/gitian-build.py
@@ -1,4 +1,7 @@
#!/usr/bin/env python3
+# Copyright (c) 2018-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
import argparse
import os
diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml
index 2b86602a82..257dd8ba30 100644
--- a/contrib/gitian-descriptors/gitian-linux.yml
+++ b/contrib/gitian-descriptors/gitian-linux.yml
@@ -5,7 +5,7 @@ distro: "ubuntu"
suites:
- "bionic"
architectures:
-- "linux64"
+- "amd64"
packages:
- "curl"
- "g++-aarch64-linux-gnu"
diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml
index 2d49493641..a4f3219c22 100644
--- a/contrib/gitian-descriptors/gitian-osx-signer.yml
+++ b/contrib/gitian-descriptors/gitian-osx-signer.yml
@@ -4,7 +4,7 @@ distro: "ubuntu"
suites:
- "bionic"
architectures:
-- "linux64"
+- "amd64"
packages:
- "faketime"
remotes:
diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml
index 75040c137f..7c5abb9018 100644
--- a/contrib/gitian-descriptors/gitian-osx.yml
+++ b/contrib/gitian-descriptors/gitian-osx.yml
@@ -5,7 +5,7 @@ distro: "ubuntu"
suites:
- "bionic"
architectures:
-- "linux64"
+- "amd64"
packages:
- "ca-certificates"
- "curl"
@@ -138,6 +138,7 @@ script: |
CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS}
make ${MAKEOPTS}
make ${MAKEOPTS} -C src check-security
+ make ${MAKEOPTS} -C src check-symbols
make install-strip DESTDIR=${INSTALLPATH}
make osx_volname
diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml
index 70b7bb111d..9d96465742 100644
--- a/contrib/gitian-descriptors/gitian-win-signer.yml
+++ b/contrib/gitian-descriptors/gitian-win-signer.yml
@@ -4,7 +4,7 @@ distro: "ubuntu"
suites:
- "bionic"
architectures:
-- "linux64"
+- "amd64"
packages:
- "libssl-dev"
- "autoconf"
diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml
index b772404ae5..de2e45190a 100644
--- a/contrib/gitian-descriptors/gitian-win.yml
+++ b/contrib/gitian-descriptors/gitian-win.yml
@@ -5,7 +5,7 @@ distro: "ubuntu"
suites:
- "bionic"
architectures:
-- "linux64"
+- "amd64"
packages:
- "curl"
- "g++"
diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh
index 4b848dda99..e9130a21de 100755
--- a/contrib/install_db4.sh
+++ b/contrib/install_db4.sh
@@ -1,4 +1,7 @@
#!/bin/sh
+# Copyright (c) 2017-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
# Install libdb4.8 (Berkeley DB).
diff --git a/contrib/macdeploy/extract-osx-sdk.sh b/contrib/macdeploy/extract-osx-sdk.sh
index 3fa05cafac..21243ada04 100755
--- a/contrib/macdeploy/extract-osx-sdk.sh
+++ b/contrib/macdeploy/extract-osx-sdk.sh
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-# Copyright (c) 2016-2018 The Bitcoin Core developers
+# Copyright (c) 2016-2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py
index b90a628781..d516ca10c1 100755
--- a/contrib/seeds/makeseeds.py
+++ b/contrib/seeds/makeseeds.py
@@ -19,13 +19,9 @@ MIN_BLOCKS = 337600
# These are hosts that have been observed to be behaving strangely (e.g.
# aggressively connecting to every node).
-SUSPICIOUS_HOSTS = {
- "130.211.129.106", "178.63.107.226",
- "83.81.130.26", "88.198.17.7", "148.251.238.178", "176.9.46.6",
- "54.173.72.127", "54.174.10.182", "54.183.64.54", "54.194.231.211",
- "54.66.214.167", "54.66.220.137", "54.67.33.14", "54.77.251.214",
- "54.94.195.96", "54.94.200.247"
-}
+with open("suspicious_hosts.txt", mode="r", encoding="utf-8") as f:
+ SUSPICIOUS_HOSTS = {s.strip() for s in f if s.strip()}
+
PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$")
PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$")
diff --git a/contrib/seeds/suspicious_hosts.txt b/contrib/seeds/suspicious_hosts.txt
new file mode 100644
index 0000000000..13385cc816
--- /dev/null
+++ b/contrib/seeds/suspicious_hosts.txt
@@ -0,0 +1,16 @@
+130.211.129.106
+148.251.238.178
+176.9.46.6
+178.63.107.226
+54.173.72.127
+54.174.10.182
+54.183.64.54
+54.194.231.211
+54.66.214.167
+54.66.220.137
+54.67.33.14
+54.77.251.214
+54.94.195.96
+54.94.200.247
+83.81.130.26
+88.198.17.7 \ No newline at end of file
diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp
index f232bb62c2..744b8ee70f 100644
--- a/contrib/valgrind.supp
+++ b/contrib/valgrind.supp
@@ -184,3 +184,16 @@
...
fun:_ZN5BCLog6Logger12StartLoggingEv
}
+{
+ Suppress BCLog::Logger::StartLogging() still reachable memory warning
+ Memcheck:Leak
+ match-leak-kinds: reachable
+ fun:malloc
+ ...
+ fun:_ZN5BCLog6Logger12StartLoggingEv
+}
+{
+ Suppress rest_blockhash_by_height Conditional jump or move depends on uninitialised value(s)
+ Memcheck:Cond
+ fun:_ZL24rest_blockhash_by_heightP11HTTPRequestRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE
+}
diff --git a/contrib/verify-commits/pre-push-hook.sh b/contrib/verify-commits/pre-push-hook.sh
index 2e15a6326d..a26791f0d1 100755
--- a/contrib/verify-commits/pre-push-hook.sh
+++ b/contrib/verify-commits/pre-push-hook.sh
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-# Copyright (c) 2014-2018 The Bitcoin Core developers
+# Copyright (c) 2014-2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
generated by cgit v1.2.3 (git 2.26.2) at 2024-10-06 21:25:59 +0000