Merge #13998: Scripts and tools: gitian-build.py improvements and corrections

0f22a0cf2f Fix gitian-build.py --verify option (Hennadii Stepanov)
4c56a798c0 Set/unset USE_LXC, USE_VBOX, USE_DOCKER explicitly (Hennadii Stepanov)
cbbd98863b Fix Docker related issues for gitian-build.py (Hennadii Stepanov)

Pull request description:

  1. The Docker does not depend on `apt-cacher-ng` package. Ref: #14002.

  2. Do not try to install the Docker if `docker.service` is detected on the system (e.g., the Docker was installed manually). Fix https://github.com/bitcoin/bitcoin/pull/13623#issuecomment-405684241 by **Sjors**.

  3. Prevent the setting of more than one environment variable for the `gitian-builder` (an alternative to #13999). E.g., USE_LXC being set shadows USE_DOCKER; for details see [`gitian-builder/libexec/make-clean-vm`](https://github.com/devrandom/gitian-builder/blob/93a62c7d7d018c66c02a19bac3d751144043cfec/libexec/make-clean-vm#L7):
  ```sh
  VMSW=KVM
  if [ -n "$USE_LXC" ]; then
      VMSW=LXC
  elif [ -n "$USE_VBOX" ]; then
      VMSW=VBOX
  elif [ -n "$USE_DOCKER" ]; then
      VMSW=DOCKER
  fi
  ```
  4. The [`gitian-builder/bin/gverify`](https://github.com/devrandom/gitian-builder/blob/master/bin/gverify) script returns the exit code 1 if a signature verification ends with 'BAD SIGNATURE' or 'MISMATCH' by design. This PR allows to see the verification results for all signatures without a premature fail of the `gitian-build.py` script. Ref: #14014.

ACKs for commit 0f22a0:

Tree-SHA512: 55f8a5cffa20d0c745f51a687f3199cea015fa616e56a0aee4c25b5ca0985036c61e8cf1922515338d8c6a85f873674ebe7a9a56a5069d65a187e383150f1a83

1 files changed, 41 insertions, 21 deletions

diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py
index 62d78bf164..07aeabc018 100755
--- a/contrib/gitian-build.py
+++ b/contrib/gitian-build.py
@@ -7,20 +7,20 @@ import sys
 
 def setup():
     global args, workdir
-    programs = ['ruby', 'git', 'apt-cacher-ng', 'make', 'wget']
+    programs = ['ruby', 'git', 'make', 'wget']
     if args.kvm:
-        programs += ['python-vm-builder', 'qemu-kvm', 'qemu-utils']
-    elif args.docker:
+        programs += ['apt-cacher-ng', 'python-vm-builder', 'qemu-kvm', 'qemu-utils']
+    elif args.docker and not os.path.isfile('/lib/systemd/system/docker.service'):
         dockers = ['docker.io', 'docker-ce']
         for i in dockers:
             return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i])
             if return_code == 0:
                 break
         if return_code != 0:
-            print('Cannot find any way to install docker', file=sys.stderr)
-            exit(1)
+            print('Cannot find any way to install Docker.', file=sys.stderr)
+            sys.exit(1)
     else:
-        programs += ['lxc', 'debootstrap']
+        programs += ['apt-cacher-ng', 'lxc', 'debootstrap']
     subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs)
     if not os.path.isdir('gitian.sigs'):
         subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/gitian.sigs.git'])
@@ -41,7 +41,7 @@ def setup():
     if args.is_bionic and not args.kvm and not args.docker:
         subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net'])
         print('Reboot is required')
-        exit(0)
+        sys.exit(0)
 
 def build():
     global args, workdir
@@ -118,20 +118,36 @@ def sign():
 
 def verify():
     global args, workdir
+    rc = 0
     os.chdir('gitian-builder')
 
     print('\nVerifying v'+args.version+' Linux\n')
-    subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml'])
+    if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml']):
+        print('Verifying v'+args.version+' Linux FAILED\n')
+        rc = 1
+
     print('\nVerifying v'+args.version+' Windows\n')
-    subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-win.yml'])
+    if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-win.yml']):
+        print('Verifying v'+args.version+' Windows FAILED\n')
+        rc = 1
+
     print('\nVerifying v'+args.version+' MacOS\n')
-    subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml'])
+    if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml']):
+        print('Verifying v'+args.version+' MacOS FAILED\n')
+        rc = 1
+
     print('\nVerifying v'+args.version+' Signed Windows\n')
-    subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-signed', '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml'])
+    if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-signed', '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml']):
+        print('Verifying v'+args.version+' Signed Windows FAILED\n')
+        rc = 1
+
     print('\nVerifying v'+args.version+' Signed MacOS\n')
-    subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-signed', '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml'])
+    if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-signed', '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml']):
+        print('Verifying v'+args.version+' Signed MacOS FAILED\n')
+        rc = 1
 
     os.chdir(workdir)
+    return rc
 
 def main():
     global args, workdir
@@ -149,7 +165,7 @@ def main():
     parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s')
     parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC')
     parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC')
-    parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Uses LXC. If you want to use KVM, use the --kvm option. Only works on Debian-based systems (Ubuntu, Debian)')
+    parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Only works on Debian-based systems (Ubuntu, Debian)')
     parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.')
     parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git')
     parser.add_argument('signer', help='GPG signer to sign each build assert file')
@@ -165,22 +181,26 @@ def main():
     args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs'])
 
     if args.buildsign:
-        args.build=True
-        args.sign=True
+        args.build = True
+        args.sign = True
 
     if args.kvm and args.docker:
         raise Exception('Error: cannot have both kvm and docker')
 
     args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign'
 
-    # Set environment variable USE_LXC or USE_DOCKER, let gitian-builder know that we use lxc or docker
+    # Ensure no more than one environment variable for gitian-builder (USE_LXC, USE_VBOX, USE_DOCKER) is set as they
+    # can interfere (e.g., USE_LXC being set shadows USE_DOCKER; for details see gitian-builder/libexec/make-clean-vm).
+    os.environ['USE_LXC'] = ''
+    os.environ['USE_VBOX'] = ''
+    os.environ['USE_DOCKER'] = ''
     if args.docker:
         os.environ['USE_DOCKER'] = '1'
     elif not args.kvm:
         os.environ['USE_LXC'] = '1'
-        if not 'GITIAN_HOST_IP' in os.environ.keys():
+        if 'GITIAN_HOST_IP' not in os.environ.keys():
             os.environ['GITIAN_HOST_IP'] = '10.0.3.1'
-        if not 'LXC_GUEST_IP' in os.environ.keys():
+        if 'LXC_GUEST_IP' not in os.environ.keys():
             os.environ['LXC_GUEST_IP'] = '10.0.3.5'
 
     # Disable for MacOS if no SDK found
@@ -193,11 +213,11 @@ def main():
     if args.signer == '':
         print(script_name+': Missing signer.')
         print('Try '+script_name+' --help for more information')
-        exit(1)
+        sys.exit(1)
     if args.version == '':
         print(script_name+': Missing version.')
         print('Try '+script_name+' --help for more information')
-        exit(1)
+        sys.exit(1)
 
     # Add leading 'v' for tags
     if args.commit and args.pull:
@@ -233,7 +253,7 @@ def main():
         os.chdir('gitian.sigs')
         subprocess.check_call(['git', 'pull'])
         os.chdir(workdir)
-        verify()
+        sys.exit(verify())
 
 if __name__ == '__main__':
     main()