Master pull-tester-driver python script

1 files changed, 174 insertions, 0 deletions

diff --git a/qa/pull-tester/pull-tester.py b/qa/pull-tester/pull-tester.py
new file mode 100755
index 0000000000..7ca2bcf04d
--- /dev/null
+++ b/qa/pull-tester/pull-tester.py
@@ -0,0 +1,174 @@
+#!/usr/bin/python
+import json
+from urllib import urlopen
+import requests
+import getpass
+from string import Template
+import sys
+import os
+import subprocess
+
+class RunError(Exception):
+    def __init__(self, value):
+        self.value = value
+    def __str__(self):
+        return repr(self.value)
+
+def run(command, **kwargs):
+    fail_hard = kwargs.pop("fail_hard", True)
+    # output to /dev/null by default:
+    kwargs.setdefault("stdout", open('/dev/null', 'w'))
+    kwargs.setdefault("stderr", open('/dev/null', 'w'))
+    command = Template(command).substitute(os.environ)
+    if "TRACE" in os.environ:
+        print(command)
+    process = subprocess.Popen(command.split(' '), **kwargs)
+    process.wait()
+    if process.returncode != 0 and fail_hard:
+        raise RunError("Failed: "+command)
+    return process.returncode
+
+def checkout_pull(clone_url, commit, out):
+    # Init
+    build_dir=os.environ["BUILD_DIR"]
+    run("umount ${CHROOT_COPY}/proc", fail_hard=False)
+    run("rsync --delete -apv ${CHROOT_MASTER} ${CHROOT_COPY}")
+    run("cp -a ${SCRIPTS_DIR} ${CHROOT_COPY}${SCRIPTS_DIR}")
+    # Merge onto upstream/master
+    run("rm -rf ${BUILD_DIR}")
+    run("mkdir -p ${BUILD_DIR}")
+    run("git clone ${CLONE_URL} ${BUILD_DIR}")
+    run("git remote add pull "+clone_url, cwd=build_dir, stdout=out, stderr=out)
+    run("git fetch pull", cwd=build_dir, stdout=out, stderr=out)
+    if run("git merge "+ commit, fail_hard=False, cwd=build_dir, stdout=out, stderr=out) != 0:
+        return False
+    run("chown -R ${BUILD_USER}:${BUILD_GROUP} ${BUILD_DIR}", stdout=out, stderr=out)
+    run("mount --bind /proc ${CHROOT_COPY}/proc")
+    return True
+
+def commentOn(commentUrl, success, inMerge, needTests, linkUrl):
+    common_message = """
+This test script verifies pulls every time they are updated. It, however, dies sometimes and fails to test properly.  If you are waiting on a test, please check timestamps to verify that the test.log is moving at http://jenkins.bluematt.me/pull-tester/current/
+Contact BlueMatt on freenode if something looks broken."""
+
+    # Remove old BitcoinPullTester comments (I'm being lazy and not paginating here)
+    recentcomments = requests.get(commentUrl+"?sort=created&direction=desc",
+                                  auth=(os.environ['GITHUB_USER'], os.environ["GITHUB_AUTH_TOKEN"])).json
+    for comment in recentcomments:
+        if comment["user"]["login"] == os.environ["GITHUB_USER"] and common_message in comment["body"]:
+            requests.delete(comment["url"],
+                                  auth=(os.environ['GITHUB_USER'], os.environ["GITHUB_AUTH_TOKEN"]))
+
+    if success == True:
+        post_data = { "body" : "Automatic sanity-testing: PASSED, see " + linkUrl + " for binaries and test log." + common_message}
+    elif inMerge:
+        post_data = { "body" : "Automatic sanity-testing: FAILED MERGE, see " + linkUrl + " for test log." + """
+
+This pull does not merge cleanly onto current master""" + common_message}
+    else:
+        post_data = { "body" : "Automatic sanity-testing: FAILED BUILD/TEST, see " + linkUrl + " for binaries and test log." + """
+
+This could happen for one of several reasons:
+1. It chanages paths in makefile.linux-mingw or otherwise changes build scripts in a way that made them incompatible with the automated testing scripts (please tweak those patches in qa/pull-tester)
+2. It adds/modifies tests which test network rules (thanks for doing that), which conflicts with a patch applied at test time
+3. It does not build on either Linux i386 or Win32 (via MinGW cross compile)
+4. The test suite fails on either Linux i386 or Win32
+5. The block test-cases failed (lookup the first bNN identifier which failed in https://github.com/TheBlueMatt/test-scripts/blob/master/FullBlockTestGenerator.java)
+
+If you believe this to be in error, please ping BlueMatt on freenode or TheBlueMatt here.
+""" + common_message}
+
+    resp = requests.post(commentUrl, json.dumps(post_data), auth=(os.environ['GITHUB_USER'], os.environ["GITHUB_AUTH_TOKEN"]))
+
+def testpull(number, comment_url, clone_url, commit):
+    print("Testing pull %d: %s : %s"%(number, clone_url,commit))
+
+    dir = os.environ["RESULTS_DIR"] + "/" + commit + "/"
+    if os.path.exists(dir):
+        os.system("rm -r " + dir)
+    os.makedirs(dir)
+    currentdir = os.environ["RESULTS_DIR"] + "/current"
+    os.system("rm -r "+currentdir)
+    os.system("ln -s " + dir + " " + currentdir)
+    out = open(dir + "test.log", 'w+')
+
+    resultsurl = os.environ["RESULTS_URL"] + commit
+    checkedout = checkout_pull(clone_url, commit, out)
+    if checkedout != True:
+        print("Failed to test pull - sending comment to: " + comment_url)
+        commentOn(comment_url, False, True, False, resultsurl)
+        open(os.environ["TESTED_DB"], "a").write(commit + "\n")
+        return
+
+    # New: pull-tester.sh script(s) are in the tree:
+    script = os.environ["BUILD_PATH"]+"/qa/pull-tester/pull-tester.sh"
+    script += " ${BUILD_PATH} ${MINGW_DEPS_DIR} ${SCRIPTS_DIR}/BitcoinjBitcoindComparisonTool.jar 6"
+    returncode = run("chroot ${CHROOT_COPY} sudo -u ${BUILD_USER} -H timeout ${TEST_TIMEOUT} "+script,
+                     fail_hard=False, stdout=out, stderr=out)
+
+    run("mv ${BUILD_DIR} " + dir)
+    # TODO: FIXME 
+    # Idea: have run-script save interesting output...
+    #    run("cp /mnt/chroot-tmp/home/ubuntu/.bitcoin/regtest/debug.log " + dir)
+    #    os.system("chmod +r " + dir + "/debug.log")
+    if returncode == 42:
+        print("Successfully tested pull (needs tests) - sending comment to: " + comment_url)
+        commentOn(comment_url, True, False, True, resultsurl)
+    elif returncode != 0:
+        print("Failed to test pull - sending comment to: " + comment_url)
+        commentOn(comment_url, False, False, False, resultsurl)
+    else:
+        print("Successfully tested pull - sending comment to: " + comment_url)
+        commentOn(comment_url, True, False, False, resultsurl)
+    open(os.environ["TESTED_DB"], "a").write(commit + "\n")
+
+def environ_default(setting, value):
+    if not setting in os.environ:
+        os.environ[setting] = value
+
+if getpass.getuser() != "root":
+	print("Run me as root!")
+	sys.exit(1)
+
+if "GITHUB_USER" not in os.environ or "GITHUB_AUTH_TOKEN" not in os.environ:
+    print("GITHUB_USER and/or GITHUB_AUTH_TOKEN environment variables not set")
+    sys.exit(1)
+
+environ_default("CLONE_URL", "https://github.com/bitcoin/bitcoin.git")
+environ_default("MINGW_DEPS_DIR", "/mnt/w32deps")
+environ_default("SCRIPTS_DIR", "/mnt/test-scripts")
+environ_default("CHROOT_COPY", "/mnt/chroot-tmp")
+environ_default("CHROOT_MASTER", "/mnt/chroot")
+environ_default("BUILD_PATH", "/mnt/bitcoin")
+os.environ["BUILD_DIR"] = os.environ["CHROOT_COPY"] + os.environ["BUILD_PATH"]
+environ_default("RESULTS_DIR", "/mnt/www/pull-tester")
+environ_default("RESULTS_URL", "http://jenkins.bluematt.me/pull-tester/")
+environ_default("GITHUB_REPO", "bitcoin/bitcoin")
+environ_default("TESTED_DB", "/mnt/commits-tested.txt")
+environ_default("BUILD_USER", "matt")
+environ_default("BUILD_GROUP", "matt")
+environ_default("TEST_TIMEOUT", str(60*60*2))
+
+print("Optional usage: pull-tester.py 2112")
+
+f = open(os.environ["TESTED_DB"])
+tested = set( line.rstrip() for line in f.readlines() )
+f.close()
+
+if len(sys.argv) > 1:
+    pull = requests.get("https://api.github.com/repos/"+os.environ["GITHUB_REPO"]+"/pulls/"+sys.argv[1],
+                        auth=(os.environ['GITHUB_USER'], os.environ["GITHUB_AUTH_TOKEN"])).json
+    testpull(pull["number"], pull["_links"]["comments"]["href"],
+             pull["head"]["repo"]["clone_url"], pull["head"]["sha"])
+
+else:
+    for page in range(1,100):
+        result = requests.get("https://api.github.com/repos/"+os.environ["GITHUB_REPO"]+"/pulls?state=open&page=%d"%(page,),
+                              auth=(os.environ['GITHUB_USER'], os.environ["GITHUB_AUTH_TOKEN"])).json
+        if len(result) == 0: break;
+        for pull in result:
+            if pull["head"]["sha"] in tested:
+                print("Pull %d already tested"%(pull["number"],))
+                continue
+            testpull(pull["number"], pull["_links"]["comments"]["href"],
+                     pull["head"]["repo"]["clone_url"], pull["head"]["sha"])