Untangle precommit and travis test scripts (#288)

8 files changed, 89 insertions, 36 deletions

diff --git a/.travis.yml b/.travis.yml
index ab2492d2..dd7359af 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,8 +24,8 @@ before_script:
  - openssl req -x509 -newkey rsa:4096 -keyout server.key -out server.crt -days 365 -nodes -subj /CN=localhost
 
 script:
- - ./travis-install-kafka.sh
- - ./travis-test.sh
+ - ./scripts/install-local-kafka.sh
+ - ./scripts/travis-test.sh
 
 notifications:
     webhooks:
diff --git a/INSTALL.md b/INSTALL.md
index 1d002c5f..54addd28 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -17,7 +17,7 @@ Dendrite can be run in one of two configurations:
  - For Kafka (optional if using the monolith server):
    - Unix-based system (https://kafka.apache.org/documentation/#os)
    - JDK 1.8+ / OpenJDK 1.8+
-   - Apache Kafka 0.10.2+ (see [travis-install-kafka.sh](travis-install-kafka.sh) for up-to-date version numbers)
+   - Apache Kafka 0.10.2+ (see [scripts/install-local-kafka.sh](scripts/install-local-kafka.sh) for up-to-date version numbers)
 
 
 ## Setting up a development environment
@@ -34,7 +34,7 @@ go get github.com/constabulary/gb/...
 gb build
 ```
 
-If using Kafka, install and start it (c.f. [travis-install-kafka.sh](travis-install-kafka.sh)):
+If using Kafka, install and start it (c.f. [scripts/install-local-kafka.sh](scripts/install-local-kafka.sh)):
 ```bash
 MIRROR=http://apache.mirror.anlx.net/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz
 
diff --git a/hooks/pre-commit b/hooks/pre-commit
index 904d38dc..ec7e3e1e 100755
--- a/hooks/pre-commit
+++ b/hooks/pre-commit
@@ -2,33 +2,4 @@
 
 set -eu
 
-# Tune the GC to use more memory to reduce the number of garbage collections
-export GOGC=400
-export GOPATH="$(pwd):$(pwd)/vendor"
-export PATH="$PATH:$(pwd)/vendor/bin:$(pwd)/bin"
-
-echo "Checking that it builds"
-gb build
-
-# Check that all the packages can build.
-# When `go build` is given multiple packages it won't output anything, and just
-# checks that everything builds. This seems to do a better job of handling
-# missing imports than `gb build` does.
-echo "Double checking it builds..."
-go build github.com/matrix-org/dendrite/cmd/...
-
-echo "Installing lint search engine..."
-go install github.com/alecthomas/gometalinter/
-gometalinter --config=linter.json ./... --install
-
-echo "Looking for lint..."
-gometalinter --config=linter.json ./... --enable-gc
-
-echo "Double checking spelling..."
-misspell -error src *.md
-
-echo "Testing..."
-gb test
-
-
-echo "Done!"
+./scripts/build-test-lint.sh
diff --git a/scripts/README.md b/scripts/README.md
new file mode 100644
index 00000000..4d855ba4
--- /dev/null
+++ b/scripts/README.md
@@ -0,0 +1,7 @@
+# Dev Scripts
+
+These are a collection of scripts that should be helpful for those developing
+on dendrite.
+
+See `find-lint.sh` for environment variables that control linter resource
+usage.
diff --git a/scripts/build-test-lint.sh b/scripts/build-test-lint.sh
new file mode 100755
index 00000000..cf1f37b7
--- /dev/null
+++ b/scripts/build-test-lint.sh
@@ -0,0 +1,26 @@
+#! /bin/bash
+
+# Builds, tests and lints dendrite, and should be run before pushing commits
+
+set -eu
+
+export GOPATH="$(pwd):$(pwd)/vendor"
+export PATH="$PATH:$(pwd)/vendor/bin:$(pwd)/bin"
+
+echo "Checking that it builds"
+gb build
+
+# Check that all the packages can build.
+# When `go build` is given multiple packages it won't output anything, and just
+# checks that everything builds. This seems to do a better job of handling
+# missing imports than `gb build` does.
+echo "Double checking it builds..."
+go build github.com/matrix-org/dendrite/cmd/...
+
+./scripts/find-lint.sh
+
+echo "Double checking spelling..."
+misspell -error src *.md
+
+echo "Testing..."
+gb test
diff --git a/scripts/find-lint.sh b/scripts/find-lint.sh
new file mode 100755
index 00000000..81ca806b
--- /dev/null
+++ b/scripts/find-lint.sh
@@ -0,0 +1,41 @@
+#! /bin/bash
+
+# Runs the linters against dendrite
+
+# The linters can take a lot of resources and are slow, so they can be
+# configured using two environment variables:
+#
+# - `DENDRITE_LINT_CONCURRENCY` - number of concurrent linters to run,
+#   gometalinter defaults this to 8
+# - `DENDRITE_LINT_DISABLE_GC` - if set then the the go gc will be disabled
+#   when running the linters, speeding them up but using much more memory.
+
+
+set -eu
+
+export GOPATH="$(pwd):$(pwd)/vendor"
+export PATH="$PATH:$(pwd)/vendor/bin:$(pwd)/bin"
+
+args=""
+if [ ${1:-""} = "fast" ]
+then args="--config=linter-fast.json"
+else args="--config=linter.json"
+fi
+
+if [ -n "${DENDRITE_LINT_CONCURRENCY:-}" ]
+then args="$args --concurrency=$DENDRITE_LINT_CONCURRENCY"
+fi
+
+if [ -z "${DENDRITE_LINT_DISABLE_GC:-}" ]
+then args="$args --enable-gc"
+fi
+
+echo "Installing lint search engine..."
+go install github.com/alecthomas/gometalinter/
+gometalinter --config=linter.json ./... --install
+
+echo "Looking for lint..."
+gometalinter ./... $args
+
+echo "Double checking spelling..."
+misspell -error src *.md
diff --git a/travis-install-kafka.sh b/scripts/install-local-kafka.sh
index 32f95234..869d2891 100755
--- a/travis-install-kafka.sh
+++ b/scripts/install-local-kafka.sh
@@ -1,5 +1,7 @@
 # /bin/bash
 
+# Downloads, installs and runs a kafka instance
+
 set -eu
 
 # The mirror to download kafka from is picked from the list of mirrors at
diff --git a/travis-test.sh b/scripts/travis-test.sh
index 20d5f6fa..5851c3b0 100755
--- a/travis-test.sh
+++ b/scripts/travis-test.sh
@@ -1,7 +1,13 @@
 #! /bin/bash
 
+# The entry point for travis tests
+
 set -eu
 
+# Tune the GC to use more memory to reduce the number of garbage collections
+export GOGC=400
+export DENDRITE_LINT_DISABLE_GC=1
+
 # Check that the servers build (this is done explicitly because `gb build` can silently fail (exit 0) and then we'd test a stale binary)
 gb build github.com/matrix-org/dendrite/cmd/dendrite-room-server
 gb build github.com/matrix-org/dendrite/cmd/roomserver-integration-tests
@@ -12,8 +18,8 @@ gb build github.com/matrix-org/dendrite/cmd/dendrite-media-api-server
 gb build github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests
 gb build github.com/matrix-org/dendrite/cmd/client-api-proxy
 
-# Run the pre commit hooks
-./hooks/pre-commit
+# Run unit tests and linters
+./scripts/build-test-lint.sh
 
 # Run the integration tests
 bin/roomserver-integration-tests