bugfix: E2EE device keys could sometimes not be sent to remote servers (#2466)

* Fix flakey sytest 'Local device key changes get to remote servers'

* Debug logs

* Remove internal/test and use /test only

Remove a lot of ancient code too.

* Use FederationRoomserverAPI in more places

* Use more interfaces in federationapi; begin adding regression test

* Linting

* Add regression test

* Unbreak tests

* ALL THE LOGS

* Fix a race condition which could cause events to not be sent to servers

If a new room event which rewrites state arrives, we remove all joined hosts
then re-calculate them. This wasn't done in a transaction so for a brief period
we would have no joined hosts. During this interim, key change events which arrive
would not be sent to destination servers. This would sporadically fail on sytest.

* Unbreak new tests

* Linting

7 files changed, 1 insertions, 725 deletions

diff --git a/internal/caching/cache_typing_test.go b/internal/caching/cache_typing_test.go
index c03d89bc..2cef32d3 100644
--- a/internal/caching/cache_typing_test.go
+++ b/internal/caching/cache_typing_test.go
@@ -20,7 +20,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/matrix-org/dendrite/internal/test"
+	"github.com/matrix-org/dendrite/test"
 )
 
 func TestEDUCache(t *testing.T) {
diff --git a/internal/test/client.go b/internal/test/client.go
deleted file mode 100644
index a38540ac..00000000
--- a/internal/test/client.go
+++ /dev/null
@@ -1,158 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package test
-
-import (
-	"crypto/tls"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// Request contains the information necessary to issue a request and test its result
-type Request struct {
-	Req              *http.Request
-	WantedBody       string
-	WantedStatusCode int
-	LastErr          *LastRequestErr
-}
-
-// LastRequestErr is a synchronised error wrapper
-// Useful for obtaining the last error from a set of requests
-type LastRequestErr struct {
-	sync.Mutex
-	Err error
-}
-
-// Set sets the error
-func (r *LastRequestErr) Set(err error) {
-	r.Lock()
-	defer r.Unlock()
-	r.Err = err
-}
-
-// Get gets the error
-func (r *LastRequestErr) Get() error {
-	r.Lock()
-	defer r.Unlock()
-	return r.Err
-}
-
-// CanonicalJSONInput canonicalises a slice of JSON strings
-// Useful for test input
-func CanonicalJSONInput(jsonData []string) []string {
-	for i := range jsonData {
-		jsonBytes, err := gomatrixserverlib.CanonicalJSON([]byte(jsonData[i]))
-		if err != nil && err != io.EOF {
-			panic(err)
-		}
-		jsonData[i] = string(jsonBytes)
-	}
-	return jsonData
-}
-
-// Do issues a request and checks the status code and body of the response
-func (r *Request) Do() (err error) {
-	client := &http.Client{
-		Timeout: 5 * time.Second,
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: true,
-			},
-		},
-	}
-	res, err := client.Do(r.Req)
-	if err != nil {
-		return err
-	}
-	defer (func() { err = res.Body.Close() })()
-
-	if res.StatusCode != r.WantedStatusCode {
-		return fmt.Errorf("incorrect status code. Expected: %d  Got: %d", r.WantedStatusCode, res.StatusCode)
-	}
-
-	if r.WantedBody != "" {
-		resBytes, err := ioutil.ReadAll(res.Body)
-		if err != nil {
-			return err
-		}
-		jsonBytes, err := gomatrixserverlib.CanonicalJSON(resBytes)
-		if err != nil {
-			return err
-		}
-		if string(jsonBytes) != r.WantedBody {
-			return fmt.Errorf("returned wrong bytes. Expected:\n%s\n\nGot:\n%s", r.WantedBody, string(jsonBytes))
-		}
-	}
-
-	return nil
-}
-
-// DoUntilSuccess blocks and repeats the same request until the response returns the desired status code and body.
-// It then closes the given channel and returns.
-func (r *Request) DoUntilSuccess(done chan error) {
-	r.LastErr = &LastRequestErr{}
-	for {
-		if err := r.Do(); err != nil {
-			r.LastErr.Set(err)
-			time.Sleep(1 * time.Second) // don't tightloop
-			continue
-		}
-		close(done)
-		return
-	}
-}
-
-// Run repeatedly issues a request until success, error or a timeout is reached
-func (r *Request) Run(label string, timeout time.Duration, serverCmdChan chan error) {
-	fmt.Printf("==TESTING== %v (timeout: %v)\n", label, timeout)
-	done := make(chan error, 1)
-
-	// We need to wait for the server to:
-	// - have connected to the database
-	// - have created the tables
-	// - be listening on the given port
-	go r.DoUntilSuccess(done)
-
-	// wait for one of:
-	// - the test to pass (done channel is closed)
-	// - the server to exit with an error (error sent on serverCmdChan)
-	// - our test timeout to expire
-	// We don't need to clean up since the main() function handles that in the event we panic
-	select {
-	case <-time.After(timeout):
-		fmt.Printf("==TESTING== %v TIMEOUT\n", label)
-		if reqErr := r.LastErr.Get(); reqErr != nil {
-			fmt.Println("Last /sync request error:")
-			fmt.Println(reqErr)
-		}
-		panic(fmt.Sprintf("%v server timed out", label))
-	case err := <-serverCmdChan:
-		if err != nil {
-			fmt.Println("=============================================================================================")
-			fmt.Printf("%v server failed to run. If failing with 'pq: password authentication failed for user' try:", label)
-			fmt.Println("    export PGHOST=/var/run/postgresql")
-			fmt.Println("=============================================================================================")
-			panic(err)
-		}
-	case <-done:
-		fmt.Printf("==TESTING== %v PASSED\n", label)
-	}
-}
diff --git a/internal/test/config.go b/internal/test/config.go
deleted file mode 100644
index d8e0c453..00000000
--- a/internal/test/config.go
+++ /dev/null
@@ -1,273 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package test
-
-import (
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/x509"
-	"encoding/base64"
-	"encoding/pem"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"math/big"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
-	"gopkg.in/yaml.v2"
-)
-
-const (
-	// ConfigFile is the name of the config file for a server.
-	ConfigFile = "dendrite.yaml"
-	// ServerKeyFile is the name of the file holding the matrix server private key.
-	ServerKeyFile = "server_key.pem"
-	// TLSCertFile is the name of the file holding the TLS certificate used for federation.
-	TLSCertFile = "tls_cert.pem"
-	// TLSKeyFile is the name of the file holding the TLS key used for federation.
-	TLSKeyFile = "tls_key.pem"
-	// MediaDir is the name of the directory used to store media.
-	MediaDir = "media"
-)
-
-// MakeConfig makes a config suitable for running integration tests.
-// Generates new matrix and TLS keys for the server.
-func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*config.Dendrite, int, error) {
-	var cfg config.Dendrite
-	cfg.Defaults(true)
-
-	port := startPort
-	assignAddress := func() config.HTTPAddress {
-		result := config.HTTPAddress(fmt.Sprintf("http://%s:%d", host, port))
-		port++
-		return result
-	}
-
-	serverKeyPath := filepath.Join(configDir, ServerKeyFile)
-	tlsCertPath := filepath.Join(configDir, TLSKeyFile)
-	tlsKeyPath := filepath.Join(configDir, TLSCertFile)
-	mediaBasePath := filepath.Join(configDir, MediaDir)
-
-	if err := NewMatrixKey(serverKeyPath); err != nil {
-		return nil, 0, err
-	}
-
-	if err := NewTLSKey(tlsKeyPath, tlsCertPath); err != nil {
-		return nil, 0, err
-	}
-
-	cfg.Version = config.Version
-
-	cfg.Global.ServerName = gomatrixserverlib.ServerName(assignAddress())
-	cfg.Global.PrivateKeyPath = config.Path(serverKeyPath)
-
-	cfg.MediaAPI.BasePath = config.Path(mediaBasePath)
-
-	cfg.Global.JetStream.Addresses = []string{kafkaURI}
-
-	// TODO: Use different databases for the different schemas.
-	// Using the same database for every schema currently works because
-	// the table names are globally unique. But we might not want to
-	// rely on that in the future.
-	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(database)
-	cfg.FederationAPI.Database.ConnectionString = config.DataSource(database)
-	cfg.KeyServer.Database.ConnectionString = config.DataSource(database)
-	cfg.MediaAPI.Database.ConnectionString = config.DataSource(database)
-	cfg.RoomServer.Database.ConnectionString = config.DataSource(database)
-	cfg.SyncAPI.Database.ConnectionString = config.DataSource(database)
-	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(database)
-
-	cfg.AppServiceAPI.InternalAPI.Listen = assignAddress()
-	cfg.FederationAPI.InternalAPI.Listen = assignAddress()
-	cfg.KeyServer.InternalAPI.Listen = assignAddress()
-	cfg.MediaAPI.InternalAPI.Listen = assignAddress()
-	cfg.RoomServer.InternalAPI.Listen = assignAddress()
-	cfg.SyncAPI.InternalAPI.Listen = assignAddress()
-	cfg.UserAPI.InternalAPI.Listen = assignAddress()
-
-	cfg.AppServiceAPI.InternalAPI.Connect = cfg.AppServiceAPI.InternalAPI.Listen
-	cfg.FederationAPI.InternalAPI.Connect = cfg.FederationAPI.InternalAPI.Listen
-	cfg.KeyServer.InternalAPI.Connect = cfg.KeyServer.InternalAPI.Listen
-	cfg.MediaAPI.InternalAPI.Connect = cfg.MediaAPI.InternalAPI.Listen
-	cfg.RoomServer.InternalAPI.Connect = cfg.RoomServer.InternalAPI.Listen
-	cfg.SyncAPI.InternalAPI.Connect = cfg.SyncAPI.InternalAPI.Listen
-	cfg.UserAPI.InternalAPI.Connect = cfg.UserAPI.InternalAPI.Listen
-
-	return &cfg, port, nil
-}
-
-// WriteConfig writes the config file to the directory.
-func WriteConfig(cfg *config.Dendrite, configDir string) error {
-	data, err := yaml.Marshal(cfg)
-	if err != nil {
-		return err
-	}
-	return ioutil.WriteFile(filepath.Join(configDir, ConfigFile), data, 0666)
-}
-
-// NewMatrixKey generates a new ed25519 matrix server key and writes it to a file.
-func NewMatrixKey(matrixKeyPath string) (err error) {
-	var data [35]byte
-	_, err = rand.Read(data[:])
-	if err != nil {
-		return err
-	}
-	keyOut, err := os.OpenFile(matrixKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
-	if err != nil {
-		return err
-	}
-
-	defer (func() {
-		err = keyOut.Close()
-	})()
-
-	keyID := base64.RawURLEncoding.EncodeToString(data[:])
-	keyID = strings.ReplaceAll(keyID, "-", "")
-	keyID = strings.ReplaceAll(keyID, "_", "")
-
-	err = pem.Encode(keyOut, &pem.Block{
-		Type: "MATRIX PRIVATE KEY",
-		Headers: map[string]string{
-			"Key-ID": fmt.Sprintf("ed25519:%s", keyID[:6]),
-		},
-		Bytes: data[3:],
-	})
-	return err
-}
-
-const certificateDuration = time.Hour * 24 * 365 * 10
-
-func generateTLSTemplate(dnsNames []string) (*rsa.PrivateKey, *x509.Certificate, error) {
-	priv, err := rsa.GenerateKey(rand.Reader, 4096)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	notBefore := time.Now()
-	notAfter := notBefore.Add(certificateDuration)
-	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
-	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	template := x509.Certificate{
-		SerialNumber:          serialNumber,
-		NotBefore:             notBefore,
-		NotAfter:              notAfter,
-		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		BasicConstraintsValid: true,
-		DNSNames:              dnsNames,
-	}
-	return priv, &template, nil
-}
-
-func writeCertificate(tlsCertPath string, derBytes []byte) error {
-	certOut, err := os.Create(tlsCertPath)
-	if err != nil {
-		return err
-	}
-	defer certOut.Close() // nolint: errcheck
-	return pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
-}
-
-func writePrivateKey(tlsKeyPath string, priv *rsa.PrivateKey) error {
-	keyOut, err := os.OpenFile(tlsKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
-	if err != nil {
-		return err
-	}
-	defer keyOut.Close() // nolint: errcheck
-	err = pem.Encode(keyOut, &pem.Block{
-		Type:  "RSA PRIVATE KEY",
-		Bytes: x509.MarshalPKCS1PrivateKey(priv),
-	})
-	return err
-}
-
-// NewTLSKey generates a new RSA TLS key and certificate and writes it to a file.
-func NewTLSKey(tlsKeyPath, tlsCertPath string) error {
-	priv, template, err := generateTLSTemplate(nil)
-	if err != nil {
-		return err
-	}
-
-	// Self-signed certificate: template == parent
-	derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv)
-	if err != nil {
-		return err
-	}
-
-	if err = writeCertificate(tlsCertPath, derBytes); err != nil {
-		return err
-	}
-	return writePrivateKey(tlsKeyPath, priv)
-}
-
-func NewTLSKeyWithAuthority(serverName, tlsKeyPath, tlsCertPath, authorityKeyPath, authorityCertPath string) error {
-	priv, template, err := generateTLSTemplate([]string{serverName})
-	if err != nil {
-		return err
-	}
-
-	// load the authority key
-	dat, err := ioutil.ReadFile(authorityKeyPath)
-	if err != nil {
-		return err
-	}
-	block, _ := pem.Decode([]byte(dat))
-	if block == nil || block.Type != "RSA PRIVATE KEY" {
-		return errors.New("authority .key is not a valid pem encoded rsa private key")
-	}
-	authorityPriv, err := x509.ParsePKCS1PrivateKey(block.Bytes)
-	if err != nil {
-		return err
-	}
-
-	// load the authority certificate
-	dat, err = ioutil.ReadFile(authorityCertPath)
-	if err != nil {
-		return err
-	}
-	block, _ = pem.Decode([]byte(dat))
-	if block == nil || block.Type != "CERTIFICATE" {
-		return errors.New("authority .crt is not a valid pem encoded x509 cert")
-	}
-	var caCerts []*x509.Certificate
-	caCerts, err = x509.ParseCertificates(block.Bytes)
-	if err != nil {
-		return err
-	}
-	if len(caCerts) != 1 {
-		return errors.New("authority .crt contains none or more than one cert")
-	}
-	authorityCert := caCerts[0]
-
-	// Sign the new certificate using the authority's key/cert
-	derBytes, err := x509.CreateCertificate(rand.Reader, template, authorityCert, &priv.PublicKey, authorityPriv)
-	if err != nil {
-		return err
-	}
-
-	if err = writeCertificate(tlsCertPath, derBytes); err != nil {
-		return err
-	}
-	return writePrivateKey(tlsKeyPath, priv)
-}
diff --git a/internal/test/kafka.go b/internal/test/kafka.go
deleted file mode 100644
index cbf24630..00000000
--- a/internal/test/kafka.go
+++ /dev/null
@@ -1,76 +0,0 @@
-// Copyright 2017 Vector Creations Ltd
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package test
-
-import (
-	"io"
-	"os/exec"
-	"path/filepath"
-	"strings"
-)
-
-// KafkaExecutor executes kafka scripts.
-type KafkaExecutor struct {
-	// The location of Zookeeper. Typically this is `localhost:2181`.
-	ZookeeperURI string
-	// The directory where Kafka is installed to. Used to locate kafka scripts.
-	KafkaDirectory string
-	// The location of the Kafka logs. Typically this is `localhost:9092`.
-	KafkaURI string
-	// Where stdout and stderr should be written to. Typically this is `os.Stderr`.
-	OutputWriter io.Writer
-}
-
-// CreateTopic creates a new kafka topic. This is created with a single partition.
-func (e *KafkaExecutor) CreateTopic(topic string) error {
-	cmd := exec.Command(
-		filepath.Join(e.KafkaDirectory, "bin", "kafka-topics.sh"),
-		"--create",
-		"--zookeeper", e.ZookeeperURI,
-		"--replication-factor", "1",
-		"--partitions", "1",
-		"--topic", topic,
-	)
-	cmd.Stdout = e.OutputWriter
-	cmd.Stderr = e.OutputWriter
-	return cmd.Run()
-}
-
-// WriteToTopic writes data to a kafka topic.
-func (e *KafkaExecutor) WriteToTopic(topic string, data []string) error {
-	cmd := exec.Command(
-		filepath.Join(e.KafkaDirectory, "bin", "kafka-console-producer.sh"),
-		"--broker-list", e.KafkaURI,
-		"--topic", topic,
-	)
-	cmd.Stdout = e.OutputWriter
-	cmd.Stderr = e.OutputWriter
-	cmd.Stdin = strings.NewReader(strings.Join(data, "\n"))
-	return cmd.Run()
-}
-
-// DeleteTopic deletes a given kafka topic if it exists.
-func (e *KafkaExecutor) DeleteTopic(topic string) error {
-	cmd := exec.Command(
-		filepath.Join(e.KafkaDirectory, "bin", "kafka-topics.sh"),
-		"--delete",
-		"--if-exists",
-		"--zookeeper", e.ZookeeperURI,
-		"--topic", topic,
-	)
-	cmd.Stderr = e.OutputWriter
-	cmd.Stdout = e.OutputWriter
-	return cmd.Run()
-}
diff --git a/internal/test/keyring.go b/internal/test/keyring.go
deleted file mode 100644
index ed9c3484..00000000
--- a/internal/test/keyring.go
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package test
-
-import (
-	"context"
-
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-// NopJSONVerifier is a JSONVerifier that verifies nothing and returns no errors.
-type NopJSONVerifier struct {
-	// this verifier verifies nothing
-}
-
-func (t *NopJSONVerifier) VerifyJSONs(ctx context.Context, requests []gomatrixserverlib.VerifyJSONRequest) ([]gomatrixserverlib.VerifyJSONResult, error) {
-	result := make([]gomatrixserverlib.VerifyJSONResult, len(requests))
-	return result, nil
-}
diff --git a/internal/test/server.go b/internal/test/server.go
deleted file mode 100644
index ca14ea1b..00000000
--- a/internal/test/server.go
+++ /dev/null
@@ -1,152 +0,0 @@
-// Copyright 2020 The Matrix.org Foundation C.I.C.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package test
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"net/http"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"sync"
-	"testing"
-
-	"github.com/matrix-org/dendrite/setup/config"
-)
-
-// Defaulting allows assignment of string variables with a fallback default value
-// Useful for use with os.Getenv() for example
-func Defaulting(value, defaultValue string) string {
-	if value == "" {
-		value = defaultValue
-	}
-	return value
-}
-
-// CreateDatabase creates a new database, dropping it first if it exists
-func CreateDatabase(command string, args []string, database string) error {
-	cmd := exec.Command(command, args...)
-	cmd.Stdin = strings.NewReader(
-		fmt.Sprintf("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s;", database, database),
-	)
-	// Send stdout and stderr to our stderr so that we see error messages from
-	// the psql process
-	cmd.Stdout = os.Stderr
-	cmd.Stderr = os.Stderr
-	return cmd.Run()
-}
-
-// CreateBackgroundCommand creates an executable command
-// The Cmd being executed is returned. A channel is also returned,
-// which will have any termination errors sent down it, followed immediately by the channel being closed.
-func CreateBackgroundCommand(command string, args []string) (*exec.Cmd, chan error) {
-	cmd := exec.Command(command, args...)
-	cmd.Stderr = os.Stderr
-	cmd.Stdout = os.Stderr
-
-	if err := cmd.Start(); err != nil {
-		panic("failed to start server: " + err.Error())
-	}
-	cmdChan := make(chan error, 1)
-	go func() {
-		cmdChan <- cmd.Wait()
-		close(cmdChan)
-	}()
-	return cmd, cmdChan
-}
-
-// InitDatabase creates the database and config file needed for the server to run
-func InitDatabase(postgresDatabase, postgresContainerName string, databases []string) {
-	if len(databases) > 0 {
-		var dbCmd string
-		var dbArgs []string
-		if postgresContainerName == "" {
-			dbCmd = "psql"
-			dbArgs = []string{postgresDatabase}
-		} else {
-			dbCmd = "docker"
-			dbArgs = []string{
-				"exec", "-i", postgresContainerName, "psql", "-U", "postgres", postgresDatabase,
-			}
-		}
-		for _, database := range databases {
-			if err := CreateDatabase(dbCmd, dbArgs, database); err != nil {
-				panic(err)
-			}
-		}
-	}
-}
-
-// StartProxy creates a reverse proxy
-func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) {
-	proxyArgs := []string{
-		"--bind-address", bindAddr,
-		"--sync-api-server-url", "http://" + string(cfg.SyncAPI.InternalAPI.Connect),
-		"--client-api-server-url", "http://" + string(cfg.ClientAPI.InternalAPI.Connect),
-		"--media-api-server-url", "http://" + string(cfg.MediaAPI.InternalAPI.Connect),
-		"--tls-cert", "server.crt",
-		"--tls-key", "server.key",
-	}
-	return CreateBackgroundCommand(
-		filepath.Join(filepath.Dir(os.Args[0]), "client-api-proxy"),
-		proxyArgs,
-	)
-}
-
-// ListenAndServe will listen on a random high-numbered port and attach the given router.
-// Returns the base URL to send requests to. Call `cancel` to shutdown the server, which will block until it has closed.
-func ListenAndServe(t *testing.T, router http.Handler, useTLS bool) (apiURL string, cancel func()) {
-	listener, err := net.Listen("tcp", ":0")
-	if err != nil {
-		t.Fatalf("failed to listen: %s", err)
-	}
-	port := listener.Addr().(*net.TCPAddr).Port
-	srv := http.Server{}
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		srv.Handler = router
-		var err error
-		if useTLS {
-			certFile := filepath.Join(os.TempDir(), "dendrite.cert")
-			keyFile := filepath.Join(os.TempDir(), "dendrite.key")
-			err = NewTLSKey(keyFile, certFile)
-			if err != nil {
-				t.Logf("failed to generate tls key/cert: %s", err)
-				return
-			}
-			err = srv.ServeTLS(listener, certFile, keyFile)
-		} else {
-			err = srv.Serve(listener)
-		}
-		if err != nil && err != http.ErrServerClosed {
-			t.Logf("Listen failed: %s", err)
-		}
-	}()
-
-	secure := ""
-	if useTLS {
-		secure = "s"
-	}
-	return fmt.Sprintf("http%s://localhost:%d", secure, port), func() {
-		_ = srv.Shutdown(context.Background())
-		wg.Wait()
-	}
-}
diff --git a/internal/test/slice.go b/internal/test/slice.go
deleted file mode 100644
index 00c740db..00000000
--- a/internal/test/slice.go
+++ /dev/null
@@ -1,34 +0,0 @@
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package test
-
-import "sort"
-
-// UnsortedStringSliceEqual returns true if the slices have same length & elements.
-// Does not modify the given slice.
-func UnsortedStringSliceEqual(first, second []string) bool {
-	if len(first) != len(second) {
-		return false
-	}
-
-	a, b := first[:], second[:]
-	sort.Strings(a)
-	sort.Strings(b)
-	for i := range a {
-		if a[i] != b[i] {
-			return false
-		}
-	}
-
-	return true
-}