Preparations for removing `BaseDendrite` (#3016)

Preparations to actually remove/replace `BaseDendrite`.
Quite a few changes:
- SyncAPI accepts an `fulltext.Indexer` interface (fulltext is removed
from `BaseDendrite`)
- Caches are removed from `BaseDendrite`
- Introduces a `Router` struct (likely to change)
  - also fixes #2903
- Introduces a `sqlutil.ConnectionManager`, which should remove
`base.DatabaseConnection` later on
- probably more

12 files changed, 67 insertions, 50 deletions

diff --git a/userapi/consumers/roomserver_test.go b/userapi/consumers/roomserver_test.go
index bc5ae652..a54706db 100644
--- a/userapi/consumers/roomserver_test.go
+++ b/userapi/consumers/roomserver_test.go
@@ -7,22 +7,22 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/matrix-org/dendrite/internal/pushrules"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/test"
-	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/dendrite/userapi/storage"
 	userAPITypes "github.com/matrix-org/dendrite/userapi/types"
 )
 
 func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.UserDatabase, func()) {
-	base, baseclose := testrig.CreateBaseDendrite(t, dbType)
 	t.Helper()
 	connStr, close := test.PrepareDBConnectionString(t, dbType)
-	db, err := storage.NewUserDatabase(base, &config.DatabaseOptions{
+	cm := sqlutil.NewConnectionManager()
+	db, err := storage.NewUserDatabase(context.Background(), cm, &config.DatabaseOptions{
 		ConnectionString: config.DataSource(connStr),
 	}, "", 4, 0, 0, "")
 	if err != nil {
@@ -30,7 +30,6 @@ func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.UserDatabase,
 	}
 	return db, func() {
 		close()
-		baseclose()
 	}
 }
 
diff --git a/userapi/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go
index 868fc9be..6ccc4b5f 100644
--- a/userapi/internal/device_list_update_test.go
+++ b/userapi/internal/device_list_update_test.go
@@ -27,13 +27,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	roomserver "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/process"
 	"github.com/matrix-org/dendrite/test"
-	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage"
 )
@@ -363,9 +363,9 @@ func TestDebounce(t *testing.T) {
 func mustCreateKeyserverDB(t *testing.T, dbType test.DBType) (storage.KeyDatabase, func()) {
 	t.Helper()
 
-	base, _, _ := testrig.Base(nil)
 	connStr, clearDB := test.PrepareDBConnectionString(t, dbType)
-	db, err := storage.NewKeyDatabase(base, &config.DatabaseOptions{ConnectionString: config.DataSource(connStr)})
+	cm := sqlutil.NewConnectionManager()
+	db, err := storage.NewKeyDatabase(cm, &config.DatabaseOptions{ConnectionString: config.DataSource(connStr)})
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/userapi/internal/key_api_test.go b/userapi/internal/key_api_test.go
index fc7e7e0d..ec315c73 100644
--- a/userapi/internal/key_api_test.go
+++ b/userapi/internal/key_api_test.go
@@ -5,9 +5,9 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/test"
-	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/internal"
 	"github.com/matrix-org/dendrite/userapi/storage"
@@ -16,15 +16,14 @@ import (
 func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.KeyDatabase, func()) {
 	t.Helper()
 	connStr, close := test.PrepareDBConnectionString(t, dbType)
-	base, _, _ := testrig.Base(nil)
-	db, err := storage.NewKeyDatabase(base, &config.DatabaseOptions{
+	cm := sqlutil.NewConnectionManager()
+	db, err := storage.NewKeyDatabase(cm, &config.DatabaseOptions{
 		ConnectionString: config.DataSource(connStr),
 	})
 	if err != nil {
 		t.Fatalf("failed to create new user db: %v", err)
 	}
 	return db, func() {
-		base.Close()
 		close()
 	}
 }
diff --git a/userapi/storage/postgres/storage.go b/userapi/storage/postgres/storage.go
index 673d123b..7bfae7b2 100644
--- a/userapi/storage/postgres/storage.go
+++ b/userapi/storage/postgres/storage.go
@@ -23,7 +23,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/userapi/storage/shared"
@@ -33,8 +32,8 @@ import (
 )
 
 // NewDatabase creates a new accounts and profiles database
-func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) {
-	db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter())
+func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) {
+	db, writer, err := conMan.Connection(dbProperties)
 	if err != nil {
 		return nil, err
 	}
@@ -51,7 +50,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 			return deltas.UpServerNames(ctx, txn, serverName)
 		},
 	})
-	if err = m.Up(base.Context()); err != nil {
+	if err = m.Up(ctx); err != nil {
 		return nil, err
 	}
 
@@ -111,7 +110,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 			return deltas.UpServerNamesPopulate(ctx, txn, serverName)
 		},
 	})
-	if err = m.Up(base.Context()); err != nil {
+	if err = m.Up(ctx); err != nil {
 		return nil, err
 	}
 
@@ -137,8 +136,8 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 	}, nil
 }
 
-func NewKeyDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*shared.KeyDatabase, error) {
-	db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter())
+func NewKeyDatabase(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (*shared.KeyDatabase, error) {
+	db, writer, err := conMan.Connection(dbProperties)
 	if err != nil {
 		return nil, err
 	}
diff --git a/userapi/storage/sqlite3/storage.go b/userapi/storage/sqlite3/storage.go
index 0f3eeed1..3742eeba 100644
--- a/userapi/storage/sqlite3/storage.go
+++ b/userapi/storage/sqlite3/storage.go
@@ -23,7 +23,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/dendrite/internal/sqlutil"
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 
 	"github.com/matrix-org/dendrite/userapi/storage/shared"
@@ -31,8 +30,8 @@ import (
 )
 
 // NewUserDatabase creates a new accounts and profiles database
-func NewUserDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) {
-	db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter())
+func NewUserDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) {
+	db, writer, err := conMan.Connection(dbProperties)
 	if err != nil {
 		return nil, err
 	}
@@ -49,7 +48,7 @@ func NewUserDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptio
 			return deltas.UpServerNames(ctx, txn, serverName)
 		},
 	})
-	if err = m.Up(base.Context()); err != nil {
+	if err = m.Up(ctx); err != nil {
 		return nil, err
 	}
 
@@ -109,7 +108,7 @@ func NewUserDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptio
 			return deltas.UpServerNamesPopulate(ctx, txn, serverName)
 		},
 	})
-	if err = m.Up(base.Context()); err != nil {
+	if err = m.Up(ctx); err != nil {
 		return nil, err
 	}
 
@@ -135,8 +134,8 @@ func NewUserDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptio
 	}, nil
 }
 
-func NewKeyDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*shared.KeyDatabase, error) {
-	db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter())
+func NewKeyDatabase(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (*shared.KeyDatabase, error) {
+	db, writer, err := conMan.Connection(dbProperties)
 	if err != nil {
 		return nil, err
 	}
diff --git a/userapi/storage/storage.go b/userapi/storage/storage.go
index 0329fb46..6981765f 100644
--- a/userapi/storage/storage.go
+++ b/userapi/storage/storage.go
@@ -18,12 +18,13 @@
 package storage
 
 import (
+	"context"
 	"fmt"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 
-	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/storage/postgres"
 	"github.com/matrix-org/dendrite/userapi/storage/sqlite3"
@@ -32,7 +33,8 @@ import (
 // NewUserDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
 // and sets postgres connection parameters
 func NewUserDatabase(
-	base *base.BaseDendrite,
+	ctx context.Context,
+	conMan sqlutil.Connections,
 	dbProperties *config.DatabaseOptions,
 	serverName gomatrixserverlib.ServerName,
 	bcryptCost int,
@@ -42,9 +44,9 @@ func NewUserDatabase(
 ) (UserDatabase, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewUserDatabase(base, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart)
+		return sqlite3.NewUserDatabase(ctx, conMan, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart)
 	case dbProperties.ConnectionString.IsPostgres():
-		return postgres.NewDatabase(base, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart)
+		return postgres.NewDatabase(ctx, conMan, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart)
 	default:
 		return nil, fmt.Errorf("unexpected database type")
 	}
@@ -52,12 +54,12 @@ func NewUserDatabase(
 
 // NewKeyDatabase opens a new Postgres or Sqlite database (base on dataSourceName) scheme)
 // and sets postgres connection parameters.
-func NewKeyDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (KeyDatabase, error) {
+func NewKeyDatabase(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (KeyDatabase, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewKeyDatabase(base, dbProperties)
+		return sqlite3.NewKeyDatabase(conMan, dbProperties)
 	case dbProperties.ConnectionString.IsPostgres():
-		return postgres.NewKeyDatabase(base, dbProperties)
+		return postgres.NewKeyDatabase(conMan, dbProperties)
 	default:
 		return nil, fmt.Errorf("unexpected database type")
 	}
diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go
index f52e7e17..c6369ef9 100644
--- a/userapi/storage/storage_test.go
+++ b/userapi/storage/storage_test.go
@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -33,9 +34,9 @@ var (
 )
 
 func mustCreateUserDatabase(t *testing.T, dbType test.DBType) (storage.UserDatabase, func()) {
-	base, baseclose := testrig.CreateBaseDendrite(t, dbType)
 	connStr, close := test.PrepareDBConnectionString(t, dbType)
-	db, err := storage.NewUserDatabase(base, &config.DatabaseOptions{
+	cm := sqlutil.NewConnectionManager()
+	db, err := storage.NewUserDatabase(context.Background(), cm, &config.DatabaseOptions{
 		ConnectionString: config.DataSource(connStr),
 	}, "localhost", bcrypt.MinCost, openIDLifetimeMS, loginTokenLifetime, "_server")
 	if err != nil {
@@ -43,7 +44,6 @@ func mustCreateUserDatabase(t *testing.T, dbType test.DBType) (storage.UserDatab
 	}
 	return db, func() {
 		close()
-		baseclose()
 	}
 }
 
@@ -577,7 +577,7 @@ func Test_Notification(t *testing.T) {
 
 func mustCreateKeyDatabase(t *testing.T, dbType test.DBType) (storage.KeyDatabase, func()) {
 	base, close := testrig.CreateBaseDendrite(t, dbType)
-	db, err := storage.NewKeyDatabase(base, &base.Cfg.KeyServer.Database)
+	db, err := storage.NewKeyDatabase(base.ConnectionManager, &base.Cfg.KeyServer.Database)
 	if err != nil {
 		t.Fatalf("failed to create new database: %v", err)
 	}
diff --git a/userapi/storage/storage_wasm.go b/userapi/storage/storage_wasm.go
index 163e3e17..19e5f23c 100644
--- a/userapi/storage/storage_wasm.go
+++ b/userapi/storage/storage_wasm.go
@@ -15,17 +15,19 @@
 package storage
 
 import (
+	"context"
 	"fmt"
 	"time"
 
-	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/storage/sqlite3"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-func NewUserAPIDatabase(
-	base *base.BaseDendrite,
+func NewUserDatabase(
+	ctx context.Context,
+	conMan sqlutil.Connections,
 	dbProperties *config.DatabaseOptions,
 	serverName gomatrixserverlib.ServerName,
 	bcryptCost int,
@@ -35,7 +37,20 @@ func NewUserAPIDatabase(
 ) (UserDatabase, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewUserDatabase(base, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart)
+		return sqlite3.NewUserDatabase(ctx, conMan, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart)
+	case dbProperties.ConnectionString.IsPostgres():
+		return nil, fmt.Errorf("can't use Postgres implementation")
+	default:
+		return nil, fmt.Errorf("unexpected database type")
+	}
+}
+
+// NewKeyDatabase opens a new Postgres or Sqlite database (base on dataSourceName) scheme)
+// and sets postgres connection parameters.
+func NewKeyDatabase(conMan sqlutil.Connections, dbProperties *config.DatabaseOptions) (KeyDatabase, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.NewKeyDatabase(conMan, dbProperties)
 	case dbProperties.ConnectionString.IsPostgres():
 		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
diff --git a/userapi/userapi.go b/userapi/userapi.go
index 826bd721..3ada8020 100644
--- a/userapi/userapi.go
+++ b/userapi/userapi.go
@@ -18,6 +18,7 @@ import (
 	"time"
 
 	fedsenderapi "github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/internal/pushgateway"
 	"github.com/sirupsen/logrus"
 
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
@@ -42,10 +43,11 @@ func NewInternalAPI(
 	js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
 	appServices := base.Cfg.Derived.ApplicationServices
 
-	pgClient := base.PushGatewayHTTPClient()
+	pgClient := pushgateway.NewHTTPClient(cfg.PushGatewayDisableTLSValidation)
 
 	db, err := storage.NewUserDatabase(
-		base,
+		base.ProcessContext.Context(),
+		base.ConnectionManager,
 		&cfg.AccountDatabase,
 		cfg.Matrix.ServerName,
 		cfg.BCryptCost,
@@ -57,7 +59,7 @@ func NewInternalAPI(
 		logrus.WithError(err).Panicf("failed to connect to accounts db")
 	}
 
-	keyDB, err := storage.NewKeyDatabase(base, &base.Cfg.KeyServer.Database)
+	keyDB, err := storage.NewKeyDatabase(base.ConnectionManager, &base.Cfg.KeyServer.Database)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to key db")
 	}
diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go
index 01e491cb..c2d4e5a2 100644
--- a/userapi/userapi_test.go
+++ b/userapi/userapi_test.go
@@ -22,6 +22,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/producers"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -73,14 +74,16 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType, pub
 	if opts.serverName != "" {
 		sName = gomatrixserverlib.ServerName(opts.serverName)
 	}
-	accountDB, err := storage.NewUserDatabase(base, &config.DatabaseOptions{
+	cm := sqlutil.NewConnectionManager()
+	ctx := context.Background()
+	accountDB, err := storage.NewUserDatabase(ctx, cm, &config.DatabaseOptions{
 		ConnectionString: config.DataSource(connStr),
 	}, sName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "")
 	if err != nil {
 		t.Fatalf("failed to create account DB: %s", err)
 	}
 
-	keyDB, err := storage.NewKeyDatabase(base, &config.DatabaseOptions{
+	keyDB, err := storage.NewKeyDatabase(base.ConnectionManager, &config.DatabaseOptions{
 		ConnectionString: config.DataSource(connStr),
 	})
 	if err != nil {
diff --git a/userapi/util/notify_test.go b/userapi/util/notify_test.go
index 421852d3..c899e3a7 100644
--- a/userapi/util/notify_test.go
+++ b/userapi/util/notify_test.go
@@ -8,6 +8,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"golang.org/x/crypto/bcrypt"
@@ -15,7 +16,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/pushgateway"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/test"
-	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage"
 	userUtil "github.com/matrix-org/dendrite/userapi/util"
@@ -77,9 +77,8 @@ func TestNotifyUserCountsAsync(t *testing.T) {
 		// Create DB and Dendrite base
 		connStr, close := test.PrepareDBConnectionString(t, dbType)
 		defer close()
-		base, _, _ := testrig.Base(nil)
-		defer base.Close()
-		db, err := storage.NewUserDatabase(base, &config.DatabaseOptions{
+		cm := sqlutil.NewConnectionManager()
+		db, err := storage.NewUserDatabase(ctx, cm, &config.DatabaseOptions{
 			ConnectionString: config.DataSource(connStr),
 		}, "test", bcrypt.MinCost, 0, 0, "")
 		if err != nil {
diff --git a/userapi/util/phonehomestats_test.go b/userapi/util/phonehomestats_test.go
index 5f626b5b..4e931a1b 100644
--- a/userapi/util/phonehomestats_test.go
+++ b/userapi/util/phonehomestats_test.go
@@ -21,7 +21,7 @@ func TestCollect(t *testing.T) {
 		b, _, _ := testrig.Base(nil)
 		connStr, closeDB := test.PrepareDBConnectionString(t, dbType)
 		defer closeDB()
-		db, err := storage.NewUserDatabase(b, &config.DatabaseOptions{
+		db, err := storage.NewUserDatabase(b.Context(), b.ConnectionManager, &config.DatabaseOptions{
 			ConnectionString: config.DataSource(connStr),
 		}, "localhost", bcrypt.MinCost, 1000, 1000, "")
 		if err != nil {