Merge both user API databases into one (#2186)

* Merge user API databases into one

* Remove DeviceDatabase from config

* Fix tests

* Try that again

* Clean up keyserver device keys when the devices no longer exist in the user API

* Tweak ordering

* Fix UserExists flag, device check

* Allow including empty entries so we can clean them up

* Remove logging

8 files changed, 18 insertions, 25 deletions

diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go
index d9202eb0..3003896c 100644
--- a/cmd/create-account/main.go
+++ b/cmd/create-account/main.go
@@ -30,7 +30,7 @@ import (
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/dendrite/userapi/storage/accounts"
+	userdb "github.com/matrix-org/dendrite/userapi/storage"
 )
 
 const usage = `Usage: %s
@@ -77,9 +77,14 @@ func main() {
 
 	pass := getPassword(password, pwdFile, pwdStdin, askPass, os.Stdin)
 
-	accountDB, err := accounts.NewDatabase(&config.DatabaseOptions{
-		ConnectionString: cfg.UserAPI.AccountDatabase.ConnectionString,
-	}, cfg.Global.ServerName, bcrypt.DefaultCost, cfg.UserAPI.OpenIDTokenLifetimeMS)
+	accountDB, err := userdb.NewDatabase(
+		&config.DatabaseOptions{
+			ConnectionString: cfg.UserAPI.AccountDatabase.ConnectionString,
+		},
+		cfg.Global.ServerName, bcrypt.DefaultCost,
+		cfg.UserAPI.OpenIDTokenLifetimeMS,
+		api.DefaultLoginTokenLifetime,
+	)
 	if err != nil {
 		logrus.Fatalln("Failed to connect to the database:", err.Error())
 	}
diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go
index 7cbd0b6d..78536901 100644
--- a/cmd/dendrite-demo-libp2p/main.go
+++ b/cmd/dendrite-demo-libp2p/main.go
@@ -126,7 +126,6 @@ func main() {
 	cfg.FederationAPI.FederationMaxRetries = 6
 	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
 	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-	cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
 	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
 	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
 	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go
index a897dcd1..5810a7f1 100644
--- a/cmd/dendrite-demo-pinecone/main.go
+++ b/cmd/dendrite-demo-pinecone/main.go
@@ -160,7 +160,6 @@ func main() {
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
 	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-	cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
 	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
 	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
 	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index 52e69ee5..49e096bd 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -79,7 +79,6 @@ func main() {
 	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
 	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
 	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-	cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
 	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
 	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
 	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go
index 62eea78f..664f644f 100644
--- a/cmd/dendritejs-pinecone/main.go
+++ b/cmd/dendritejs-pinecone/main.go
@@ -164,7 +164,6 @@ func startup() {
 	cfg.Defaults(true)
 	cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db"
 	cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db"
-	cfg.UserAPI.DeviceDatabase.ConnectionString = "file:/idb/dendritejs_device.db"
 	cfg.FederationAPI.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db"
 	cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db"
 	cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db"
diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go
index 59de07cd..0ea41b4c 100644
--- a/cmd/dendritejs/main.go
+++ b/cmd/dendritejs/main.go
@@ -167,7 +167,6 @@ func main() {
 	cfg.Defaults(true)
 	cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db"
 	cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db"
-	cfg.UserAPI.DeviceDatabase.ConnectionString = "file:/idb/dendritejs_device.db"
 	cfg.FederationAPI.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db"
 	cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db"
 	cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db"
diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go
index f87665fb..ba5a87a7 100644
--- a/cmd/generate-config/main.go
+++ b/cmd/generate-config/main.go
@@ -32,7 +32,6 @@ func main() {
 		cfg.RoomServer.Database.ConnectionString = config.DataSource(*dbURI)
 		cfg.SyncAPI.Database.ConnectionString = config.DataSource(*dbURI)
 		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(*dbURI)
-		cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(*dbURI)
 	}
 	cfg.Global.TrustedIDServers = []string{
 		"matrix.org",
diff --git a/cmd/goose/main.go b/cmd/goose/main.go
index 8ed5cbd9..31a5b005 100644
--- a/cmd/goose/main.go
+++ b/cmd/goose/main.go
@@ -8,12 +8,11 @@ import (
 	"log"
 	"os"
 
-	pgaccounts "github.com/matrix-org/dendrite/userapi/storage/accounts/postgres/deltas"
-	slaccounts "github.com/matrix-org/dendrite/userapi/storage/accounts/sqlite3/deltas"
-	pgdevices "github.com/matrix-org/dendrite/userapi/storage/devices/postgres/deltas"
-	sldevices "github.com/matrix-org/dendrite/userapi/storage/devices/sqlite3/deltas"
 	"github.com/pressly/goose"
 
+	pgusers "github.com/matrix-org/dendrite/userapi/storage/postgres/deltas"
+	slusers "github.com/matrix-org/dendrite/userapi/storage/sqlite3/deltas"
+
 	_ "github.com/lib/pq"
 	_ "github.com/mattn/go-sqlite3"
 )
@@ -26,8 +25,7 @@ const (
 	RoomServer       = "roomserver"
 	SigningKeyServer = "signingkeyserver"
 	SyncAPI          = "syncapi"
-	UserAPIAccounts  = "userapi_accounts"
-	UserAPIDevices   = "userapi_devices"
+	UserAPI          = "userapi"
 )
 
 var (
@@ -35,7 +33,7 @@ var (
 	flags     = flag.NewFlagSet("goose", flag.ExitOnError)
 	component = flags.String("component", "", "dendrite component name")
 	knownDBs  = []string{
-		AppService, FederationSender, KeyServer, MediaAPI, RoomServer, SigningKeyServer, SyncAPI, UserAPIAccounts, UserAPIDevices,
+		AppService, FederationSender, KeyServer, MediaAPI, RoomServer, SigningKeyServer, SyncAPI, UserAPI,
 	}
 )
 
@@ -143,18 +141,14 @@ Commands:
 
 func loadSQLiteDeltas(component string) {
 	switch component {
-	case UserAPIAccounts:
-		slaccounts.LoadFromGoose()
-	case UserAPIDevices:
-		sldevices.LoadFromGoose()
+	case UserAPI:
+		slusers.LoadFromGoose()
 	}
 }
 
 func loadPostgresDeltas(component string) {
 	switch component {
-	case UserAPIAccounts:
-		pgaccounts.LoadFromGoose()
-	case UserAPIDevices:
-		pgdevices.LoadFromGoose()
+	case UserAPI:
+		pgusers.LoadFromGoose()
 	}
 }