Update database migrations, remove goose (#2264)

* Add new db migration

* Update migrations
Remove goose

* Add possibility to test direct upgrades

* Try to fix WASM test

* Add checks for specific migrations

* Remove AddMigration
Use WithTransaction
Add Dendrite version to table

* Fix linter issues

* Update tests

* Update comments, outdent if

* Namespace migrations

* Add direct upgrade tests, skipping over one version

* Split migrations

* Update go version in CI

* Fix copy&paste mistake

* Use contexts in migrations

Co-authored-by: kegsay <kegan@matrix.org>
Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>

12 files changed, 86 insertions, 111 deletions

diff --git a/userapi/storage/postgres/accounts_table.go b/userapi/storage/postgres/accounts_table.go
index e3cab56e..33fb6dd4 100644
--- a/userapi/storage/postgres/accounts_table.go
+++ b/userapi/storage/postgres/accounts_table.go
@@ -24,6 +24,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
 
 	log "github.com/sirupsen/logrus"
@@ -85,6 +86,23 @@ func NewPostgresAccountsTable(db *sql.DB, serverName gomatrixserverlib.ServerNam
 	if err != nil {
 		return nil, err
 	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations([]sqlutil.Migration{
+		{
+			Version: "userapi: add is active",
+			Up:      deltas.UpIsActive,
+			Down:    deltas.DownIsActive,
+		},
+		{
+			Version: "userapi: add account type",
+			Up:      deltas.UpAddAccountType,
+			Down:    deltas.DownAddAccountType,
+		},
+	}...)
+	err = m.Up(context.Background())
+	if err != nil {
+		return nil, err
+	}
 	return s, sqlutil.StatementList{
 		{&s.insertAccountStmt, insertAccountSQL},
 		{&s.updatePasswordStmt, updatePasswordSQL},
diff --git a/userapi/storage/postgres/deltas/20200929203058_is_active.go b/userapi/storage/postgres/deltas/20200929203058_is_active.go
index 32d3235b..24f87e07 100644
--- a/userapi/storage/postgres/deltas/20200929203058_is_active.go
+++ b/userapi/storage/postgres/deltas/20200929203058_is_active.go
@@ -1,33 +1,21 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/pressly/goose"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadFromGoose() {
-	goose.AddMigration(UpIsActive, DownIsActive)
-	goose.AddMigration(UpAddAccountType, DownAddAccountType)
-}
-
-func LoadIsActive(m *sqlutil.Migrations) {
-	m.AddMigration(UpIsActive, DownIsActive)
-}
-
-func UpIsActive(tx *sql.Tx) error {
-	_, err := tx.Exec("ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS is_deactivated BOOLEAN DEFAULT FALSE;")
+func UpIsActive(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, "ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS is_deactivated BOOLEAN DEFAULT FALSE;")
 	if err != nil {
 		return fmt.Errorf("failed to execute upgrade: %w", err)
 	}
 	return nil
 }
 
-func DownIsActive(tx *sql.Tx) error {
-	_, err := tx.Exec("ALTER TABLE account_accounts DROP COLUMN is_deactivated;")
+func DownIsActive(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, "ALTER TABLE account_accounts DROP COLUMN is_deactivated;")
 	if err != nil {
 		return fmt.Errorf("failed to execute downgrade: %w", err)
 	}
diff --git a/userapi/storage/postgres/deltas/20201001204705_last_seen_ts_ip.go b/userapi/storage/postgres/deltas/20201001204705_last_seen_ts_ip.go
index 1bbb0a9d..edd3353f 100644
--- a/userapi/storage/postgres/deltas/20201001204705_last_seen_ts_ip.go
+++ b/userapi/storage/postgres/deltas/20201001204705_last_seen_ts_ip.go
@@ -1,18 +1,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadLastSeenTSIP(m *sqlutil.Migrations) {
-	m.AddMigration(UpLastSeenTSIP, DownLastSeenTSIP)
-}
-
-func UpLastSeenTSIP(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func UpLastSeenTSIP(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS last_seen_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)*1000;
 ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS ip TEXT;
 ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS user_agent TEXT;`)
@@ -22,8 +17,8 @@ ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS user_agent TEXT;`)
 	return nil
 }
 
-func DownLastSeenTSIP(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func DownLastSeenTSIP(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 	ALTER TABLE device_devices DROP COLUMN last_seen_ts;
 	ALTER TABLE device_devices DROP COLUMN ip;
 	ALTER TABLE device_devices DROP COLUMN user_agent;`)
diff --git a/userapi/storage/postgres/deltas/2022021013023800_add_account_type.go b/userapi/storage/postgres/deltas/2022021013023800_add_account_type.go
index 2fae00cb..eb7c3a95 100644
--- a/userapi/storage/postgres/deltas/2022021013023800_add_account_type.go
+++ b/userapi/storage/postgres/deltas/2022021013023800_add_account_type.go
@@ -1,20 +1,15 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadAddAccountType(m *sqlutil.Migrations) {
-	m.AddMigration(UpAddAccountType, DownAddAccountType)
-}
-
-func UpAddAccountType(tx *sql.Tx) error {
+func UpAddAccountType(ctx context.Context, tx *sql.Tx) error {
 	// initially set every account to useraccount, change appservice and guest accounts afterwards
 	// (user = 1, guest = 2, admin = 3, appservice = 4)
-	_, err := tx.Exec(`ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS account_type SMALLINT NOT NULL DEFAULT 1;
+	_, err := tx.ExecContext(ctx, `ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS account_type SMALLINT NOT NULL DEFAULT 1;
 UPDATE account_accounts SET account_type = 4 WHERE appservice_id <> '';
 UPDATE account_accounts SET account_type = 2 WHERE localpart ~ '^[0-9]+$';
 ALTER TABLE account_accounts ALTER COLUMN account_type DROP DEFAULT;`,
@@ -25,8 +20,8 @@ ALTER TABLE account_accounts ALTER COLUMN account_type DROP DEFAULT;`,
 	return nil
 }
 
-func DownAddAccountType(tx *sql.Tx) error {
-	_, err := tx.Exec("ALTER TABLE account_accounts DROP COLUMN account_type;")
+func DownAddAccountType(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, "ALTER TABLE account_accounts DROP COLUMN account_type;")
 	if err != nil {
 		return fmt.Errorf("failed to execute downgrade: %w", err)
 	}
diff --git a/userapi/storage/postgres/devices_table.go b/userapi/storage/postgres/devices_table.go
index ccb77667..f65681aa 100644
--- a/userapi/storage/postgres/devices_table.go
+++ b/userapi/storage/postgres/devices_table.go
@@ -24,6 +24,7 @@ import (
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -120,6 +121,15 @@ func NewPostgresDevicesTable(db *sql.DB, serverName gomatrixserverlib.ServerName
 	if err != nil {
 		return nil, err
 	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "userapi: add last_seen_ts",
+		Up:      deltas.UpLastSeenTSIP,
+	})
+	err = m.Up(context.Background())
+	if err != nil {
+		return nil, err
+	}
 	return s, sqlutil.StatementList{
 		{&s.insertDeviceStmt, insertDeviceSQL},
 		{&s.selectDeviceByTokenStmt, selectDeviceByTokenSQL},
diff --git a/userapi/storage/postgres/storage.go b/userapi/storage/postgres/storage.go
index b9afb5a5..7d3b9b6a 100644
--- a/userapi/storage/postgres/storage.go
+++ b/userapi/storage/postgres/storage.go
@@ -23,7 +23,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/dendrite/userapi/storage/postgres/deltas"
 	"github.com/matrix-org/dendrite/userapi/storage/shared"
 
 	// Import the postgres database driver.
@@ -37,19 +36,6 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 		return nil, err
 	}
 
-	m := sqlutil.NewMigrations()
-	if _, err = db.Exec(accountsSchema); err != nil {
-		// do this so that the migration can and we don't fail on
-		// preparing statements for columns that don't exist yet
-		return nil, err
-	}
-	deltas.LoadIsActive(m)
-	//deltas.LoadLastSeenTSIP(m)
-	deltas.LoadAddAccountType(m)
-	if err = m.RunDeltas(db, dbProperties); err != nil {
-		return nil, err
-	}
-
 	accountDataTable, err := NewPostgresAccountDataTable(db)
 	if err != nil {
 		return nil, fmt.Errorf("NewPostgresAccountDataTable: %w", err)
diff --git a/userapi/storage/sqlite3/accounts_table.go b/userapi/storage/sqlite3/accounts_table.go
index 6c5fe307..484e9005 100644
--- a/userapi/storage/sqlite3/accounts_table.go
+++ b/userapi/storage/sqlite3/accounts_table.go
@@ -24,6 +24,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
 
 	log "github.com/sirupsen/logrus"
@@ -87,6 +88,23 @@ func NewSQLiteAccountsTable(db *sql.DB, serverName gomatrixserverlib.ServerName)
 	if err != nil {
 		return nil, err
 	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations([]sqlutil.Migration{
+		{
+			Version: "userapi: add is active",
+			Up:      deltas.UpIsActive,
+			Down:    deltas.DownIsActive,
+		},
+		{
+			Version: "userapi: add account type",
+			Up:      deltas.UpAddAccountType,
+			Down:    deltas.DownAddAccountType,
+		},
+	}...)
+	err = m.Up(context.Background())
+	if err != nil {
+		return nil, err
+	}
 	return s, sqlutil.StatementList{
 		{&s.insertAccountStmt, insertAccountSQL},
 		{&s.updatePasswordStmt, updatePasswordSQL},
diff --git a/userapi/storage/sqlite3/deltas/20200929203058_is_active.go b/userapi/storage/sqlite3/deltas/20200929203058_is_active.go
index c69614e8..e25efc69 100644
--- a/userapi/storage/sqlite3/deltas/20200929203058_is_active.go
+++ b/userapi/storage/sqlite3/deltas/20200929203058_is_active.go
@@ -1,25 +1,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/pressly/goose"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadFromGoose() {
-	goose.AddMigration(UpIsActive, DownIsActive)
-	goose.AddMigration(UpAddAccountType, DownAddAccountType)
-}
-
-func LoadIsActive(m *sqlutil.Migrations) {
-	m.AddMigration(UpIsActive, DownIsActive)
-}
-
-func UpIsActive(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func UpIsActive(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 	ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
 CREATE TABLE account_accounts (
     localpart TEXT NOT NULL PRIMARY KEY,
@@ -42,8 +30,8 @@ DROP TABLE account_accounts_tmp;`)
 	return nil
 }
 
-func DownIsActive(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func DownIsActive(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 	ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
 CREATE TABLE account_accounts (
     localpart TEXT NOT NULL PRIMARY KEY,
diff --git a/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go b/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go
index ebf90800..7f7e95d2 100644
--- a/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go
+++ b/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go
@@ -1,18 +1,13 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func LoadLastSeenTSIP(m *sqlutil.Migrations) {
-	m.AddMigration(UpLastSeenTSIP, DownLastSeenTSIP)
-}
-
-func UpLastSeenTSIP(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func UpLastSeenTSIP(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
     ALTER TABLE device_devices RENAME TO device_devices_tmp;
     CREATE TABLE device_devices (
         access_token TEXT PRIMARY KEY,
@@ -39,8 +34,8 @@ func UpLastSeenTSIP(tx *sql.Tx) error {
 	return nil
 }
 
-func DownLastSeenTSIP(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+func DownLastSeenTSIP(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
 ALTER TABLE device_devices RENAME TO device_devices_tmp;
 CREATE TABLE IF NOT EXISTS device_devices (
     access_token TEXT PRIMARY KEY,
diff --git a/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go b/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go
index 9b058ded..46532698 100644
--- a/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go
+++ b/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go
@@ -1,26 +1,15 @@
 package deltas
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
-
-	"github.com/pressly/goose"
-
-	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
-func init() {
-	goose.AddMigration(UpAddAccountType, DownAddAccountType)
-}
-
-func LoadAddAccountType(m *sqlutil.Migrations) {
-	m.AddMigration(UpAddAccountType, DownAddAccountType)
-}
-
-func UpAddAccountType(tx *sql.Tx) error {
+func UpAddAccountType(ctx context.Context, tx *sql.Tx) error {
 	// initially set every account to useraccount, change appservice and guest accounts afterwards
 	// (user = 1, guest = 2, admin = 3, appservice = 4)
-	_, err := tx.Exec(`ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
+	_, err := tx.ExecContext(ctx, `ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
 CREATE TABLE account_accounts (
     localpart TEXT NOT NULL PRIMARY KEY,
     created_ts BIGINT NOT NULL,
@@ -45,8 +34,8 @@ DROP TABLE account_accounts_tmp;`)
 	return nil
 }
 
-func DownAddAccountType(tx *sql.Tx) error {
-	_, err := tx.Exec(`ALTER TABLE account_accounts DROP COLUMN account_type;`)
+func DownAddAccountType(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `ALTER TABLE account_accounts DROP COLUMN account_type;`)
 	if err != nil {
 		return fmt.Errorf("failed to execute downgrade: %w", err)
 	}
diff --git a/userapi/storage/sqlite3/devices_table.go b/userapi/storage/sqlite3/devices_table.go
index 93291e6a..27a7524d 100644
--- a/userapi/storage/sqlite3/devices_table.go
+++ b/userapi/storage/sqlite3/devices_table.go
@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/userapi/storage/tables"
 
 	"github.com/matrix-org/dendrite/clientapi/userutil"
@@ -107,6 +108,15 @@ func NewSQLiteDevicesTable(db *sql.DB, serverName gomatrixserverlib.ServerName)
 	if err != nil {
 		return nil, err
 	}
+	m := sqlutil.NewMigrator(db)
+	m.AddMigrations(sqlutil.Migration{
+		Version: "userapi: add last_seen_ts",
+		Up:      deltas.UpLastSeenTSIP,
+	})
+	if err = m.Up(context.Background()); err != nil {
+		return nil, err
+	}
+
 	return s, sqlutil.StatementList{
 		{&s.insertDeviceStmt, insertDeviceSQL},
 		{&s.selectDevicesCountStmt, selectDevicesCountSQL},
diff --git a/userapi/storage/sqlite3/storage.go b/userapi/storage/sqlite3/storage.go
index a822f687..78b7ce58 100644
--- a/userapi/storage/sqlite3/storage.go
+++ b/userapi/storage/sqlite3/storage.go
@@ -25,10 +25,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 
 	"github.com/matrix-org/dendrite/userapi/storage/shared"
-	"github.com/matrix-org/dendrite/userapi/storage/sqlite3/deltas"
-
-	// Import the postgres database driver.
-	_ "github.com/lib/pq"
 )
 
 // NewDatabase creates a new accounts and profiles database
@@ -38,19 +34,6 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
 		return nil, err
 	}
 
-	m := sqlutil.NewMigrations()
-	if _, err = db.Exec(accountsSchema); err != nil {
-		// do this so that the migration can and we don't fail on
-		// preparing statements for columns that don't exist yet
-		return nil, err
-	}
-	deltas.LoadIsActive(m)
-	//deltas.LoadLastSeenTSIP(m)
-	deltas.LoadAddAccountType(m)
-	if err = m.RunDeltas(db, dbProperties); err != nil {
-		return nil, err
-	}
-
 	accountDataTable, err := NewSQLiteAccountDataTable(db)
 	if err != nil {
 		return nil, fmt.Errorf("NewSQLiteAccountDataTable: %w", err)