Configuration format v1 (#1230)

* Initial pass at refactoring config (not finished)

* Don't forget current state and EDU servers

* More shifting around

* Update server key API tests

* Fix roomserver test

* Fix more tests

* Further tweaks

* Fix current state server test (sort of)

* Maybe fix appservices

* Fix client API test

* Include database connection string in database options

* Fix sync API build

* Update config test

* Fix unit tests

* Fix federation sender build

* Fix gobind build

* Set Listen address for all services in HTTP monolith mode

* Validate config, reinstate appservice derived in directory, tweaks

* Tweak federation API test

* Set MaxOpenConnections/MaxIdleConnections to previous values

* Update generate-config

8 files changed, 48 insertions, 76 deletions

diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go
index f56fb6d8..9653c019 100644
--- a/userapi/storage/accounts/postgres/storage.go
+++ b/userapi/storage/accounts/postgres/storage.go
@@ -22,6 +22,7 @@ import (
 	"strconv"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -43,10 +44,9 @@ type Database struct {
 }
 
 // NewDatabase creates a new accounts and profiles database
-func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) {
-	var db *sql.DB
-	var err error
-	if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
+func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName) (*Database, error) {
+	db, err := sqlutil.Open(dbProperties)
+	if err != nil {
 		return nil, err
 	}
 	partitions := sqlutil.PartitionOffsetStatements{}
diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go
index 72239014..4d2c5e51 100644
--- a/userapi/storage/accounts/sqlite3/storage.go
+++ b/userapi/storage/accounts/sqlite3/storage.go
@@ -23,6 +23,7 @@ import (
 	"sync"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -47,16 +48,11 @@ type Database struct {
 }
 
 // NewDatabase creates a new accounts and profiles database
-func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
-	var db *sql.DB
-	var err error
-	cs, err := sqlutil.ParseFileURI(dataSourceName)
+func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName) (*Database, error) {
+	db, err := sqlutil.Open(dbProperties)
 	if err != nil {
 		return nil, err
 	}
-	if db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil {
-		return nil, err
-	}
 	partitions := sqlutil.PartitionOffsetStatements{}
 	if err = partitions.Prepare(db, "account"); err != nil {
 		return nil, err
diff --git a/userapi/storage/accounts/storage.go b/userapi/storage/accounts/storage.go
index 87f626bf..57d5f703 100644
--- a/userapi/storage/accounts/storage.go
+++ b/userapi/storage/accounts/storage.go
@@ -17,9 +17,9 @@
 package accounts
 
 import (
-	"net/url"
+	"fmt"
 
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts/postgres"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts/sqlite3"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -27,17 +27,13 @@ import (
 
 // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
 // and sets postgres connection parameters
-func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
-	case "file":
-		return sqlite3.NewDatabase(dataSourceName, serverName)
+func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.NewDatabase(dbProperties, serverName)
+	case dbProperties.ConnectionString.IsPostgres():
+		return postgres.NewDatabase(dbProperties, serverName)
 	default:
-		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }
diff --git a/userapi/storage/accounts/storage_wasm.go b/userapi/storage/accounts/storage_wasm.go
index 69256705..ade32b68 100644
--- a/userapi/storage/accounts/storage_wasm.go
+++ b/userapi/storage/accounts/storage_wasm.go
@@ -16,28 +16,22 @@ package accounts
 
 import (
 	"fmt"
-	"net/url"
 
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts/sqlite3"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 func NewDatabase(
-	dataSourceName string,
-	dbProperties sqlutil.DbProperties, // nolint:unparam
+	dbProperties *config.DatabaseOptions,
 	serverName gomatrixserverlib.ServerName,
 ) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	case "file":
-		return sqlite3.NewDatabase(dataSourceName, serverName)
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.NewDatabase(dbProperties, serverName)
+	case dbProperties.ConnectionString.IsPostgres():
+		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
-		return nil, fmt.Errorf("Cannot use postgres implementation")
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }
diff --git a/userapi/storage/devices/postgres/storage.go b/userapi/storage/devices/postgres/storage.go
index 6ac802bb..4a7c7f97 100644
--- a/userapi/storage/devices/postgres/storage.go
+++ b/userapi/storage/devices/postgres/storage.go
@@ -20,6 +20,7 @@ import (
 	"database/sql"
 	"encoding/base64"
 
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -35,10 +36,9 @@ type Database struct {
 }
 
 // NewDatabase creates a new device database
-func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) {
-	var db *sql.DB
-	var err error
-	if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
+func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName) (*Database, error) {
+	db, err := sqlutil.Open(dbProperties)
+	if err != nil {
 		return nil, err
 	}
 	d := devicesStatements{}
diff --git a/userapi/storage/devices/sqlite3/storage.go b/userapi/storage/devices/sqlite3/storage.go
index b9f08ca1..1f2b59f3 100644
--- a/userapi/storage/devices/sqlite3/storage.go
+++ b/userapi/storage/devices/sqlite3/storage.go
@@ -20,6 +20,7 @@ import (
 	"database/sql"
 	"encoding/base64"
 
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -37,16 +38,11 @@ type Database struct {
 }
 
 // NewDatabase creates a new device database
-func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
-	var db *sql.DB
-	var err error
-	cs, err := sqlutil.ParseFileURI(dataSourceName)
+func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName) (*Database, error) {
+	db, err := sqlutil.Open(dbProperties)
 	if err != nil {
 		return nil, err
 	}
-	if db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil {
-		return nil, err
-	}
 	d := devicesStatements{}
 	if err = d.prepare(db, serverName); err != nil {
 		return nil, err
diff --git a/userapi/storage/devices/storage.go b/userapi/storage/devices/storage.go
index e094d202..1bd73a9f 100644
--- a/userapi/storage/devices/storage.go
+++ b/userapi/storage/devices/storage.go
@@ -17,9 +17,9 @@
 package devices
 
 import (
-	"net/url"
+	"fmt"
 
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/userapi/storage/devices/postgres"
 	"github.com/matrix-org/dendrite/userapi/storage/devices/sqlite3"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -27,17 +27,13 @@ import (
 
 // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
 // and sets postgres connection parameters
-func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
-	case "file":
-		return sqlite3.NewDatabase(dataSourceName, serverName)
+func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.NewDatabase(dbProperties, serverName)
+	case dbProperties.ConnectionString.IsPostgres():
+		return postgres.NewDatabase(dbProperties, serverName)
 	default:
-		return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }
diff --git a/userapi/storage/devices/storage_wasm.go b/userapi/storage/devices/storage_wasm.go
index a5a515ef..e966c37f 100644
--- a/userapi/storage/devices/storage_wasm.go
+++ b/userapi/storage/devices/storage_wasm.go
@@ -16,28 +16,22 @@ package devices
 
 import (
 	"fmt"
-	"net/url"
 
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/userapi/storage/devices/sqlite3"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
 func NewDatabase(
-	dataSourceName string,
-	dbProperties sqlutil.DbProperties, // nolint:unparam
+	dbProperties *config.DatabaseOptions,
 	serverName gomatrixserverlib.ServerName,
 ) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	case "file":
-		return sqlite3.NewDatabase(dataSourceName, serverName)
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.NewDatabase(dbProperties, serverName)
+	case dbProperties.ConnectionString.IsPostgres():
+		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
-		return nil, fmt.Errorf("Cannot use postgres implementation")
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }