Configuration format v1 (#1230)

* Initial pass at refactoring config (not finished)

* Don't forget current state and EDU servers

* More shifting around

* Update server key API tests

* Fix roomserver test

* Fix more tests

* Further tweaks

* Fix current state server test (sort of)

* Maybe fix appservices

* Fix client API test

* Include database connection string in database options

* Fix sync API build

* Update config test

* Fix unit tests

* Fix federation sender build

* Fix gobind build

* Set Listen address for all services in HTTP monolith mode

* Validate config, reinstate appservice derived in directory, tweaks

* Tweak federation API test

* Set MaxOpenConnections/MaxIdleConnections to previous values

* Update generate-config

5 files changed, 30 insertions, 36 deletions

diff --git a/serverkeyapi/serverkeyapi.go b/serverkeyapi/serverkeyapi.go
index cddd392e..fbaaefad 100644
--- a/serverkeyapi/serverkeyapi.go
+++ b/serverkeyapi/serverkeyapi.go
@@ -25,13 +25,12 @@ func AddInternalRoutes(router *mux.Router, intAPI api.ServerKeyInternalAPI, cach
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
-	cfg *config.Dendrite,
+	cfg *config.ServerKeyAPI,
 	fedClient *gomatrixserverlib.FederationClient,
 	caches *caching.Caches,
 ) api.ServerKeyInternalAPI {
 	innerDB, err := storage.NewDatabase(
-		string(cfg.Database.ServerKey),
-		cfg.DbProperties(),
+		&cfg.Database,
 		cfg.Matrix.ServerName,
 		cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey),
 		cfg.Matrix.KeyID,
@@ -62,7 +61,7 @@ func NewInternalAPI(
 	}
 
 	var b64e = base64.StdEncoding.WithPadding(base64.NoPadding)
-	for _, ps := range cfg.Matrix.KeyPerspectives {
+	for _, ps := range cfg.KeyPerspectives {
 		perspective := &gomatrixserverlib.PerspectiveKeyFetcher{
 			PerspectiveServerName: ps.ServerName,
 			PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{},
diff --git a/serverkeyapi/serverkeyapi_test.go b/serverkeyapi/serverkeyapi_test.go
index c53575bb..152a853e 100644
--- a/serverkeyapi/serverkeyapi_test.go
+++ b/serverkeyapi/serverkeyapi_test.go
@@ -23,7 +23,8 @@ import (
 type server struct {
 	name      gomatrixserverlib.ServerName        // server name
 	validity  time.Duration                       // key validity duration from now
-	config    *config.Dendrite                    // skeleton config, from TestMain
+	config    *config.ServerKeyAPI                // skeleton config, from TestMain
+	fedconfig *config.FederationAPI               //
 	fedclient *gomatrixserverlib.FederationClient // uses MockRoundTripper
 	cache     *caching.Caches                     // server-specific cache
 	api       api.ServerKeyInternalAPI            // server-specific server key API
@@ -69,13 +70,15 @@ func TestMain(m *testing.M) {
 
 		// Draw up just enough Dendrite config for the server key
 		// API to work.
-		s.config = &config.Dendrite{}
-		s.config.SetDefaults()
-		s.config.Matrix.ServerName = gomatrixserverlib.ServerName(s.name)
-		s.config.Matrix.PrivateKey = testPriv
-		s.config.Matrix.KeyID = serverKeyID
-		s.config.Matrix.KeyValidityPeriod = s.validity
-		s.config.Database.ServerKey = config.DataSource("file::memory:")
+		cfg := &config.Dendrite{}
+		cfg.Defaults()
+		cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name)
+		cfg.Global.PrivateKey = testPriv
+		cfg.Global.KeyID = serverKeyID
+		cfg.Global.KeyValidityPeriod = s.validity
+		cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource("file::memory:")
+		s.config = &cfg.ServerKeyAPI
+		s.fedconfig = &cfg.FederationAPI
 
 		// Create a transport which redirects federation requests to
 		// the mock round tripper. Since we're not *really* listening for
@@ -115,7 +118,7 @@ func (m *MockRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err
 	}
 
 	// Get the keys and JSON-ify them.
-	keys := routing.LocalKeys(s.config)
+	keys := routing.LocalKeys(s.fedconfig)
 	body, err := json.MarshalIndent(keys.JSON, "", "  ")
 	if err != nil {
 		return nil, err
diff --git a/serverkeyapi/storage/keydb.go b/serverkeyapi/storage/keydb.go
index c28c4de1..3d3a0c30 100644
--- a/serverkeyapi/storage/keydb.go
+++ b/serverkeyapi/storage/keydb.go
@@ -17,11 +17,11 @@
 package storage
 
 import (
-	"net/url"
+	"fmt"
 
 	"golang.org/x/crypto/ed25519"
 
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/serverkeyapi/storage/postgres"
 	"github.com/matrix-org/dendrite/serverkeyapi/storage/sqlite3"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -29,22 +29,17 @@ import (
 
 // NewDatabase opens a database connection.
 func NewDatabase(
-	dataSourceName string,
-	dbProperties sqlutil.DbProperties,
+	dbProperties *config.DatabaseOptions,
 	serverName gomatrixserverlib.ServerName,
 	serverKey ed25519.PublicKey,
 	serverKeyID gomatrixserverlib.KeyID,
 ) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID)
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID)
-	case "file":
-		return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.NewDatabase(dbProperties, serverName, serverKey, serverKeyID)
+	case dbProperties.ConnectionString.IsPostgres():
+		return postgres.NewDatabase(dbProperties, serverName, serverKey, serverKeyID)
 	default:
-		return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID)
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }
diff --git a/serverkeyapi/storage/postgres/keydb.go b/serverkeyapi/storage/postgres/keydb.go
index aaa4409b..63444085 100644
--- a/serverkeyapi/storage/postgres/keydb.go
+++ b/serverkeyapi/storage/postgres/keydb.go
@@ -20,6 +20,7 @@ import (
 
 	"golang.org/x/crypto/ed25519"
 
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -35,13 +36,12 @@ type Database struct {
 // It prepares all the SQL statements that it will use.
 // Returns an error if there was a problem talking to the database.
 func NewDatabase(
-	dataSourceName string,
-	dbProperties sqlutil.DbProperties,
+	dbProperties *config.DatabaseOptions,
 	serverName gomatrixserverlib.ServerName,
 	serverKey ed25519.PublicKey,
 	serverKeyID gomatrixserverlib.KeyID,
 ) (*Database, error) {
-	db, err := sqlutil.Open("postgres", dataSourceName, dbProperties)
+	db, err := sqlutil.Open(dbProperties)
 	if err != nil {
 		return nil, err
 	}
diff --git a/serverkeyapi/storage/sqlite3/keydb.go b/serverkeyapi/storage/sqlite3/keydb.go
index dc72b79e..5174ece1 100644
--- a/serverkeyapi/storage/sqlite3/keydb.go
+++ b/serverkeyapi/storage/sqlite3/keydb.go
@@ -20,6 +20,7 @@ import (
 
 	"golang.org/x/crypto/ed25519"
 
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 
@@ -37,16 +38,12 @@ type Database struct {
 // It prepares all the SQL statements that it will use.
 // Returns an error if there was a problem talking to the database.
 func NewDatabase(
-	dataSourceName string,
+	dbProperties *config.DatabaseOptions,
 	serverName gomatrixserverlib.ServerName,
 	serverKey ed25519.PublicKey,
 	serverKeyID gomatrixserverlib.KeyID,
 ) (*Database, error) {
-	cs, err := sqlutil.ParseFileURI(dataSourceName)
-	if err != nil {
-		return nil, err
-	}
-	db, err := sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil)
+	db, err := sqlutil.Open(dbProperties)
 	if err != nil {
 		return nil, err
 	}