Configuration format v1 (#1230)

* Initial pass at refactoring config (not finished)

* Don't forget current state and EDU servers

* More shifting around

* Update server key API tests

* Fix roomserver test

* Fix more tests

* Further tweaks

* Fix current state server test (sort of)

* Maybe fix appservices

* Fix client API test

* Include database connection string in database options

* Fix sync API build

* Update config test

* Fix unit tests

* Fix federation sender build

* Fix gobind build

* Set Listen address for all services in HTTP monolith mode

* Validate config, reinstate appservice derived in directory, tweaks

* Tweak federation API test

* Set MaxOpenConnections/MaxIdleConnections to previous values

* Update generate-config

8 files changed, 44 insertions, 58 deletions

diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go
index 290ef46e..1c14559f 100644
--- a/mediaapi/mediaapi.go
+++ b/mediaapi/mediaapi.go
@@ -26,11 +26,11 @@ import (
 
 // AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component.
 func AddPublicRoutes(
-	router *mux.Router, cfg *config.Dendrite,
+	router *mux.Router, cfg *config.MediaAPI,
 	userAPI userapi.UserInternalAPI,
 	client *gomatrixserverlib.Client,
 ) {
-	mediaDB, err := storage.Open(string(cfg.Database.MediaAPI), cfg.DbProperties())
+	mediaDB, err := storage.Open(&cfg.Database)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to media db")
 	}
diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go
index 7e121de3..e61fa82b 100644
--- a/mediaapi/routing/download.go
+++ b/mediaapi/routing/download.go
@@ -73,7 +73,7 @@ func Download(
 	req *http.Request,
 	origin gomatrixserverlib.ServerName,
 	mediaID types.MediaID,
-	cfg *config.Dendrite,
+	cfg *config.MediaAPI,
 	db storage.Database,
 	client *gomatrixserverlib.Client,
 	activeRemoteRequests *types.ActiveRemoteRequests,
@@ -203,7 +203,7 @@ func (r *downloadRequest) Validate() *util.JSONResponse {
 func (r *downloadRequest) doDownload(
 	ctx context.Context,
 	w http.ResponseWriter,
-	cfg *config.Dendrite,
+	cfg *config.MediaAPI,
 	db storage.Database,
 	client *gomatrixserverlib.Client,
 	activeRemoteRequests *types.ActiveRemoteRequests,
@@ -233,9 +233,9 @@ func (r *downloadRequest) doDownload(
 		r.MediaMetadata = mediaMetadata
 	}
 	return r.respondFromLocalFile(
-		ctx, w, cfg.Media.AbsBasePath, activeThumbnailGeneration,
-		cfg.Media.MaxThumbnailGenerators, db,
-		cfg.Media.DynamicThumbnails, cfg.Media.ThumbnailSizes,
+		ctx, w, cfg.AbsBasePath, activeThumbnailGeneration,
+		cfg.MaxThumbnailGenerators, db,
+		cfg.DynamicThumbnails, cfg.ThumbnailSizes,
 	)
 }
 
@@ -514,7 +514,7 @@ func (r *downloadRequest) generateThumbnail(
 func (r *downloadRequest) getRemoteFile(
 	ctx context.Context,
 	client *gomatrixserverlib.Client,
-	cfg *config.Dendrite,
+	cfg *config.MediaAPI,
 	db storage.Database,
 	activeRemoteRequests *types.ActiveRemoteRequests,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
@@ -550,9 +550,9 @@ func (r *downloadRequest) getRemoteFile(
 			// If we do not have a record, we need to fetch the remote file first and then respond from the local file
 			err := r.fetchRemoteFileAndStoreMetadata(
 				ctx, client,
-				cfg.Media.AbsBasePath, *cfg.Media.MaxFileSizeBytes, db,
-				cfg.Media.ThumbnailSizes, activeThumbnailGeneration,
-				cfg.Media.MaxThumbnailGenerators,
+				cfg.AbsBasePath, *cfg.MaxFileSizeBytes, db,
+				cfg.ThumbnailSizes, activeThumbnailGeneration,
+				cfg.MaxThumbnailGenerators,
 			)
 			if err != nil {
 				return errors.Wrap(err, "error querying the database.")
diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go
index f4a8b157..ee5f28f9 100644
--- a/mediaapi/routing/routing.go
+++ b/mediaapi/routing/routing.go
@@ -42,7 +42,7 @@ const pathPrefixV1 = "/media/v1" // TODO: remove when synapse is fixed
 // nolint: gocyclo
 func Setup(
 	publicAPIMux *mux.Router,
-	cfg *config.Dendrite,
+	cfg *config.MediaAPI,
 	db storage.Database,
 	userAPI userapi.UserInternalAPI,
 	client *gomatrixserverlib.Client,
@@ -81,7 +81,7 @@ func Setup(
 
 func makeDownloadAPI(
 	name string,
-	cfg *config.Dendrite,
+	cfg *config.MediaAPI,
 	db storage.Database,
 	client *gomatrixserverlib.Client,
 	activeRemoteRequests *types.ActiveRemoteRequests,
diff --git a/mediaapi/routing/upload.go b/mediaapi/routing/upload.go
index 9b5dc3df..9f35e90c 100644
--- a/mediaapi/routing/upload.go
+++ b/mediaapi/routing/upload.go
@@ -54,7 +54,7 @@ type uploadResponse struct {
 // This implementation supports a configurable maximum file size limit in bytes. If a user tries to upload more than this, they will receive an error that their upload is too large.
 // Uploaded files are processed piece-wise to avoid DoS attacks which would starve the server of memory.
 // TODO: We should time out requests if they have not received any data within a configured timeout period.
-func Upload(req *http.Request, cfg *config.Dendrite, db storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse {
+func Upload(req *http.Request, cfg *config.MediaAPI, db storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse {
 	r, resErr := parseAndValidateRequest(req, cfg)
 	if resErr != nil {
 		return *resErr
@@ -75,7 +75,7 @@ func Upload(req *http.Request, cfg *config.Dendrite, db storage.Database, active
 // parseAndValidateRequest parses the incoming upload request to validate and extract
 // all the metadata about the media being uploaded.
 // Returns either an uploadRequest or an error formatted as a util.JSONResponse
-func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRequest, *util.JSONResponse) {
+func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRequest, *util.JSONResponse) {
 	r := &uploadRequest{
 		MediaMetadata: &types.MediaMetadata{
 			Origin:        cfg.Matrix.ServerName,
@@ -86,7 +86,7 @@ func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRe
 		Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.Matrix.ServerName),
 	}
 
-	if resErr := r.Validate(*cfg.Media.MaxFileSizeBytes); resErr != nil {
+	if resErr := r.Validate(*cfg.MaxFileSizeBytes); resErr != nil {
 		return nil, resErr
 	}
 
@@ -96,7 +96,7 @@ func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRe
 func (r *uploadRequest) doUpload(
 	ctx context.Context,
 	reqReader io.Reader,
-	cfg *config.Dendrite,
+	cfg *config.MediaAPI,
 	db storage.Database,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
 ) *util.JSONResponse {
@@ -110,10 +110,10 @@ func (r *uploadRequest) doUpload(
 	// method of deduplicating files to save storage, as well as a way to conduct
 	// integrity checks on the file data in the repository.
 	// Data is truncated to maxFileSizeBytes. Content-Length was reported as 0 < Content-Length <= maxFileSizeBytes so this is OK.
-	hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.Media.MaxFileSizeBytes, cfg.Media.AbsBasePath)
+	hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.MaxFileSizeBytes, cfg.AbsBasePath)
 	if err != nil {
 		r.Logger.WithError(err).WithFields(log.Fields{
-			"MaxFileSizeBytes": *cfg.Media.MaxFileSizeBytes,
+			"MaxFileSizeBytes": *cfg.MaxFileSizeBytes,
 		}).Warn("Error while transferring file")
 		fileutils.RemoveDir(tmpDir, r.Logger)
 		return &util.JSONResponse{
@@ -159,8 +159,8 @@ func (r *uploadRequest) doUpload(
 	}
 
 	return r.storeFileAndMetadata(
-		ctx, tmpDir, cfg.Media.AbsBasePath, db, cfg.Media.ThumbnailSizes,
-		activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators,
+		ctx, tmpDir, cfg.AbsBasePath, db, cfg.ThumbnailSizes,
+		activeThumbnailGeneration, cfg.MaxThumbnailGenerators,
 	)
 }
 
diff --git a/mediaapi/storage/postgres/storage.go b/mediaapi/storage/postgres/storage.go
index e45e0841..756913d3 100644
--- a/mediaapi/storage/postgres/storage.go
+++ b/mediaapi/storage/postgres/storage.go
@@ -21,6 +21,7 @@ import (
 
 	// Import the postgres database driver.
 	_ "github.com/lib/pq"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -33,10 +34,10 @@ type Database struct {
 }
 
 // Open opens a postgres database.
-func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) {
+func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
 	var d Database
 	var err error
-	if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
+	if d.db, err = sqlutil.Open(dbProperties); err != nil {
 		return nil, err
 	}
 	if err = d.statements.prepare(d.db); err != nil {
diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go
index 010c0a66..a1e7fec7 100644
--- a/mediaapi/storage/sqlite3/storage.go
+++ b/mediaapi/storage/sqlite3/storage.go
@@ -20,6 +20,7 @@ import (
 	"database/sql"
 
 	// Import the postgres database driver.
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -33,14 +34,10 @@ type Database struct {
 }
 
 // Open opens a postgres database.
-func Open(dataSourceName string) (*Database, error) {
+func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
 	var d Database
 	var err error
-	cs, err := sqlutil.ParseFileURI(dataSourceName)
-	if err != nil {
-		return nil, err
-	}
-	if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil {
+	if d.db, err = sqlutil.Open(dbProperties); err != nil {
 		return nil, err
 	}
 	if err = d.statements.prepare(d.db); err != nil {
diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go
index 5ff114db..829d47b3 100644
--- a/mediaapi/storage/storage.go
+++ b/mediaapi/storage/storage.go
@@ -17,25 +17,21 @@
 package storage
 
 import (
-	"net/url"
+	"fmt"
 
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/mediaapi/storage/postgres"
 	"github.com/matrix-org/dendrite/mediaapi/storage/sqlite3"
 )
 
 // Open opens a postgres database.
-func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return postgres.Open(dataSourceName, dbProperties)
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return postgres.Open(dataSourceName, dbProperties)
-	case "file":
-		return sqlite3.Open(dataSourceName)
+func Open(dbProperties *config.DatabaseOptions) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.Open(dbProperties)
+	case dbProperties.ConnectionString.IsPostgres():
+		return postgres.Open(dbProperties)
 	default:
-		return postgres.Open(dataSourceName, dbProperties)
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }
diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go
index a672271f..6b5de681 100644
--- a/mediaapi/storage/storage_wasm.go
+++ b/mediaapi/storage/storage_wasm.go
@@ -16,27 +16,19 @@ package storage
 
 import (
 	"fmt"
-	"net/url"
 
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/mediaapi/storage/sqlite3"
 )
 
 // Open opens a postgres database.
-func Open(
-	dataSourceName string,
-	dbProperties sqlutil.DbProperties, // nolint:unparam
-) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	case "file":
-		return sqlite3.Open(dataSourceName)
+func Open(dbProperties *config.DatabaseOptions) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.Open(dbProperties)
+	case dbProperties.ConnectionString.IsPostgres():
+		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
-		return nil, fmt.Errorf("Cannot use postgres implementation")
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }