Configuration format v1 (#1230)

* Initial pass at refactoring config (not finished)

* Don't forget current state and EDU servers

* More shifting around

* Update server key API tests

* Fix roomserver test

* Fix more tests

* Further tweaks

* Fix current state server test (sort of)

* Maybe fix appservices

* Fix client API test

* Include database connection string in database options

* Fix sync API build

* Update config test

* Fix unit tests

* Fix federation sender build

* Fix gobind build

* Set Listen address for all services in HTTP monolith mode

* Validate config, reinstate appservice derived in directory, tweaks

* Tweak federation API test

* Set MaxOpenConnections/MaxIdleConnections to previous values

* Update generate-config

12 files changed, 34 insertions, 33 deletions

diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go
index 079f333a..e9a8e40a 100644
--- a/federationapi/federationapi.go
+++ b/federationapi/federationapi.go
@@ -31,7 +31,7 @@ import (
 // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component.
 func AddPublicRoutes(
 	router *mux.Router,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	userAPI userapi.UserInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 	keyRing gomatrixserverlib.JSONVerifier,
diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go
index 0e0d7740..331d3329 100644
--- a/federationapi/federationapi_test.go
+++ b/federationapi/federationapi_test.go
@@ -20,30 +20,31 @@ import (
 func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 	_, privKey, _ := ed25519.GenerateKey(nil)
 	cfg := &config.Dendrite{}
-	cfg.Matrix.KeyID = gomatrixserverlib.KeyID("ed25519:auto")
-	cfg.Matrix.ServerName = gomatrixserverlib.ServerName("localhost")
-	cfg.Matrix.PrivateKey = privKey
-	cfg.Kafka.UseNaffka = true
-	cfg.Database.Naffka = "file::memory:"
-	cfg.SetDefaults()
-	base := setup.NewBaseDendrite(cfg, "Test", false)
+	cfg.Defaults()
+	cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto")
+	cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost")
+	cfg.Global.PrivateKey = privKey
+	cfg.Global.Kafka.UseNaffka = true
+	cfg.Global.Kafka.Database.ConnectionString = config.DataSource("file::memory:")
+	cfg.FederationSender.Database.ConnectionString = config.DataSource("file::memory:")
+	base := setup.NewBaseDendrite(cfg, "Monolith", false)
 	keyRing := &test.NopJSONVerifier{}
 	fsAPI := base.FederationSenderHTTPClient()
 	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
 	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
-	federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil, nil, nil)
+	federationapi.AddPublicRoutes(base.PublicAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, nil, nil)
 	httputil.SetupHTTPAPI(
 		base.BaseMux,
 		base.PublicAPIMux,
 		base.InternalAPIMux,
-		cfg,
+		&cfg.Global,
 		base.UseHTTPAPIs,
 	)
 	baseURL, cancel := test.ListenAndServe(t, base.BaseMux, true)
 	defer cancel()
 	serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))
 
-	fedCli := gomatrixserverlib.NewFederationClient(serverName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, true)
+	fedCli := gomatrixserverlib.NewFederationClient(serverName, cfg.Global.KeyID, cfg.Global.PrivateKey, true)
 
 	testCases := []struct {
 		roomVer   gomatrixserverlib.RoomVersion
diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go
index f906c73c..ea77c947 100644
--- a/federationapi/routing/backfill.go
+++ b/federationapi/routing/backfill.go
@@ -35,7 +35,7 @@ func Backfill(
 	request *gomatrixserverlib.FederationRequest,
 	rsAPI api.RoomserverInternalAPI,
 	roomID string,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 ) util.JSONResponse {
 	var res api.PerformBackfillResponse
 	var eIDs []string
diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go
index 4a49463a..3f9661ee 100644
--- a/federationapi/routing/invite.go
+++ b/federationapi/routing/invite.go
@@ -34,7 +34,7 @@ func InviteV2(
 	request *gomatrixserverlib.FederationRequest,
 	roomID string,
 	eventID string,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 ) util.JSONResponse {
@@ -56,7 +56,7 @@ func InviteV1(
 	request *gomatrixserverlib.FederationRequest,
 	roomID string,
 	eventID string,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 ) util.JSONResponse {
@@ -86,7 +86,7 @@ func processInvite(
 	strippedState []gomatrixserverlib.InviteV2StrippedState,
 	roomID string,
 	eventID string,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 ) util.JSONResponse {
diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go
index 17981f53..4874f4d1 100644
--- a/federationapi/routing/join.go
+++ b/federationapi/routing/join.go
@@ -32,7 +32,7 @@ import (
 func MakeJoin(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	roomID, userID string,
 	remoteVersions []gomatrixserverlib.RoomVersion,
@@ -95,7 +95,7 @@ func MakeJoin(
 	queryRes := api.QueryLatestEventsAndStateResponse{
 		RoomVersion: verRes.RoomVersion,
 	}
-	event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes)
+	event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg.Matrix, time.Now(), rsAPI, &queryRes)
 	if err == eventutil.ErrRoomNoExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
@@ -141,7 +141,7 @@ func MakeJoin(
 func SendJoin(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 	roomID, eventID string,
diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go
index 90eec9e0..f1ed4176 100644
--- a/federationapi/routing/keys.go
+++ b/federationapi/routing/keys.go
@@ -121,7 +121,7 @@ func ClaimOneTimeKeys(
 
 // LocalKeys returns the local keys for the server.
 // See https://matrix.org/docs/spec/server_server/unstable.html#publishing-keys
-func LocalKeys(cfg *config.Dendrite) util.JSONResponse {
+func LocalKeys(cfg *config.FederationAPI) util.JSONResponse {
 	keys, err := localKeys(cfg, time.Now().Add(cfg.Matrix.KeyValidityPeriod))
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -129,7 +129,7 @@ func LocalKeys(cfg *config.Dendrite) util.JSONResponse {
 	return util.JSONResponse{Code: http.StatusOK, JSON: keys}
 }
 
-func localKeys(cfg *config.Dendrite, validUntil time.Time) (*gomatrixserverlib.ServerKeys, error) {
+func localKeys(cfg *config.FederationAPI, validUntil time.Time) (*gomatrixserverlib.ServerKeys, error) {
 	var keys gomatrixserverlib.ServerKeys
 
 	keys.ServerName = cfg.Matrix.ServerName
@@ -142,7 +142,7 @@ func localKeys(cfg *config.Dendrite, validUntil time.Time) (*gomatrixserverlib.S
 		},
 	}
 
-	keys.TLSFingerprints = cfg.Matrix.TLSFingerPrints
+	keys.TLSFingerprints = cfg.TLSFingerPrints
 	keys.OldVerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.OldVerifyKey{}
 	keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(validUntil)
 
diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go
index 56f1b05a..314b4c72 100644
--- a/federationapi/routing/leave.go
+++ b/federationapi/routing/leave.go
@@ -28,7 +28,7 @@ import (
 func MakeLeave(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	roomID, userID string,
 ) util.JSONResponse {
@@ -60,7 +60,7 @@ func MakeLeave(
 	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
-	event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes)
+	event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg.Matrix, time.Now(), rsAPI, &queryRes)
 	if err == eventutil.ErrRoomNoExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
@@ -102,7 +102,7 @@ func MakeLeave(
 func SendLeave(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 	roomID, eventID string,
diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go
index a6180ae6..f1d90bbf 100644
--- a/federationapi/routing/profile.go
+++ b/federationapi/routing/profile.go
@@ -30,7 +30,7 @@ import (
 func GetProfile(
 	httpReq *http.Request,
 	userAPI userapi.UserInternalAPI,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 ) util.JSONResponse {
 	userID, field := httpReq.FormValue("user_id"), httpReq.FormValue("field")
 
diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go
index 39fd6d2e..99b5460b 100644
--- a/federationapi/routing/query.go
+++ b/federationapi/routing/query.go
@@ -31,7 +31,7 @@ import (
 func RoomAliasToID(
 	httpReq *http.Request,
 	federation *gomatrixserverlib.FederationClient,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	senderAPI federationSenderAPI.FederationSenderInternalAPI,
 ) util.JSONResponse {
diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go
index 30a65271..88fd8757 100644
--- a/federationapi/routing/routing.go
+++ b/federationapi/routing/routing.go
@@ -47,7 +47,7 @@ const (
 // nolint: gocyclo
 func Setup(
 	publicAPIMux *mux.Router,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	eduAPI eduserverAPI.EDUServerInputAPI,
 	fsAPI federationSenderAPI.FederationSenderInternalAPI,
diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go
index f85da148..d1aa728c 100644
--- a/federationapi/routing/send.go
+++ b/federationapi/routing/send.go
@@ -35,7 +35,7 @@ func Send(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	txnID gomatrixserverlib.TransactionID,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	eduAPI eduserverAPI.EDUServerInputAPI,
 	keyAPI keyapi.KeyInternalAPI,
diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go
index 61788010..e8d9a939 100644
--- a/federationapi/routing/threepid.go
+++ b/federationapi/routing/threepid.go
@@ -56,7 +56,7 @@ var (
 // CreateInvitesFrom3PIDInvites implements POST /_matrix/federation/v1/3pid/onbind
 func CreateInvitesFrom3PIDInvites(
 	req *http.Request, rsAPI api.RoomserverInternalAPI,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	federation *gomatrixserverlib.FederationClient,
 	userAPI userapi.UserInternalAPI,
 ) util.JSONResponse {
@@ -106,7 +106,7 @@ func ExchangeThirdPartyInvite(
 	request *gomatrixserverlib.FederationRequest,
 	roomID string,
 	rsAPI api.RoomserverInternalAPI,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
 	var builder gomatrixserverlib.EventBuilder
@@ -196,7 +196,7 @@ func ExchangeThirdPartyInvite(
 // necessary data to do so.
 func createInviteFrom3PIDInvite(
 	ctx context.Context, rsAPI api.RoomserverInternalAPI,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	inv invite, federation *gomatrixserverlib.FederationClient,
 	userAPI userapi.UserInternalAPI,
 ) (*gomatrixserverlib.Event, error) {
@@ -263,7 +263,7 @@ func createInviteFrom3PIDInvite(
 func buildMembershipEvent(
 	ctx context.Context,
 	builder *gomatrixserverlib.EventBuilder, rsAPI api.RoomserverInternalAPI,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 ) (*gomatrixserverlib.Event, error) {
 	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
 	if err != nil {
@@ -327,7 +327,7 @@ func buildMembershipEvent(
 // them responded with an error.
 func sendToRemoteServer(
 	ctx context.Context, inv invite,
-	federation *gomatrixserverlib.FederationClient, _ *config.Dendrite,
+	federation *gomatrixserverlib.FederationClient, _ *config.FederationAPI,
 	builder gomatrixserverlib.EventBuilder,
 ) (err error) {
 	remoteServers := make([]gomatrixserverlib.ServerName, 2)