Configuration format v1 (#1230)

* Initial pass at refactoring config (not finished)

* Don't forget current state and EDU servers

* More shifting around

* Update server key API tests

* Fix roomserver test

* Fix more tests

* Further tweaks

* Fix current state server test (sort of)

* Maybe fix appservices

* Fix client API test

* Include database connection string in database options

* Fix sync API build

* Update config test

* Fix unit tests

* Fix federation sender build

* Fix gobind build

* Set Listen address for all services in HTTP monolith mode

* Validate config, reinstate appservice derived in directory, tweaks

* Tweak federation API test

* Set MaxOpenConnections/MaxIdleConnections to previous values

* Update generate-config

23 files changed, 170 insertions, 127 deletions

diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go
index ff022ec3..73e223d6 100644
--- a/cmd/create-account/main.go
+++ b/cmd/create-account/main.go
@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -63,7 +64,9 @@ func main() {
 
 	serverName := gomatrixserverlib.ServerName(*serverNameStr)
 
-	accountDB, err := accounts.NewDatabase(*database, nil, serverName)
+	accountDB, err := accounts.NewDatabase(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(*database),
+	}, serverName)
 	if err != nil {
 		fmt.Println(err.Error())
 		os.Exit(1)
@@ -75,7 +78,9 @@ func main() {
 		os.Exit(1)
 	}
 
-	deviceDB, err := devices.NewDatabase(*database, nil, serverName)
+	deviceDB, err := devices.NewDatabase(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(*database),
+	}, serverName)
 	if err != nil {
 		fmt.Println(err.Error())
 		os.Exit(1)
diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go
index 6719d047..632b45e6 100644
--- a/cmd/dendrite-appservice-server/main.go
+++ b/cmd/dendrite-appservice-server/main.go
@@ -30,6 +30,6 @@ func main() {
 	intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	appservice.AddInternalRoutes(base.InternalAPIMux, intAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI))
+	base.SetupAndServeHTTP(string(base.Cfg.AppServiceAPI.Bind), string(base.Cfg.AppServiceAPI.Listen))
 
 }
diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go
index 367f27d1..2b6c39b6 100644
--- a/cmd/dendrite-client-api-server/main.go
+++ b/cmd/dendrite-client-api-server/main.go
@@ -39,10 +39,10 @@ func main() {
 	keyAPI := base.KeyServerHTTPClient()
 
 	clientapi.AddPublicRoutes(
-		base.PublicAPIMux, base.Cfg, base.KafkaProducer, deviceDB, accountDB, federation,
+		base.PublicAPIMux, &base.Cfg.ClientAPI, base.KafkaProducer, deviceDB, accountDB, federation,
 		rsAPI, eduInputAPI, asQuery, stateAPI, transactions.New(), fsAPI, userAPI, keyAPI, nil,
 	)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI))
+	base.SetupAndServeHTTP(string(base.Cfg.ClientAPI.Bind), string(base.Cfg.ClientAPI.Listen))
 
 }
diff --git a/cmd/dendrite-current-state-server/main.go b/cmd/dendrite-current-state-server/main.go
index 0d4eae7b..a8c0813f 100644
--- a/cmd/dendrite-current-state-server/main.go
+++ b/cmd/dendrite-current-state-server/main.go
@@ -24,10 +24,10 @@ func main() {
 	base := setup.NewBaseDendrite(cfg, "CurrentStateServer", true)
 	defer base.Close() // nolint: errcheck
 
-	stateAPI := currentstateserver.NewInternalAPI(cfg, base.KafkaConsumer)
+	stateAPI := currentstateserver.NewInternalAPI(&cfg.CurrentStateServer, base.KafkaConsumer)
 
 	currentstateserver.AddInternalRoutes(base.InternalAPIMux, stateAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.CurrentState), string(base.Cfg.Listen.CurrentState))
+	base.SetupAndServeHTTP(string(base.Cfg.CurrentStateServer.Bind), string(base.Cfg.CurrentStateServer.Listen))
 
 }
diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go
index 93d54f21..8c28014a 100644
--- a/cmd/dendrite-demo-libp2p/main.go
+++ b/cmd/dendrite-demo-libp2p/main.go
@@ -75,8 +75,8 @@ func createFederationClient(
 		p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")),
 	)
 	return gomatrixserverlib.NewFederationClientWithTransport(
-		base.Base.Cfg.Matrix.ServerName, base.Base.Cfg.Matrix.KeyID,
-		base.Base.Cfg.Matrix.PrivateKey, true, tr,
+		base.Base.Cfg.Global.ServerName, base.Base.Cfg.Global.KeyID,
+		base.Base.Cfg.Global.PrivateKey, true, tr,
 	)
 }
 
@@ -113,25 +113,28 @@ func main() {
 	}
 
 	cfg := config.Dendrite{}
-	cfg.SetDefaults()
-	cfg.Matrix.ServerName = "p2p"
-	cfg.Matrix.PrivateKey = privKey
-	cfg.Matrix.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName))
-	cfg.Kafka.UseNaffka = true
-	cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
-	cfg.Kafka.Topics.OutputClientData = "clientapiOutput"
-	cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
-	cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-	cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
-	cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
-	cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
-	cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
-	cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
-	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
-	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
-	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
-	cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName))
-	cfg.Database.E2EKey = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName))
+	cfg.Defaults()
+	cfg.Global.ServerName = "p2p"
+	cfg.Global.PrivateKey = privKey
+	cfg.Global.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName))
+	cfg.Global.Kafka.UseNaffka = true
+	cfg.Global.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
+	cfg.Global.Kafka.Topics.OutputClientData = "clientapiOutput"
+	cfg.Global.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
+	cfg.Global.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput"
+	cfg.Global.Kafka.Topics.OutputKeyChangeEvent = "keyChangeOutput"
+	cfg.FederationSender.FederationMaxRetries = 6
+	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
+	cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
+	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
+	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
+	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
+	cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
+	cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
+	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
+	cfg.CurrentStateServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName))
+	cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
+	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName))
 	if err = cfg.Derive(); err != nil {
 		panic(err)
 	}
@@ -142,19 +145,19 @@ func main() {
 	accountDB := base.Base.CreateAccountsDB()
 	deviceDB := base.Base.CreateDeviceDB()
 	federation := createFederationClient(base)
-	keyAPI := keyserver.NewInternalAPI(base.Base.Cfg, federation, base.Base.KafkaProducer)
-	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil, keyAPI)
+	keyAPI := keyserver.NewInternalAPI(&base.Base.Cfg.KeyServer, federation, base.Base.KafkaProducer)
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, nil, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
 	serverKeyAPI := serverkeyapi.NewInternalAPI(
-		base.Base.Cfg, federation, base.Base.Caches,
+		&base.Base.Cfg.ServerKeyAPI, federation, base.Base.Caches,
 	)
 	keyRing := serverKeyAPI.KeyRing()
 	createKeyDB(
 		base, serverKeyAPI,
 	)
 
-	stateAPI := currentstateserver.NewInternalAPI(base.Base.Cfg, base.Base.KafkaConsumer)
+	stateAPI := currentstateserver.NewInternalAPI(&base.Base.Cfg.CurrentStateServer, base.Base.KafkaConsumer)
 	rsAPI := roomserver.NewInternalAPI(
 		&base.Base, keyRing, federation,
 	)
@@ -198,7 +201,7 @@ func main() {
 		base.Base.BaseMux,
 		base.Base.PublicAPIMux,
 		base.Base.InternalAPIMux,
-		&cfg,
+		&cfg.Global,
 		base.Base.UseHTTPAPIs,
 	)
 
diff --git a/cmd/dendrite-demo-libp2p/p2pdendrite.go b/cmd/dendrite-demo-libp2p/p2pdendrite.go
index 4270143f..8fff46af 100644
--- a/cmd/dendrite-demo-libp2p/p2pdendrite.go
+++ b/cmd/dendrite-demo-libp2p/p2pdendrite.go
@@ -58,7 +58,7 @@ func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite {
 
 	ctx, cancel := context.WithCancel(context.Background())
 
-	privKey, err := crypto.UnmarshalEd25519PrivateKey(cfg.Matrix.PrivateKey[:])
+	privKey, err := crypto.UnmarshalEd25519PrivateKey(cfg.Global.PrivateKey[:])
 	if err != nil {
 		panic(err)
 	}
@@ -97,7 +97,7 @@ func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite {
 	fmt.Println("Our node ID:", libp2p.ID())
 	fmt.Println("Our addresses:", libp2p.Addrs())
 
-	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(libp2p.ID().String())
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(libp2p.ID().String())
 
 	return &P2PDendrite{
 		Base:          *baseDendrite,
diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go
index 81bf994b..cf4c7735 100644
--- a/cmd/dendrite-demo-yggdrasil/main.go
+++ b/cmd/dendrite-demo-yggdrasil/main.go
@@ -68,27 +68,28 @@ func main() {
 	}
 
 	cfg := &config.Dendrite{}
-	cfg.SetDefaults()
-	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
-	cfg.Matrix.PrivateKey = ygg.SigningPrivateKey()
-	cfg.Matrix.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
-	cfg.Matrix.FederationMaxRetries = 8
-	cfg.Kafka.UseNaffka = true
-	cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
-	cfg.Kafka.Topics.OutputClientData = "clientapiOutput"
-	cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
-	cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-	cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
-	cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
-	cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
-	cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
-	cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
-	cfg.Database.E2EKey = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
-	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
-	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
-	cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName))
-	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
-	cfg.Database.E2EKey = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName))
+	cfg.Defaults()
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
+	cfg.Global.PrivateKey = ygg.SigningPrivateKey()
+	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
+	cfg.Global.Kafka.UseNaffka = true
+	cfg.Global.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
+	cfg.Global.Kafka.Topics.OutputClientData = "clientapiOutput"
+	cfg.Global.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
+	cfg.Global.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput"
+	cfg.Global.Kafka.Topics.OutputKeyChangeEvent = "keyChangeOutput"
+	cfg.FederationSender.FederationMaxRetries = 8
+	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
+	cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
+	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
+	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
+	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
+	cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
+	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
+	cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
+	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
+	cfg.CurrentStateServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName))
+	cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
 	if err = cfg.Derive(); err != nil {
 		panic(err)
 	}
@@ -103,8 +104,8 @@ func main() {
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	keyAPI := keyserver.NewInternalAPI(base.Cfg, federation, base.KafkaProducer)
-	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil, keyAPI)
+	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation, base.KafkaProducer)
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, nil, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
 	rsComponent := roomserver.NewInternalAPI(
@@ -117,7 +118,7 @@ func main() {
 	)
 
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
-	stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
+	stateAPI := currentstateserver.NewInternalAPI(&base.Cfg.CurrentStateServer, base.KafkaConsumer)
 	fsAPI := federationsender.NewInternalAPI(
 		base, federation, rsAPI, stateAPI, keyRing,
 	)
@@ -155,7 +156,6 @@ func main() {
 		UserAPI:             userAPI,
 		StateAPI:            stateAPI,
 		KeyAPI:              keyAPI,
-		//ServerKeyAPI:        serverKeyAPI,
 		ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider(
 			ygg, fsAPI, federation,
 		),
@@ -166,7 +166,7 @@ func main() {
 		base.BaseMux,
 		base.PublicAPIMux,
 		base.InternalAPIMux,
-		cfg,
+		&cfg.Global,
 		base.UseHTTPAPIs,
 	)
 
diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/client.go b/cmd/dendrite-demo-yggdrasil/yggconn/client.go
index 9cb6f975..1236c553 100644
--- a/cmd/dendrite-demo-yggdrasil/yggconn/client.go
+++ b/cmd/dendrite-demo-yggdrasil/yggconn/client.go
@@ -54,7 +54,7 @@ func (n *Node) CreateFederationClient(
 		},
 	)
 	return gomatrixserverlib.NewFederationClientWithTransport(
-		base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID,
-		base.Cfg.Matrix.PrivateKey, true, tr,
+		base.Cfg.Global.ServerName, base.Cfg.Global.KeyID,
+		base.Cfg.Global.PrivateKey, true, tr,
 	)
 }
diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-edu-server/main.go
index 6704ebd0..d3e4e0a0 100644
--- a/cmd/dendrite-edu-server/main.go
+++ b/cmd/dendrite-edu-server/main.go
@@ -33,6 +33,6 @@ func main() {
 	intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient())
 	eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.EDUServer), string(base.Cfg.Listen.EDUServer))
+	base.SetupAndServeHTTP(string(base.Cfg.EDUServer.Bind), string(base.Cfg.EDUServer.Listen))
 
 }
diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go
index 70d8394f..3b12a295 100644
--- a/cmd/dendrite-federation-api-server/main.go
+++ b/cmd/dendrite-federation-api-server/main.go
@@ -33,10 +33,10 @@ func main() {
 	keyAPI := base.KeyServerHTTPClient()
 
 	federationapi.AddPublicRoutes(
-		base.PublicAPIMux, base.Cfg, userAPI, federation, keyRing,
+		base.PublicAPIMux, &base.Cfg.FederationAPI, userAPI, federation, keyRing,
 		rsAPI, fsAPI, base.EDUServerClient(), base.CurrentStateAPIClient(), keyAPI,
 	)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI))
+	base.SetupAndServeHTTP(string(base.Cfg.FederationAPI.Bind), string(base.Cfg.FederationAPI.Listen))
 
 }
diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go
index fa6cf7ab..152c798a 100644
--- a/cmd/dendrite-federation-sender-server/main.go
+++ b/cmd/dendrite-federation-sender-server/main.go
@@ -35,6 +35,6 @@ func main() {
 	)
 	federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender))
+	base.SetupAndServeHTTP(string(base.Cfg.FederationSender.Bind), string(base.Cfg.FederationSender.Listen))
 
 }
diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-key-server/main.go
index 94ea819f..f3110a1e 100644
--- a/cmd/dendrite-key-server/main.go
+++ b/cmd/dendrite-key-server/main.go
@@ -24,11 +24,11 @@ func main() {
 	base := setup.NewBaseDendrite(cfg, "KeyServer", true)
 	defer base.Close() // nolint: errcheck
 
-	intAPI := keyserver.NewInternalAPI(base.Cfg, base.CreateFederationClient(), base.KafkaProducer)
+	intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient(), base.KafkaProducer)
 	intAPI.SetUserAPI(base.UserAPIClient())
 
 	keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.KeyServer), string(base.Cfg.Listen.KeyServer))
+	base.SetupAndServeHTTP(string(base.Cfg.KeyServer.Bind), string(base.Cfg.KeyServer.Listen))
 
 }
diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go
index 2b9c5090..1bbb62bd 100644
--- a/cmd/dendrite-media-api-server/main.go
+++ b/cmd/dendrite-media-api-server/main.go
@@ -26,10 +26,10 @@ func main() {
 	defer base.Close() // nolint: errcheck
 
 	userAPI := base.UserAPIClient()
-	client := gomatrixserverlib.NewClient(cfg.Matrix.FederationDisableTLSValidation)
+	client := gomatrixserverlib.NewClient(cfg.FederationSender.DisableTLSValidation)
 
-	mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, userAPI, client)
+	mediaapi.AddPublicRoutes(base.PublicAPIMux, &base.Cfg.MediaAPI, userAPI, client)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.MediaAPI), string(base.Cfg.Listen.MediaAPI))
+	base.SetupAndServeHTTP(string(base.Cfg.MediaAPI.Bind), string(base.Cfg.MediaAPI.Listen))
 
 }
diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go
index ed30685b..8f98cdd0 100644
--- a/cmd/dendrite-monolith-server/main.go
+++ b/cmd/dendrite-monolith-server/main.go
@@ -54,11 +54,17 @@ func main() {
 		// the API endpoints. They'll listen on the same port as the monolith
 		// itself.
 		addr := config.Address(*httpBindAddr)
-		cfg.Listen.RoomServer = addr
-		cfg.Listen.EDUServer = addr
-		cfg.Listen.AppServiceAPI = addr
-		cfg.Listen.FederationSender = addr
-		cfg.Listen.ServerKeyAPI = addr
+		cfg.AppServiceAPI.Listen = addr
+		cfg.ClientAPI.Listen = addr
+		cfg.CurrentStateServer.Listen = addr
+		cfg.EDUServer.Listen = addr
+		cfg.FederationAPI.Listen = addr
+		cfg.FederationSender.Listen = addr
+		cfg.KeyServer.Listen = addr
+		cfg.MediaAPI.Listen = addr
+		cfg.RoomServer.Listen = addr
+		cfg.ServerKeyAPI.Listen = addr
+		cfg.SyncAPI.Listen = addr
 	}
 
 	base := setup.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs)
@@ -69,15 +75,15 @@ func main() {
 	federation := base.CreateFederationClient()
 
 	serverKeyAPI := serverkeyapi.NewInternalAPI(
-		base.Cfg, federation, base.Caches,
+		&base.Cfg.ServerKeyAPI, federation, base.Caches,
 	)
 	if base.UseHTTPAPIs {
 		serverkeyapi.AddInternalRoutes(base.InternalAPIMux, serverKeyAPI, base.Caches)
 		serverKeyAPI = base.ServerKeyAPIClient()
 	}
 	keyRing := serverKeyAPI.KeyRing()
-	keyAPI := keyserver.NewInternalAPI(base.Cfg, federation, base.KafkaProducer)
-	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices, keyAPI)
+	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation, base.KafkaProducer)
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, cfg.Derived.ApplicationServices, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
 	rsImpl := roomserver.NewInternalAPI(
@@ -109,7 +115,7 @@ func main() {
 		asAPI = base.AppserviceHTTPClient()
 	}
 
-	stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
+	stateAPI := currentstateserver.NewInternalAPI(&base.Cfg.CurrentStateServer, base.KafkaConsumer)
 
 	fsAPI := federationsender.NewInternalAPI(
 		base, federation, rsAPI, stateAPI, keyRing,
@@ -126,7 +132,7 @@ func main() {
 		Config:        base.Cfg,
 		AccountDB:     accountDB,
 		DeviceDB:      deviceDB,
-		Client:        gomatrixserverlib.NewClient(cfg.Matrix.FederationDisableTLSValidation),
+		Client:        gomatrixserverlib.NewClient(cfg.FederationSender.DisableTLSValidation),
 		FedClient:     federation,
 		KeyRing:       keyRing,
 		KafkaConsumer: base.KafkaConsumer,
@@ -147,7 +153,7 @@ func main() {
 		base.BaseMux,
 		base.PublicAPIMux,
 		base.InternalAPIMux,
-		cfg,
+		&cfg.Global,
 		base.UseHTTPAPIs,
 	)
 
diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go
index 627a6867..e1d4b16b 100644
--- a/cmd/dendrite-room-server/main.go
+++ b/cmd/dendrite-room-server/main.go
@@ -33,6 +33,6 @@ func main() {
 	rsAPI.SetFederationSenderAPI(fsAPI)
 	roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer))
+	base.SetupAndServeHTTP(string(base.Cfg.RoomServer.Bind), string(base.Cfg.RoomServer.Listen))
 
 }
diff --git a/cmd/dendrite-server-key-api-server/main.go b/cmd/dendrite-server-key-api-server/main.go
index 9ffaeee3..36f34890 100644
--- a/cmd/dendrite-server-key-api-server/main.go
+++ b/cmd/dendrite-server-key-api-server/main.go
@@ -26,8 +26,8 @@ func main() {
 
 	federation := base.CreateFederationClient()
 
-	intAPI := serverkeyapi.NewInternalAPI(base.Cfg, federation, base.Caches)
+	intAPI := serverkeyapi.NewInternalAPI(&base.Cfg.ServerKeyAPI, federation, base.Caches)
 	serverkeyapi.AddInternalRoutes(base.InternalAPIMux, intAPI, base.Caches)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.ServerKeyAPI), string(base.Cfg.Listen.ServerKeyAPI))
+	base.SetupAndServeHTTP(string(base.Cfg.ServerKeyAPI.Bind), string(base.Cfg.ServerKeyAPI.Listen))
 }
diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go
index 0761a1d1..c25f8ec1 100644
--- a/cmd/dendrite-sync-api-server/main.go
+++ b/cmd/dendrite-sync-api-server/main.go
@@ -31,8 +31,8 @@ func main() {
 
 	syncapi.AddPublicRoutes(
 		base.PublicAPIMux, base.KafkaConsumer, userAPI, rsAPI, base.KeyServerHTTPClient(), base.CurrentStateAPIClient(),
-		federation, cfg)
+		federation, &cfg.SyncAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI))
+	base.SetupAndServeHTTP(string(base.Cfg.SyncAPI.Bind), string(base.Cfg.SyncAPI.Listen))
 
 }
diff --git a/cmd/dendrite-user-api-server/main.go b/cmd/dendrite-user-api-server/main.go
index e6d61da1..22b6255e 100644
--- a/cmd/dendrite-user-api-server/main.go
+++ b/cmd/dendrite-user-api-server/main.go
@@ -27,9 +27,9 @@ func main() {
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
 
-	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient())
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient())
 
 	userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.UserAPI), string(base.Cfg.Listen.UserAPI))
+	base.SetupAndServeHTTP(string(base.Cfg.UserAPI.Bind), string(base.Cfg.UserAPI.Listen))
 }
diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go
index 274d170e..ce7812fa 100644
--- a/cmd/dendritejs/main.go
+++ b/cmd/dendritejs/main.go
@@ -139,7 +139,7 @@ func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLoc
 	tr := go_http_js_libp2p.NewP2pTransport(node)
 
 	fed := gomatrixserverlib.NewFederationClient(
-		cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, true,
+		cfg.Global.ServerName, cfg.Global.KeyID, cfg.Global.PrivateKey, true,
 	)
 	fed.Client = *gomatrixserverlib.NewClientWithTransport(true, tr)
 
@@ -161,31 +161,31 @@ func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http
 
 func main() {
 	cfg := &config.Dendrite{}
-	cfg.SetDefaults()
-	cfg.Kafka.UseNaffka = true
-	cfg.Database.Account = "file:/idb/dendritejs_account.db"
-	cfg.Database.AppService = "file:/idb/dendritejs_appservice.db"
-	cfg.Database.Device = "file:/idb/dendritejs_device.db"
-	cfg.Database.FederationSender = "file:/idb/dendritejs_fedsender.db"
-	cfg.Database.MediaAPI = "file:/idb/dendritejs_mediaapi.db"
-	cfg.Database.Naffka = "file:/idb/dendritejs_naffka.db"
-	cfg.Database.RoomServer = "file:/idb/dendritejs_roomserver.db"
-	cfg.Database.ServerKey = "file:/idb/dendritejs_serverkey.db"
-	cfg.Database.SyncAPI = "file:/idb/dendritejs_syncapi.db"
-	cfg.Database.CurrentState = "file:/idb/dendritejs_currentstate.db"
-	cfg.Database.E2EKey = "file:/idb/dendritejs_e2ekey.db"
-	cfg.Kafka.Topics.OutputTypingEvent = "output_typing_event"
-	cfg.Kafka.Topics.OutputSendToDeviceEvent = "output_send_to_device_event"
-	cfg.Kafka.Topics.OutputClientData = "output_client_data"
-	cfg.Kafka.Topics.OutputRoomEvent = "output_room_event"
-	cfg.Matrix.TrustedIDServers = []string{
+	cfg.Defaults()
+	cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db"
+	cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db"
+	cfg.UserAPI.DeviceDatabase.ConnectionString = "file:/idb/dendritejs_device.db"
+	cfg.FederationSender.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db"
+	cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db"
+	cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db"
+	cfg.ServerKeyAPI.Database.ConnectionString = "file:/idb/dendritejs_serverkey.db"
+	cfg.SyncAPI.Database.ConnectionString = "file:/idb/dendritejs_syncapi.db"
+	cfg.CurrentStateServer.Database.ConnectionString = "file:/idb/dendritejs_currentstate.db"
+	cfg.KeyServer.Database.ConnectionString = "file:/idb/dendritejs_e2ekey.db"
+	cfg.Global.Kafka.UseNaffka = true
+	cfg.Global.Kafka.Database.ConnectionString = "file:/idb/dendritejs_naffka.db"
+	cfg.Global.Kafka.Topics.OutputTypingEvent = "output_typing_event"
+	cfg.Global.Kafka.Topics.OutputSendToDeviceEvent = "output_send_to_device_event"
+	cfg.Global.Kafka.Topics.OutputClientData = "output_client_data"
+	cfg.Global.Kafka.Topics.OutputRoomEvent = "output_room_event"
+	cfg.Global.TrustedIDServers = []string{
 		"matrix.org", "vector.im",
 	}
-	cfg.Matrix.KeyID = libp2pMatrixKeyID
-	cfg.Matrix.PrivateKey = generateKey()
+	cfg.Global.KeyID = libp2pMatrixKeyID
+	cfg.Global.PrivateKey = generateKey()
 
-	serverName, node := createP2PNode(cfg.Matrix.PrivateKey)
-	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(serverName)
+	serverName, node := createP2PNode(cfg.Global.PrivateKey)
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(serverName)
 
 	if err := cfg.Derive(); err != nil {
 		logrus.Fatalf("Failed to derive values from config: %s", err)
@@ -196,8 +196,8 @@ func main() {
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
 	federation := createFederationClient(cfg, node)
-	keyAPI := keyserver.NewInternalAPI(base.Cfg, federation, base.KafkaProducer)
-	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil, keyAPI)
+	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation, base.KafkaProducer)
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, nil, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
 	fetcher := &libp2pKeyFetcher{}
@@ -208,7 +208,7 @@ func main() {
 		KeyDatabase: fetcher,
 	}
 
-	stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
+	stateAPI := currentstateserver.NewInternalAPI(&base.Cfg.CurrentStateServer, base.KafkaConsumer)
 	rsAPI := roomserver.NewInternalAPI(base, keyRing, federation)
 	eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI)
 	asQuery := appservice.NewInternalAPI(
@@ -244,7 +244,7 @@ func main() {
 		base.BaseMux,
 		base.PublicAPIMux,
 		base.InternalAPIMux,
-		cfg,
+		&cfg.Global,
 		base.UseHTTPAPIs,
 	)
 
diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go
new file mode 100644
index 00000000..4dd12593
--- /dev/null
+++ b/cmd/generate-config/main.go
@@ -0,0 +1,29 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/config"
+	"gopkg.in/yaml.v2"
+)
+
+func main() {
+	cfg := &config.Dendrite{}
+	cfg.Defaults()
+	cfg.Logging = []config.LogrusHook{
+		{
+			Type:  "file",
+			Level: "info",
+			Params: map[string]interface{}{
+				"path": "/var/log/dendrite",
+			},
+		},
+	}
+
+	j, err := yaml.Marshal(cfg)
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Println(string(j))
+}
diff --git a/cmd/mediaapi-integration-tests/main.go b/cmd/mediaapi-integration-tests/main.go
index e6ce14d2..4c584979 100644
--- a/cmd/mediaapi-integration-tests/main.go
+++ b/cmd/mediaapi-integration-tests/main.go
@@ -88,9 +88,9 @@ func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error
 	if err != nil {
 		panic(err)
 	}
-	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(proxyAddr)
-	cfg.Media.DynamicThumbnails = dynamicThumbnails
-	if err = yaml.Unmarshal([]byte(thumbnailSizes), &cfg.Media.ThumbnailSizes); err != nil {
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(proxyAddr)
+	cfg.MediaAPI.DynamicThumbnails = dynamicThumbnails
+	if err = yaml.Unmarshal([]byte(thumbnailSizes), &cfg.MediaAPI.ThumbnailSizes); err != nil {
 		panic(err)
 	}
 
@@ -120,7 +120,7 @@ func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error
 		serverArgs,
 	)
 
-	fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, cfg.Listen.MediaAPI, dir)
+	fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, cfg.MediaAPI.Listen, dir)
 	return cmd, cmdChan, proxyCmd, proxyAddr, dir
 }
 
diff --git a/cmd/roomserver-integration-tests/main.go b/cmd/roomserver-integration-tests/main.go
index 3860ca1f..4d3095be 100644
--- a/cmd/roomserver-integration-tests/main.go
+++ b/cmd/roomserver-integration-tests/main.go
@@ -240,7 +240,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R
 		panic(err)
 	}
 
-	outputTopic := string(cfg.Kafka.Topics.OutputRoomEvent)
+	outputTopic := string(cfg.Global.Kafka.Topics.OutputRoomEvent)
 
 	err = exe.DeleteTopic(outputTopic)
 	if err != nil {
@@ -277,7 +277,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R
 	cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)}
 
 	gotOutput, err := runAndReadFromTopic(cmd, cfg.RoomServerURL()+"/metrics", doInput, outputTopic, len(wantOutput), func() {
-		queryAPI, _ := inthttp.NewRoomserverClient("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache)
+		queryAPI, _ := inthttp.NewRoomserverClient("http://"+string(cfg.RoomServer.Listen), &http.Client{Timeout: timeoutHTTP}, cache)
 		checkQueries(queryAPI)
 	})
 	if err != nil {
diff --git a/cmd/syncserver-integration-tests/main.go b/cmd/syncserver-integration-tests/main.go
index cfe8cc16..76fa0eba 100644
--- a/cmd/syncserver-integration-tests/main.go
+++ b/cmd/syncserver-integration-tests/main.go
@@ -132,10 +132,10 @@ func startSyncServer() (*exec.Cmd, chan error) {
 		panic(err)
 	}
 	// TODO use the address assigned by the config generator rather than clobbering.
-	cfg.Matrix.ServerName = "localhost"
-	cfg.Listen.SyncAPI = config.Address(syncserverAddr)
-	cfg.Kafka.Topics.OutputRoomEvent = config.Topic(inputTopic)
-	cfg.Kafka.Topics.OutputClientData = config.Topic(clientTopic)
+	cfg.Global.ServerName = "localhost"
+	cfg.SyncAPI.Listen = config.Address(syncserverAddr)
+	cfg.Global.Kafka.Topics.OutputRoomEvent = config.Topic(inputTopic)
+	cfg.Global.Kafka.Topics.OutputClientData = config.Topic(clientTopic)
 
 	if err := test.WriteConfig(cfg, dir); err != nil {
 		panic(err)