diff options
| author | Neil Alexander <neilalexander@users.noreply.github.com> | 2020-08-10 14:18:04 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-08-10 14:18:04 +0100 |
| commit | 4b09f445c992fd0a389efc34d75aaa7e5bd50e9c (patch) | |
| tree | 18d6168718ac06e569eb271f25ed4dc064010b50 | |
| parent | fdabba1851c489d801ea4029bce9dec7d415b2df (diff) | |
Configuration format v1 (#1230)
* Initial pass at refactoring config (not finished)
* Don't forget current state and EDU servers
* More shifting around
* Update server key API tests
* Fix roomserver test
* Fix more tests
* Further tweaks
* Fix current state server test (sort of)
* Maybe fix appservices
* Fix client API test
* Include database connection string in database options
* Fix sync API build
* Update config test
* Fix unit tests
* Fix federation sender build
* Fix gobind build
* Set Listen address for all services in HTTP monolith mode
* Validate config, reinstate appservice derived in directory, tweaks
* Tweak federation API test
* Set MaxOpenConnections/MaxIdleConnections to previous values
* Update generate-config
155 files changed, 1726 insertions, 1513 deletions
diff --git a/appservice/appservice.go b/appservice/appservice.go index 72869041..e356f68e 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -48,7 +48,7 @@ func NewInternalAPI( rsAPI roomserverAPI.RoomserverInternalAPI, ) appserviceAPI.AppServiceQueryAPI { // Create a connection to the appservice postgres DB - appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService), base.Cfg.DbProperties()) + appserviceDB, err := storage.NewDatabase(&base.Cfg.AppServiceAPI.Database) if err != nil { logrus.WithError(err).Panicf("failed to connect to appservice db") } diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 4c0156b2..84f2b9ab 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -48,7 +48,7 @@ func NewOutputRoomEventConsumer( workerStates []types.ApplicationServiceWorkerState, ) *OutputRoomEventConsumer { consumer := internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputRoomEvent), + Topic: string(cfg.Global.Kafka.Topics.OutputRoomEvent), Consumer: kafkaConsumer, PartitionStore: appserviceDB, } @@ -56,7 +56,7 @@ func NewOutputRoomEventConsumer( roomServerConsumer: &consumer, asDB: appserviceDB, rsAPI: rsAPI, - serverName: string(cfg.Matrix.ServerName), + serverName: string(cfg.Global.ServerName), workerStates: workerStates, } consumer.ProcessMessage = s.onMessage diff --git a/appservice/storage/postgres/storage.go b/appservice/storage/postgres/storage.go index 03f331d6..9fda87ae 100644 --- a/appservice/storage/postgres/storage.go +++ b/appservice/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( // Import postgres database driver _ "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -34,10 +35,10 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { + if result.db, err = sqlutil.Open(dbProperties); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index cb55c8d9..59af9016 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -20,6 +20,7 @@ import ( "database/sql" // Import SQLite database driver + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -34,14 +35,10 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dataSourceName string) (*Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) { var result Database var err error - cs, err := sqlutil.ParseFileURI(dataSourceName) - if err != nil { - return nil, err - } - if result.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { + if result.db, err = sqlutil.Open(dbProperties); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go index c848d15d..e2d7e4e5 100644 --- a/appservice/storage/storage.go +++ b/appservice/storage/storage.go @@ -17,26 +17,22 @@ package storage import ( - "net/url" + "fmt" "github.com/matrix-org/dendrite/appservice/storage/postgres" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" ) // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets DB connection parameters -func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgres.NewDatabase(dataSourceName, dbProperties) - } - switch uri.Scheme { - case "postgres": - return postgres.NewDatabase(dataSourceName, dbProperties) - case "file": - return sqlite3.NewDatabase(dataSourceName) +func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(dbProperties) + case dbProperties.ConnectionString.IsPostgres(): + return postgres.NewDatabase(dbProperties) default: - return postgres.NewDatabase(dataSourceName, dbProperties) + return nil, fmt.Errorf("unexpected database type") } } diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go index 1d6c4b4a..7eb7da26 100644 --- a/appservice/storage/storage_wasm.go +++ b/appservice/storage/storage_wasm.go @@ -16,26 +16,18 @@ package storage import ( "fmt" - "net/url" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" ) -func NewDatabase( - dataSourceName string, - dbProperties sqlutil.DbProperties, // nolint:unparam -) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return nil, fmt.Errorf("Cannot use postgres implementation") - } - switch uri.Scheme { - case "postgres": - return nil, fmt.Errorf("Cannot use postgres implementation") - case "file": - return sqlite3.NewDatabase(dataSourceName) +func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(dbProperties) + case dbProperties.ConnectionString.IsPostgres(): + return nil, fmt.Errorf("can't use Postgres implementation") default: - return nil, fmt.Errorf("Cannot use postgres implementation") + return nil, fmt.Errorf("unexpected database type") } } diff --git a/build/gobind/monolith.go b/build/gobind/monolith.go index c0c0ccb4..9c3880b1 100644 --- a/build/gobind/monolith.go +++ b/build/gobind/monolith.go @@ -83,29 +83,29 @@ func (m *DendriteMonolith) Start() { m.YggdrasilNode = ygg cfg := &config.Dendrite{} - cfg.SetDefaults() - cfg.Matrix.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName()) - cfg.Matrix.PrivateKey = ygg.SigningPrivateKey() - cfg.Matrix.KeyID = gomatrixserverlib.KeyID(signing.KeyID) - cfg.Matrix.FederationMaxRetries = 8 - cfg.Kafka.UseNaffka = true - cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput" - cfg.Kafka.Topics.OutputClientData = "clientapiOutput" - cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput" - cfg.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput" - cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s/dendrite-account.db", m.StorageDirectory)) - cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s/dendrite-device.db", m.StorageDirectory)) - cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-mediaapi.db", m.StorageDirectory)) - cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-syncapi.db", m.StorageDirectory)) - cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s/dendrite-roomserver.db", m.StorageDirectory)) - cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s/dendrite-serverkey.db", m.StorageDirectory)) - cfg.Database.E2EKey = config.DataSource(fmt.Sprintf("file:%s/dendrite-keyserver.db", m.StorageDirectory)) - cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory)) - cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory)) - cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s/dendrite-currentstate.db", m.StorageDirectory)) - cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory)) - cfg.Media.BasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) - cfg.Media.AbsBasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) + cfg.Defaults() + cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName()) + cfg.Global.PrivateKey = ygg.SigningPrivateKey() + cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) + cfg.Global.Kafka.UseNaffka = true + cfg.Global.Kafka.Topics.OutputRoomEvent = "roomserverOutput" + cfg.Global.Kafka.Topics.OutputClientData = "clientapiOutput" + cfg.Global.Kafka.Topics.OutputTypingEvent = "typingServerOutput" + cfg.Global.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput" + cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory)) + cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-account.db", m.StorageDirectory)) + cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-device.db", m.StorageDirectory)) + cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-mediaapi.db", m.StorageDirectory)) + cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-syncapi.db", m.StorageDirectory)) + cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-roomserver.db", m.StorageDirectory)) + cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-serverkey.db", m.StorageDirectory)) + cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-keyserver.db", m.StorageDirectory)) + cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory)) + cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory)) + cfg.CurrentStateServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-currentstate.db", m.StorageDirectory)) + cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) + cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) + cfg.FederationSender.FederationMaxRetries = 8 if err = cfg.Derive(); err != nil { panic(err) } @@ -119,8 +119,8 @@ func (m *DendriteMonolith) Start() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - keyAPI := keyserver.NewInternalAPI(base.Cfg, federation, base.KafkaProducer) - userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices, keyAPI) + keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation, base.KafkaProducer) + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, cfg.Derived.ApplicationServices, keyAPI) keyAPI.SetUserAPI(userAPI) rsAPI := roomserver.NewInternalAPI( @@ -132,7 +132,7 @@ func (m *DendriteMonolith) Start() { ) asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) - stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer) + stateAPI := currentstateserver.NewInternalAPI(&base.Cfg.CurrentStateServer, base.KafkaConsumer) fsAPI := federationsender.NewInternalAPI( base, federation, rsAPI, stateAPI, keyRing, ) @@ -180,7 +180,7 @@ func (m *DendriteMonolith) Start() { base.BaseMux, base.PublicAPIMux, base.InternalAPIMux, - cfg, + &cfg.Global, base.UseHTTPAPIs, ) diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index f4814925..d9801955 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -35,7 +35,7 @@ type PasswordRequest struct { // LoginTypePassword implements https://matrix.org/docs/spec/client_server/r0.6.1#password-based type LoginTypePassword struct { GetAccountByPassword GetAccountByPassword - Config *config.Dendrite + Config *config.ClientAPI } func (t *LoginTypePassword) Name() string { diff --git a/clientapi/auth/user_interactive.go b/clientapi/auth/user_interactive.go index 581a85f0..c67eba15 100644 --- a/clientapi/auth/user_interactive.go +++ b/clientapi/auth/user_interactive.go @@ -110,7 +110,7 @@ type UserInteractive struct { Sessions map[string][]string } -func NewUserInteractive(getAccByPass GetAccountByPassword, cfg *config.Dendrite) *UserInteractive { +func NewUserInteractive(getAccByPass GetAccountByPassword, cfg *config.ClientAPI) *UserInteractive { typePassword := &LoginTypePassword{ GetAccountByPassword: getAccByPass, Config: cfg, diff --git a/clientapi/auth/user_interactive_test.go b/clientapi/auth/user_interactive_test.go index d12652c0..47d1cad3 100644 --- a/clientapi/auth/user_interactive_test.go +++ b/clientapi/auth/user_interactive_test.go @@ -33,8 +33,11 @@ func getAccountByPassword(ctx context.Context, localpart, plaintextPassword stri } func setup() *UserInteractive { - cfg := &config.Dendrite{} - cfg.Matrix.ServerName = serverName + cfg := &config.ClientAPI{ + Matrix: &config.Global{ + ServerName: serverName, + }, + } return NewUserInteractive(getAccountByPassword, cfg) } diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 9ed285a8..f3789521 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -37,7 +37,7 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component. func AddPublicRoutes( router *mux.Router, - cfg *config.Dendrite, + cfg *config.ClientAPI, producer sarama.SyncProducer, deviceDB devices.Database, accountsDB accounts.Database, @@ -54,7 +54,7 @@ func AddPublicRoutes( ) { syncProducer := &producers.SyncAPIProducer{ Producer: producer, - Topic: string(cfg.Kafka.Topics.OutputClientData), + Topic: string(cfg.Matrix.Kafka.Topics.OutputClientData), } routing.Setup( diff --git a/clientapi/routing/auth_fallback.go b/clientapi/routing/auth_fallback.go index b7f2cd6d..e639b101 100644 --- a/clientapi/routing/auth_fallback.go +++ b/clientapi/routing/auth_fallback.go @@ -101,7 +101,7 @@ func serveTemplate(w http.ResponseWriter, templateHTML string, data map[string]s // AuthFallback implements GET and POST /auth/{authType}/fallback/web?session={sessionID} func AuthFallback( w http.ResponseWriter, req *http.Request, authType string, - cfg *config.Dendrite, + cfg *config.ClientAPI, ) *util.JSONResponse { sessionID := req.URL.Query().Get("session") @@ -116,7 +116,7 @@ func AuthFallback( data := map[string]string{ "myUrl": req.URL.String(), "session": sessionID, - "siteKey": cfg.Matrix.RecaptchaPublicKey, + "siteKey": cfg.RecaptchaPublicKey, } serveTemplate(w, recaptchaTemplate, data) } @@ -181,11 +181,11 @@ func AuthFallback( // checkRecaptchaEnabled creates an error response if recaptcha is not usable on homeserver. func checkRecaptchaEnabled( - cfg *config.Dendrite, + cfg *config.ClientAPI, w http.ResponseWriter, req *http.Request, ) *util.JSONResponse { - if !cfg.Matrix.RecaptchaEnabled { + if !cfg.RecaptchaEnabled { return writeHTTPMessage(w, req, "Recaptcha login is disabled on this Homeserver", http.StatusBadRequest, diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 027a21e7..5412c222 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -135,7 +135,7 @@ type fledglingEvent struct { // CreateRoom implements /createRoom func CreateRoom( req *http.Request, device *api.Device, - cfg *config.Dendrite, + cfg *config.ClientAPI, accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { @@ -149,7 +149,7 @@ func CreateRoom( // nolint: gocyclo func createRoom( req *http.Request, device *api.Device, - cfg *config.Dendrite, roomID string, + cfg *config.ClientAPI, roomID string, accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { @@ -438,7 +438,7 @@ func createRoom( func buildEvent( builder *gomatrixserverlib.EventBuilder, provider gomatrixserverlib.AuthEventProvider, - cfg *config.Dendrite, + cfg *config.ClientAPI, evTime time.Time, roomVersion gomatrixserverlib.RoomVersion, ) (*gomatrixserverlib.Event, error) { diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 0f78f4a2..62f295fe 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -47,7 +47,7 @@ func DirectoryRoom( req *http.Request, roomAlias string, federation *gomatrixserverlib.FederationClient, - cfg *config.Dendrite, + cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, fedSenderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { @@ -116,7 +116,7 @@ func SetLocalAlias( req *http.Request, device *api.Device, alias string, - cfg *config.Dendrite, + cfg *config.ClientAPI, aliasAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', alias) @@ -139,6 +139,7 @@ func SetLocalAlias( // TODO: This code should eventually be refactored with: // 1. The new method for checking for things matching an AS's namespace // 2. Using an overall Regex object for all AS's just like we did for usernames + for _, appservice := range cfg.Derived.ApplicationServices { // Don't prevent AS from creating aliases in its own namespace // Note that Dendrite uses SenderLocalpart as UserID for AS users diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go index 2a51db73..c74509f0 100644 --- a/clientapi/routing/getevent.go +++ b/clientapi/routing/getevent.go @@ -30,7 +30,7 @@ type getEventRequest struct { device *userapi.Device roomID string eventID string - cfg *config.Dendrite + cfg *config.ClientAPI federation *gomatrixserverlib.FederationClient requestedEvent gomatrixserverlib.Event } @@ -42,7 +42,7 @@ func GetEvent( device *userapi.Device, roomID string, eventID string, - cfg *config.Dendrite, + cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index 42f828f6..d2bc9337 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -58,7 +58,7 @@ func passwordLogin() flows { // Login implements GET and POST /login func Login( req *http.Request, accountDB accounts.Database, userAPI userapi.UserInternalAPI, - cfg *config.Dendrite, + cfg *config.ClientAPI, ) util.JSONResponse { if req.Method == http.MethodGet { // TODO: support other forms of login other than password, depending on config options diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 90ddb699..8303a68e 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -41,7 +41,7 @@ var errMissingUserID = errors.New("'user_id' must be supplied") func SendBan( req *http.Request, accountDB accounts.Database, device *userapi.Device, - roomID string, cfg *config.Dendrite, + roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI) @@ -52,7 +52,7 @@ func SendBan( } func sendMembership(ctx context.Context, accountDB accounts.Database, device *userapi.Device, - roomID, membership, reason string, cfg *config.Dendrite, targetUserID string, evTime time.Time, + roomID, membership, reason string, cfg *config.ClientAPI, targetUserID string, evTime time.Time, roomVer gomatrixserverlib.RoomVersion, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI) util.JSONResponse { @@ -94,7 +94,7 @@ func sendMembership(ctx context.Context, accountDB accounts.Database, device *us func SendKick( req *http.Request, accountDB accounts.Database, device *userapi.Device, - roomID string, cfg *config.Dendrite, + roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, stateAPI currentstateAPI.CurrentStateInternalAPI, ) util.JSONResponse { @@ -135,7 +135,7 @@ func SendKick( func SendUnban( req *http.Request, accountDB accounts.Database, device *userapi.Device, - roomID string, cfg *config.Dendrite, + roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI) @@ -170,7 +170,7 @@ func SendUnban( func SendInvite( req *http.Request, accountDB accounts.Database, device *userapi.Device, - roomID string, cfg *config.Dendrite, + roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI) @@ -236,7 +236,7 @@ func buildMembershipEvent( targetUserID, reason string, accountDB accounts.Database, device *userapi.Device, membership, roomID string, isDirect bool, - cfg *config.Dendrite, evTime time.Time, + cfg *config.ClientAPI, evTime time.Time, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) (*gomatrixserverlib.HeaderedEvent, error) { profile, err := loadProfile(ctx, targetUserID, cfg, accountDB, asAPI) @@ -263,7 +263,7 @@ func buildMembershipEvent( return nil, err } - return eventutil.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) + return eventutil.BuildEvent(ctx, &builder, cfg.Matrix, evTime, rsAPI, nil) } // loadProfile lookups the profile of a given user from the database and returns @@ -273,7 +273,7 @@ func buildMembershipEvent( func loadProfile( ctx context.Context, userID string, - cfg *config.Dendrite, + cfg *config.ClientAPI, accountDB accounts.Database, asAPI appserviceAPI.AppServiceQueryAPI, ) (*authtypes.Profile, error) { @@ -326,7 +326,7 @@ func checkAndProcessThreepid( req *http.Request, device *userapi.Device, body *threepid.MembershipRequest, - cfg *config.Dendrite, + cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, accountDB accounts.Database, roomID string, diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index 9c4cf749..56059350 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -48,7 +48,7 @@ type joinedMember struct { // GetMemberships implements GET /rooms/{roomId}/members func GetMemberships( req *http.Request, device *userapi.Device, roomID string, joinedOnly bool, - _ *config.Dendrite, + _ *config.ClientAPI, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { queryReq := api.QueryMembershipsForRoomRequest{ diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 1df4c9b3..faf92451 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -37,7 +37,7 @@ import ( // GetProfile implements GET /profile/{userID} func GetProfile( - req *http.Request, accountDB accounts.Database, cfg *config.Dendrite, + req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, @@ -66,7 +66,7 @@ func GetProfile( // GetAvatarURL implements GET /profile/{userID}/avatar_url func GetAvatarURL( - req *http.Request, accountDB accounts.Database, cfg *config.Dendrite, + req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { @@ -95,7 +95,7 @@ func GetAvatarURL( // nolint:gocyclo func SetAvatarURL( req *http.Request, accountDB accounts.Database, stateAPI currentstateAPI.CurrentStateInternalAPI, - device *userapi.Device, userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, + device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -184,7 +184,7 @@ func SetAvatarURL( // GetDisplayName implements GET /profile/{userID}/displayname func GetDisplayName( - req *http.Request, accountDB accounts.Database, cfg *config.Dendrite, + req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { @@ -213,7 +213,7 @@ func GetDisplayName( // nolint:gocyclo func SetDisplayName( req *http.Request, accountDB accounts.Database, stateAPI currentstateAPI.CurrentStateInternalAPI, - device *userapi.Device, userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, + device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -305,7 +305,7 @@ func SetDisplayName( // Returns an error when something goes wrong or specifically // eventutil.ErrProfileNoExists when the profile doesn't exist. func getProfile( - ctx context.Context, accountDB accounts.Database, cfg *config.Dendrite, + ctx context.Context, accountDB accounts.Database, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, @@ -345,7 +345,7 @@ func getProfile( func buildMembershipEvents( ctx context.Context, roomIDs []string, - newProfile authtypes.Profile, userID string, cfg *config.Dendrite, + newProfile authtypes.Profile, userID string, cfg *config.ClientAPI, evTime time.Time, rsAPI api.RoomserverInternalAPI, ) ([]gomatrixserverlib.HeaderedEvent, error) { evs := []gomatrixserverlib.HeaderedEvent{} @@ -375,7 +375,7 @@ func buildMembershipEvents( return nil, err } - event, err := eventutil.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) + event, err := eventutil.BuildEvent(ctx, &builder, cfg.Matrix, evTime, rsAPI, nil) if err != nil { return nil, err } diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go index fd80e0ab..bb526513 100644 --- a/clientapi/routing/redaction.go +++ b/clientapi/routing/redaction.go @@ -40,7 +40,7 @@ type redactionResponse struct { } func SendRedaction( - req *http.Request, device *userapi.Device, roomID, eventID string, cfg *config.Dendrite, + req *http.Request, device *userapi.Device, roomID, eventID string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI, ) util.JSONResponse { resErr := checkMemberInRoom(req.Context(), stateAPI, device.UserID, roomID) @@ -115,7 +115,7 @@ func SendRedaction( } var queryRes api.QueryLatestEventsAndStateResponse - e, err := eventutil.BuildEvent(req.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) + e, err := eventutil.BuildEvent(req.Context(), &builder, cfg.Matrix, time.Now(), rsAPI, &queryRes) if err == eventutil.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 69ebdfd7..937abc83 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -255,11 +255,11 @@ func validatePassword(password string) *util.JSONResponse { // validateRecaptcha returns an error response if the captcha response is invalid func validateRecaptcha( - cfg *config.Dendrite, + cfg *config.ClientAPI, response string, clientip string, ) *util.JSONResponse { - if !cfg.Matrix.RecaptchaEnabled { + if !cfg.RecaptchaEnabled { return &util.JSONResponse{ Code: http.StatusConflict, JSON: jsonerror.Unknown("Captcha registration is disabled"), @@ -274,9 +274,9 @@ func validateRecaptcha( } // Make a POST request to Google's API to check the captcha response - resp, err := http.PostForm(cfg.Matrix.RecaptchaSiteVerifyAPI, + resp, err := http.PostForm(cfg.RecaptchaSiteVerifyAPI, url.Values{ - "secret": {cfg.Matrix.RecaptchaPrivateKey}, + "secret": {cfg.RecaptchaPrivateKey}, "response": {response}, "remoteip": {clientip}, }, @@ -324,7 +324,7 @@ func validateRecaptcha( // Application Service is given, it will check to see if it matches any // Application Service's namespace. func UserIDIsWithinApplicationServiceNamespace( - cfg *config.Dendrite, + cfg *config.ClientAPI, userID string, appservice *config.ApplicationService, ) bool { @@ -354,7 +354,7 @@ func UserIDIsWithinApplicationServiceNamespace( // UsernameMatchesMultipleExclusiveNamespaces will check if a given username matches // more than one exclusive namespace. More than one is not allowed func UsernameMatchesMultipleExclusiveNamespaces( - cfg *config.Dendrite, + cfg *config.ClientAPI, username string, ) bool { userID := userutil.MakeUserID(username, cfg.Matrix.ServerName) @@ -374,7 +374,7 @@ func UsernameMatchesMultipleExclusiveNamespaces( // UsernameMatchesExclusiveNamespaces will check if a given username matches any // application service's exclusive users namespace func UsernameMatchesExclusiveNamespaces( - cfg *config.Dendrite, + cfg *config.ClientAPI, username string, ) bool { userID := userutil.MakeUserID(username, cfg.Matrix.ServerName) @@ -386,7 +386,7 @@ func UsernameMatchesExclusiveNamespaces( // username is within that application service's namespace. As long as these // two requirements are met, no error will be returned. func validateApplicationService( - cfg *config.Dendrite, + cfg *config.ClientAPI, username string, accessToken string, ) (string, *util.JSONResponse) { @@ -442,7 +442,7 @@ func Register( req *http.Request, userAPI userapi.UserInternalAPI, accountDB accounts.Database, - cfg *config.Dendrite, + cfg *config.ClientAPI, ) util.JSONResponse { var r registerRequest resErr := httputil.UnmarshalJSONRequest(req, &r) @@ -512,7 +512,7 @@ func Register( func handleGuestRegistration( req *http.Request, r registerRequest, - cfg *config.Dendrite, + cfg *config.ClientAPI, userAPI userapi.UserInternalAPI, ) util.JSONResponse { var res userapi.PerformAccountCreationResponse @@ -568,7 +568,7 @@ func handleRegistrationFlow( req *http.Request, r registerRequest, sessionID string, - cfg *config.Dendrite, + cfg *config.ClientAPI, userAPI userapi.UserInternalAPI, ) util.JSONResponse { // TODO: Shared secret registration (create new user scripts) @@ -580,7 +580,7 @@ func handleRegistrationFlow( // TODO: email / msisdn auth types. - if cfg.Matrix.RegistrationDisabled && r.Auth.Type != authtypes.LoginTypeSharedSecret { + if cfg.RegistrationDisabled && r.Auth.Type != authtypes.LoginTypeSharedSecret { return util.MessageResponse(http.StatusForbidden, "Registration has been disabled") } @@ -666,7 +666,7 @@ func handleApplicationServiceRegistration( tokenErr error, req *http.Request, r registerRequest, - cfg *config.Dendrite, + cfg *config.ClientAPI, userAPI userapi.UserInternalAPI, ) util.JSONResponse { // Check if we previously had issues extracting the access token from the @@ -704,7 +704,7 @@ func checkAndCompleteFlow( req *http.Request, r registerRequest, sessionID string, - cfg *config.Dendrite, + cfg *config.ClientAPI, userAPI userapi.UserInternalAPI, ) util.JSONResponse { if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) { @@ -728,7 +728,7 @@ func checkAndCompleteFlow( func LegacyRegister( req *http.Request, userAPI userapi.UserInternalAPI, - cfg *config.Dendrite, + cfg *config.ClientAPI, ) util.JSONResponse { var r legacyRegisterRequest resErr := parseAndValidateLegacyLogin(req, &r) @@ -742,13 +742,13 @@ func LegacyRegister( "auth.type": r.Type, }).Info("Processing registration request") - if cfg.Matrix.RegistrationDisabled && r.Type != authtypes.LoginTypeSharedSecret { + if cfg.RegistrationDisabled && r.Type != authtypes.LoginTypeSharedSecret { return util.MessageResponse(http.StatusForbidden, "Registration has been disabled") } switch r.Type { case authtypes.LoginTypeSharedSecret: - if cfg.Matrix.RegistrationSharedSecret == "" { + if cfg.RegistrationSharedSecret == "" { return util.MessageResponse(http.StatusBadRequest, "Shared secret registration is disabled") } @@ -902,15 +902,15 @@ func completeRegistration( // Used for shared secret registration. // Checks if the username, password and isAdmin flag matches the given mac. func isValidMacLogin( - cfg *config.Dendrite, + cfg *config.ClientAPI, username, password string, isAdmin bool, givenMac []byte, ) (bool, error) { - sharedSecret := cfg.Matrix.RegistrationSharedSecret + sharedSecret := cfg.RegistrationSharedSecret // Check that shared secret registration isn't disabled. - if cfg.Matrix.RegistrationSharedSecret == "" { + if cfg.RegistrationSharedSecret == "" { return false, errors.New("Shared secret registration is disabled") } @@ -1001,7 +1001,7 @@ type availableResponse struct { // RegisterAvailable checks if the username is already taken or invalid. func RegisterAvailable( req *http.Request, - cfg *config.Dendrite, + cfg *config.ClientAPI, accountDB accounts.Database, ) util.JSONResponse { username := req.URL.Query().Get("username") diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index a44389f9..0a91ae0f 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -179,30 +179,31 @@ func TestValidationOfApplicationServices(t *testing.T) { } // Set up a config - fakeConfig := config.Dendrite{} - fakeConfig.Matrix.ServerName = "localhost" - fakeConfig.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService} + fakeConfig := &config.Dendrite{} + fakeConfig.Defaults() + fakeConfig.Global.ServerName = "localhost" + fakeConfig.ClientAPI.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService} // Access token is correct, user_id omitted so we are acting as SenderLocalpart - asID, resp := validateApplicationService(&fakeConfig, fakeSenderLocalpart, "1234") + asID, resp := validateApplicationService(&fakeConfig.ClientAPI, fakeSenderLocalpart, "1234") if resp != nil || asID != fakeID { t.Errorf("appservice should have validated and returned correct ID: %s", resp.JSON) } // Access token is incorrect, user_id omitted so we are acting as SenderLocalpart - asID, resp = validateApplicationService(&fakeConfig, fakeSenderLocalpart, "xxxx") + asID, resp = validateApplicationService(&fakeConfig.ClientAPI, fakeSenderLocalpart, "xxxx") if resp == nil || asID == fakeID { t.Errorf("access_token should have been marked as invalid") } // Access token is correct, acting as valid user_id - asID, resp = validateApplicationService(&fakeConfig, "_appservice_bob", "1234") + asID, resp = validateApplicationService(&fakeConfig.ClientAPI, "_appservice_bob", "1234") if resp != nil || asID != fakeID { t.Errorf("access_token and user_id should've been valid: %s", resp.JSON) } // Access token is correct, acting as invalid user_id - asID, resp = validateApplicationService(&fakeConfig, "_something_else", "1234") + asID, resp = validateApplicationService(&fakeConfig.ClientAPI, "_something_else", "1234") if resp == nil || asID == fakeID { t.Errorf("user_id should not have been valid: @_something_else:localhost") } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 0e58129e..883b473b 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -51,7 +51,7 @@ const pathPrefixUnstable = "/client/unstable" // applied: // nolint: gocyclo func Setup( - publicAPIMux *mux.Router, cfg *config.Dendrite, + publicAPIMux *mux.Router, cfg *config.ClientAPI, eduAPI eduServerAPI.EDUServerInputAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index bf32992f..e0cd7eb5 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -43,7 +43,7 @@ func SendEvent( req *http.Request, device *userapi.Device, roomID, eventType string, txnID, stateKey *string, - cfg *config.Dendrite, + cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, txnCache *transactions.Cache, ) util.JSONResponse { @@ -112,7 +112,7 @@ func generateSendEvent( req *http.Request, device *userapi.Device, roomID, eventType string, stateKey *string, - cfg *config.Dendrite, + cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, ) (*gomatrixserverlib.Event, *util.JSONResponse) { // parse the incoming http request @@ -146,7 +146,7 @@ func generateSendEvent( } var queryRes api.QueryLatestEventsAndStateResponse - e, err := eventutil.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes) + e, err := eventutil.BuildEvent(req.Context(), &builder, cfg.Matrix, evTime, rsAPI, &queryRes) if err == eventutil.ErrRoomNoExists { return nil, &util.JSONResponse{ Code: http.StatusNotFound, diff --git a/clientapi/routing/threepid.go b/clientapi/routing/threepid.go index e7aaadf5..54ffa53f 100644 --- a/clientapi/routing/threepid.go +++ b/clientapi/routing/threepid.go @@ -40,7 +40,7 @@ type threePIDsResponse struct { // RequestEmailToken implements: // POST /account/3pid/email/requestToken // POST /register/email/requestToken -func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *config.Dendrite) util.JSONResponse { +func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI) util.JSONResponse { var body threepid.EmailAssociationRequest if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { return *reqErr @@ -86,7 +86,7 @@ func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *conf // CheckAndSave3PIDAssociation implements POST /account/3pid func CheckAndSave3PIDAssociation( req *http.Request, accountDB accounts.Database, device *api.Device, - cfg *config.Dendrite, + cfg *config.ClientAPI, ) util.JSONResponse { var body threepid.EmailAssociationCheckRequest if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { diff --git a/clientapi/routing/voip.go b/clientapi/routing/voip.go index 046e8781..536c69fb 100644 --- a/clientapi/routing/voip.go +++ b/clientapi/routing/voip.go @@ -31,7 +31,7 @@ import ( // RequestTurnServer implements: // GET /voip/turnServer -func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.Dendrite) util.JSONResponse { +func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.ClientAPI) util.JSONResponse { turnConfig := cfg.TURN // TODO Guest Support diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 89bc8606..f1d54a47 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -86,7 +86,7 @@ var ( // can be emitted. func CheckAndProcessInvite( ctx context.Context, - device *userapi.Device, body *MembershipRequest, cfg *config.Dendrite, + device *userapi.Device, body *MembershipRequest, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, db accounts.Database, roomID string, evTime time.Time, @@ -137,7 +137,7 @@ func CheckAndProcessInvite( // Returns an error if a check or a request failed. func queryIDServer( ctx context.Context, - db accounts.Database, cfg *config.Dendrite, device *userapi.Device, + db accounts.Database, cfg *config.ClientAPI, device *userapi.Device, body *MembershipRequest, roomID string, ) (lookupRes *idServerLookupResponse, storeInviteRes *idServerStoreInviteResponse, err error) { if err = isTrusted(body.IDServer, cfg); err != nil { @@ -206,7 +206,7 @@ func queryIDServerLookup(ctx context.Context, body *MembershipRequest) (*idServe // Returns an error if the request failed to send or if the response couldn't be parsed. func queryIDServerStoreInvite( ctx context.Context, - db accounts.Database, cfg *config.Dendrite, device *userapi.Device, + db accounts.Database, cfg *config.ClientAPI, device *userapi.Device, body *MembershipRequest, roomID string, ) (*idServerStoreInviteResponse, error) { // Retrieve the sender's profile to get their display name @@ -330,7 +330,7 @@ func checkIDServerSignatures( func emit3PIDInviteEvent( ctx context.Context, body *MembershipRequest, res *idServerStoreInviteResponse, - device *userapi.Device, roomID string, cfg *config.Dendrite, + device *userapi.Device, roomID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, evTime time.Time, ) error { @@ -354,7 +354,7 @@ func emit3PIDInviteEvent( } queryRes := api.QueryLatestEventsAndStateResponse{} - event, err := eventutil.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes) + event, err := eventutil.BuildEvent(ctx, builder, cfg.Matrix, evTime, rsAPI, &queryRes) if err != nil { return err } diff --git a/clientapi/threepid/threepid.go b/clientapi/threepid/threepid.go index bffe31ad..40fd161d 100644 --- a/clientapi/threepid/threepid.go +++ b/clientapi/threepid/threepid.go @@ -53,7 +53,7 @@ type Credentials struct { // Returns an error if there was a problem sending the request or decoding the // response, or if the identity server responded with a non-OK status. func CreateSession( - ctx context.Context, req EmailAssociationRequest, cfg *config.Dendrite, + ctx context.Context, req EmailAssociationRequest, cfg *config.ClientAPI, ) (string, error) { if err := isTrusted(req.IDServer, cfg); err != nil { return "", err @@ -101,7 +101,7 @@ func CreateSession( // Returns an error if there was a problem sending the request or decoding the // response, or if the identity server responded with a non-OK status. func CheckAssociation( - ctx context.Context, creds Credentials, cfg *config.Dendrite, + ctx context.Context, creds Credentials, cfg *config.ClientAPI, ) (bool, string, string, error) { if err := isTrusted(creds.IDServer, cfg); err != nil { return false, "", "", err @@ -142,7 +142,7 @@ func CheckAssociation( // identifier and a Matrix ID. // Returns an error if there was a problem sending the request or decoding the // response, or if the identity server responded with a non-OK status. -func PublishAssociation(creds Credentials, userID string, cfg *config.Dendrite) error { +func PublishAssociation(creds Credentials, userID string, cfg *config.ClientAPI) error { if err := isTrusted(creds.IDServer, cfg); err != nil { return err } @@ -177,7 +177,7 @@ func PublishAssociation(creds Credentials, userID string, cfg *config.Dendrite) // isTrusted checks if a given identity server is part of the list of trusted // identity servers in the configuration file. // Returns an error if the server isn't trusted. -func isTrusted(idServer string, cfg *config.Dendrite) error { +func isTrusted(idServer string, cfg *config.ClientAPI) error { for _, server := range cfg.Matrix.TrustedIDServers { if idServer == server { return nil diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index ff022ec3..73e223d6 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -20,6 +20,7 @@ import ( "fmt" "os" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" @@ -63,7 +64,9 @@ func main() { serverName := gomatrixserverlib.ServerName(*serverNameStr) - accountDB, err := accounts.NewDatabase(*database, nil, serverName) + accountDB, err := accounts.NewDatabase(&config.DatabaseOptions{ + ConnectionString: config.DataSource(*database), + }, serverName) if err != nil { fmt.Println(err.Error()) os.Exit(1) @@ -75,7 +78,9 @@ func main() { os.Exit(1) } - deviceDB, err := devices.NewDatabase(*database, nil, serverName) + deviceDB, err := devices.NewDatabase(&config.DatabaseOptions{ + ConnectionString: config.DataSource(*database), + }, serverName) if err != nil { fmt.Println(err.Error()) os.Exit(1) diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index 6719d047..632b45e6 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -30,6 +30,6 @@ func main() { intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) appservice.AddInternalRoutes(base.InternalAPIMux, intAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI)) + base.SetupAndServeHTTP(string(base.Cfg.AppServiceAPI.Bind), string(base.Cfg.AppServiceAPI.Listen)) } diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 367f27d1..2b6c39b6 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -39,10 +39,10 @@ func main() { keyAPI := base.KeyServerHTTPClient() clientapi.AddPublicRoutes( - base.PublicAPIMux, base.Cfg, base.KafkaProducer, deviceDB, accountDB, federation, + base.PublicAPIMux, &base.Cfg.ClientAPI, base.KafkaProducer, deviceDB, accountDB, federation, rsAPI, eduInputAPI, asQuery, stateAPI, transactions.New(), fsAPI, userAPI, keyAPI, nil, ) - base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI)) + base.SetupAndServeHTTP(string(base.Cfg.ClientAPI.Bind), string(base.Cfg.ClientAPI.Listen)) } diff --git a/cmd/dendrite-current-state-server/main.go b/cmd/dendrite-current-state-server/main.go index 0d4eae7b..a8c0813f 100644 --- a/cmd/dendrite-current-state-server/main.go +++ b/cmd/dendrite-current-state-server/main.go @@ -24,10 +24,10 @@ func main() { base := setup.NewBaseDendrite(cfg, "CurrentStateServer", true) defer base.Close() // nolint: errcheck - stateAPI := currentstateserver.NewInternalAPI(cfg, base.KafkaConsumer) + stateAPI := currentstateserver.NewInternalAPI(&cfg.CurrentStateServer, base.KafkaConsumer) currentstateserver.AddInternalRoutes(base.InternalAPIMux, stateAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.CurrentState), string(base.Cfg.Listen.CurrentState)) + base.SetupAndServeHTTP(string(base.Cfg.CurrentStateServer.Bind), string(base.Cfg.CurrentStateServer.Listen)) } diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 93d54f21..8c28014a 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -75,8 +75,8 @@ func createFederationClient( p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")), ) return gomatrixserverlib.NewFederationClientWithTransport( - base.Base.Cfg.Matrix.ServerName, base.Base.Cfg.Matrix.KeyID, - base.Base.Cfg.Matrix.PrivateKey, true, tr, + base.Base.Cfg.Global.ServerName, base.Base.Cfg.Global.KeyID, + base.Base.Cfg.Global.PrivateKey, true, tr, ) } @@ -113,25 +113,28 @@ func main() { } cfg := config.Dendrite{} - cfg.SetDefaults() - cfg.Matrix.ServerName = "p2p" - cfg.Matrix.PrivateKey = privKey - cfg.Matrix.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName)) - cfg.Kafka.UseNaffka = true - cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput" - cfg.Kafka.Topics.OutputClientData = "clientapiOutput" - cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput" - cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) - cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName)) - cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) - cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) - cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) - cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName)) - cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) - cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) - cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) - cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName)) - cfg.Database.E2EKey = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName)) + cfg.Defaults() + cfg.Global.ServerName = "p2p" + cfg.Global.PrivateKey = privKey + cfg.Global.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName)) + cfg.Global.Kafka.UseNaffka = true + cfg.Global.Kafka.Topics.OutputRoomEvent = "roomserverOutput" + cfg.Global.Kafka.Topics.OutputClientData = "clientapiOutput" + cfg.Global.Kafka.Topics.OutputTypingEvent = "typingServerOutput" + cfg.Global.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput" + cfg.Global.Kafka.Topics.OutputKeyChangeEvent = "keyChangeOutput" + cfg.FederationSender.FederationMaxRetries = 6 + cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) + cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName)) + cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) + cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) + cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) + cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName)) + cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) + cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) + cfg.CurrentStateServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName)) + cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) + cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName)) if err = cfg.Derive(); err != nil { panic(err) } @@ -142,19 +145,19 @@ func main() { accountDB := base.Base.CreateAccountsDB() deviceDB := base.Base.CreateDeviceDB() federation := createFederationClient(base) - keyAPI := keyserver.NewInternalAPI(base.Base.Cfg, federation, base.Base.KafkaProducer) - userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil, keyAPI) + keyAPI := keyserver.NewInternalAPI(&base.Base.Cfg.KeyServer, federation, base.Base.KafkaProducer) + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, nil, keyAPI) keyAPI.SetUserAPI(userAPI) serverKeyAPI := serverkeyapi.NewInternalAPI( - base.Base.Cfg, federation, base.Base.Caches, + &base.Base.Cfg.ServerKeyAPI, federation, base.Base.Caches, ) keyRing := serverKeyAPI.KeyRing() createKeyDB( base, serverKeyAPI, ) - stateAPI := currentstateserver.NewInternalAPI(base.Base.Cfg, base.Base.KafkaConsumer) + stateAPI := currentstateserver.NewInternalAPI(&base.Base.Cfg.CurrentStateServer, base.Base.KafkaConsumer) rsAPI := roomserver.NewInternalAPI( &base.Base, keyRing, federation, ) @@ -198,7 +201,7 @@ func main() { base.Base.BaseMux, base.Base.PublicAPIMux, base.Base.InternalAPIMux, - &cfg, + &cfg.Global, base.Base.UseHTTPAPIs, ) diff --git a/cmd/dendrite-demo-libp2p/p2pdendrite.go b/cmd/dendrite-demo-libp2p/p2pdendrite.go index 4270143f..8fff46af 100644 --- a/cmd/dendrite-demo-libp2p/p2pdendrite.go +++ b/cmd/dendrite-demo-libp2p/p2pdendrite.go @@ -58,7 +58,7 @@ func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite { ctx, cancel := context.WithCancel(context.Background()) - privKey, err := crypto.UnmarshalEd25519PrivateKey(cfg.Matrix.PrivateKey[:]) + privKey, err := crypto.UnmarshalEd25519PrivateKey(cfg.Global.PrivateKey[:]) if err != nil { panic(err) } @@ -97,7 +97,7 @@ func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite { fmt.Println("Our node ID:", libp2p.ID()) fmt.Println("Our addresses:", libp2p.Addrs()) - cfg.Matrix.ServerName = gomatrixserverlib.ServerName(libp2p.ID().String()) + cfg.Global.ServerName = gomatrixserverlib.ServerName(libp2p.ID().String()) return &P2PDendrite{ Base: *baseDendrite, diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 81bf994b..cf4c7735 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -68,27 +68,28 @@ func main() { } cfg := &config.Dendrite{} - cfg.SetDefaults() - cfg.Matrix.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName()) - cfg.Matrix.PrivateKey = ygg.SigningPrivateKey() - cfg.Matrix.KeyID = gomatrixserverlib.KeyID(signing.KeyID) - cfg.Matrix.FederationMaxRetries = 8 - cfg.Kafka.UseNaffka = true - cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput" - cfg.Kafka.Topics.OutputClientData = "clientapiOutput" - cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput" - cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) - cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName)) - cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) - cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) - cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) - cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName)) - cfg.Database.E2EKey = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName)) - cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) - cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) - cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName)) - cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) - cfg.Database.E2EKey = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName)) + cfg.Defaults() + cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName()) + cfg.Global.PrivateKey = ygg.SigningPrivateKey() + cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) + cfg.Global.Kafka.UseNaffka = true + cfg.Global.Kafka.Topics.OutputRoomEvent = "roomserverOutput" + cfg.Global.Kafka.Topics.OutputClientData = "clientapiOutput" + cfg.Global.Kafka.Topics.OutputTypingEvent = "typingServerOutput" + cfg.Global.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput" + cfg.Global.Kafka.Topics.OutputKeyChangeEvent = "keyChangeOutput" + cfg.FederationSender.FederationMaxRetries = 8 + cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) + cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName)) + cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) + cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) + cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) + cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName)) + cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName)) + cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) + cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) + cfg.CurrentStateServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName)) + cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) if err = cfg.Derive(); err != nil { panic(err) } @@ -103,8 +104,8 @@ func main() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - keyAPI := keyserver.NewInternalAPI(base.Cfg, federation, base.KafkaProducer) - userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil, keyAPI) + keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation, base.KafkaProducer) + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, nil, keyAPI) keyAPI.SetUserAPI(userAPI) rsComponent := roomserver.NewInternalAPI( @@ -117,7 +118,7 @@ func main() { ) asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) - stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer) + stateAPI := currentstateserver.NewInternalAPI(&base.Cfg.CurrentStateServer, base.KafkaConsumer) fsAPI := federationsender.NewInternalAPI( base, federation, rsAPI, stateAPI, keyRing, ) @@ -155,7 +156,6 @@ func main() { UserAPI: userAPI, StateAPI: stateAPI, KeyAPI: keyAPI, - //ServerKeyAPI: serverKeyAPI, ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider( ygg, fsAPI, federation, ), @@ -166,7 +166,7 @@ func main() { base.BaseMux, base.PublicAPIMux, base.InternalAPIMux, - cfg, + &cfg.Global, base.UseHTTPAPIs, ) diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/client.go b/cmd/dendrite-demo-yggdrasil/yggconn/client.go index 9cb6f975..1236c553 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/client.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/client.go @@ -54,7 +54,7 @@ func (n *Node) CreateFederationClient( }, ) return gomatrixserverlib.NewFederationClientWithTransport( - base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, - base.Cfg.Matrix.PrivateKey, true, tr, + base.Cfg.Global.ServerName, base.Cfg.Global.KeyID, + base.Cfg.Global.PrivateKey, true, tr, ) } diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-edu-server/main.go index 6704ebd0..d3e4e0a0 100644 --- a/cmd/dendrite-edu-server/main.go +++ b/cmd/dendrite-edu-server/main.go @@ -33,6 +33,6 @@ func main() { intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient()) eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.EDUServer), string(base.Cfg.Listen.EDUServer)) + base.SetupAndServeHTTP(string(base.Cfg.EDUServer.Bind), string(base.Cfg.EDUServer.Listen)) } diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index 70d8394f..3b12a295 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -33,10 +33,10 @@ func main() { keyAPI := base.KeyServerHTTPClient() federationapi.AddPublicRoutes( - base.PublicAPIMux, base.Cfg, userAPI, federation, keyRing, + base.PublicAPIMux, &base.Cfg.FederationAPI, userAPI, federation, keyRing, rsAPI, fsAPI, base.EDUServerClient(), base.CurrentStateAPIClient(), keyAPI, ) - base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) + base.SetupAndServeHTTP(string(base.Cfg.FederationAPI.Bind), string(base.Cfg.FederationAPI.Listen)) } diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index fa6cf7ab..152c798a 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -35,6 +35,6 @@ func main() { ) federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender)) + base.SetupAndServeHTTP(string(base.Cfg.FederationSender.Bind), string(base.Cfg.FederationSender.Listen)) } diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-key-server/main.go index 94ea819f..f3110a1e 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-key-server/main.go @@ -24,11 +24,11 @@ func main() { base := setup.NewBaseDendrite(cfg, "KeyServer", true) defer base.Close() // nolint: errcheck - intAPI := keyserver.NewInternalAPI(base.Cfg, base.CreateFederationClient(), base.KafkaProducer) + intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient(), base.KafkaProducer) intAPI.SetUserAPI(base.UserAPIClient()) keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.KeyServer), string(base.Cfg.Listen.KeyServer)) + base.SetupAndServeHTTP(string(base.Cfg.KeyServer.Bind), string(base.Cfg.KeyServer.Listen)) } diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index 2b9c5090..1bbb62bd 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -26,10 +26,10 @@ func main() { defer base.Close() // nolint: errcheck userAPI := base.UserAPIClient() - client := gomatrixserverlib.NewClient(cfg.Matrix.FederationDisableTLSValidation) + client := gomatrixserverlib.NewClient(cfg.FederationSender.DisableTLSValidation) - mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, userAPI, client) + mediaapi.AddPublicRoutes(base.PublicAPIMux, &base.Cfg.MediaAPI, userAPI, client) - base.SetupAndServeHTTP(string(base.Cfg.Bind.MediaAPI), string(base.Cfg.Listen.MediaAPI)) + base.SetupAndServeHTTP(string(base.Cfg.MediaAPI.Bind), string(base.Cfg.MediaAPI.Listen)) } diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index ed30685b..8f98cdd0 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -54,11 +54,17 @@ func main() { // the API endpoints. They'll listen on the same port as the monolith // itself. addr := config.Address(*httpBindAddr) - cfg.Listen.RoomServer = addr - cfg.Listen.EDUServer = addr - cfg.Listen.AppServiceAPI = addr - cfg.Listen.FederationSender = addr - cfg.Listen.ServerKeyAPI = addr + cfg.AppServiceAPI.Listen = addr + cfg.ClientAPI.Listen = addr + cfg.CurrentStateServer.Listen = addr + cfg.EDUServer.Listen = addr + cfg.FederationAPI.Listen = addr + cfg.FederationSender.Listen = addr + cfg.KeyServer.Listen = addr + cfg.MediaAPI.Listen = addr + cfg.RoomServer.Listen = addr + cfg.ServerKeyAPI.Listen = addr + cfg.SyncAPI.Listen = addr } base := setup.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs) @@ -69,15 +75,15 @@ func main() { federation := base.CreateFederationClient() serverKeyAPI := serverkeyapi.NewInternalAPI( - base.Cfg, federation, base.Caches, + &base.Cfg.ServerKeyAPI, federation, base.Caches, ) if base.UseHTTPAPIs { serverkeyapi.AddInternalRoutes(base.InternalAPIMux, serverKeyAPI, base.Caches) serverKeyAPI = base.ServerKeyAPIClient() } keyRing := serverKeyAPI.KeyRing() - keyAPI := keyserver.NewInternalAPI(base.Cfg, federation, base.KafkaProducer) - userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices, keyAPI) + keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation, base.KafkaProducer) + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, cfg.Derived.ApplicationServices, keyAPI) keyAPI.SetUserAPI(userAPI) rsImpl := roomserver.NewInternalAPI( @@ -109,7 +115,7 @@ func main() { asAPI = base.AppserviceHTTPClient() } - stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer) + stateAPI := currentstateserver.NewInternalAPI(&base.Cfg.CurrentStateServer, base.KafkaConsumer) fsAPI := federationsender.NewInternalAPI( base, federation, rsAPI, stateAPI, keyRing, @@ -126,7 +132,7 @@ func main() { Config: base.Cfg, AccountDB: accountDB, DeviceDB: deviceDB, - Client: gomatrixserverlib.NewClient(cfg.Matrix.FederationDisableTLSValidation), + Client: gomatrixserverlib.NewClient(cfg.FederationSender.DisableTLSValidation), FedClient: federation, KeyRing: keyRing, KafkaConsumer: base.KafkaConsumer, @@ -147,7 +153,7 @@ func main() { base.BaseMux, base.PublicAPIMux, base.InternalAPIMux, - cfg, + &cfg.Global, base.UseHTTPAPIs, ) diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 627a6867..e1d4b16b 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -33,6 +33,6 @@ func main() { rsAPI.SetFederationSenderAPI(fsAPI) roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer)) + base.SetupAndServeHTTP(string(base.Cfg.RoomServer.Bind), string(base.Cfg.RoomServer.Listen)) } diff --git a/cmd/dendrite-server-key-api-server/main.go b/cmd/dendrite-server-key-api-server/main.go index 9ffaeee3..36f34890 100644 --- a/cmd/dendrite-server-key-api-server/main.go +++ b/cmd/dendrite-server-key-api-server/main.go @@ -26,8 +26,8 @@ func main() { federation := base.CreateFederationClient() - intAPI := serverkeyapi.NewInternalAPI(base.Cfg, federation, base.Caches) + intAPI := serverkeyapi.NewInternalAPI(&base.Cfg.ServerKeyAPI, federation, base.Caches) serverkeyapi.AddInternalRoutes(base.InternalAPIMux, intAPI, base.Caches) - base.SetupAndServeHTTP(string(base.Cfg.Bind.ServerKeyAPI), string(base.Cfg.Listen.ServerKeyAPI)) + base.SetupAndServeHTTP(string(base.Cfg.ServerKeyAPI.Bind), string(base.Cfg.ServerKeyAPI.Listen)) } diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index 0761a1d1..c25f8ec1 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -31,8 +31,8 @@ func main() { syncapi.AddPublicRoutes( base.PublicAPIMux, base.KafkaConsumer, userAPI, rsAPI, base.KeyServerHTTPClient(), base.CurrentStateAPIClient(), - federation, cfg) + federation, &cfg.SyncAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI)) + base.SetupAndServeHTTP(string(base.Cfg.SyncAPI.Bind), string(base.Cfg.SyncAPI.Listen)) } diff --git a/cmd/dendrite-user-api-server/main.go b/cmd/dendrite-user-api-server/main.go index e6d61da1..22b6255e 100644 --- a/cmd/dendrite-user-api-server/main.go +++ b/cmd/dendrite-user-api-server/main.go @@ -27,9 +27,9 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() - userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient()) + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient()) userapi.AddInternalRoutes(base.InternalAPIMux, userAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.UserAPI), string(base.Cfg.Listen.UserAPI)) + base.SetupAndServeHTTP(string(base.Cfg.UserAPI.Bind), string(base.Cfg.UserAPI.Listen)) } diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 274d170e..ce7812fa 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -139,7 +139,7 @@ func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLoc tr := go_http_js_libp2p.NewP2pTransport(node) fed := gomatrixserverlib.NewFederationClient( - cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, true, + cfg.Global.ServerName, cfg.Global.KeyID, cfg.Global.PrivateKey, true, ) fed.Client = *gomatrixserverlib.NewClientWithTransport(true, tr) @@ -161,31 +161,31 @@ func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http func main() { cfg := &config.Dendrite{} - cfg.SetDefaults() - cfg.Kafka.UseNaffka = true - cfg.Database.Account = "file:/idb/dendritejs_account.db" - cfg.Database.AppService = "file:/idb/dendritejs_appservice.db" - cfg.Database.Device = "file:/idb/dendritejs_device.db" - cfg.Database.FederationSender = "file:/idb/dendritejs_fedsender.db" - cfg.Database.MediaAPI = "file:/idb/dendritejs_mediaapi.db" - cfg.Database.Naffka = "file:/idb/dendritejs_naffka.db" - cfg.Database.RoomServer = "file:/idb/dendritejs_roomserver.db" - cfg.Database.ServerKey = "file:/idb/dendritejs_serverkey.db" - cfg.Database.SyncAPI = "file:/idb/dendritejs_syncapi.db" - cfg.Database.CurrentState = "file:/idb/dendritejs_currentstate.db" - cfg.Database.E2EKey = "file:/idb/dendritejs_e2ekey.db" - cfg.Kafka.Topics.OutputTypingEvent = "output_typing_event" - cfg.Kafka.Topics.OutputSendToDeviceEvent = "output_send_to_device_event" - cfg.Kafka.Topics.OutputClientData = "output_client_data" - cfg.Kafka.Topics.OutputRoomEvent = "output_room_event" - cfg.Matrix.TrustedIDServers = []string{ + cfg.Defaults() + cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db" + cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db" + cfg.UserAPI.DeviceDatabase.ConnectionString = "file:/idb/dendritejs_device.db" + cfg.FederationSender.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db" + cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db" + cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db" + cfg.ServerKeyAPI.Database.ConnectionString = "file:/idb/dendritejs_serverkey.db" + cfg.SyncAPI.Database.ConnectionString = "file:/idb/dendritejs_syncapi.db" + cfg.CurrentStateServer.Database.ConnectionString = "file:/idb/dendritejs_currentstate.db" + cfg.KeyServer.Database.ConnectionString = "file:/idb/dendritejs_e2ekey.db" + cfg.Global.Kafka.UseNaffka = true + cfg.Global.Kafka.Database.ConnectionString = "file:/idb/dendritejs_naffka.db" + cfg.Global.Kafka.Topics.OutputTypingEvent = "output_typing_event" + cfg.Global.Kafka.Topics.OutputSendToDeviceEvent = "output_send_to_device_event" + cfg.Global.Kafka.Topics.OutputClientData = "output_client_data" + cfg.Global.Kafka.Topics.OutputRoomEvent = "output_room_event" + cfg.Global.TrustedIDServers = []string{ "matrix.org", "vector.im", } - cfg.Matrix.KeyID = libp2pMatrixKeyID - cfg.Matrix.PrivateKey = generateKey() + cfg.Global.KeyID = libp2pMatrixKeyID + cfg.Global.PrivateKey = generateKey() - serverName, node := createP2PNode(cfg.Matrix.PrivateKey) - cfg.Matrix.ServerName = gomatrixserverlib.ServerName(serverName) + serverName, node := createP2PNode(cfg.Global.PrivateKey) + cfg.Global.ServerName = gomatrixserverlib.ServerName(serverName) if err := cfg.Derive(); err != nil { logrus.Fatalf("Failed to derive values from config: %s", err) @@ -196,8 +196,8 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() federation := createFederationClient(cfg, node) - keyAPI := keyserver.NewInternalAPI(base.Cfg, federation, base.KafkaProducer) - userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil, keyAPI) + keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation, base.KafkaProducer) + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, nil, keyAPI) keyAPI.SetUserAPI(userAPI) fetcher := &libp2pKeyFetcher{} @@ -208,7 +208,7 @@ func main() { KeyDatabase: fetcher, } - stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer) + stateAPI := currentstateserver.NewInternalAPI(&base.Cfg.CurrentStateServer, base.KafkaConsumer) rsAPI := roomserver.NewInternalAPI(base, keyRing, federation) eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI) asQuery := appservice.NewInternalAPI( @@ -244,7 +244,7 @@ func main() { base.BaseMux, base.PublicAPIMux, base.InternalAPIMux, - cfg, + &cfg.Global, base.UseHTTPAPIs, ) diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go new file mode 100644 index 00000000..4dd12593 --- /dev/null +++ b/cmd/generate-config/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "fmt" + + "github.com/matrix-org/dendrite/internal/config" + "gopkg.in/yaml.v2" +) + +func main() { + cfg := &config.Dendrite{} + cfg.Defaults() + cfg.Logging = []config.LogrusHook{ + { + Type: "file", + Level: "info", + Params: map[string]interface{}{ + "path": "/var/log/dendrite", + }, + }, + } + + j, err := yaml.Marshal(cfg) + if err != nil { + panic(err) + } + + fmt.Println(string(j)) +} diff --git a/cmd/mediaapi-integration-tests/main.go b/cmd/mediaapi-integration-tests/main.go index e6ce14d2..4c584979 100644 --- a/cmd/mediaapi-integration-tests/main.go +++ b/cmd/mediaapi-integration-tests/main.go @@ -88,9 +88,9 @@ func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error if err != nil { panic(err) } - cfg.Matrix.ServerName = gomatrixserverlib.ServerName(proxyAddr) - cfg.Media.DynamicThumbnails = dynamicThumbnails - if err = yaml.Unmarshal([]byte(thumbnailSizes), &cfg.Media.ThumbnailSizes); err != nil { + cfg.Global.ServerName = gomatrixserverlib.ServerName(proxyAddr) + cfg.MediaAPI.DynamicThumbnails = dynamicThumbnails + if err = yaml.Unmarshal([]byte(thumbnailSizes), &cfg.MediaAPI.ThumbnailSizes); err != nil { panic(err) } @@ -120,7 +120,7 @@ func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error serverArgs, ) - fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, cfg.Listen.MediaAPI, dir) + fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, cfg.MediaAPI.Listen, dir) return cmd, cmdChan, proxyCmd, proxyAddr, dir } diff --git a/cmd/roomserver-integration-tests/main.go b/cmd/roomserver-integration-tests/main.go index 3860ca1f..4d3095be 100644 --- a/cmd/roomserver-integration-tests/main.go +++ b/cmd/roomserver-integration-tests/main.go @@ -240,7 +240,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R panic(err) } - outputTopic := string(cfg.Kafka.Topics.OutputRoomEvent) + outputTopic := string(cfg.Global.Kafka.Topics.OutputRoomEvent) err = exe.DeleteTopic(outputTopic) if err != nil { @@ -277,7 +277,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)} gotOutput, err := runAndReadFromTopic(cmd, cfg.RoomServerURL()+"/metrics", doInput, outputTopic, len(wantOutput), func() { - queryAPI, _ := inthttp.NewRoomserverClient("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache) + queryAPI, _ := inthttp.NewRoomserverClient("http://"+string(cfg.RoomServer.Listen), &http.Client{Timeout: timeoutHTTP}, cache) checkQueries(queryAPI) }) if err != nil { diff --git a/cmd/syncserver-integration-tests/main.go b/cmd/syncserver-integration-tests/main.go index cfe8cc16..76fa0eba 100644 --- a/cmd/syncserver-integration-tests/main.go +++ b/cmd/syncserver-integration-tests/main.go @@ -132,10 +132,10 @@ func startSyncServer() (*exec.Cmd, chan error) { panic(err) } // TODO use the address assigned by the config generator rather than clobbering. - cfg.Matrix.ServerName = "localhost" - cfg.Listen.SyncAPI = config.Address(syncserverAddr) - cfg.Kafka.Topics.OutputRoomEvent = config.Topic(inputTopic) - cfg.Kafka.Topics.OutputClientData = config.Topic(clientTopic) + cfg.Global.ServerName = "localhost" + cfg.SyncAPI.Listen = config.Address(syncserverAddr) + cfg.Global.Kafka.Topics.OutputRoomEvent = config.Topic(inputTopic) + cfg.Global.Kafka.Topics.OutputClientData = config.Topic(clientTopic) if err := test.WriteConfig(cfg, dir); err != nil { panic(err) diff --git a/currentstateserver/currentstateserver.go b/currentstateserver/currentstateserver.go index 07d5e54a..8e985e84 100644 --- a/currentstateserver/currentstateserver.go +++ b/currentstateserver/currentstateserver.go @@ -34,13 +34,13 @@ func AddInternalRoutes(router *mux.Router, intAPI api.CurrentStateInternalAPI) { // NewInternalAPI returns a concrete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. -func NewInternalAPI(cfg *config.Dendrite, consumer sarama.Consumer) api.CurrentStateInternalAPI { - csDB, err := storage.NewDatabase(string(cfg.Database.CurrentState), cfg.DbProperties()) +func NewInternalAPI(cfg *config.CurrentStateServer, consumer sarama.Consumer) api.CurrentStateInternalAPI { + csDB, err := storage.NewDatabase(&cfg.Database) if err != nil { logrus.WithError(err).Panicf("failed to open database") } roomConsumer := consumers.NewOutputRoomEventConsumer( - string(cfg.Kafka.Topics.OutputRoomEvent), consumer, csDB, + string(cfg.Matrix.Kafka.Topics.OutputRoomEvent), consumer, csDB, ) if err = roomConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") diff --git a/currentstateserver/currentstateserver_test.go b/currentstateserver/currentstateserver_test.go index 19317390..2b0e40c2 100644 --- a/currentstateserver/currentstateserver_test.go +++ b/currentstateserver/currentstateserver_test.go @@ -15,6 +15,7 @@ package currentstateserver import ( + "bytes" "context" "crypto/ed25519" "encoding/json" @@ -94,11 +95,15 @@ func MustWriteOutputEvent(t *testing.T, producer sarama.SyncProducer, out *rooms func MustMakeInternalAPI(t *testing.T) (api.CurrentStateInternalAPI, sarama.SyncProducer, func()) { cfg := &config.Dendrite{} + cfg.Defaults() stateDBName := "test_state.db" naffkaDBName := "test_naffka.db" - cfg.Kafka.Topics.OutputRoomEvent = config.Topic(kafkaTopic) - cfg.Database.CurrentState = config.DataSource("file:" + stateDBName) - db, err := sqlutil.Open(sqlutil.SQLiteDriverName(), "file:"+naffkaDBName, nil) + cfg.Global.ServerName = "kaer.morhen" + cfg.Global.Kafka.Topics.OutputRoomEvent = config.Topic(kafkaTopic) + cfg.CurrentStateServer.Database.ConnectionString = config.DataSource("file:" + stateDBName) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource("file:" + naffkaDBName), + }) if err != nil { t.Fatalf("Failed to open naffka database: %s", err) } @@ -110,7 +115,7 @@ func MustMakeInternalAPI(t *testing.T) (api.CurrentStateInternalAPI, sarama.Sync if err != nil { t.Fatalf("Failed to create naffka consumer: %s", err) } - return NewInternalAPI(cfg, naff), naff, func() { + return NewInternalAPI(&cfg.CurrentStateServer, naff), naff, func() { os.Remove(naffkaDBName) os.Remove(stateDBName) } @@ -165,8 +170,13 @@ func TestQueryCurrentState(t *testing.T) { t.Errorf("QueryCurrentState want tuple %+v but it is missing from the response", tuple) continue } - if !reflect.DeepEqual(gotEvent.JSON(), wantEvent.JSON()) { - t.Errorf("QueryCurrentState tuple %+v got event JSON %s want %s", tuple, string(gotEvent.JSON()), string(wantEvent.JSON())) + gotCanon, err := gomatrixserverlib.CanonicalJSON(gotEvent.JSON()) + if err != nil { + t.Errorf("CanonicalJSON failed: %w", err) + continue + } + if !bytes.Equal(gotCanon, wantEvent.JSON()) { + t.Errorf("QueryCurrentState tuple %+v got event JSON %s want %s", tuple, string(gotCanon), string(wantEvent.JSON())) } } } diff --git a/currentstateserver/storage/postgres/storage.go b/currentstateserver/storage/postgres/storage.go index f8edb94e..0cd7e555 100644 --- a/currentstateserver/storage/postgres/storage.go +++ b/currentstateserver/storage/postgres/storage.go @@ -4,6 +4,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/currentstateserver/storage/shared" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" ) @@ -14,10 +15,10 @@ type Database struct { } // NewDatabase creates a new sync server database -func NewDatabase(dbDataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) { var d Database var err error - if d.db, err = sqlutil.Open("postgres", dbDataSourceName, dbProperties); err != nil { + if d.db, err = sqlutil.Open(dbProperties); err != nil { return nil, err } if err = d.PartitionOffsetStatements.Prepare(d.db, "currentstate"); err != nil { diff --git a/currentstateserver/storage/sqlite3/storage.go b/currentstateserver/storage/sqlite3/storage.go index 6975e40b..4454c9ed 100644 --- a/currentstateserver/storage/sqlite3/storage.go +++ b/currentstateserver/storage/sqlite3/storage.go @@ -4,6 +4,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/currentstateserver/storage/shared" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" ) @@ -15,13 +16,10 @@ type Database struct { // NewDatabase creates a new sync server database // nolint: gocyclo -func NewDatabase(dataSourceName string) (*Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) { var d Database - cs, err := sqlutil.ParseFileURI(dataSourceName) - if err != nil { - return nil, err - } - if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { + var err error + if d.db, err = sqlutil.Open(dbProperties); err != nil { return nil, err } if err = d.PartitionOffsetStatements.Prepare(d.db, "currentstate"); err != nil { diff --git a/currentstateserver/storage/storage.go b/currentstateserver/storage/storage.go index ad04cf41..e0707def 100644 --- a/currentstateserver/storage/storage.go +++ b/currentstateserver/storage/storage.go @@ -17,25 +17,21 @@ package storage import ( - "net/url" + "fmt" "github.com/matrix-org/dendrite/currentstateserver/storage/postgres" "github.com/matrix-org/dendrite/currentstateserver/storage/sqlite3" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" ) // NewDatabase opens a database connection. -func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgres.NewDatabase(dataSourceName, dbProperties) - } - switch uri.Scheme { - case "postgres": - return postgres.NewDatabase(dataSourceName, dbProperties) - case "file": - return sqlite3.NewDatabase(dataSourceName) +func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(dbProperties) + case dbProperties.ConnectionString.IsPostgres(): + return postgres.NewDatabase(dbProperties) default: - return postgres.NewDatabase(dataSourceName, dbProperties) + return nil, fmt.Errorf("unexpected database type") } } diff --git a/currentstateserver/storage/storage_wasm.go b/currentstateserver/storage/storage_wasm.go index aa46c44d..46a5abd6 100644 --- a/currentstateserver/storage/storage_wasm.go +++ b/currentstateserver/storage/storage_wasm.go @@ -16,27 +16,19 @@ package storage import ( "fmt" - "net/url" "github.com/matrix-org/dendrite/currentstateserver/storage/sqlite3" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" ) // NewDatabase opens a database connection. -func NewDatabase( - dataSourceName string, - dbProperties sqlutil.DbProperties, // nolint:unparam -) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return nil, fmt.Errorf("Cannot use postgres implementation") - } - switch uri.Scheme { - case "postgres": - return nil, fmt.Errorf("Cannot use postgres implementation") - case "file": - return sqlite3.NewDatabase(dataSourceName) +func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(dbProperties) + case dbProperties.ConnectionString.IsPostgres(): + return nil, fmt.Errorf("can't use Postgres implementation") default: - return nil, fmt.Errorf("Cannot use postgres implementation") + return nil, fmt.Errorf("unexpected database type") } } diff --git a/dendrite-config.yaml b/dendrite-config.yaml deleted file mode 100644 index 8f144875..00000000 --- a/dendrite-config.yaml +++ /dev/null @@ -1,178 +0,0 @@ -# The config file format version -# This is used by dendrite to tell if it understands the config format. -# This will change if the structure of the config file changes or if the meaning -# of an existing config key changes. -version: 0 - -# The matrix specific config -matrix: - # The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. - server_name: "example.com" - # The path to the PEM formatted matrix private key. - private_key: "/etc/dendrite/matrix_key.pem" - # The x509 certificates used by the federation listeners for this server - federation_certificates: ["/etc/dendrite/server.crt"] - # The list of identity servers trusted to verify third party identifiers by this server. - # Defaults to no trusted servers. - trusted_third_party_id_servers: - - vector.im - - matrix.org - # Perspective key servers which are used when direct key requests fail - #key_perspectives: - # - server_name: matrix.org - # keys: - # - key_id: ed25519:auto - # public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw - # - key_id: ed25519:a_RXGa - # public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ - # Disables new users from registering (except via shared secrets) - registration_disabled: false - # Whether to disable TLS certificate validation. Warning: this reduces federation - # security and should not be enabled in production! - federation_disable_tls_validation: false - -# The media repository config -media: - # The base path to where the media files will be stored. May be relative or absolute. - base_path: /var/dendrite/media - - # The maximum file size in bytes that is allowed to be stored on this server. - # Note: if max_file_size_bytes is set to 0, the size is unlimited. - # Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) - max_file_size_bytes: 10485760 - - # Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated - # NOTE: This is a possible denial-of-service attack vector - use at your own risk - dynamic_thumbnails: false - - # A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content - # method is one of crop or scale. If omitted, it will default to scale. - # crop scales to fill the requested dimensions and crops the excess. - # scale scales to fit the requested dimensions and one dimension may be smaller than requested. - thumbnail_sizes: - - width: 32 - height: 32 - method: crop - - width: 96 - height: 96 - method: crop - - width: 320 - height: 240 - method: scale - - width: 640 - height: 480 - method: scale - - width: 800 - height: 600 - method: scale - -# Metrics config for Prometheus -metrics: - # Whether or not metrics are enabled - enabled: false - # Use basic auth to protect the metrics. Uncomment to the complete block to enable. - #basic_auth: - # username: prometheusUser - # password: y0ursecr3tPa$$w0rd - -# The config for the TURN server -turn: - # Whether or not guests can request TURN credentials - turn_allow_guests: true - # How long the authorization should last - turn_user_lifetime: "1h" - # The list of TURN URIs to pass to clients - turn_uris: [] - - # Authorization via Shared Secret - # The shared secret from coturn - turn_shared_secret: "<SECRET STRING GOES HERE>" - - # Authorization via Static Username & Password - # Hardcoded Username and Password - turn_username: "" - turn_password: "" - -# The config for communicating with kafka -kafka: - # Where the kafka servers are running. - addresses: ["localhost:9092"] - # Whether to use naffka instead of kafka. - # Naffka can only be used when running dendrite as a single monolithic server. - # Kafka can be used both with a monolithic server and when running the - # components as separate servers. - # If enabled database.naffka must also be specified. - use_naffka: false - # The names of the kafka topics to use. - topics: - output_room_event: roomserverOutput - output_client_data: clientapiOutput - output_typing_event: eduServerTypingOutput - output_send_to_device_event: eduServerSendToDeviceOutput - user_updates: userUpdates - -# The postgres connection configs for connecting to the databases, e.g. -# for Postgres: postgres://username:password@hostname/database -# for SQLite: file:filename.db or file:///path/to/filename.db -database: - account: "postgres://dendrite:itsasecret@localhost/dendrite_account?sslmode=disable" - device: "postgres://dendrite:itsasecret@localhost/dendrite_device?sslmode=disable" - media_api: "postgres://dendrite:itsasecret@localhost/dendrite_mediaapi?sslmode=disable" - sync_api: "postgres://dendrite:itsasecret@localhost/dendrite_syncapi?sslmode=disable" - room_server: "postgres://dendrite:itsasecret@localhost/dendrite_roomserver?sslmode=disable" - server_key: "postgres://dendrite:itsasecret@localhost/dendrite_serverkey?sslmode=disable" - federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable" - appservice: "postgres://dendrite:itsasecret@localhost/dendrite_appservice?sslmode=disable" - current_state: "postgres://dendrite:itsasecret@localhost/dendrite_currentstate?sslmode=disable" - e2e_key: "postgres://dendrite:itsasecret@localhost/dendrite_e2ekey?sslmode=disable" - max_open_conns: 100 - max_idle_conns: 2 - conn_max_lifetime: -1 - # If 'use_naffka: true' set above then you need to specify a naffka database - # naffka: "postgres://dendrite:itsasecret@localhost/dendrite_naffka?sslmode=disable" - -# The TCP host:port pairs to bind the internal HTTP APIs to. -# These shouldn't be exposed to the public internet. -# These aren't needed when running dendrite as a monolithic server. -listen: - room_server: "localhost:7770" - client_api: "localhost:7771" - federation_api: "localhost:7772" - sync_api: "localhost:7773" - media_api: "localhost:7774" - federation_sender: "localhost:7776" - appservice_api: "localhost:7777" - edu_server: "localhost:7778" - key_server: "localhost:7779" - server_key_api: "localhost:7780" - user_api: "localhost:7781" - current_state_server: "localhost:7782" - -# The configuration for tracing the dendrite components. -tracing: - # Config for the jaeger opentracing reporter. - # See https://godoc.org/github.com/uber/jaeger-client-go/config#Configuration - # for documentation. - jaeger: - disabled: true - -# A list of application service config files to use -application_services: - config_files: [] - -# The configuration for dendrite logs -logging: - # The logging type, only "file" is supported at the moment - - type: "file" - # The logging level, must be one of debug, info, warn, error, fatal, panic. - level: "info" - # Parameters for this type of log - params: - # File logging must be given a path to a directory. Each component will write to a different file. Logs are rotated each day and gzipped - path: "/var/log/dendrite" - # It is possible to have multiple logging hooks at the same time. - # To save only errors in a different directory, uncomment the following. - # - type: "file" - # level: "error" - # params: - # path: "/var/log/dendrite/errors" diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index 2e6ba0c8..e0e61b1a 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -39,12 +39,13 @@ func NewInternalAPI( eduCache *cache.EDUCache, userAPI userapi.UserInternalAPI, ) api.EDUServerInputAPI { + cfg := &base.Cfg.EDUServer return &input.EDUServerInputAPI{ Cache: eduCache, UserAPI: userAPI, Producer: base.KafkaProducer, - OutputTypingEventTopic: string(base.Cfg.Kafka.Topics.OutputTypingEvent), - OutputSendToDeviceEventTopic: string(base.Cfg.Kafka.Topics.OutputSendToDeviceEvent), - ServerName: base.Cfg.Matrix.ServerName, + OutputTypingEventTopic: string(cfg.Matrix.Kafka.Topics.OutputTypingEvent), + OutputSendToDeviceEventTopic: string(cfg.Matrix.Kafka.Topics.OutputSendToDeviceEvent), + ServerName: cfg.Matrix.ServerName, } } diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index 079f333a..e9a8e40a 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -31,7 +31,7 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component. func AddPublicRoutes( router *mux.Router, - cfg *config.Dendrite, + cfg *config.FederationAPI, userAPI userapi.UserInternalAPI, federation *gomatrixserverlib.FederationClient, keyRing gomatrixserverlib.JSONVerifier, diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index 0e0d7740..331d3329 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -20,30 +20,31 @@ import ( func TestRoomsV3URLEscapeDoNot404(t *testing.T) { _, privKey, _ := ed25519.GenerateKey(nil) cfg := &config.Dendrite{} - cfg.Matrix.KeyID = gomatrixserverlib.KeyID("ed25519:auto") - cfg.Matrix.ServerName = gomatrixserverlib.ServerName("localhost") - cfg.Matrix.PrivateKey = privKey - cfg.Kafka.UseNaffka = true - cfg.Database.Naffka = "file::memory:" - cfg.SetDefaults() - base := setup.NewBaseDendrite(cfg, "Test", false) + cfg.Defaults() + cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto") + cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost") + cfg.Global.PrivateKey = privKey + cfg.Global.Kafka.UseNaffka = true + cfg.Global.Kafka.Database.ConnectionString = config.DataSource("file::memory:") + cfg.FederationSender.Database.ConnectionString = config.DataSource("file::memory:") + base := setup.NewBaseDendrite(cfg, "Monolith", false) keyRing := &test.NopJSONVerifier{} fsAPI := base.FederationSenderHTTPClient() // TODO: This is pretty fragile, as if anything calls anything on these nils this test will break. // Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing. - federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil, nil, nil) + federationapi.AddPublicRoutes(base.PublicAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, nil, nil) httputil.SetupHTTPAPI( base.BaseMux, base.PublicAPIMux, base.InternalAPIMux, - cfg, + &cfg.Global, base.UseHTTPAPIs, ) baseURL, cancel := test.ListenAndServe(t, base.BaseMux, true) defer cancel() serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://")) - fedCli := gomatrixserverlib.NewFederationClient(serverName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, true) + fedCli := gomatrixserverlib.NewFederationClient(serverName, cfg.Global.KeyID, cfg.Global.PrivateKey, true) testCases := []struct { roomVer gomatrixserverlib.RoomVersion diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index f906c73c..ea77c947 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -35,7 +35,7 @@ func Backfill( request *gomatrixserverlib.FederationRequest, rsAPI api.RoomserverInternalAPI, roomID string, - cfg *config.Dendrite, + cfg *config.FederationAPI, ) util.JSONResponse { var res api.PerformBackfillResponse var eIDs []string diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 4a49463a..3f9661ee 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -34,7 +34,7 @@ func InviteV2( request *gomatrixserverlib.FederationRequest, roomID string, eventID string, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, keys gomatrixserverlib.JSONVerifier, ) util.JSONResponse { @@ -56,7 +56,7 @@ func InviteV1( request *gomatrixserverlib.FederationRequest, roomID string, eventID string, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, keys gomatrixserverlib.JSONVerifier, ) util.JSONResponse { @@ -86,7 +86,7 @@ func processInvite( strippedState []gomatrixserverlib.InviteV2StrippedState, roomID string, eventID string, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, keys gomatrixserverlib.JSONVerifier, ) util.JSONResponse { diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 17981f53..4874f4d1 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -32,7 +32,7 @@ import ( func MakeJoin( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, roomID, userID string, remoteVersions []gomatrixserverlib.RoomVersion, @@ -95,7 +95,7 @@ func MakeJoin( queryRes := api.QueryLatestEventsAndStateResponse{ RoomVersion: verRes.RoomVersion, } - event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) + event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg.Matrix, time.Now(), rsAPI, &queryRes) if err == eventutil.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, @@ -141,7 +141,7 @@ func MakeJoin( func SendJoin( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, keys gomatrixserverlib.JSONVerifier, roomID, eventID string, diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go index 90eec9e0..f1ed4176 100644 --- a/federationapi/routing/keys.go +++ b/federationapi/routing/keys.go @@ -121,7 +121,7 @@ func ClaimOneTimeKeys( // LocalKeys returns the local keys for the server. // See https://matrix.org/docs/spec/server_server/unstable.html#publishing-keys -func LocalKeys(cfg *config.Dendrite) util.JSONResponse { +func LocalKeys(cfg *config.FederationAPI) util.JSONResponse { keys, err := localKeys(cfg, time.Now().Add(cfg.Matrix.KeyValidityPeriod)) if err != nil { return util.ErrorResponse(err) @@ -129,7 +129,7 @@ func LocalKeys(cfg *config.Dendrite) util.JSONResponse { return util.JSONResponse{Code: http.StatusOK, JSON: keys} } -func localKeys(cfg *config.Dendrite, validUntil time.Time) (*gomatrixserverlib.ServerKeys, error) { +func localKeys(cfg *config.FederationAPI, validUntil time.Time) (*gomatrixserverlib.ServerKeys, error) { var keys gomatrixserverlib.ServerKeys keys.ServerName = cfg.Matrix.ServerName @@ -142,7 +142,7 @@ func localKeys(cfg *config.Dendrite, validUntil time.Time) (*gomatrixserverlib.S }, } - keys.TLSFingerprints = cfg.Matrix.TLSFingerPrints + keys.TLSFingerprints = cfg.TLSFingerPrints keys.OldVerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.OldVerifyKey{} keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(validUntil) diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 56f1b05a..314b4c72 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -28,7 +28,7 @@ import ( func MakeLeave( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, roomID, userID string, ) util.JSONResponse { @@ -60,7 +60,7 @@ func MakeLeave( } var queryRes api.QueryLatestEventsAndStateResponse - event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) + event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg.Matrix, time.Now(), rsAPI, &queryRes) if err == eventutil.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, @@ -102,7 +102,7 @@ func MakeLeave( func SendLeave( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, keys gomatrixserverlib.JSONVerifier, roomID, eventID string, diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index a6180ae6..f1d90bbf 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -30,7 +30,7 @@ import ( func GetProfile( httpReq *http.Request, userAPI userapi.UserInternalAPI, - cfg *config.Dendrite, + cfg *config.FederationAPI, ) util.JSONResponse { userID, field := httpReq.FormValue("user_id"), httpReq.FormValue("field") diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 39fd6d2e..99b5460b 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -31,7 +31,7 @@ import ( func RoomAliasToID( httpReq *http.Request, federation *gomatrixserverlib.FederationClient, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI roomserverAPI.RoomserverInternalAPI, senderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 30a65271..88fd8757 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -47,7 +47,7 @@ const ( // nolint: gocyclo func Setup( publicAPIMux *mux.Router, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI roomserverAPI.RoomserverInternalAPI, eduAPI eduserverAPI.EDUServerInputAPI, fsAPI federationSenderAPI.FederationSenderInternalAPI, diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index f85da148..d1aa728c 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -35,7 +35,7 @@ func Send( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, txnID gomatrixserverlib.TransactionID, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, eduAPI eduserverAPI.EDUServerInputAPI, keyAPI keyapi.KeyInternalAPI, diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 61788010..e8d9a939 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -56,7 +56,7 @@ var ( // CreateInvitesFrom3PIDInvites implements POST /_matrix/federation/v1/3pid/onbind func CreateInvitesFrom3PIDInvites( req *http.Request, rsAPI api.RoomserverInternalAPI, - cfg *config.Dendrite, + cfg *config.FederationAPI, federation *gomatrixserverlib.FederationClient, userAPI userapi.UserInternalAPI, ) util.JSONResponse { @@ -106,7 +106,7 @@ func ExchangeThirdPartyInvite( request *gomatrixserverlib.FederationRequest, roomID string, rsAPI api.RoomserverInternalAPI, - cfg *config.Dendrite, + cfg *config.FederationAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { var builder gomatrixserverlib.EventBuilder @@ -196,7 +196,7 @@ func ExchangeThirdPartyInvite( // necessary data to do so. func createInviteFrom3PIDInvite( ctx context.Context, rsAPI api.RoomserverInternalAPI, - cfg *config.Dendrite, + cfg *config.FederationAPI, inv invite, federation *gomatrixserverlib.FederationClient, userAPI userapi.UserInternalAPI, ) (*gomatrixserverlib.Event, error) { @@ -263,7 +263,7 @@ func createInviteFrom3PIDInvite( func buildMembershipEvent( ctx context.Context, builder *gomatrixserverlib.EventBuilder, rsAPI api.RoomserverInternalAPI, - cfg *config.Dendrite, + cfg *config.FederationAPI, ) (*gomatrixserverlib.Event, error) { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) if err != nil { @@ -327,7 +327,7 @@ func buildMembershipEvent( // them responded with an error. func sendToRemoteServer( ctx context.Context, inv invite, - federation *gomatrixserverlib.FederationClient, _ *config.Dendrite, + federation *gomatrixserverlib.FederationClient, _ *config.FederationAPI, builder gomatrixserverlib.EventBuilder, ) (err error) { remoteServers := make([]gomatrixserverlib.ServerName, 2) diff --git a/federationsender/consumers/eduserver.go b/federationsender/consumers/eduserver.go index 7b539530..74ce65db 100644 --- a/federationsender/consumers/eduserver.go +++ b/federationsender/consumers/eduserver.go @@ -43,27 +43,27 @@ type OutputEDUConsumer struct { // NewOutputEDUConsumer creates a new OutputEDUConsumer. Call Start() to begin consuming from EDU servers. func NewOutputEDUConsumer( - cfg *config.Dendrite, + cfg *config.FederationSender, kafkaConsumer sarama.Consumer, queues *queue.OutgoingQueues, store storage.Database, ) *OutputEDUConsumer { c := &OutputEDUConsumer{ typingConsumer: &internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputTypingEvent), + Topic: string(cfg.Matrix.Kafka.Topics.OutputTypingEvent), Consumer: kafkaConsumer, PartitionStore: store, }, sendToDeviceConsumer: &internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputSendToDeviceEvent), + Topic: string(cfg.Matrix.Kafka.Topics.OutputSendToDeviceEvent), Consumer: kafkaConsumer, PartitionStore: store, }, queues: queues, db: store, ServerName: cfg.Matrix.ServerName, - TypingTopic: string(cfg.Kafka.Topics.OutputTypingEvent), - SendToDeviceTopic: string(cfg.Kafka.Topics.OutputSendToDeviceEvent), + TypingTopic: string(cfg.Matrix.Kafka.Topics.OutputTypingEvent), + SendToDeviceTopic: string(cfg.Matrix.Kafka.Topics.OutputSendToDeviceEvent), } c.typingConsumer.ProcessMessage = c.onTypingEvent c.sendToDeviceConsumer.ProcessMessage = c.onSendToDeviceEvent diff --git a/federationsender/consumers/keychange.go b/federationsender/consumers/keychange.go index d33a6e07..8060125e 100644 --- a/federationsender/consumers/keychange.go +++ b/federationsender/consumers/keychange.go @@ -41,7 +41,7 @@ type KeyChangeConsumer struct { // NewKeyChangeConsumer creates a new KeyChangeConsumer. Call Start() to begin consuming from key servers. func NewKeyChangeConsumer( - cfg *config.Dendrite, + cfg *config.KeyServer, kafkaConsumer sarama.Consumer, queues *queue.OutgoingQueues, store storage.Database, @@ -49,7 +49,7 @@ func NewKeyChangeConsumer( ) *KeyChangeConsumer { c := &KeyChangeConsumer{ consumer: &internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputKeyChangeEvent), + Topic: string(cfg.Matrix.Kafka.Topics.OutputKeyChangeEvent), Consumer: kafkaConsumer, PartitionStore: store, }, diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 299c7b37..b3a4cde3 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -33,7 +33,7 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - cfg *config.Dendrite + cfg *config.FederationSender rsAPI api.RoomserverInternalAPI rsConsumer *internal.ContinualConsumer db storage.Database @@ -42,14 +42,14 @@ type OutputRoomEventConsumer struct { // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. func NewOutputRoomEventConsumer( - cfg *config.Dendrite, + cfg *config.FederationSender, kafkaConsumer sarama.Consumer, queues *queue.OutgoingQueues, store storage.Database, rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputRoomEvent), + Topic: string(cfg.Matrix.Kafka.Topics.OutputRoomEvent), Consumer: kafkaConsumer, PartitionStore: store, } diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index fbf506aa..b02686fe 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -45,27 +45,29 @@ func NewInternalAPI( stateAPI stateapi.CurrentStateInternalAPI, keyRing *gomatrixserverlib.KeyRing, ) api.FederationSenderInternalAPI { - federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender), base.Cfg.DbProperties()) + cfg := &base.Cfg.FederationSender + + federationSenderDB, err := storage.NewDatabase(&cfg.Database) if err != nil { logrus.WithError(err).Panic("failed to connect to federation sender db") } stats := &statistics.Statistics{ DB: federationSenderDB, - FailuresUntilBlacklist: base.Cfg.Matrix.FederationMaxRetries, + FailuresUntilBlacklist: cfg.FederationMaxRetries, } queues := queue.NewOutgoingQueues( - federationSenderDB, base.Cfg.Matrix.ServerName, federation, rsAPI, stats, + federationSenderDB, cfg.Matrix.ServerName, federation, rsAPI, stats, &queue.SigningInfo{ - KeyID: base.Cfg.Matrix.KeyID, - PrivateKey: base.Cfg.Matrix.PrivateKey, - ServerName: base.Cfg.Matrix.ServerName, + KeyID: cfg.Matrix.KeyID, + PrivateKey: cfg.Matrix.PrivateKey, + ServerName: cfg.Matrix.ServerName, }, ) rsConsumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, queues, + cfg, base.KafkaConsumer, queues, federationSenderDB, rsAPI, ) if err = rsConsumer.Start(); err != nil { @@ -73,17 +75,17 @@ func NewInternalAPI( } tsConsumer := consumers.NewOutputEDUConsumer( - base.Cfg, base.KafkaConsumer, queues, federationSenderDB, + cfg, base.KafkaConsumer, queues, federationSenderDB, ) if err := tsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start typing server consumer") } keyConsumer := consumers.NewKeyChangeConsumer( - base.Cfg, base.KafkaConsumer, queues, federationSenderDB, stateAPI, + &base.Cfg.KeyServer, base.KafkaConsumer, queues, federationSenderDB, stateAPI, ) if err := keyConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start key server consumer") } - return internal.NewFederationSenderInternalAPI(federationSenderDB, base.Cfg, rsAPI, federation, keyRing, stats, queues) + return internal.NewFederationSenderInternalAPI(federationSenderDB, cfg, rsAPI, federation, keyRing, stats, queues) } diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go index 9a9880ce..647e3fcb 100644 --- a/federationsender/internal/api.go +++ b/federationsender/internal/api.go @@ -12,7 +12,7 @@ import ( // FederationSenderInternalAPI is an implementation of api.FederationSenderInternalAPI type FederationSenderInternalAPI struct { db storage.Database - cfg *config.Dendrite + cfg *config.FederationSender statistics *statistics.Statistics rsAPI api.RoomserverInternalAPI federation *gomatrixserverlib.FederationClient @@ -21,7 +21,7 @@ type FederationSenderInternalAPI struct { } func NewFederationSenderInternalAPI( - db storage.Database, cfg *config.Dendrite, + db storage.Database, cfg *config.FederationSender, rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, keyRing *gomatrixserverlib.KeyRing, diff --git a/federationsender/storage/postgres/storage.go b/federationsender/storage/postgres/storage.go index a3094bda..b65ff0b6 100644 --- a/federationsender/storage/postgres/storage.go +++ b/federationsender/storage/postgres/storage.go @@ -19,6 +19,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/federationsender/storage/shared" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" ) @@ -30,10 +31,10 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) { var d Database var err error - if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { + if d.db, err = sqlutil.Open(dbProperties); err != nil { return nil, err } joinedHosts, err := NewPostgresJoinedHostsTable(d.db) diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index c303d094..41b91871 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -21,6 +21,7 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/federationsender/storage/shared" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" ) @@ -32,14 +33,10 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dataSourceName string) (*Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) { var d Database var err error - cs, err := sqlutil.ParseFileURI(dataSourceName) - if err != nil { - return nil, err - } - if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { + if d.db, err = sqlutil.Open(dbProperties); err != nil { return nil, err } joinedHosts, err := NewSQLiteJoinedHostsTable(d.db) diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index d3736005..1380fefd 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -17,25 +17,21 @@ package storage import ( - "net/url" + "fmt" "github.com/matrix-org/dendrite/federationsender/storage/postgres" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" ) // NewDatabase opens a new database -func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgres.NewDatabase(dataSourceName, dbProperties) - } - switch uri.Scheme { - case "file": - return sqlite3.NewDatabase(dataSourceName) - case "postgres": - return postgres.NewDatabase(dataSourceName, dbProperties) +func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(dbProperties) + case dbProperties.ConnectionString.IsPostgres(): + return postgres.NewDatabase(dbProperties) default: - return postgres.NewDatabase(dataSourceName, dbProperties) + return nil, fmt.Errorf("unexpected database type") } } diff --git a/federationsender/storage/storage_wasm.go b/federationsender/storage/storage_wasm.go index e5c8f293..459329e9 100644 --- a/federationsender/storage/storage_wasm.go +++ b/federationsender/storage/storage_wasm.go @@ -16,27 +16,19 @@ package storage import ( "fmt" - "net/url" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" ) // NewDatabase opens a new database -func NewDatabase( - dataSourceName string, - dbProperties sqlutil.DbProperties, // nolint:unparam -) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return nil, fmt.Errorf("Cannot use postgres implementation") - } - switch uri.Scheme { - case "file": - return sqlite3.NewDatabase(dataSourceName) - case "postgres": - return nil, fmt.Errorf("Cannot use postgres implementation") +func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(dbProperties) + case dbProperties.ConnectionString.IsPostgres(): + return nil, fmt.Errorf("can't use Postgres implementation") default: - return nil, fmt.Errorf("Cannot use postgres implementation") + return nil, fmt.Errorf("unexpected database type") } } @@ -2,7 +2,9 @@ module github.com/matrix-org/dendrite require ( github.com/Shopify/sarama v1.26.1 + github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect + github.com/ghodss/yaml v1.0.0 github.com/gologme/log v1.2.0 github.com/gorilla/mux v1.7.3 github.com/hashicorp/golang-lru v0.5.4 @@ -40,7 +42,7 @@ require ( go.uber.org/atomic v1.4.0 golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 gopkg.in/h2non/bimg.v1 v1.0.18 - gopkg.in/yaml.v2 v2.2.8 + gopkg.in/yaml.v2 v2.3.0 ) go 1.13 @@ -51,6 +51,9 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/cheggaaa/pb/v3 v3.0.4/go.mod h1:7rgWxLrAUcFMkvJuv09+DYi7mMUYi8nO9iOWcvGJPfw= +github.com/circonus-labs/circonus-gometrics v1.2.0 h1:Kqa/+nIJhqFJ12B07aeekgC6F95J7yYgEtpD57NQzrE= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible h1:C29Ae4G5GtYyYMm1aztcyj/J5ckgJm2zwdDajFbx1NY= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= @@ -91,6 +94,7 @@ github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3B github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= @@ -857,6 +861,8 @@ gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099 h1:XJP7lxbSxWLOMNdBE4B/STaqVy6L73o0knwj2vIlxnw= diff --git a/internal/config/config.go b/internal/config/config.go index 900d3b14..cf9168f7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -24,7 +24,6 @@ import ( "path/filepath" "regexp" "strings" - "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/gomatrixserverlib" @@ -38,7 +37,7 @@ import ( // Version is the current version of the config format. // This will change whenever we make breaking changes to the config format. -const Version = 0 +const Version = 1 // Dendrite contains all the config used by a dendrite process. // Relative paths are resolved relative to the current working directory @@ -51,217 +50,19 @@ type Dendrite struct { // been a breaking change to the config file format. Version int `yaml:"version"` - // The configuration required for a matrix server. - Matrix struct { - // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. - ServerName gomatrixserverlib.ServerName `yaml:"server_name"` - // Path to the private key which will be used to sign requests and events. - PrivateKeyPath Path `yaml:"private_key"` - // The private key which will be used to sign requests and events. - PrivateKey ed25519.PrivateKey `yaml:"-"` - // An arbitrary string used to uniquely identify the PrivateKey. Must start with the - // prefix "ed25519:". - KeyID gomatrixserverlib.KeyID `yaml:"-"` - // List of paths to X509 certificates used by the external federation listeners. - // These are used to calculate the TLS fingerprints to publish for this server. - // Other matrix servers talking to this server will expect the x509 certificate - // to match one of these certificates. - // The certificates should be in PEM format. - FederationCertificatePaths []Path `yaml:"federation_certificates"` - // A list of SHA256 TLS fingerprints for the X509 certificates used by the - // federation listener for this server. - TLSFingerPrints []gomatrixserverlib.TLSFingerprint `yaml:"-"` - // How long a remote server can cache our server key for before requesting it again. - // Increasing this number will reduce the number of requests made by remote servers - // for our key, but increases the period a compromised key will be considered valid - // by remote servers. - // Defaults to 24 hours. - KeyValidityPeriod time.Duration `yaml:"key_validity_period"` - // List of domains that the server will trust as identity servers to - // verify third-party identifiers. - // Defaults to an empty array. - TrustedIDServers []string `yaml:"trusted_third_party_id_servers"` - // If set, allows registration by anyone who also has the shared - // secret, even if registration is otherwise disabled. - RegistrationSharedSecret string `yaml:"registration_shared_secret"` - // This Home Server's ReCAPTCHA public key. - RecaptchaPublicKey string `yaml:"recaptcha_public_key"` - // This Home Server's ReCAPTCHA private key. - RecaptchaPrivateKey string `yaml:"recaptcha_private_key"` - // Boolean stating whether catpcha registration is enabled - // and required - RecaptchaEnabled bool `yaml:"enable_registration_captcha"` - // Secret used to bypass the captcha registration entirely - RecaptchaBypassSecret string `yaml:"captcha_bypass_secret"` - // HTTP API endpoint used to verify whether the captcha response - // was successful - RecaptchaSiteVerifyAPI string `yaml:"recaptcha_siteverify_api"` - // If set disables new users from registering (except via shared - // secrets) - RegistrationDisabled bool `yaml:"registration_disabled"` - // Perspective keyservers, to use as a backup when direct key fetch - // requests don't succeed - KeyPerspectives KeyPerspectives `yaml:"key_perspectives"` - // Federation failure threshold. How many consecutive failures that we should - // tolerate when sending federation requests to a specific server. The backoff - // is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. - // The default value is 16 if not specified, which is circa 18 hours. - FederationMaxRetries uint32 `yaml:"federation_max_retries"` - // FederationDisableTLSValidation disables the validation of X.509 TLS certs - // on remote federation endpoints. This is not recommended in production! - FederationDisableTLSValidation bool `yaml:"federation_disable_tls_validation"` - } `yaml:"matrix"` - - // The configuration specific to the media repostitory. - Media struct { - // The base path to where the media files will be stored. May be relative or absolute. - BasePath Path `yaml:"base_path"` - // The absolute base path to where media files will be stored. - AbsBasePath Path `yaml:"-"` - // The maximum file size in bytes that is allowed to be stored on this server. - // Note: if max_file_size_bytes is set to 0, the size is unlimited. - // Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) - MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"` - // Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated - DynamicThumbnails bool `yaml:"dynamic_thumbnails"` - // The maximum number of simultaneous thumbnail generators. default: 10 - MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"` - // A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content - ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"` - } `yaml:"media"` - - // The configuration to use for Prometheus metrics - Metrics struct { - // Whether or not the metrics are enabled - Enabled bool `yaml:"enabled"` - // Use BasicAuth for Authorization - BasicAuth struct { - // Authorization via Static Username & Password - // Hardcoded Username and Password - Username string `yaml:"username"` - Password string `yaml:"password"` - } `yaml:"basic_auth"` - } `yaml:"metrics"` - - // The configuration for talking to kafka. - Kafka struct { - // A list of kafka addresses to connect to. - Addresses []string `yaml:"addresses"` - // Whether to use naffka instead of kafka. - // Naffka can only be used when running dendrite as a single monolithic server. - // Kafka can be used both with a monolithic server and when running the - // components as separate servers. - UseNaffka bool `yaml:"use_naffka,omitempty"` - // The names of the topics to use when reading and writing from kafka. - Topics struct { - // Topic for roomserver/api.OutputRoomEvent events. - OutputRoomEvent Topic `yaml:"output_room_event"` - // Topic for sending account data from client API to sync API - OutputClientData Topic `yaml:"output_client_data"` - // Topic for eduserver/api.OutputTypingEvent events. - OutputTypingEvent Topic `yaml:"output_typing_event"` - // Topic for eduserver/api.OutputSendToDeviceEvent events. - OutputSendToDeviceEvent Topic `yaml:"output_send_to_device_event"` - // Topic for keyserver when new device keys are added. - OutputKeyChangeEvent Topic `yaml:"output_key_change_event"` - } - } `yaml:"kafka"` - - // Postgres Config - Database struct { - // The Account database stores the login details and account information - // for local users. It is accessed by the UserAPI. - Account DataSource `yaml:"account"` - // The CurrentState database stores the current state of all rooms. - // It is accessed by the CurrentStateServer. - CurrentState DataSource `yaml:"current_state"` - // The Device database stores session information for the devices of logged - // in local users. It is accessed by the UserAPI. - Device DataSource `yaml:"device"` - // The MediaAPI database stores information about files uploaded and downloaded - // by local users. It is only accessed by the MediaAPI. - MediaAPI DataSource `yaml:"media_api"` - // The ServerKey database caches the public keys of remote servers. - // It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI. - ServerKey DataSource `yaml:"server_key"` - // The E2EKey database stores one-time public keys for devices in addition to - // signed device keys. Used for E2E. - E2EKey DataSource `yaml:"e2e_key"` - // The SyncAPI stores information used by the SyncAPI server. - // It is only accessed by the SyncAPI server. - SyncAPI DataSource `yaml:"sync_api"` - // The RoomServer database stores information about matrix rooms. - // It is only accessed by the RoomServer. - RoomServer DataSource `yaml:"room_server"` - // The FederationSender database stores information used by the FederationSender - // It is only accessed by the FederationSender. - FederationSender DataSource `yaml:"federation_sender"` - // The AppServices database stores information used by the AppService component. - // It is only accessed by the AppService component. - AppService DataSource `yaml:"appservice"` - // The Naffka database is used internally by the naffka library, if used. - Naffka DataSource `yaml:"naffka,omitempty"` - // Maximum open connections to the DB (0 = use default, negative means unlimited) - MaxOpenConns int `yaml:"max_open_conns"` - // Maximum idle connections to the DB (0 = use default, negative means unlimited) - MaxIdleConns int `yaml:"max_idle_conns"` - // maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited) - ConnMaxLifetimeSec int `yaml:"conn_max_lifetime"` - } `yaml:"database"` - - // TURN Server Config - TURN struct { - // TODO Guest Support - // Whether or not guests can request TURN credentials - //AllowGuests bool `yaml:"turn_allow_guests"` - // How long the authorization should last - UserLifetime string `yaml:"turn_user_lifetime"` - // The list of TURN URIs to pass to clients - URIs []string `yaml:"turn_uris"` - - // Authorization via Shared Secret - // The shared secret from coturn - SharedSecret string `yaml:"turn_shared_secret"` - - // Authorization via Static Username & Password - // Hardcoded Username and Password - Username string `yaml:"turn_username"` - Password string `yaml:"turn_password"` - } `yaml:"turn"` - - // The internal addresses the components will listen on. - // These should not be exposed externally as they expose metrics and debugging APIs. - // Falls back to addresses listed in Listen if not specified - Bind struct { - MediaAPI Address `yaml:"media_api"` - ClientAPI Address `yaml:"client_api"` - CurrentState Address `yaml:"current_state_server"` - FederationAPI Address `yaml:"federation_api"` - ServerKeyAPI Address `yaml:"server_key_api"` - AppServiceAPI Address `yaml:"appservice_api"` - SyncAPI Address `yaml:"sync_api"` - UserAPI Address `yaml:"user_api"` - RoomServer Address `yaml:"room_server"` - FederationSender Address `yaml:"federation_sender"` - EDUServer Address `yaml:"edu_server"` - KeyServer Address `yaml:"key_server"` - } `yaml:"bind"` - - // The addresses for talking to other microservices. - Listen struct { - MediaAPI Address `yaml:"media_api"` - ClientAPI Address `yaml:"client_api"` - CurrentState Address `yaml:"current_state_server"` - FederationAPI Address `yaml:"federation_api"` - ServerKeyAPI Address `yaml:"server_key_api"` - AppServiceAPI Address `yaml:"appservice_api"` - SyncAPI Address `yaml:"sync_api"` - UserAPI Address `yaml:"user_api"` - RoomServer Address `yaml:"room_server"` - FederationSender Address `yaml:"federation_sender"` - EDUServer Address `yaml:"edu_server"` - KeyServer Address `yaml:"key_server"` - } `yaml:"listen"` + Global Global `yaml:"global"` + AppServiceAPI AppServiceAPI `yaml:"app_service_api"` + ClientAPI ClientAPI `yaml:"client_api"` + CurrentStateServer CurrentStateServer `yaml:"current_state_server"` + EDUServer EDUServer `yaml:"edu_server"` + FederationAPI FederationAPI `yaml:"federation_api"` + FederationSender FederationSender `yaml:"federation_sender"` + KeyServer KeyServer `yaml:"key_server"` + MediaAPI MediaAPI `yaml:"media_api"` + RoomServer RoomServer `yaml:"room_server"` + ServerKeyAPI ServerKeyAPI `yaml:"server_key_api"` + SyncAPI SyncAPI `yaml:"sync_api"` + UserAPI UserAPI `yaml:"user_api"` // The config for tracing the dendrite servers. Tracing struct { @@ -271,56 +72,42 @@ type Dendrite struct { Jaeger jaegerconfig.Configuration `yaml:"jaeger"` } `yaml:"tracing"` - // Application Services - // https://matrix.org/docs/spec/application_service/unstable.html - ApplicationServices struct { - // Configuration files for various application services - ConfigFiles []string `yaml:"config_files"` - } `yaml:"application_services"` - // The config for logging informations. Each hook will be added to logrus. Logging []LogrusHook `yaml:"logging"` - // The config for setting a proxy to use for server->server requests - Proxy *struct { - // The protocol for the proxy (http / https / socks5) - Protocol string `yaml:"protocol"` - // The host where the proxy is listening - Host string `yaml:"host"` - // The port on which the proxy is listening - Port uint16 `yaml:"port"` - } `yaml:"proxy"` - // Any information derived from the configuration options for later use. - Derived struct { - Registration struct { - // Flows is a slice of flows, which represent one possible way that the client can authenticate a request. - // http://matrix.org/docs/spec/HEAD/client_server/r0.3.0.html#user-interactive-authentication-api - // As long as the generated flows only rely on config file options, - // we can generate them on startup and store them until needed - Flows []authtypes.Flow `json:"flows"` - - // Params that need to be returned to the client during - // registration in order to complete registration stages. - Params map[string]interface{} `json:"params"` - } + Derived Derived `yaml:"-"` +} - // Application services parsed from their config files - // The paths of which were given above in the main config file - ApplicationServices []ApplicationService +// TODO: Kill Derived +type Derived struct { + Registration struct { + // Flows is a slice of flows, which represent one possible way that the client can authenticate a request. + // http://matrix.org/docs/spec/HEAD/client_server/r0.3.0.html#user-interactive-authentication-api + // As long as the generated flows only rely on config file options, + // we can generate them on startup and store them until needed + Flows []authtypes.Flow `json:"flows"` + + // Params that need to be returned to the client during + // registration in order to complete registration stages. + Params map[string]interface{} `json:"params"` + } - // Meta-regexes compiled from all exclusive application service - // Regexes. - // - // When a user registers, we check that their username does not match any - // exclusive application service namespaces - ExclusiveApplicationServicesUsernameRegexp *regexp.Regexp - // When a user creates a room alias, we check that it isn't already - // reserved by an application service - ExclusiveApplicationServicesAliasRegexp *regexp.Regexp - // Note: An Exclusive Regex for room ID isn't necessary as we aren't blocking - // servers from creating RoomIDs in exclusive application service namespaces - } `yaml:"-"` + // Application services parsed from their config files + // The paths of which were given above in the main config file + ApplicationServices []ApplicationService + + // Meta-regexes compiled from all exclusive application service + // Regexes. + // + // When a user registers, we check that their username does not match any + // exclusive application service namespaces + ExclusiveApplicationServicesUsernameRegexp *regexp.Regexp + // When a user creates a room alias, we check that it isn't already + // reserved by an application service + ExclusiveApplicationServicesAliasRegexp *regexp.Regexp + // Note: An Exclusive Regex for room ID isn't necessary as we aren't blocking + // servers from creating RoomIDs in exclusive application service namespaces } // KeyPerspectives are used to configure perspective key servers for @@ -344,6 +131,16 @@ type Path string // A DataSource for opening a postgresql database using lib/pq. type DataSource string +func (d DataSource) IsSQLite() bool { + return strings.HasPrefix(string(d), "file:") +} + +func (d DataSource) IsPostgres() bool { + // commented line may not always be true? + // return strings.HasPrefix(string(d), "postgres:") + return !d.IsSQLite() +} + // A Topic in kafka. type Topic string @@ -379,9 +176,9 @@ type LogrusHook struct { Params map[string]interface{} `yaml:"params"` } -// configErrors stores problems encountered when parsing a config file. +// ConfigErrors stores problems encountered when parsing a config file. // It implements the error interface. -type configErrors []string +type ConfigErrors []string // Load a yaml config file for a server run as multiple processes or as a monolith. // Checks the config to ensure that it is valid. @@ -405,29 +202,29 @@ func loadConfig( readFile func(string) ([]byte, error), monolithic bool, ) (*Dendrite, error) { - var config Dendrite + var c Dendrite + c.Defaults() + var err error - if err = yaml.Unmarshal(configData, &config); err != nil { + if err = yaml.Unmarshal(configData, &c); err != nil { return nil, err } - config.SetDefaults() - - if err = config.check(monolithic); err != nil { + if err = c.check(monolithic); err != nil { return nil, err } - privateKeyPath := absPath(basePath, config.Matrix.PrivateKeyPath) + privateKeyPath := absPath(basePath, c.Global.PrivateKeyPath) privateKeyData, err := readFile(privateKeyPath) if err != nil { return nil, err } - if config.Matrix.KeyID, config.Matrix.PrivateKey, err = readKeyPEM(privateKeyPath, privateKeyData); err != nil { + if c.Global.KeyID, c.Global.PrivateKey, err = readKeyPEM(privateKeyPath, privateKeyData); err != nil { return nil, err } - for _, certPath := range config.Matrix.FederationCertificatePaths { + for _, certPath := range c.FederationAPI.FederationCertificatePaths { absCertPath := absPath(basePath, certPath) var pemData []byte pemData, err = readFile(absCertPath) @@ -438,18 +235,19 @@ func loadConfig( if fingerprint == nil { return nil, fmt.Errorf("no certificate PEM data in %q", absCertPath) } - config.Matrix.TLSFingerPrints = append(config.Matrix.TLSFingerPrints, *fingerprint) + c.FederationAPI.TLSFingerPrints = append(c.FederationAPI.TLSFingerPrints, *fingerprint) } - config.Media.AbsBasePath = Path(absPath(basePath, config.Media.BasePath)) + c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath)) // Generate data from config options - err = config.Derive() + err = c.Derive() if err != nil { return nil, err } - return &config, nil + c.Wiring() + return &c, nil } // Derive generates data that is derived from various values provided in @@ -462,8 +260,8 @@ func (config *Dendrite) Derive() error { // TODO: Add email auth type // TODO: Add MSISDN auth type - if config.Matrix.RecaptchaEnabled { - config.Derived.Registration.Params[authtypes.LoginTypeRecaptcha] = map[string]string{"public_key": config.Matrix.RecaptchaPublicKey} + if config.ClientAPI.RecaptchaEnabled { + config.Derived.Registration.Params[authtypes.LoginTypeRecaptcha] = map[string]string{"public_key": config.ClientAPI.RecaptchaPublicKey} config.Derived.Registration.Flows = append(config.Derived.Registration.Flows, authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeRecaptcha}}) } else { @@ -472,7 +270,7 @@ func (config *Dendrite) Derive() error { } // Load application service configuration files - if err := loadAppServices(config); err != nil { + if err := loadAppServices(&config.AppServiceAPI, &config.Derived); err != nil { return err } @@ -480,41 +278,62 @@ func (config *Dendrite) Derive() error { } // SetDefaults sets default config values if they are not explicitly set. -func (config *Dendrite) SetDefaults() { - if config.Matrix.KeyValidityPeriod == 0 { - config.Matrix.KeyValidityPeriod = 24 * time.Hour - } - - if config.Matrix.TrustedIDServers == nil { - config.Matrix.TrustedIDServers = []string{} - } - - if config.Matrix.FederationMaxRetries == 0 { - config.Matrix.FederationMaxRetries = 16 - } - - if config.Media.MaxThumbnailGenerators == 0 { - config.Media.MaxThumbnailGenerators = 10 - } - - if config.Media.MaxFileSizeBytes == nil { - defaultMaxFileSizeBytes := FileSizeBytes(10485760) - config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes - } - - if config.Database.MaxIdleConns == 0 { - config.Database.MaxIdleConns = 2 - } - - if config.Database.MaxOpenConns == 0 { - config.Database.MaxOpenConns = 100 - } - +func (c *Dendrite) Defaults() { + c.Version = 1 + + c.Global.Defaults() + c.ClientAPI.Defaults() + c.CurrentStateServer.Defaults() + c.EDUServer.Defaults() + c.FederationAPI.Defaults() + c.FederationSender.Defaults() + c.KeyServer.Defaults() + c.MediaAPI.Defaults() + c.RoomServer.Defaults() + c.ServerKeyAPI.Defaults() + c.SyncAPI.Defaults() + c.UserAPI.Defaults() + c.AppServiceAPI.Defaults() + + c.Wiring() +} + +func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) { + type verifiable interface { + Verify(configErrs *ConfigErrors, isMonolith bool) + } + for _, c := range []verifiable{ + &c.Global, &c.ClientAPI, &c.CurrentStateServer, + &c.EDUServer, &c.FederationAPI, &c.FederationSender, + &c.KeyServer, &c.MediaAPI, &c.RoomServer, + &c.ServerKeyAPI, &c.SyncAPI, &c.UserAPI, + &c.AppServiceAPI, + } { + c.Verify(configErrs, isMonolith) + } +} + +func (c *Dendrite) Wiring() { + c.ClientAPI.Matrix = &c.Global + c.CurrentStateServer.Matrix = &c.Global + c.EDUServer.Matrix = &c.Global + c.FederationAPI.Matrix = &c.Global + c.FederationSender.Matrix = &c.Global + c.KeyServer.Matrix = &c.Global + c.MediaAPI.Matrix = &c.Global + c.RoomServer.Matrix = &c.Global + c.ServerKeyAPI.Matrix = &c.Global + c.SyncAPI.Matrix = &c.Global + c.UserAPI.Matrix = &c.Global + c.AppServiceAPI.Matrix = &c.Global + + c.ClientAPI.Derived = &c.Derived + c.AppServiceAPI.Derived = &c.Derived } // Error returns a string detailing how many errors were contained within a // configErrors type. -func (errs configErrors) Error() string { +func (errs ConfigErrors) Error() string { if len(errs) == 1 { return errs[0] } @@ -528,13 +347,13 @@ func (errs configErrors) Error() string { // the client code. // This method is safe to use with an uninitialized configErrors because // if it is nil, it will be properly allocated. -func (errs *configErrors) Add(str string) { +func (errs *ConfigErrors) Add(str string) { *errs = append(*errs, str) } // checkNotEmpty verifies the given value is not empty in the configuration. // If it is, adds an error to the list. -func checkNotEmpty(configErrs *configErrors, key, value string) { +func checkNotEmpty(configErrs *ConfigErrors, key, value string) { if value == "" { configErrs.Add(fmt.Sprintf("missing config key %q", key)) } @@ -542,7 +361,7 @@ func checkNotEmpty(configErrs *configErrors, key, value string) { // checkNotZero verifies the given value is not zero in the configuration. // If it is, adds an error to the list. -func checkNotZero(configErrs *configErrors, key string, value int64) { +func checkNotZero(configErrs *ConfigErrors, key string, value int64) { if value == 0 { configErrs.Add(fmt.Sprintf("missing config key %q", key)) } @@ -550,96 +369,14 @@ func checkNotZero(configErrs *configErrors, key string, value int64) { // checkPositive verifies the given value is positive (zero included) // in the configuration. If it is not, adds an error to the list. -func checkPositive(configErrs *configErrors, key string, value int64) { +func checkPositive(configErrs *ConfigErrors, key string, value int64) { if value < 0 { configErrs.Add(fmt.Sprintf("invalid value for config key %q: %d", key, value)) } } -// checkTurn verifies the parameters turn.* are valid. -func (config *Dendrite) checkTurn(configErrs *configErrors) { - value := config.TURN.UserLifetime - if value != "" { - if _, err := time.ParseDuration(value); err != nil { - configErrs.Add(fmt.Sprintf("invalid duration for config key %q: %s", "turn.turn_user_lifetime", value)) - } - } -} - -// checkMatrix verifies the parameters matrix.* are valid. -func (config *Dendrite) checkMatrix(configErrs *configErrors) { - checkNotEmpty(configErrs, "matrix.server_name", string(config.Matrix.ServerName)) - checkNotEmpty(configErrs, "matrix.private_key", string(config.Matrix.PrivateKeyPath)) - checkNotZero(configErrs, "matrix.federation_certificates", int64(len(config.Matrix.FederationCertificatePaths))) - if config.Matrix.RecaptchaEnabled { - checkNotEmpty(configErrs, "matrix.recaptcha_public_key", string(config.Matrix.RecaptchaPublicKey)) - checkNotEmpty(configErrs, "matrix.recaptcha_private_key", string(config.Matrix.RecaptchaPrivateKey)) - checkNotEmpty(configErrs, "matrix.recaptcha_siteverify_api", string(config.Matrix.RecaptchaSiteVerifyAPI)) - } -} - -// checkMedia verifies the parameters media.* are valid. -func (config *Dendrite) checkMedia(configErrs *configErrors) { - checkNotEmpty(configErrs, "media.base_path", string(config.Media.BasePath)) - checkPositive(configErrs, "media.max_file_size_bytes", int64(*config.Media.MaxFileSizeBytes)) - checkPositive(configErrs, "media.max_thumbnail_generators", int64(config.Media.MaxThumbnailGenerators)) - - for i, size := range config.Media.ThumbnailSizes { - checkPositive(configErrs, fmt.Sprintf("media.thumbnail_sizes[%d].width", i), int64(size.Width)) - checkPositive(configErrs, fmt.Sprintf("media.thumbnail_sizes[%d].height", i), int64(size.Height)) - } -} - -// checkKafka verifies the parameters kafka.* and the related -// database.naffka are valid. -func (config *Dendrite) checkKafka(configErrs *configErrors, monolithic bool) { - - if config.Kafka.UseNaffka { - if !monolithic { - configErrs.Add(fmt.Sprintf("naffka can only be used in a monolithic server")) - } - - checkNotEmpty(configErrs, "database.naffka", string(config.Database.Naffka)) - } else { - // If we aren't using naffka then we need to have at least one kafka - // server to talk to. - checkNotZero(configErrs, "kafka.addresses", int64(len(config.Kafka.Addresses))) - } - checkNotEmpty(configErrs, "kafka.topics.output_room_event", string(config.Kafka.Topics.OutputRoomEvent)) - checkNotEmpty(configErrs, "kafka.topics.output_client_data", string(config.Kafka.Topics.OutputClientData)) - checkNotEmpty(configErrs, "kafka.topics.output_typing_event", string(config.Kafka.Topics.OutputTypingEvent)) - checkNotEmpty(configErrs, "kafka.topics.output_send_to_device_event", string(config.Kafka.Topics.OutputSendToDeviceEvent)) - checkNotEmpty(configErrs, "kafka.topics.output_key_change_event", string(config.Kafka.Topics.OutputKeyChangeEvent)) -} - -// checkDatabase verifies the parameters database.* are valid. -func (config *Dendrite) checkDatabase(configErrs *configErrors) { - checkNotEmpty(configErrs, "database.account", string(config.Database.Account)) - checkNotEmpty(configErrs, "database.device", string(config.Database.Device)) - checkNotEmpty(configErrs, "database.server_key", string(config.Database.ServerKey)) - checkNotEmpty(configErrs, "database.media_api", string(config.Database.MediaAPI)) - checkNotEmpty(configErrs, "database.sync_api", string(config.Database.SyncAPI)) - checkNotEmpty(configErrs, "database.room_server", string(config.Database.RoomServer)) - checkNotEmpty(configErrs, "database.current_state", string(config.Database.CurrentState)) - checkNotEmpty(configErrs, "database.e2e_key", string(config.Database.E2EKey)) -} - -// checkListen verifies the parameters listen.* are valid. -func (config *Dendrite) checkListen(configErrs *configErrors) { - checkNotEmpty(configErrs, "listen.media_api", string(config.Listen.MediaAPI)) - checkNotEmpty(configErrs, "listen.client_api", string(config.Listen.ClientAPI)) - checkNotEmpty(configErrs, "listen.federation_api", string(config.Listen.FederationAPI)) - checkNotEmpty(configErrs, "listen.sync_api", string(config.Listen.SyncAPI)) - checkNotEmpty(configErrs, "listen.room_server", string(config.Listen.RoomServer)) - checkNotEmpty(configErrs, "listen.edu_server", string(config.Listen.EDUServer)) - checkNotEmpty(configErrs, "listen.server_key_api", string(config.Listen.EDUServer)) - checkNotEmpty(configErrs, "listen.user_api", string(config.Listen.UserAPI)) - checkNotEmpty(configErrs, "listen.current_state_server", string(config.Listen.CurrentState)) - checkNotEmpty(configErrs, "listen.key_server", string(config.Listen.KeyServer)) -} - // checkLogging verifies the parameters logging.* are valid. -func (config *Dendrite) checkLogging(configErrs *configErrors) { +func (config *Dendrite) checkLogging(configErrs *ConfigErrors) { for _, logrusHook := range config.Logging { checkNotEmpty(configErrs, "logging.type", string(logrusHook.Type)) checkNotEmpty(configErrs, "logging.level", string(logrusHook.Level)) @@ -648,8 +385,8 @@ func (config *Dendrite) checkLogging(configErrs *configErrors) { // check returns an error type containing all errors found within the config // file. -func (config *Dendrite) check(monolithic bool) error { - var configErrs configErrors +func (config *Dendrite) check(_ bool) error { // monolithic + var configErrs ConfigErrors if config.Version != Version { configErrs.Add(fmt.Sprintf( @@ -658,17 +395,8 @@ func (config *Dendrite) check(monolithic bool) error { return configErrs } - config.checkMatrix(&configErrs) - config.checkMedia(&configErrs) - config.checkTurn(&configErrs) - config.checkKafka(&configErrs, monolithic) - config.checkDatabase(&configErrs) config.checkLogging(&configErrs) - if !monolithic { - config.checkListen(&configErrs) - } - // Due to how Golang manages its interface types, this condition is not redundant. // In order to get the proper behaviour, it is necessary to return an explicit nil // and not a nil configErrors. @@ -737,7 +465,7 @@ func (config *Dendrite) AppServiceURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.AppServiceAPI) + return "http://" + string(config.AppServiceAPI.Listen) } // RoomServerURL returns an HTTP URL for where the roomserver is listening. @@ -746,7 +474,7 @@ func (config *Dendrite) RoomServerURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.RoomServer) + return "http://" + string(config.RoomServer.Listen) } // UserAPIURL returns an HTTP URL for where the userapi is listening. @@ -755,7 +483,7 @@ func (config *Dendrite) UserAPIURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.UserAPI) + return "http://" + string(config.UserAPI.Listen) } // CurrentStateAPIURL returns an HTTP URL for where the currentstateserver is listening. @@ -764,7 +492,7 @@ func (config *Dendrite) CurrentStateAPIURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.CurrentState) + return "http://" + string(config.CurrentStateServer.Listen) } // EDUServerURL returns an HTTP URL for where the EDU server is listening. @@ -773,7 +501,7 @@ func (config *Dendrite) EDUServerURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.EDUServer) + return "http://" + string(config.EDUServer.Listen) } // FederationSenderURL returns an HTTP URL for where the federation sender is listening. @@ -782,7 +510,7 @@ func (config *Dendrite) FederationSenderURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.FederationSender) + return "http://" + string(config.FederationSender.Listen) } // ServerKeyAPIURL returns an HTTP URL for where the server key API is listening. @@ -791,7 +519,7 @@ func (config *Dendrite) ServerKeyAPIURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.ServerKeyAPI) + return "http://" + string(config.ServerKeyAPI.Listen) } // KeyServerURL returns an HTTP URL for where the key server is listening. @@ -800,7 +528,7 @@ func (config *Dendrite) KeyServerURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.KeyServer) + return "http://" + string(config.KeyServer.Listen) } // SetupTracing configures the opentracing using the supplied configuration. @@ -815,33 +543,6 @@ func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err ) } -// MaxIdleConns returns maximum idle connections to the DB -func (config Dendrite) MaxIdleConns() int { - return config.Database.MaxIdleConns -} - -// MaxOpenConns returns maximum open connections to the DB -func (config Dendrite) MaxOpenConns() int { - return config.Database.MaxOpenConns -} - -// ConnMaxLifetime returns maximum amount of time a connection may be reused -func (config Dendrite) ConnMaxLifetime() time.Duration { - return time.Duration(config.Database.ConnMaxLifetimeSec) * time.Second -} - -// DbProperties functions return properties used by database/sql/DB -type DbProperties interface { - MaxIdleConns() int - MaxOpenConns() int - ConnMaxLifetime() time.Duration -} - -// DbProperties returns cfg as a DbProperties interface -func (config Dendrite) DbProperties() DbProperties { - return config -} - // logrusLogger is a small wrapper that implements jaeger.Logger using logrus. type logrusLogger struct { l *logrus.Logger diff --git a/internal/config/appservice.go b/internal/config/config_appservice.go index bf5f089b..b8962ded 100644 --- a/internal/config/appservice.go +++ b/internal/config/config_appservice.go @@ -25,6 +25,31 @@ import ( yaml "gopkg.in/yaml.v2" ) +type AppServiceAPI struct { + Matrix *Global `yaml:"-"` + Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + Database DatabaseOptions `yaml:"database"` + + ConfigFiles []string `yaml:"config_files"` +} + +func (c *AppServiceAPI) Defaults() { + c.Listen = "localhost:7777" + c.Bind = "localhost:7777" + c.Database.Defaults() + c.Database.ConnectionString = "file:appservice.db" +} + +func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "app_service_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "app_service_api.bind", string(c.Bind)) + checkNotEmpty(configErrs, "app_service_api.database.connection_string", string(c.Database.ConnectionString)) +} + // ApplicationServiceNamespace is the namespace that a specific application // service has management over. type ApplicationServiceNamespace struct { @@ -132,8 +157,8 @@ func (a *ApplicationService) IsInterestedInRoomAlias( // loadAppServices iterates through all application service config files // and loads their data into the config object for later access. -func loadAppServices(config *Dendrite) error { - for _, configPath := range config.ApplicationServices.ConfigFiles { +func loadAppServices(config *AppServiceAPI, derived *Derived) error { + for _, configPath := range config.ConfigFiles { // Create a new application service with default options appservice := ApplicationService{ RateLimited: true, @@ -157,26 +182,26 @@ func loadAppServices(config *Dendrite) error { } // Append the parsed application service to the global config - config.Derived.ApplicationServices = append( - config.Derived.ApplicationServices, appservice, + derived.ApplicationServices = append( + derived.ApplicationServices, appservice, ) } // Check for any errors in the loaded application services - return checkErrors(config) + return checkErrors(config, derived) } // setupRegexps will create regex objects for exclusive and non-exclusive // usernames, aliases and rooms of all application services, so that other // methods can quickly check if a particular string matches any of them. -func setupRegexps(cfg *Dendrite) (err error) { +func setupRegexps(_ *AppServiceAPI, derived *Derived) (err error) { // Combine all exclusive namespaces for later string checking var exclusiveUsernameStrings, exclusiveAliasStrings []string // If an application service's regex is marked as exclusive, add // its contents to the overall exlusive regex string. Room regex // not necessary as we aren't denying exclusive room ID creation - for _, appservice := range cfg.Derived.ApplicationServices { + for _, appservice := range derived.ApplicationServices { for key, namespaceSlice := range appservice.NamespaceMap { switch key { case "users": @@ -204,10 +229,10 @@ func setupRegexps(cfg *Dendrite) (err error) { } // Store compiled Regex - if cfg.Derived.ExclusiveApplicationServicesUsernameRegexp, err = regexp.Compile(exclusiveUsernames); err != nil { + if derived.ExclusiveApplicationServicesUsernameRegexp, err = regexp.Compile(exclusiveUsernames); err != nil { return err } - if cfg.Derived.ExclusiveApplicationServicesAliasRegexp, err = regexp.Compile(exclusiveAliases); err != nil { + if derived.ExclusiveApplicationServicesAliasRegexp, err = regexp.Compile(exclusiveAliases); err != nil { return err } @@ -234,7 +259,7 @@ func appendExclusiveNamespaceRegexs( // checkErrors checks for any configuration errors amongst the loaded // application services according to the application service spec. -func checkErrors(config *Dendrite) (err error) { +func checkErrors(config *AppServiceAPI, derived *Derived) (err error) { var idMap = make(map[string]bool) var tokenMap = make(map[string]bool) @@ -242,7 +267,7 @@ func checkErrors(config *Dendrite) (err error) { groupIDRegexp := regexp.MustCompile(`\+.*:.*`) // Check each application service for any config errors - for _, appservice := range config.Derived.ApplicationServices { + for _, appservice := range derived.ApplicationServices { // Namespace-related checks for key, namespaceSlice := range appservice.NamespaceMap { for _, namespace := range namespaceSlice { @@ -258,13 +283,13 @@ func checkErrors(config *Dendrite) (err error) { // Check if we've already seen this ID. No two application services // can have the same ID or token. if idMap[appservice.ID] { - return configErrors([]string{fmt.Sprintf( + return ConfigErrors([]string{fmt.Sprintf( "Application service ID %s must be unique", appservice.ID, )}) } // Check if we've already seen this token if tokenMap[appservice.ASToken] { - return configErrors([]string{fmt.Sprintf( + return ConfigErrors([]string{fmt.Sprintf( "Application service Token %s must be unique", appservice.ASToken, )}) } @@ -284,7 +309,7 @@ func checkErrors(config *Dendrite) (err error) { } } - return setupRegexps(config) + return setupRegexps(config, derived) } // validateNamespace returns nil or an error based on whether a given @@ -298,7 +323,7 @@ func validateNamespace( ) error { // Check that namespace(s) are valid regex if !IsValidRegex(namespace.Regex) { - return configErrors([]string{fmt.Sprintf( + return ConfigErrors([]string{fmt.Sprintf( "Invalid regex string for Application Service %s", appservice.ID, )}) } @@ -310,7 +335,7 @@ func validateNamespace( correctFormat := groupIDRegexp.MatchString(namespace.GroupID) if !correctFormat { - return configErrors([]string{fmt.Sprintf( + return ConfigErrors([]string{fmt.Sprintf( "Invalid user group_id field for application service %s.", appservice.ID, )}) diff --git a/internal/config/config_clientapi.go b/internal/config/config_clientapi.go new file mode 100644 index 00000000..c441a9c0 --- /dev/null +++ b/internal/config/config_clientapi.go @@ -0,0 +1,87 @@ +package config + +import ( + "fmt" + "time" +) + +type ClientAPI struct { + Matrix *Global `yaml:"-"` + Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // If set, allows registration by anyone who also has the shared + // secret, even if registration is otherwise disabled. + RegistrationSharedSecret string `yaml:"registration_shared_secret"` + // This Home Server's ReCAPTCHA public key. + RecaptchaPublicKey string `yaml:"recaptcha_public_key"` + // This Home Server's ReCAPTCHA private key. + RecaptchaPrivateKey string `yaml:"recaptcha_private_key"` + // Boolean stating whether catpcha registration is enabled + // and required + RecaptchaEnabled bool `yaml:"enable_registration_captcha"` + // Secret used to bypass the captcha registration entirely + RecaptchaBypassSecret string `yaml:"captcha_bypass_secret"` + // HTTP API endpoint used to verify whether the captcha response + // was successful + RecaptchaSiteVerifyAPI string `yaml:"recaptcha_siteverify_api"` + // If set disables new users from registering (except via shared + // secrets) + RegistrationDisabled bool `yaml:"registration_disabled"` + + // TURN options + TURN TURN `yaml:"turn"` +} + +func (c *ClientAPI) Defaults() { + c.Listen = "localhost:7771" + c.Bind = "localhost:7771" + c.RegistrationSharedSecret = "" + c.RecaptchaPublicKey = "" + c.RecaptchaPrivateKey = "" + c.RecaptchaEnabled = false + c.RecaptchaBypassSecret = "" + c.RecaptchaSiteVerifyAPI = "" + c.RegistrationDisabled = false +} + +func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "client_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "client_api.bind", string(c.Bind)) + if c.RecaptchaEnabled { + checkNotEmpty(configErrs, "client_api.recaptcha_public_key", string(c.RecaptchaPublicKey)) + checkNotEmpty(configErrs, "client_api.recaptcha_private_key", string(c.RecaptchaPrivateKey)) + checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", string(c.RecaptchaSiteVerifyAPI)) + } + c.TURN.Verify(configErrs) +} + +type TURN struct { + // TODO Guest Support + // Whether or not guests can request TURN credentials + // AllowGuests bool `yaml:"turn_allow_guests"` + // How long the authorization should last + UserLifetime string `yaml:"turn_user_lifetime"` + // The list of TURN URIs to pass to clients + URIs []string `yaml:"turn_uris"` + + // Authorization via Shared Secret + // The shared secret from coturn + SharedSecret string `yaml:"turn_shared_secret"` + + // Authorization via Static Username & Password + // Hardcoded Username and Password + Username string `yaml:"turn_username"` + Password string `yaml:"turn_password"` +} + +func (c *TURN) Verify(configErrs *ConfigErrors) { + value := c.UserLifetime + if value != "" { + if _, err := time.ParseDuration(value); err != nil { + configErrs.Add(fmt.Sprintf("invalid duration for config key %q: %s", "client_api.turn.turn_user_lifetime", value)) + } + } +} diff --git a/internal/config/config_currentstate.go b/internal/config/config_currentstate.go new file mode 100644 index 00000000..2687f7f5 --- /dev/null +++ b/internal/config/config_currentstate.go @@ -0,0 +1,25 @@ +package config + +type CurrentStateServer struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // The CurrentState database stores the current state of all rooms. + // It is accessed by the CurrentStateServer. + Database DatabaseOptions `yaml:"database"` +} + +func (c *CurrentStateServer) Defaults() { + c.Listen = "localhost:7782" + c.Bind = "localhost:7782" + c.Database.Defaults() + c.Database.ConnectionString = "file:currentstate.db" +} + +func (c *CurrentStateServer) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "current_state_server.listen", string(c.Listen)) + checkNotEmpty(configErrs, "current_state_server.bind", string(c.Bind)) + checkNotEmpty(configErrs, "current_state_server.database.connection_string", string(c.Database.ConnectionString)) +} diff --git a/internal/config/config_eduserver.go b/internal/config/config_eduserver.go new file mode 100644 index 00000000..02743041 --- /dev/null +++ b/internal/config/config_eduserver.go @@ -0,0 +1,18 @@ +package config + +type EDUServer struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` +} + +func (c *EDUServer) Defaults() { + c.Listen = "localhost:7778" + c.Bind = "localhost:7778" +} + +func (c *EDUServer) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "edu_server.listen", string(c.Listen)) + checkNotEmpty(configErrs, "edu_server.bind", string(c.Bind)) +} diff --git a/internal/config/config_federationapi.go b/internal/config/config_federationapi.go new file mode 100644 index 00000000..d155ef25 --- /dev/null +++ b/internal/config/config_federationapi.go @@ -0,0 +1,33 @@ +package config + +import "github.com/matrix-org/gomatrixserverlib" + +type FederationAPI struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // List of paths to X509 certificates used by the external federation listeners. + // These are used to calculate the TLS fingerprints to publish for this server. + // Other matrix servers talking to this server will expect the x509 certificate + // to match one of these certificates. + // The certificates should be in PEM format. + FederationCertificatePaths []Path `yaml:"federation_certificates"` + + // A list of SHA256 TLS fingerprints for the X509 certificates used by the + // federation listener for this server. + TLSFingerPrints []gomatrixserverlib.TLSFingerprint `yaml:"-"` +} + +func (c *FederationAPI) Defaults() { + c.Listen = "localhost:7772" + c.Bind = "localhost:7772" +} + +func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "federation_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "federation_api.bind", string(c.Bind)) + // TODO: not applicable always, e.g. in demos + //checkNotZero(configErrs, "federation_api.federation_certificates", int64(len(c.FederationCertificatePaths))) +} diff --git a/internal/config/config_federationsender.go b/internal/config/config_federationsender.go new file mode 100644 index 00000000..09d8287b --- /dev/null +++ b/internal/config/config_federationsender.go @@ -0,0 +1,64 @@ +package config + +type FederationSender struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // The FederationSender database stores information used by the FederationSender + // It is only accessed by the FederationSender. + Database DatabaseOptions `yaml:"database"` + + // Federation failure threshold. How many consecutive failures that we should + // tolerate when sending federation requests to a specific server. The backoff + // is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. + // The default value is 16 if not specified, which is circa 18 hours. + FederationMaxRetries uint32 `yaml:"send_max_retries"` + + // FederationDisableTLSValidation disables the validation of X.509 TLS certs + // on remote federation endpoints. This is not recommended in production! + DisableTLSValidation bool `yaml:"disable_tls_validation"` + + Proxy Proxy `yaml:"proxy_outbound"` +} + +func (c *FederationSender) Defaults() { + c.Listen = "localhost:7775" + c.Bind = "localhost:7775" + c.Database.Defaults() + c.Database.ConnectionString = "file:federationsender.db" + + c.FederationMaxRetries = 16 + c.DisableTLSValidation = false + + c.Proxy.Defaults() +} + +func (c *FederationSender) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "federation_sender.listen", string(c.Listen)) + checkNotEmpty(configErrs, "federation_sender.bind", string(c.Bind)) + checkNotEmpty(configErrs, "federation_sender.database.connection_string", string(c.Database.ConnectionString)) +} + +// The config for setting a proxy to use for server->server requests +type Proxy struct { + // Is the proxy enabled? + Enabled bool `yaml:"enabled"` + // The protocol for the proxy (http / https / socks5) + Protocol string `yaml:"protocol"` + // The host where the proxy is listening + Host string `yaml:"host"` + // The port on which the proxy is listening + Port uint16 `yaml:"port"` +} + +func (c *Proxy) Defaults() { + c.Enabled = false + c.Protocol = "http" + c.Host = "localhost" + c.Port = 8080 +} + +func (c *Proxy) Verify(configErrs *ConfigErrors) { +} diff --git a/internal/config/config_global.go b/internal/config/config_global.go new file mode 100644 index 00000000..9456dd3f --- /dev/null +++ b/internal/config/config_global.go @@ -0,0 +1,172 @@ +package config + +import ( + "math/rand" + "time" + + "github.com/matrix-org/gomatrixserverlib" + "golang.org/x/crypto/ed25519" +) + +type Global struct { + // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. + ServerName gomatrixserverlib.ServerName `yaml:"server_name"` + + // Path to the private key which will be used to sign requests and events. + PrivateKeyPath Path `yaml:"private_key"` + + // The private key which will be used to sign requests and events. + PrivateKey ed25519.PrivateKey `yaml:"-"` + + // An arbitrary string used to uniquely identify the PrivateKey. Must start with the + // prefix "ed25519:". + KeyID gomatrixserverlib.KeyID `yaml:"-"` + + // How long a remote server can cache our server key for before requesting it again. + // Increasing this number will reduce the number of requests made by remote servers + // for our key, but increases the period a compromised key will be considered valid + // by remote servers. + // Defaults to 24 hours. + KeyValidityPeriod time.Duration `yaml:"key_validity_period"` + + // List of domains that the server will trust as identity servers to + // verify third-party identifiers. + // Defaults to an empty array. + TrustedIDServers []string `yaml:"trusted_third_party_id_servers"` + + // Kafka/Naffka configuration + Kafka Kafka `yaml:"kafka"` + + // Metrics configuration + Metrics Metrics `yaml:"metrics"` +} + +func (c *Global) Defaults() { + c.ServerName = "localhost" + c.PrivateKeyPath = "matrix.pem" + _, c.PrivateKey, _ = ed25519.GenerateKey(rand.New(rand.NewSource(0))) + c.KeyID = "ed25519:auto" + c.KeyValidityPeriod = time.Hour * 24 * 7 + + c.Kafka.Defaults() + c.Metrics.Defaults() +} + +func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "global.server_name", string(c.ServerName)) + checkNotEmpty(configErrs, "global.private_key", string(c.PrivateKeyPath)) + + c.Kafka.Verify(configErrs, isMonolith) + c.Metrics.Verify(configErrs, isMonolith) +} + +type Kafka struct { + // A list of kafka addresses to connect to. + Addresses []string `yaml:"addresses"` + // Whether to use naffka instead of kafka. + // Naffka can only be used when running dendrite as a single monolithic server. + // Kafka can be used both with a monolithic server and when running the + // components as separate servers. + UseNaffka bool `yaml:"use_naffka"` + // The Naffka database is used internally by the naffka library, if used. + Database DatabaseOptions `yaml:"naffka_database"` + // The names of the topics to use when reading and writing from kafka. + Topics struct { + // Topic for roomserver/api.OutputRoomEvent events. + OutputRoomEvent Topic `yaml:"output_room_event"` + // Topic for sending account data from client API to sync API + OutputClientData Topic `yaml:"output_client_data"` + // Topic for eduserver/api.OutputTypingEvent events. + OutputTypingEvent Topic `yaml:"output_typing_event"` + // Topic for eduserver/api.OutputSendToDeviceEvent events. + OutputSendToDeviceEvent Topic `yaml:"output_send_to_device_event"` + // Topic for keyserver when new device keys are added. + OutputKeyChangeEvent Topic `yaml:"output_key_change_event"` + } +} + +func (c *Kafka) Defaults() { + c.UseNaffka = true + c.Database.Defaults() + c.Database.ConnectionString = DataSource("file:naffka.db") + c.Topics.OutputRoomEvent = "OutputRoomEventTopic" + c.Topics.OutputClientData = "OutputClientDataTopic" + c.Topics.OutputTypingEvent = "OutputTypingEventTopic" + c.Topics.OutputSendToDeviceEvent = "OutputSendToDeviceEventTopic" + c.Topics.OutputKeyChangeEvent = "OutputKeyChangeEventTopic" +} + +func (c *Kafka) Verify(configErrs *ConfigErrors, isMonolith bool) { + if c.UseNaffka { + if !isMonolith { + configErrs.Add("naffka can only be used in a monolithic server") + } + checkNotEmpty(configErrs, "global.kafka.database.connection_string", string(c.Database.ConnectionString)) + } else { + // If we aren't using naffka then we need to have at least one kafka + // server to talk to. + checkNotZero(configErrs, "global.kafka.addresses", int64(len(c.Addresses))) + } + checkNotEmpty(configErrs, "global.kafka.topics.output_room_event", string(c.Topics.OutputRoomEvent)) + checkNotEmpty(configErrs, "global.kafka.topics.output_client_data", string(c.Topics.OutputClientData)) + checkNotEmpty(configErrs, "global.kafka.topics.output_typing_event", string(c.Topics.OutputTypingEvent)) + checkNotEmpty(configErrs, "global.kafka.topics.output_send_to_device_event", string(c.Topics.OutputSendToDeviceEvent)) + checkNotEmpty(configErrs, "global.kafka.topics.output_key_change_event", string(c.Topics.OutputKeyChangeEvent)) +} + +// The configuration to use for Prometheus metrics +type Metrics struct { + // Whether or not the metrics are enabled + Enabled bool `yaml:"enabled"` + // Use BasicAuth for Authorization + BasicAuth struct { + // Authorization via Static Username & Password + // Hardcoded Username and Password + Username string `yaml:"username"` + Password string `yaml:"password"` + } `yaml:"basic_auth"` +} + +func (c *Metrics) Defaults() { + c.Enabled = false + c.BasicAuth.Username = "metrics" + c.BasicAuth.Password = "metrics" +} + +func (c *Metrics) Verify(configErrs *ConfigErrors, isMonolith bool) { +} + +type DatabaseOptions struct { + // The connection string, file:filename.db or postgres://server.... + ConnectionString DataSource `yaml:"connection_string"` + // Maximum open connections to the DB (0 = use default, negative means unlimited) + MaxOpenConnections int `yaml:"max_open_conns"` + // Maximum idle connections to the DB (0 = use default, negative means unlimited) + MaxIdleConnections int `yaml:"max_idle_conns"` + // maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited) + ConnMaxLifetimeSeconds int `yaml:"conn_max_lifetime"` +} + +func (c *DatabaseOptions) Defaults() { + c.MaxOpenConnections = 100 + c.MaxIdleConnections = 2 + c.ConnMaxLifetimeSeconds = -1 +} + +func (c *DatabaseOptions) Verify(configErrs *ConfigErrors, isMonolith bool) { +} + +// MaxIdleConns returns maximum idle connections to the DB +func (c DatabaseOptions) MaxIdleConns() int { + return c.MaxIdleConnections +} + +// MaxOpenConns returns maximum open connections to the DB +func (c DatabaseOptions) MaxOpenConns() int { + return c.MaxOpenConnections +} + +// ConnMaxLifetime returns maximum amount of time a connection may be reused +func (c DatabaseOptions) ConnMaxLifetime() time.Duration { + return time.Duration(c.ConnMaxLifetimeSeconds) * time.Second +} diff --git a/internal/config/config_keyserver.go b/internal/config/config_keyserver.go new file mode 100644 index 00000000..c0967a8a --- /dev/null +++ b/internal/config/config_keyserver.go @@ -0,0 +1,23 @@ +package config + +type KeyServer struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + Database DatabaseOptions `yaml:"database"` +} + +func (c *KeyServer) Defaults() { + c.Listen = "localhost:7779" + c.Bind = "localhost:7779" + c.Database.Defaults() + c.Database.ConnectionString = "file:keyserver.db" +} + +func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "key_server.listen", string(c.Listen)) + checkNotEmpty(configErrs, "key_server.bind", string(c.Bind)) + checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString)) +} diff --git a/internal/config/config_mediaapi.go b/internal/config/config_mediaapi.go new file mode 100644 index 00000000..9a4d7e0a --- /dev/null +++ b/internal/config/config_mediaapi.go @@ -0,0 +1,63 @@ +package config + +import ( + "fmt" +) + +type MediaAPI struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // The MediaAPI database stores information about files uploaded and downloaded + // by local users. It is only accessed by the MediaAPI. + Database DatabaseOptions `yaml:"database"` + + // The base path to where the media files will be stored. May be relative or absolute. + BasePath Path `yaml:"base_path"` + + // The absolute base path to where media files will be stored. + AbsBasePath Path `yaml:"-"` + + // The maximum file size in bytes that is allowed to be stored on this server. + // Note: if max_file_size_bytes is set to 0, the size is unlimited. + // Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) + MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"` + + // Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated + DynamicThumbnails bool `yaml:"dynamic_thumbnails"` + + // The maximum number of simultaneous thumbnail generators. default: 10 + MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"` + + // A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content + ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"` +} + +func (c *MediaAPI) Defaults() { + c.Listen = "localhost:7774" + c.Bind = "localhost:7774" + c.Database.Defaults() + c.Database.ConnectionString = "file:mediaapi.db" + + defaultMaxFileSizeBytes := FileSizeBytes(10485760) + c.MaxFileSizeBytes = &defaultMaxFileSizeBytes + c.MaxThumbnailGenerators = 10 + c.BasePath = "./media_store" +} + +func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "media_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "media_api.bind", string(c.Bind)) + checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString)) + + checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath)) + checkPositive(configErrs, "media_api.max_file_size_bytes", int64(*c.MaxFileSizeBytes)) + checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators)) + + for i, size := range c.ThumbnailSizes { + checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].width", i), int64(size.Width)) + checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].height", i), int64(size.Height)) + } +} diff --git a/internal/config/config_roomserver.go b/internal/config/config_roomserver.go new file mode 100644 index 00000000..1a16e2b1 --- /dev/null +++ b/internal/config/config_roomserver.go @@ -0,0 +1,23 @@ +package config + +type RoomServer struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + Database DatabaseOptions `yaml:"database"` +} + +func (c *RoomServer) Defaults() { + c.Listen = "localhost:7770" + c.Bind = "localhost:7770" + c.Database.Defaults() + c.Database.ConnectionString = "file:roomserver.db" +} + +func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "room_server.listen", string(c.Listen)) + checkNotEmpty(configErrs, "room_server.bind", string(c.Bind)) + checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString)) +} diff --git a/internal/config/config_serverkey.go b/internal/config/config_serverkey.go new file mode 100644 index 00000000..cf1f537a --- /dev/null +++ b/internal/config/config_serverkey.go @@ -0,0 +1,29 @@ +package config + +type ServerKeyAPI struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // The ServerKey database caches the public keys of remote servers. + // It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI. + Database DatabaseOptions `yaml:"database"` + + // Perspective keyservers, to use as a backup when direct key fetch + // requests don't succeed + KeyPerspectives KeyPerspectives `yaml:"key_perspectives"` +} + +func (c *ServerKeyAPI) Defaults() { + c.Listen = "localhost:7780" + c.Bind = "localhost:7780" + c.Database.Defaults() + c.Database.ConnectionString = "file:serverkeyapi.db" +} + +func (c *ServerKeyAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "server_key_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "server_key_api.bind", string(c.Bind)) + checkNotEmpty(configErrs, "server_key_api.database.connection_string", string(c.Database.ConnectionString)) +} diff --git a/internal/config/config_syncapi.go b/internal/config/config_syncapi.go new file mode 100644 index 00000000..488f6658 --- /dev/null +++ b/internal/config/config_syncapi.go @@ -0,0 +1,23 @@ +package config + +type SyncAPI struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + Database DatabaseOptions `yaml:"database"` +} + +func (c *SyncAPI) Defaults() { + c.Listen = "localhost:7773" + c.Bind = "localhost:7773" + c.Database.Defaults() + c.Database.ConnectionString = "file:syncapi.db" +} + +func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "sync_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "sync_api.bind", string(c.Bind)) + checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString)) +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 758d7552..4ff170e4 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -33,48 +33,157 @@ func TestLoadConfigRelative(t *testing.T) { } const testConfig = ` -version: 0 -matrix: +version: 1 +global: server_name: localhost private_key: matrix_key.pem - federation_certificates: [tls_cert.pem] -media: - base_path: media_store -kafka: - addresses: ["localhost:9092"] - topics: - output_room_event: output.room - output_client_data: output.client - output_typing_event: output.typing - output_send_to_device_event: output.std - output_key_change_event: output.key_change - user_updates: output.user -database: - media_api: "postgresql:///media_api" - account: "postgresql:///account" - device: "postgresql:///device" - server_key: "postgresql:///server_keys" - sync_api: "postgresql:///syn_api" - room_server: "postgresql:///room_server" - appservice: "postgresql:///appservice" - current_state: "postgresql:///current_state" - e2e_key: "postgresql:///e2e_key" -listen: - room_server: "localhost:7770" - client_api: "localhost:7771" - federation_api: "localhost:7772" - sync_api: "localhost:7773" - media_api: "localhost:7774" - appservice_api: "localhost:7777" - edu_server: "localhost:7778" - user_api: "localhost:7779" - current_state_server: "localhost:7775" - key_server: "localhost:7776" -logging: - - type: "file" - level: "info" - params: - path: "/my/log/dir" + key_validity_period: 168h0m0s + trusted_third_party_id_servers: [] + kafka: + addresses: [] + use_naffka: true + naffka_database: + connection_string: file:naffka.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 + topics: + output_room_event: OutputRoomEventTopic + output_client_data: OutputClientDataTopic + output_typing_event: OutputTypingEventTopic + output_send_to_device_event: OutputSendToDeviceEventTopic + output_key_change_event: OutputKeyChangeEventTopic + metrics: + enabled: false + basic_auth: + username: metrics + password: metrics +app_service_api: + listen: localhost:7777 + bind: localhost:7777 + database: + connection_string: file:appservice.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 + config_files: [] +client_api: + listen: localhost:7771 + bind: localhost:7771 + registration_shared_secret: "" + recaptcha_public_key: "" + recaptcha_private_key: "" + enable_registration_captcha: false + captcha_bypass_secret: "" + recaptcha_siteverify_api: "" + registration_disabled: false + turn: + turn_user_lifetime: "" + turn_uris: [] + turn_shared_secret: "" + turn_username: "" + turn_password: "" +current_state_server: + listen: localhost:7782 + bind: localhost:7782 + database: + connection_string: file:currentstate.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 +edu_server: + listen: localhost:7778 + bind: localhost:7778 +federation_api: + listen: localhost:7772 + bind: localhost:7772 + federation_certificates: [] +federation_sender: + listen: localhost:7775 + bind: localhost:7775 + database: + connection_string: file:federationsender.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 + federation_max_retries: 16 + proxy_outbound: + enabled: false + protocol: http + host: localhost + port: 8080 +key_server: + listen: localhost:7779 + bind: localhost:7779 + database: + connection_string: file:keyserver.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 +media_api: + listen: localhost:7774 + bind: localhost:7774 + database: + connection_string: file:mediaapi.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 + base_path: "" + max_file_size_bytes: 10485760 + dynamic_thumbnails: false + max_thumbnail_generators: 10 + thumbnail_sizes: [] +room_server: + listen: localhost:7770 + bind: localhost:7770 + database: + connection_string: file:roomserver.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 +server_key_api: + listen: localhost:7780 + bind: localhost:7780 + database: + connection_string: file:serverkeyapi.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 + key_perspectives: [] +sync_api: + listen: localhost:7773 + bind: localhost:7773 + database: + connection_string: file:syncapi.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 +user_api: + listen: localhost:7781 + bind: localhost:7781 + account_database: + connection_string: file:userapi_accounts.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 + device_database: + connection_string: file:userapi_devices.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 +tracing: + enabled: false + jaeger: + serviceName: "" + disabled: false + rpc_metrics: false + tags: [] + sampler: null + reporter: null + headers: null + baggage_restrictions: null + throttler: null +logging: [] ` type mockReadFile map[string]string diff --git a/internal/config/config_userapi.go b/internal/config/config_userapi.go new file mode 100644 index 00000000..f7da9e59 --- /dev/null +++ b/internal/config/config_userapi.go @@ -0,0 +1,31 @@ +package config + +type UserAPI struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // The Account database stores the login details and account information + // for local users. It is accessed by the UserAPI. + AccountDatabase DatabaseOptions `yaml:"account_database"` + // The Device database stores session information for the devices of logged + // in local users. It is accessed by the UserAPI. + DeviceDatabase DatabaseOptions `yaml:"device_database"` +} + +func (c *UserAPI) Defaults() { + c.Listen = "localhost:7781" + c.Bind = "localhost:7781" + c.AccountDatabase.Defaults() + c.DeviceDatabase.Defaults() + c.AccountDatabase.ConnectionString = "file:userapi_accounts.db" + c.DeviceDatabase.ConnectionString = "file:userapi_devices.db" +} + +func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "user_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "user_api.bind", string(c.Bind)) + checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString)) + checkNotEmpty(configErrs, "user_api.device_database.connection_string", string(c.DeviceDatabase.ConnectionString)) +} diff --git a/internal/eventutil/events.go b/internal/eventutil/events.go index 1e3afac8..35c7f33d 100644 --- a/internal/eventutil/events.go +++ b/internal/eventutil/events.go @@ -38,7 +38,7 @@ var ErrRoomNoExists = errors.New("Room does not exist") // Returns an error if something else went wrong func BuildEvent( ctx context.Context, - builder *gomatrixserverlib.EventBuilder, cfg *config.Dendrite, evTime time.Time, + builder *gomatrixserverlib.EventBuilder, cfg *config.Global, evTime time.Time, rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) (*gomatrixserverlib.HeaderedEvent, error) { if queryRes == nil { @@ -52,8 +52,8 @@ func BuildEvent( } event, err := builder.Build( - evTime, cfg.Matrix.ServerName, cfg.Matrix.KeyID, - cfg.Matrix.PrivateKey, queryRes.RoomVersion, + evTime, cfg.ServerName, cfg.KeyID, + cfg.PrivateKey, queryRes.RoomVersion, ) if err != nil { return nil, err diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index d371d172..8f7723ef 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -234,7 +234,7 @@ func (f *FederationWakeups) Wakeup(ctx context.Context, origin gomatrixserverlib } // SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics listener -func SetupHTTPAPI(servMux, publicApiMux, internalApiMux *mux.Router, cfg *config.Dendrite, enableHTTPAPIs bool) { +func SetupHTTPAPI(servMux, publicApiMux, internalApiMux *mux.Router, cfg *config.Global, enableHTTPAPIs bool) { if cfg.Metrics.Enabled { servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) } diff --git a/internal/setup/base.go b/internal/setup/base.go index 4fef0cbc..f59d136e 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -15,7 +15,6 @@ package setup import ( - "database/sql" "fmt" "io" "net/http" @@ -85,6 +84,15 @@ const HTTPClientTimeout = time.Second * 30 // The componentName is used for logging purposes, and should be a friendly name // of the compontent running, e.g. "SyncAPI" func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs bool) *BaseDendrite { + configErrors := &config.ConfigErrors{} + cfg.Verify(configErrors, componentName == "Monolith") // TODO: better way? + if len(*configErrors) > 0 { + for _, err := range *configErrors { + logrus.Errorf("Configuration error: %s", err) + } + logrus.Fatalf("Failed to start due to configuration errors") + } + internal.SetupStdLogging() internal.SetupHookLogging(cfg.Logging, componentName) internal.SetupPprof() @@ -96,7 +104,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo var kafkaConsumer sarama.Consumer var kafkaProducer sarama.SyncProducer - if cfg.Kafka.UseNaffka { + if cfg.Global.Kafka.UseNaffka { kafkaConsumer, kafkaProducer = setupNaffka(cfg) } else { kafkaConsumer, kafkaProducer = setupKafka(cfg) @@ -108,10 +116,10 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo } client := http.Client{Timeout: HTTPClientTimeout} - if cfg.Proxy != nil { + if cfg.FederationSender.Proxy.Enabled { client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{ - Scheme: cfg.Proxy.Protocol, - Host: fmt.Sprintf("%s:%d", cfg.Proxy.Host, cfg.Proxy.Port), + Scheme: cfg.FederationSender.Proxy.Protocol, + Host: fmt.Sprintf("%s:%d", cfg.FederationSender.Proxy.Host, cfg.FederationSender.Proxy.Port), })} } @@ -228,7 +236,7 @@ func (b *BaseDendrite) KeyServerHTTPClient() keyserverAPI.KeyInternalAPI { // CreateDeviceDB creates a new instance of the device database. Should only be // called once per component. func (b *BaseDendrite) CreateDeviceDB() devices.Database { - db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName) + db, err := devices.NewDatabase(&b.Cfg.UserAPI.DeviceDatabase, b.Cfg.Global.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to devices db") } @@ -239,7 +247,7 @@ func (b *BaseDendrite) CreateDeviceDB() devices.Database { // CreateAccountsDB creates a new instance of the accounts database. Should only // be called once per component. func (b *BaseDendrite) CreateAccountsDB() accounts.Database { - db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName) + db, err := accounts.NewDatabase(&b.Cfg.UserAPI.AccountDatabase, b.Cfg.Global.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to accounts db") } @@ -251,8 +259,8 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database { // once per component. func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationClient { return gomatrixserverlib.NewFederationClient( - b.Cfg.Matrix.ServerName, b.Cfg.Matrix.KeyID, b.Cfg.Matrix.PrivateKey, - b.Cfg.Matrix.FederationDisableTLSValidation, + b.Cfg.Global.ServerName, b.Cfg.Global.KeyID, b.Cfg.Global.PrivateKey, + b.Cfg.FederationSender.DisableTLSValidation, ) } @@ -277,7 +285,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { b.BaseMux, b.PublicAPIMux, b.InternalAPIMux, - b.Cfg, + &b.Cfg.Global, b.UseHTTPAPIs, ) serv.Handler = b.BaseMux @@ -293,12 +301,12 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { // setupKafka creates kafka consumer/producer pair from the config. func setupKafka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { - consumer, err := sarama.NewConsumer(cfg.Kafka.Addresses, nil) + consumer, err := sarama.NewConsumer(cfg.Global.Kafka.Addresses, nil) if err != nil { logrus.WithError(err).Panic("failed to start kafka consumer") } - producer, err := sarama.NewSyncProducer(cfg.Kafka.Addresses, nil) + producer, err := sarama.NewSyncProducer(cfg.Global.Kafka.Addresses, nil) if err != nil { logrus.WithError(err).Panic("failed to setup kafka producers") } @@ -308,36 +316,26 @@ func setupKafka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { // setupNaffka creates kafka consumer/producer pair from the config. func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { - var err error - var db *sql.DB var naffkaDB *naffka.DatabaseImpl - uri, err := url.Parse(string(cfg.Database.Naffka)) - if err != nil || uri.Scheme == "file" { - var cs string - cs, err = sqlutil.ParseFileURI(string(cfg.Database.Naffka)) - if err != nil { - logrus.WithError(err).Panic("Failed to parse naffka database file URI") - } - db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil) - if err != nil { - logrus.WithError(err).Panic("Failed to open naffka database") - } + db, err := sqlutil.Open(&cfg.Global.Kafka.Database) + if err != nil { + logrus.WithError(err).Panic("Failed to open naffka database") + } + switch { + case cfg.Global.Kafka.Database.ConnectionString.IsSQLite(): naffkaDB, err = naffka.NewSqliteDatabase(db) if err != nil { logrus.WithError(err).Panic("Failed to setup naffka database") } - } else { - db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka), nil) - if err != nil { - logrus.WithError(err).Panic("Failed to open naffka database") - } - + case cfg.Global.Kafka.Database.ConnectionString.IsPostgres(): naffkaDB, err = naffka.NewPostgresqlDatabase(db) if err != nil { logrus.WithError(err).Panic("Failed to setup naffka database") } + default: + panic("unknown naffka database type") } if naffkaDB == nil { diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index f33f97ee..92bbca72 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -65,19 +65,19 @@ type Monolith struct { // AddAllPublicRoutes attaches all public paths to the given router func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) { clientapi.AddPublicRoutes( - publicMux, m.Config, m.KafkaProducer, m.DeviceDB, m.AccountDB, + publicMux, &m.Config.ClientAPI, m.KafkaProducer, m.DeviceDB, m.AccountDB, m.FedClient, m.RoomserverAPI, m.EDUInternalAPI, m.AppserviceAPI, m.StateAPI, transactions.New(), m.FederationSenderAPI, m.UserAPI, m.KeyAPI, m.ExtPublicRoomsProvider, ) federationapi.AddPublicRoutes( - publicMux, m.Config, m.UserAPI, m.FedClient, + publicMux, &m.Config.FederationAPI, m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationSenderAPI, m.EDUInternalAPI, m.StateAPI, m.KeyAPI, ) - mediaapi.AddPublicRoutes(publicMux, m.Config, m.UserAPI, m.Client) + mediaapi.AddPublicRoutes(publicMux, &m.Config.MediaAPI, m.UserAPI, m.Client) syncapi.AddPublicRoutes( publicMux, m.KafkaConsumer, m.UserAPI, m.RoomserverAPI, - m.KeyAPI, m.StateAPI, m.FedClient, m.Config, + m.KeyAPI, m.StateAPI, m.FedClient, &m.Config.SyncAPI, ) } diff --git a/internal/sqlutil/sql.go b/internal/sqlutil/sql.go index 2ec6ce29..95467c63 100644 --- a/internal/sqlutil/sql.go +++ b/internal/sqlutil/sql.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "runtime" - "time" "go.uber.org/atomic" ) @@ -107,13 +106,6 @@ func SQLiteDriverName() string { return "sqlite3" } -// DbProperties functions return properties used by database/sql/DB -type DbProperties interface { - MaxIdleConns() int - MaxOpenConns() int - ConnMaxLifetime() time.Duration -} - // TransactionWriter allows queuing database writes so that you don't // contend on database locks in, e.g. SQLite. Only one task will run // at a time on a given TransactionWriter. diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go index f6644d59..fbd983be 100644 --- a/internal/sqlutil/trace.go +++ b/internal/sqlutil/trace.go @@ -25,6 +25,7 @@ import ( "strings" "time" + "github.com/matrix-org/dendrite/internal/config" "github.com/ngrok/sqlmw" "github.com/sirupsen/logrus" ) @@ -77,7 +78,22 @@ func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest [ // Open opens a database specified by its database driver name and a driver-specific data source name, // usually consisting of at least a database name and connection information. Includes tracing driver // if DENDRITE_TRACE_SQL=1 -func Open(driverName, dsn string, dbProperties DbProperties) (*sql.DB, error) { +func Open(dbProperties *config.DatabaseOptions) (*sql.DB, error) { + var err error + var driverName, dsn string + switch { + case dbProperties.ConnectionString.IsSQLite(): + driverName = SQLiteDriverName() + dsn, err = ParseFileURI(dbProperties.ConnectionString) + if err != nil { + return nil, fmt.Errorf("ParseFileURI: %w", err) + } + case dbProperties.ConnectionString.IsPostgres(): + driverName = "postgres" + dsn = string(dbProperties.ConnectionString) + default: + return nil, fmt.Errorf("invalid database connection string %q", dbProperties.ConnectionString) + } if tracingEnabled { // install the wrapped driver driverName += "-trace" @@ -86,11 +102,11 @@ func Open(driverName, dsn string, dbProperties DbProperties) (*sql.DB, error) { if err != nil { return nil, err } - if driverName != SQLiteDriverName() && dbProperties != nil { + if driverName != SQLiteDriverName() { logrus.WithFields(logrus.Fields{ - "MaxOpenConns": dbProperties.MaxOpenConns(), - "MaxIdleConns": dbProperties.MaxIdleConns(), - "ConnMaxLifetime": dbProperties.ConnMaxLifetime(), + "MaxOpenConns": dbProperties.MaxOpenConns, + "MaxIdleConns": dbProperties.MaxIdleConns, + "ConnMaxLifetime": dbProperties.ConnMaxLifetime, "dataSourceName": regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"), }).Debug("Setting DB connection limits") db.SetMaxOpenConns(dbProperties.MaxOpenConns()) diff --git a/internal/sqlutil/uri.go b/internal/sqlutil/uri.go index 703258e6..e2c825d9 100644 --- a/internal/sqlutil/uri.go +++ b/internal/sqlutil/uri.go @@ -15,14 +15,20 @@ package sqlutil import ( + "errors" "fmt" "net/url" + + "github.com/matrix-org/dendrite/internal/config" ) // ParseFileURI returns the filepath in the given file: URI. Specifically, this will handle // both relative (file:foo.db) and absolute (file:///path/to/foo) paths. -func ParseFileURI(dataSourceName string) (string, error) { - uri, err := url.Parse(dataSourceName) +func ParseFileURI(dataSourceName config.DataSource) (string, error) { + if !dataSourceName.IsSQLite() { + return "", errors.New("ParseFileURI expects SQLite connection string") + } + uri, err := url.Parse(string(dataSourceName)) if err != nil { return "", err } diff --git a/internal/test/config.go b/internal/test/config.go index bbcc9bed..43a5d1ff 100644 --- a/internal/test/config.go +++ b/internal/test/config.go @@ -49,6 +49,7 @@ const ( // Generates new matrix and TLS keys for the server. func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*config.Dendrite, int, error) { var cfg config.Dendrite + cfg.Defaults() port := startPort assignAddress := func() config.Address { @@ -72,51 +73,53 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Version = config.Version - cfg.Matrix.ServerName = gomatrixserverlib.ServerName(assignAddress()) - cfg.Matrix.PrivateKeyPath = config.Path(serverKeyPath) - cfg.Matrix.FederationCertificatePaths = []config.Path{config.Path(tlsCertPath)} + cfg.Global.ServerName = gomatrixserverlib.ServerName(assignAddress()) + cfg.Global.PrivateKeyPath = config.Path(serverKeyPath) - cfg.Media.BasePath = config.Path(mediaBasePath) + cfg.FederationAPI.FederationCertificatePaths = []config.Path{config.Path(tlsCertPath)} - cfg.Kafka.Addresses = []string{kafkaURI} - // TODO: Different servers should be using different topics. - // Make this configurable somehow? - cfg.Kafka.Topics.OutputRoomEvent = "test.room.output" - cfg.Kafka.Topics.OutputClientData = "test.clientapi.output" - cfg.Kafka.Topics.OutputTypingEvent = "test.typing.output" + cfg.MediaAPI.BasePath = config.Path(mediaBasePath) + + cfg.Global.Kafka.Addresses = []string{kafkaURI} // TODO: Use different databases for the different schemas. // Using the same database for every schema currently works because // the table names are globally unique. But we might not want to // rely on that in the future. - cfg.Database.Account = config.DataSource(database) - cfg.Database.AppService = config.DataSource(database) - cfg.Database.Device = config.DataSource(database) - cfg.Database.MediaAPI = config.DataSource(database) - cfg.Database.RoomServer = config.DataSource(database) - cfg.Database.ServerKey = config.DataSource(database) - cfg.Database.SyncAPI = config.DataSource(database) - cfg.Database.CurrentState = config.DataSource(database) - - cfg.Listen.ClientAPI = assignAddress() - cfg.Listen.AppServiceAPI = assignAddress() - cfg.Listen.FederationAPI = assignAddress() - cfg.Listen.MediaAPI = assignAddress() - cfg.Listen.RoomServer = assignAddress() - cfg.Listen.SyncAPI = assignAddress() - cfg.Listen.CurrentState = assignAddress() - cfg.Listen.EDUServer = assignAddress() - - // Bind to the same address as the listen address - // All microservices are run on the same host in testing - cfg.Bind.ClientAPI = cfg.Listen.ClientAPI - cfg.Bind.AppServiceAPI = cfg.Listen.AppServiceAPI - cfg.Bind.FederationAPI = cfg.Listen.FederationAPI - cfg.Bind.MediaAPI = cfg.Listen.MediaAPI - cfg.Bind.RoomServer = cfg.Listen.RoomServer - cfg.Bind.SyncAPI = cfg.Listen.SyncAPI - cfg.Bind.CurrentState = cfg.Listen.CurrentState - cfg.Bind.EDUServer = cfg.Listen.EDUServer + cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(database) + cfg.CurrentStateServer.Database.ConnectionString = config.DataSource(database) + cfg.FederationSender.Database.ConnectionString = config.DataSource(database) + cfg.KeyServer.Database.ConnectionString = config.DataSource(database) + cfg.MediaAPI.Database.ConnectionString = config.DataSource(database) + cfg.RoomServer.Database.ConnectionString = config.DataSource(database) + cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(database) + cfg.SyncAPI.Database.ConnectionString = config.DataSource(database) + cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(database) + cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(database) + + cfg.AppServiceAPI.Listen = assignAddress() + cfg.CurrentStateServer.Listen = assignAddress() + cfg.EDUServer.Listen = assignAddress() + cfg.FederationAPI.Listen = assignAddress() + cfg.FederationSender.Listen = assignAddress() + cfg.KeyServer.Listen = assignAddress() + cfg.MediaAPI.Listen = assignAddress() + cfg.RoomServer.Listen = assignAddress() + cfg.ServerKeyAPI.Listen = assignAddress() + cfg.SyncAPI.Listen = assignAddress() + cfg.UserAPI.Listen = assignAddress() + + cfg.AppServiceAPI.Bind = cfg.AppServiceAPI.Listen + cfg.CurrentStateServer.Bind = cfg.CurrentStateServer.Listen + cfg.EDUServer.Bind = cfg.EDUServer.Listen + cfg.FederationAPI.Bind = cfg.FederationAPI.Listen + cfg.FederationSender.Bind = cfg.FederationSender.Listen + cfg.KeyServer.Bind = cfg.KeyServer.Listen + cfg.MediaAPI.Bind = cfg.MediaAPI.Listen + cfg.RoomServer.Bind = cfg.RoomServer.Listen + cfg.ServerKeyAPI.Bind = cfg.ServerKeyAPI.Listen + cfg.SyncAPI.Bind = cfg.SyncAPI.Listen + cfg.UserAPI.Bind = cfg.UserAPI.Listen return &cfg, port, nil } diff --git a/internal/test/server.go b/internal/test/server.go index 2d4117f4..57df21db 100644 --- a/internal/test/server.go +++ b/internal/test/server.go @@ -96,9 +96,9 @@ func InitDatabase(postgresDatabase, postgresContainerName string, databases []st func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) { proxyArgs := []string{ "--bind-address", bindAddr, - "--sync-api-server-url", "http://" + string(cfg.Listen.SyncAPI), - "--client-api-server-url", "http://" + string(cfg.Listen.ClientAPI), - "--media-api-server-url", "http://" + string(cfg.Listen.MediaAPI), + "--sync-api-server-url", "http://" + string(cfg.SyncAPI.Listen), + "--client-api-server-url", "http://" + string(cfg.ClientAPI.Listen), + "--media-api-server-url", "http://" + string(cfg.MediaAPI.Listen), "--tls-cert", "server.crt", "--tls-key", "server.key", } diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index 4a6fbe3c..b5bbb519 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -36,17 +36,14 @@ func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI) { // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - cfg *config.Dendrite, fedClient *gomatrixserverlib.FederationClient, producer sarama.SyncProducer, + cfg *config.KeyServer, fedClient *gomatrixserverlib.FederationClient, producer sarama.SyncProducer, ) api.KeyInternalAPI { - db, err := storage.NewDatabase( - string(cfg.Database.E2EKey), - cfg.DbProperties(), - ) + db, err := storage.NewDatabase(&cfg.Database) if err != nil { logrus.WithError(err).Panicf("failed to connect to key server database") } keyChangeProducer := &producers.KeyChange{ - Topic: string(cfg.Kafka.Topics.OutputKeyChangeEvent), + Topic: string(cfg.Matrix.Kafka.Topics.OutputKeyChangeEvent), Producer: producer, DB: db, } diff --git a/keyserver/storage/postgres/storage.go b/keyserver/storage/postgres/storage.go index de2fabfd..1c693f5b 100644 --- a/keyserver/storage/postgres/storage.go +++ b/keyserver/storage/postgres/storage.go @@ -15,14 +15,15 @@ package postgres import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/keyserver/storage/shared" ) // NewDatabase creates a new sync server database -func NewDatabase(dbDataSourceName string, dbProperties sqlutil.DbProperties) (*shared.Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) { var err error - db, err := sqlutil.Open("postgres", dbDataSourceName, dbProperties) + db, err := sqlutil.Open(dbProperties) if err != nil { return nil, err } diff --git a/keyserver/storage/sqlite3/storage.go b/keyserver/storage/sqlite3/storage.go index bbfd1e79..bb293558 100644 --- a/keyserver/storage/sqlite3/storage.go +++ b/keyserver/storage/sqlite3/storage.go @@ -15,17 +15,13 @@ package sqlite3 import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/keyserver/storage/shared" ) -func NewDatabase(dataSourceName string) (*shared.Database, error) { - var err error - cs, err := sqlutil.ParseFileURI(dataSourceName) - if err != nil { - return nil, err - } - db, err := sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil) +func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) { + db, err := sqlutil.Open(dbProperties) if err != nil { return nil, err } diff --git a/keyserver/storage/storage.go b/keyserver/storage/storage.go index ffcead70..e1deaf93 100644 --- a/keyserver/storage/storage.go +++ b/keyserver/storage/storage.go @@ -17,26 +17,22 @@ package storage import ( - "net/url" + "fmt" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/keyserver/storage/postgres" "github.com/matrix-org/dendrite/keyserver/storage/sqlite3" ) // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets postgres connection parameters -func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgres.NewDatabase(dataSourceName, dbProperties) - } - switch uri.Scheme { - case "postgres": - return postgres.NewDatabase(dataSourceName, dbProperties) - case "file": - return sqlite3.NewDatabase(dataSourceName) +func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(dbProperties) + case dbProperties.ConnectionString.IsPostgres(): + return postgres.NewDatabase(dbProperties) default: - return postgres.NewDatabase(dataSourceName, dbProperties) + return nil, fmt.Errorf("unexpected database type") } } diff --git a/keyserver/storage/storage_test.go b/keyserver/storage/storage_test.go index ec1b299f..358f11e7 100644 --- a/keyserver/storage/storage_test.go +++ b/keyserver/storage/storage_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/Shopify/sarama" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/keyserver/api" ) @@ -21,7 +22,9 @@ func MustCreateDatabase(t *testing.T) (Database, func()) { log.Fatal(err) } t.Logf("Database %s", tmpfile.Name()) - db, err := NewDatabase(fmt.Sprintf("file://%s", tmpfile.Name()), nil) + db, err := NewDatabase(&config.DatabaseOptions{ + ConnectionString: config.DataSource(fmt.Sprintf("file://%s", tmpfile.Name())), + }) if err != nil { t.Fatalf("Failed to NewDatabase: %s", err) } diff --git a/keyserver/storage/storage_wasm.go b/keyserver/storage/storage_wasm.go index 233e5d29..792cd4a5 100644 --- a/keyserver/storage/storage_wasm.go +++ b/keyserver/storage/storage_wasm.go @@ -16,26 +16,18 @@ package storage import ( "fmt" - "net/url" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/keyserver/storage/sqlite3" ) -func NewDatabase( - dataSourceName string, - dbProperties sqlutil.DbProperties, // nolint:unparam -) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return nil, fmt.Errorf("Cannot use postgres implementation") - } - switch uri.Scheme { - case "postgres": - return nil, fmt.Errorf("Cannot use postgres implementation") - case "file": - return sqlite3.NewDatabase(dataSourceName) +func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(dbProperties) + case dbProperties.ConnectionString.IsPostgres(): + return nil, fmt.Errorf("can't use Postgres implementation") default: - return nil, fmt.Errorf("Cannot use postgres implementation") + return nil, fmt.Errorf("unexpected database type") } } diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index 290ef46e..1c14559f 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -26,11 +26,11 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component. func AddPublicRoutes( - router *mux.Router, cfg *config.Dendrite, + router *mux.Router, cfg *config.MediaAPI, userAPI userapi.UserInternalAPI, client *gomatrixserverlib.Client, ) { - mediaDB, err := storage.Open(string(cfg.Database.MediaAPI), cfg.DbProperties()) + mediaDB, err := storage.Open(&cfg.Database) if err != nil { logrus.WithError(err).Panicf("failed to connect to media db") } diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 7e121de3..e61fa82b 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -73,7 +73,7 @@ func Download( req *http.Request, origin gomatrixserverlib.ServerName, mediaID types.MediaID, - cfg *config.Dendrite, + cfg *config.MediaAPI, db storage.Database, client *gomatrixserverlib.Client, activeRemoteRequests *types.ActiveRemoteRequests, @@ -203,7 +203,7 @@ func (r *downloadRequest) Validate() *util.JSONResponse { func (r *downloadRequest) doDownload( ctx context.Context, w http.ResponseWriter, - cfg *config.Dendrite, + cfg *config.MediaAPI, db storage.Database, client *gomatrixserverlib.Client, activeRemoteRequests *types.ActiveRemoteRequests, @@ -233,9 +233,9 @@ func (r *downloadRequest) doDownload( r.MediaMetadata = mediaMetadata } return r.respondFromLocalFile( - ctx, w, cfg.Media.AbsBasePath, activeThumbnailGeneration, - cfg.Media.MaxThumbnailGenerators, db, - cfg.Media.DynamicThumbnails, cfg.Media.ThumbnailSizes, + ctx, w, cfg.AbsBasePath, activeThumbnailGeneration, + cfg.MaxThumbnailGenerators, db, + cfg.DynamicThumbnails, cfg.ThumbnailSizes, ) } @@ -514,7 +514,7 @@ func (r *downloadRequest) generateThumbnail( func (r *downloadRequest) getRemoteFile( ctx context.Context, client *gomatrixserverlib.Client, - cfg *config.Dendrite, + cfg *config.MediaAPI, db storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, @@ -550,9 +550,9 @@ func (r *downloadRequest) getRemoteFile( // If we do not have a record, we need to fetch the remote file first and then respond from the local file err := r.fetchRemoteFileAndStoreMetadata( ctx, client, - cfg.Media.AbsBasePath, *cfg.Media.MaxFileSizeBytes, db, - cfg.Media.ThumbnailSizes, activeThumbnailGeneration, - cfg.Media.MaxThumbnailGenerators, + cfg.AbsBasePath, *cfg.MaxFileSizeBytes, db, + cfg.ThumbnailSizes, activeThumbnailGeneration, + cfg.MaxThumbnailGenerators, ) if err != nil { return errors.Wrap(err, "error querying the database.") diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index f4a8b157..ee5f28f9 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -42,7 +42,7 @@ const pathPrefixV1 = "/media/v1" // TODO: remove when synapse is fixed // nolint: gocyclo func Setup( publicAPIMux *mux.Router, - cfg *config.Dendrite, + cfg *config.MediaAPI, db storage.Database, userAPI userapi.UserInternalAPI, client *gomatrixserverlib.Client, @@ -81,7 +81,7 @@ func Setup( func makeDownloadAPI( name string, - cfg *config.Dendrite, + cfg *config.MediaAPI, db storage.Database, client *gomatrixserverlib.Client, activeRemoteRequests *types.ActiveRemoteRequests, diff --git a/mediaapi/routing/upload.go b/mediaapi/routing/upload.go index 9b5dc3df..9f35e90c 100644 --- a/mediaapi/routing/upload.go +++ b/mediaapi/routing/upload.go @@ -54,7 +54,7 @@ type uploadResponse struct { // This implementation supports a configurable maximum file size limit in bytes. If a user tries to upload more than this, they will receive an error that their upload is too large. // Uploaded files are processed piece-wise to avoid DoS attacks which would starve the server of memory. // TODO: We should time out requests if they have not received any data within a configured timeout period. -func Upload(req *http.Request, cfg *config.Dendrite, db storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse { +func Upload(req *http.Request, cfg *config.MediaAPI, db storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse { r, resErr := parseAndValidateRequest(req, cfg) if resErr != nil { return *resErr @@ -75,7 +75,7 @@ func Upload(req *http.Request, cfg *config.Dendrite, db storage.Database, active // parseAndValidateRequest parses the incoming upload request to validate and extract // all the metadata about the media being uploaded. // Returns either an uploadRequest or an error formatted as a util.JSONResponse -func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRequest, *util.JSONResponse) { +func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRequest, *util.JSONResponse) { r := &uploadRequest{ MediaMetadata: &types.MediaMetadata{ Origin: cfg.Matrix.ServerName, @@ -86,7 +86,7 @@ func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRe Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.Matrix.ServerName), } - if resErr := r.Validate(*cfg.Media.MaxFileSizeBytes); resErr != nil { + if resErr := r.Validate(*cfg.MaxFileSizeBytes); resErr != nil { return nil, resErr } @@ -96,7 +96,7 @@ func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRe func (r *uploadRequest) doUpload( ctx context.Context, reqReader io.Reader, - cfg *config.Dendrite, + cfg *config.MediaAPI, db storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration, ) *util.JSONResponse { @@ -110,10 +110,10 @@ func (r *uploadRequest) doUpload( // method of deduplicating files to save storage, as well as a way to conduct // integrity checks on the file data in the repository. // Data is truncated to maxFileSizeBytes. Content-Length was reported as 0 < Content-Length <= maxFileSizeBytes so this is OK. - hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.Media.MaxFileSizeBytes, cfg.Media.AbsBasePath) + hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.MaxFileSizeBytes, cfg.AbsBasePath) if err != nil { r.Logger.WithError(err).WithFields(log.Fields{ - "MaxFileSizeBytes": *cfg.Media.MaxFileSizeBytes, + "MaxFileSizeBytes": *cfg.MaxFileSizeBytes, }).Warn("Error while transferring file") fileutils.RemoveDir(tmpDir, r.Logger) return &util.JSONResponse{ @@ -159,8 +159,8 @@ func (r *uploadRequest) doUpload( } return r.storeFileAndMetadata( - ctx, tmpDir, cfg.Media.AbsBasePath, db, cfg.Media.ThumbnailSizes, - activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators, + ctx, tmpDir, cfg.AbsBasePath, db, cfg.ThumbnailSizes, + activeThumbnailGeneration, cfg.MaxThumbnailGenerators, ) } diff --git a/mediaapi/storage/postgres/storage.go b/mediaapi/storage/postgres/storage.go index e45e0841..756913d3 100644 --- a/mediaapi/storage/postgres/storage.go +++ b/mediaapi/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( // Import the postgres database driver. _ "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -33,10 +34,10 @@ type Database struct { } // Open opens a postgres database. -func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) { +func Open(dbProperties *config.DatabaseOptions) (*Database, error) { var d Database var err error - if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { + if d.db, err = sqlutil.Open(dbProperties); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go index 010c0a66..a1e7fec7 100644 --- a/mediaapi/storage/sqlite3/storage.go +++ b/mediaapi/storage/sqlite3/storage.go @@ -20,6 +20,7 @@ import ( "database/sql" // Import the postgres database driver. + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -33,14 +34,10 @@ type Database struct { } // Open opens a postgres database. -func Open(dataSourceName string) (*Database, error) { +func Open(dbProperties *config.DatabaseOptions) (*Database, error) { var d Database var err error - cs, err := sqlutil.ParseFileURI(dataSourceName) - if err != nil { - return nil, err - } - if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { + if d.db, err = sqlutil.Open(dbProperties); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index 5ff114db..829d47b3 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -17,25 +17,21 @@ package storage import ( - "net/url" + "fmt" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/mediaapi/storage/postgres" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" ) // Open opens a postgres database. -func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgres.Open(dataSourceName, dbProperties) - } - switch uri.Scheme { - case "postgres": - return postgres.Open(dataSourceName, dbProperties) - case "file": - return sqlite3.Open(dataSourceName) +func Open(dbProperties *config.DatabaseOptions) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.Open(dbProperties) + case dbProperties.ConnectionString.IsPostgres(): + return postgres.Open(dbProperties) default: - return postgres.Open(dataSourceName, dbProperties) + return nil, fmt.Errorf("unexpected database type") } } diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go index a672271f..6b5de681 100644 --- a/mediaapi/storage/storage_wasm.go +++ b/mediaapi/storage/storage_wasm.go @@ -16,27 +16,19 @@ package storage import ( "fmt" - "net/url" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" ) // Open opens a postgres database. -func Open( - dataSourceName string, - dbProperties sqlutil.DbProperties, // nolint:unparam -) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return nil, fmt.Errorf("Cannot use postgres implementation") - } - switch uri.Scheme { - case "postgres": - return nil, fmt.Errorf("Cannot use postgres implementation") - case "file": - return sqlite3.Open(dataSourceName) +func Open(dbProperties *config.DatabaseOptions) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.Open(dbProperties) + case dbProperties.ConnectionString.IsPostgres(): + return nil, fmt.Errorf("can't use Postgres implementation") default: - return nil, fmt.Errorf("Cannot use postgres implementation") + return nil, fmt.Errorf("unexpected database type") } } diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 37a8a39b..6d7c8fcf 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -14,7 +14,7 @@ import ( // RoomserverInternalAPI is an implementation of api.RoomserverInternalAPI type RoomserverInternalAPI struct { DB storage.Database - Cfg *config.Dendrite + Cfg *config.RoomServer Producer sarama.SyncProducer Cache caching.RoomVersionCache ServerName gomatrixserverlib.ServerName diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go index c5480a5b..73ea008d 100644 --- a/roomserver/internal/perform_join.go +++ b/roomserver/internal/perform_join.go @@ -189,12 +189,12 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( // but everyone has since left. I suspect it does the wrong thing. buildRes := api.QueryLatestEventsAndStateResponse{} event, err := eventutil.BuildEvent( - ctx, // the request context - &eb, // the template join event - r.Cfg, // the server configuration - time.Now(), // the event timestamp to use - r, // the roomserver API to use - &buildRes, // the query response + ctx, // the request context + &eb, // the template join event + r.Cfg.Matrix, // the server configuration + time.Now(), // the event timestamp to use + r, // the roomserver API to use + &buildRes, // the query response ) switch err { diff --git a/roomserver/internal/perform_leave.go b/roomserver/internal/perform_leave.go index a19d0da9..79676530 100644 --- a/roomserver/internal/perform_leave.go +++ b/roomserver/internal/perform_leave.go @@ -99,12 +99,12 @@ func (r *RoomserverInternalAPI) performLeaveRoomByID( // but everyone has since left. I suspect it does the wrong thing. buildRes := api.QueryLatestEventsAndStateResponse{} event, err := eventutil.BuildEvent( - ctx, // the request context - &eb, // the template leave event - r.Cfg, // the server configuration - time.Now(), // the event timestamp to use - r, // the roomserver API to use - &buildRes, // the query response + ctx, // the request context + &eb, // the template leave event + r.Cfg.Matrix, // the server configuration + time.Now(), // the event timestamp to use + r, // the roomserver API to use + &buildRes, // the query response ) if err != nil { return fmt.Errorf("eventutil.BuildEvent: %w", err) diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 427d5ff3..1c226085 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -39,18 +39,20 @@ func NewInternalAPI( keyRing gomatrixserverlib.JSONVerifier, fedClient *gomatrixserverlib.FederationClient, ) api.RoomserverInternalAPI { - roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer), base.Cfg.DbProperties()) + cfg := &base.Cfg.RoomServer + + roomserverDB, err := storage.Open(&cfg.Database) if err != nil { logrus.WithError(err).Panicf("failed to connect to room server db") } return &internal.RoomserverInternalAPI{ DB: roomserverDB, - Cfg: base.Cfg, + Cfg: cfg, Producer: base.KafkaProducer, - OutputRoomEventTopic: string(base.Cfg.Kafka.Topics.OutputRoomEvent), + OutputRoomEventTopic: string(cfg.Matrix.Kafka.Topics.OutputRoomEvent), Cache: base.Caches, - ServerName: base.Cfg.Matrix.ServerName, + ServerName: cfg.Matrix.ServerName, FedClient: fedClient, KeyRing: keyRing, } diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index d553c5b7..d11c63b2 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -95,12 +95,12 @@ func mustLoadEvents(t *testing.T, ver gomatrixserverlib.RoomVersion, events []js func mustSendEvents(t *testing.T, ver gomatrixserverlib.RoomVersion, events []json.RawMessage) (api.RoomserverInternalAPI, *dummyProducer, []gomatrixserverlib.HeaderedEvent) { cfg := &config.Dendrite{} - cfg.Database.RoomServer = roomserverDBFileURI - cfg.Kafka.Topics.OutputRoomEvent = "output_room_event" - cfg.Matrix.ServerName = testOrigin - cfg.Kafka.UseNaffka = true + cfg.Defaults() + cfg.Global.ServerName = testOrigin + cfg.Global.Kafka.UseNaffka = true + cfg.RoomServer.Database.ConnectionString = config.DataSource(roomserverDBFileURI) dp := &dummyProducer{ - topic: string(cfg.Kafka.Topics.OutputRoomEvent), + topic: string(cfg.Global.Kafka.Topics.OutputRoomEvent), } cache, err := caching.NewInMemoryLRUCache(true) if err != nil { diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index c4f30f04..52ff479b 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -18,6 +18,7 @@ package postgres import ( "database/sql" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" // Import the postgres database driver. @@ -32,11 +33,11 @@ type Database struct { // Open a postgres database. // nolint: gocyclo -func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) { +func Open(dbProperties *config.DatabaseOptions) (*Database, error) { var d Database var db *sql.DB var err error - if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { + if db, err = sqlutil.Open(dbProperties); err != nil { return nil, err } eventStateKeys, err := NewPostgresEventStateKeysTable(db) diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 11781ce0..048de192 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -19,6 +19,7 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" @@ -44,13 +45,10 @@ type Database struct { // Open a sqlite database. // nolint: gocyclo -func Open(dataSourceName string) (*Database, error) { +func Open(dbProperties *config.DatabaseOptions) (*Database, error) { var d Database - cs, err := sqlutil.ParseFileURI(dataSourceName) - if err != nil { - return nil, err - } - if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { + var err error + if d.db, err = sqlutil.Open(dbProperties); err != nil { return nil, err } //d.db.Exec("PRAGMA journal_mode=WAL;") diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index d7367e4c..c6561fdc 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -17,25 +17,21 @@ package storage import ( - "net/url" + "fmt" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/storage/postgres" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" ) // Open opens a database connection. -func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgres.Open(dataSourceName, dbProperties) - } - switch uri.Scheme { - case "postgres": - return postgres.Open(dataSourceName, dbProperties) - case "file": - return sqlite3.Open(dataSourceName) +func Open(dbProperties *config.DatabaseOptions) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.Open(dbProperties) + case dbProperties.ConnectionString.IsPostgres(): + return postgres.Open(dbProperties) default: - return postgres.Open(dataSourceName, dbProperties) + return nil, fmt.Errorf("unexpected database type") } } diff --git a/roomserver/storage/storage_wasm.go b/roomserver/storage/storage_wasm.go index 78405b20..43367f36 100644 --- a/roomserver/storage/storage_wasm.go +++ b/roomserver/storage/storage_wasm.go @@ -16,27 +16,19 @@ package storage import ( "fmt" - "net/url" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" ) // NewPublicRoomsServerDatabase opens a database connection. -func Open( - dataSourceName string, - dbProperties sqlutil.DbProperties, // nolint:unparam -) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return nil, fmt.Errorf("Cannot use postgres implementation") - } - switch uri.Scheme { - case "postgres": - return nil, fmt.Errorf("Cannot use postgres implementation") - case "file": - return sqlite3.Open(dataSourceName) +func Open(dbProperties *config.DatabaseOptions) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.Open(dbProperties) + case dbProperties.ConnectionString.IsPostgres(): + return nil, fmt.Errorf("can't use Postgres implementation") default: - return nil, fmt.Errorf("Cannot use postgres implementation") + return nil, fmt.Errorf("unexpected database type") } } diff --git a/serverkeyapi/serverkeyapi.go b/serverkeyapi/serverkeyapi.go index cddd392e..fbaaefad 100644 --- a/serverkeyapi/serverkeyapi.go +++ b/serverkeyapi/serverkeyapi.go @@ -25,13 +25,12 @@ func AddInternalRoutes(router *mux.Router, intAPI api.ServerKeyInternalAPI, cach // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - cfg *config.Dendrite, + cfg *config.ServerKeyAPI, fedClient *gomatrixserverlib.FederationClient, caches *caching.Caches, ) api.ServerKeyInternalAPI { innerDB, err := storage.NewDatabase( - string(cfg.Database.ServerKey), - cfg.DbProperties(), + &cfg.Database, cfg.Matrix.ServerName, cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), cfg.Matrix.KeyID, @@ -62,7 +61,7 @@ func NewInternalAPI( } var b64e = base64.StdEncoding.WithPadding(base64.NoPadding) - for _, ps := range cfg.Matrix.KeyPerspectives { + for _, ps := range cfg.KeyPerspectives { perspective := &gomatrixserverlib.PerspectiveKeyFetcher{ PerspectiveServerName: ps.ServerName, PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{}, diff --git a/serverkeyapi/serverkeyapi_test.go b/serverkeyapi/serverkeyapi_test.go index c53575bb..152a853e 100644 --- a/serverkeyapi/serverkeyapi_test.go +++ b/serverkeyapi/serverkeyapi_test.go @@ -23,7 +23,8 @@ import ( type server struct { name gomatrixserverlib.ServerName // server name validity time.Duration // key validity duration from now - config *config.Dendrite // skeleton config, from TestMain + config *config.ServerKeyAPI // skeleton config, from TestMain + fedconfig *config.FederationAPI // fedclient *gomatrixserverlib.FederationClient // uses MockRoundTripper cache *caching.Caches // server-specific cache api api.ServerKeyInternalAPI // server-specific server key API @@ -69,13 +70,15 @@ func TestMain(m *testing.M) { // Draw up just enough Dendrite config for the server key // API to work. - s.config = &config.Dendrite{} - s.config.SetDefaults() - s.config.Matrix.ServerName = gomatrixserverlib.ServerName(s.name) - s.config.Matrix.PrivateKey = testPriv - s.config.Matrix.KeyID = serverKeyID - s.config.Matrix.KeyValidityPeriod = s.validity - s.config.Database.ServerKey = config.DataSource("file::memory:") + cfg := &config.Dendrite{} + cfg.Defaults() + cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name) + cfg.Global.PrivateKey = testPriv + cfg.Global.KeyID = serverKeyID + cfg.Global.KeyValidityPeriod = s.validity + cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource("file::memory:") + s.config = &cfg.ServerKeyAPI + s.fedconfig = &cfg.FederationAPI // Create a transport which redirects federation requests to // the mock round tripper. Since we're not *really* listening for @@ -115,7 +118,7 @@ func (m *MockRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err } // Get the keys and JSON-ify them. - keys := routing.LocalKeys(s.config) + keys := routing.LocalKeys(s.fedconfig) body, err := json.MarshalIndent(keys.JSON, "", " ") if err != nil { return nil, err diff --git a/serverkeyapi/storage/keydb.go b/serverkeyapi/storage/keydb.go index c28c4de1..3d3a0c30 100644 --- a/serverkeyapi/storage/keydb.go +++ b/serverkeyapi/storage/keydb.go @@ -17,11 +17,11 @@ package storage import ( - "net/url" + "fmt" "golang.org/x/crypto/ed25519" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/serverkeyapi/storage/postgres" "github.com/matrix-org/dendrite/serverkeyapi/storage/sqlite3" "github.com/matrix-org/gomatrixserverlib" @@ -29,22 +29,17 @@ import ( // NewDatabase opens a database connection. func NewDatabase( - dataSourceName string, - dbProperties sqlutil.DbProperties, + dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID) - } - switch uri.Scheme { - case "postgres": - return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID) - case "file": - return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(dbProperties, serverName, serverKey, serverKeyID) + case dbProperties.ConnectionString.IsPostgres(): + return postgres.NewDatabase(dbProperties, serverName, serverKey, serverKeyID) default: - return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID) + return nil, fmt.Errorf("unexpected database type") } } diff --git a/serverkeyapi/storage/postgres/keydb.go b/serverkeyapi/storage/postgres/keydb.go index aaa4409b..63444085 100644 --- a/serverkeyapi/storage/postgres/keydb.go +++ b/serverkeyapi/storage/postgres/keydb.go @@ -20,6 +20,7 @@ import ( "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -35,13 +36,12 @@ type Database struct { // It prepares all the SQL statements that it will use. // Returns an error if there was a problem talking to the database. func NewDatabase( - dataSourceName string, - dbProperties sqlutil.DbProperties, + dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sqlutil.Open("postgres", dataSourceName, dbProperties) + db, err := sqlutil.Open(dbProperties) if err != nil { return nil, err } diff --git a/serverkeyapi/storage/sqlite3/keydb.go b/serverkeyapi/storage/sqlite3/keydb.go index dc72b79e..5174ece1 100644 --- a/serverkeyapi/storage/sqlite3/keydb.go +++ b/serverkeyapi/storage/sqlite3/keydb.go @@ -20,6 +20,7 @@ import ( "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" @@ -37,16 +38,12 @@ type Database struct { // It prepares all the SQL statements that it will use. // Returns an error if there was a problem talking to the database. func NewDatabase( - dataSourceName string, + dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - cs, err := sqlutil.ParseFileURI(dataSourceName) - if err != nil { - return nil, err - } - db, err := sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil) + db, err := sqlutil.Open(dbProperties) if err != nil { return nil, err } diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index f7cf96d9..ceaa735a 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -37,14 +37,14 @@ type OutputClientDataConsumer struct { // NewOutputClientDataConsumer creates a new OutputClientData consumer. Call Start() to begin consuming from room servers. func NewOutputClientDataConsumer( - cfg *config.Dendrite, + cfg *config.SyncAPI, kafkaConsumer sarama.Consumer, n *sync.Notifier, store storage.Database, ) *OutputClientDataConsumer { consumer := internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputClientData), + Topic: string(cfg.Matrix.Kafka.Topics.OutputClientData), Consumer: kafkaConsumer, PartitionStore: store, } diff --git a/syncapi/consumers/eduserver_sendtodevice.go b/syncapi/consumers/eduserver_sendtodevice.go index 06a8928d..20dd1756 100644 --- a/syncapi/consumers/eduserver_sendtodevice.go +++ b/syncapi/consumers/eduserver_sendtodevice.go @@ -41,14 +41,14 @@ type OutputSendToDeviceEventConsumer struct { // NewOutputSendToDeviceEventConsumer creates a new OutputSendToDeviceEventConsumer. // Call Start() to begin consuming from the EDU server. func NewOutputSendToDeviceEventConsumer( - cfg *config.Dendrite, + cfg *config.SyncAPI, kafkaConsumer sarama.Consumer, n *sync.Notifier, store storage.Database, ) *OutputSendToDeviceEventConsumer { consumer := internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputSendToDeviceEvent), + Topic: string(cfg.Matrix.Kafka.Topics.OutputSendToDeviceEvent), Consumer: kafkaConsumer, PartitionStore: store, } diff --git a/syncapi/consumers/eduserver_typing.go b/syncapi/consumers/eduserver_typing.go index 0a9a9c0c..fc5703d3 100644 --- a/syncapi/consumers/eduserver_typing.go +++ b/syncapi/consumers/eduserver_typing.go @@ -37,14 +37,14 @@ type OutputTypingEventConsumer struct { // NewOutputTypingEventConsumer creates a new OutputTypingEventConsumer. // Call Start() to begin consuming from the EDU server. func NewOutputTypingEventConsumer( - cfg *config.Dendrite, + cfg *config.SyncAPI, kafkaConsumer sarama.Consumer, n *sync.Notifier, store storage.Database, ) *OutputTypingEventConsumer { consumer := internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputTypingEvent), + Topic: string(cfg.Matrix.Kafka.Topics.OutputTypingEvent), Consumer: kafkaConsumer, PartitionStore: store, } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index f8cdcd5c..06c904c3 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -40,7 +40,7 @@ type OutputRoomEventConsumer struct { // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. func NewOutputRoomEventConsumer( - cfg *config.Dendrite, + cfg *config.SyncAPI, kafkaConsumer sarama.Consumer, n *sync.Notifier, store storage.Database, @@ -49,7 +49,7 @@ func NewOutputRoomEventConsumer( ) *OutputRoomEventConsumer { consumer := internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputRoomEvent), + Topic: string(cfg.Matrix.Kafka.Topics.OutputRoomEvent), Consumer: kafkaConsumer, PartitionStore: store, } diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 15add1b4..0999d3e8 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -36,7 +36,7 @@ type messagesReq struct { db storage.Database rsAPI api.RoomserverInternalAPI federation *gomatrixserverlib.FederationClient - cfg *config.Dendrite + cfg *config.SyncAPI roomID string from *types.TopologyToken to *types.TopologyToken @@ -61,7 +61,7 @@ func OnIncomingMessagesRequest( req *http.Request, db storage.Database, roomID string, federation *gomatrixserverlib.FederationClient, rsAPI api.RoomserverInternalAPI, - cfg *config.Dendrite, + cfg *config.SyncAPI, ) util.JSONResponse { var err error diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index ed0f872e..ec21c1b4 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -39,7 +39,7 @@ func Setup( publicAPIMux *mux.Router, srp *sync.RequestPool, syncDB storage.Database, userAPI userapi.UserInternalAPI, federation *gomatrixserverlib.FederationClient, rsAPI api.RoomserverInternalAPI, - cfg *config.Dendrite, + cfg *config.SyncAPI, ) { r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 10c1b37c..26ef082f 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -21,6 +21,7 @@ import ( // Import the postgres database driver. _ "github.com/lib/pq" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/shared" ) @@ -34,10 +35,10 @@ type SyncServerDatasource struct { } // NewDatabase creates a new sync server database -func NewDatabase(dbDataSourceName string, dbProperties sqlutil.DbProperties) (*SyncServerDatasource, error) { +func NewDatabase(dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error - if d.db, err = sqlutil.Open("postgres", dbDataSourceName, dbProperties); err != nil { + if d.db, err = sqlutil.Open(dbProperties); err != nil { return nil, err } if err = d.PartitionOffsetStatements.Prepare(d.db, "syncapi"); err != nil { diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index c85db5a4..9564a23a 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -22,6 +22,7 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/shared" ) @@ -37,13 +38,10 @@ type SyncServerDatasource struct { // NewDatabase creates a new sync server database // nolint: gocyclo -func NewDatabase(dataSourceName string) (*SyncServerDatasource, error) { +func NewDatabase(dbProperties *config.DatabaseOptions) (*SyncServerDatasource, error) { var d SyncServerDatasource - cs, err := sqlutil.ParseFileURI(dataSourceName) - if err != nil { - return nil, err - } - if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { + var err error + if d.db, err = sqlutil.Open(dbProperties); err != nil { return nil, err } if err = d.prepare(); err != nil { diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index ea69da3b..c16dcd81 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -17,25 +17,21 @@ package storage import ( - "net/url" + "fmt" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) // NewSyncServerDatasource opens a database connection. -func NewSyncServerDatasource(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgres.NewDatabase(dataSourceName, dbProperties) - } - switch uri.Scheme { - case "postgres": - return postgres.NewDatabase(dataSourceName, dbProperties) - case "file": - return sqlite3.NewDatabase(dataSourceName) +func NewSyncServerDatasource(dbProperties *config.DatabaseOptions) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(dbProperties) + case dbProperties.ConnectionString.IsPostgres(): + return postgres.NewDatabase(dbProperties) default: - return postgres.NewDatabase(dataSourceName, dbProperties) + return nil, fmt.Errorf("unexpected database type") } } diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 1f679def..0e827c95 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" "github.com/matrix-org/dendrite/syncapi/types" @@ -59,7 +60,9 @@ func MustCreateDatabase(t *testing.T) storage.Database { t.Fatalf("tried to delete stale test database but failed: %s", err) } } - db, err := sqlite3.NewDatabase(fmt.Sprintf("file:%s", dbname)) + db, err := sqlite3.NewDatabase(&config.DatabaseOptions{ + ConnectionString: config.DataSource(fmt.Sprintf("file:%s", dbname)), + }) if err != nil { t.Fatalf("NewSyncServerDatasource returned %s", err) } diff --git a/syncapi/storage/storage_wasm.go b/syncapi/storage/storage_wasm.go index 0886b8c2..43b7bbea 100644 --- a/syncapi/storage/storage_wasm.go +++ b/syncapi/storage/storage_wasm.go @@ -16,27 +16,19 @@ package storage import ( "fmt" - "net/url" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) // NewPublicRoomsServerDatabase opens a database connection. -func NewSyncServerDatasource( - dataSourceName string, - dbProperties sqlutil.DbProperties, // nolint:unparam -) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return nil, fmt.Errorf("Cannot use postgres implementation") - } - switch uri.Scheme { - case "postgres": - return nil, fmt.Errorf("Cannot use postgres implementation") - case "file": - return sqlite3.NewDatabase(dataSourceName) +func NewSyncServerDatasource(dbProperties *config.DatabaseOptions) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(dbProperties) + case dbProperties.ConnectionString.IsPostgres(): + return nil, fmt.Errorf("can't use Postgres implementation") default: - return nil, fmt.Errorf("Cannot use postgres implementation") + return nil, fmt.Errorf("unexpected database type") } } diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 5198d59b..9caed7be 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -44,9 +44,9 @@ func AddPublicRoutes( keyAPI keyapi.KeyInternalAPI, currentStateAPI currentstateapi.CurrentStateInternalAPI, federation *gomatrixserverlib.FederationClient, - cfg *config.Dendrite, + cfg *config.SyncAPI, ) { - syncDB, err := storage.NewSyncServerDatasource(string(cfg.Database.SyncAPI), cfg.DbProperties()) + syncDB, err := storage.NewSyncServerDatasource(&cfg.Database) if err != nil { logrus.WithError(err).Panicf("failed to connect to sync db") } @@ -65,7 +65,7 @@ func AddPublicRoutes( requestPool := sync.NewRequestPool(syncDB, notifier, userAPI, keyAPI, currentStateAPI) keyChangeConsumer := consumers.NewOutputKeyChangeEventConsumer( - cfg.Matrix.ServerName, string(cfg.Kafka.Topics.OutputKeyChangeEvent), + cfg.Matrix.ServerName, string(cfg.Matrix.Kafka.Topics.OutputKeyChangeEvent), consumer, notifier, keyAPI, currentStateAPI, syncDB, ) if err = keyChangeConsumer.Start(); err != nil { diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go index f56fb6d8..9653c019 100644 --- a/userapi/storage/accounts/postgres/storage.go +++ b/userapi/storage/accounts/postgres/storage.go @@ -22,6 +22,7 @@ import ( "strconv" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -43,10 +44,9 @@ type Database struct { } // NewDatabase creates a new accounts and profiles database -func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { - var db *sql.DB - var err error - if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { +func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName) (*Database, error) { + db, err := sqlutil.Open(dbProperties) + if err != nil { return nil, err } partitions := sqlutil.PartitionOffsetStatements{} diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go index 72239014..4d2c5e51 100644 --- a/userapi/storage/accounts/sqlite3/storage.go +++ b/userapi/storage/accounts/sqlite3/storage.go @@ -23,6 +23,7 @@ import ( "sync" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -47,16 +48,11 @@ type Database struct { } // NewDatabase creates a new accounts and profiles database -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { - var db *sql.DB - var err error - cs, err := sqlutil.ParseFileURI(dataSourceName) +func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName) (*Database, error) { + db, err := sqlutil.Open(dbProperties) if err != nil { return nil, err } - if db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { - return nil, err - } partitions := sqlutil.PartitionOffsetStatements{} if err = partitions.Prepare(db, "account"); err != nil { return nil, err diff --git a/userapi/storage/accounts/storage.go b/userapi/storage/accounts/storage.go index 87f626bf..57d5f703 100644 --- a/userapi/storage/accounts/storage.go +++ b/userapi/storage/accounts/storage.go @@ -17,9 +17,9 @@ package accounts import ( - "net/url" + "fmt" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/userapi/storage/accounts/postgres" "github.com/matrix-org/dendrite/userapi/storage/accounts/sqlite3" "github.com/matrix-org/gomatrixserverlib" @@ -27,17 +27,13 @@ import ( // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets postgres connection parameters -func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgres.NewDatabase(dataSourceName, dbProperties, serverName) - } - switch uri.Scheme { - case "postgres": - return postgres.NewDatabase(dataSourceName, dbProperties, serverName) - case "file": - return sqlite3.NewDatabase(dataSourceName, serverName) +func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(dbProperties, serverName) + case dbProperties.ConnectionString.IsPostgres(): + return postgres.NewDatabase(dbProperties, serverName) default: - return postgres.NewDatabase(dataSourceName, dbProperties, serverName) + return nil, fmt.Errorf("unexpected database type") } } diff --git a/userapi/storage/accounts/storage_wasm.go b/userapi/storage/accounts/storage_wasm.go index 69256705..ade32b68 100644 --- a/userapi/storage/accounts/storage_wasm.go +++ b/userapi/storage/accounts/storage_wasm.go @@ -16,28 +16,22 @@ package accounts import ( "fmt" - "net/url" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/userapi/storage/accounts/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) func NewDatabase( - dataSourceName string, - dbProperties sqlutil.DbProperties, // nolint:unparam + dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, ) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return nil, fmt.Errorf("Cannot use postgres implementation") - } - switch uri.Scheme { - case "postgres": - return nil, fmt.Errorf("Cannot use postgres implementation") - case "file": - return sqlite3.NewDatabase(dataSourceName, serverName) + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(dbProperties, serverName) + case dbProperties.ConnectionString.IsPostgres(): + return nil, fmt.Errorf("can't use Postgres implementation") default: - return nil, fmt.Errorf("Cannot use postgres implementation") + return nil, fmt.Errorf("unexpected database type") } } diff --git a/userapi/storage/devices/postgres/storage.go b/userapi/storage/devices/postgres/storage.go index 6ac802bb..4a7c7f97 100644 --- a/userapi/storage/devices/postgres/storage.go +++ b/userapi/storage/devices/postgres/storage.go @@ -20,6 +20,7 @@ import ( "database/sql" "encoding/base64" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -35,10 +36,9 @@ type Database struct { } // NewDatabase creates a new device database -func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { - var db *sql.DB - var err error - if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { +func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName) (*Database, error) { + db, err := sqlutil.Open(dbProperties) + if err != nil { return nil, err } d := devicesStatements{} diff --git a/userapi/storage/devices/sqlite3/storage.go b/userapi/storage/devices/sqlite3/storage.go index b9f08ca1..1f2b59f3 100644 --- a/userapi/storage/devices/sqlite3/storage.go +++ b/userapi/storage/devices/sqlite3/storage.go @@ -20,6 +20,7 @@ import ( "database/sql" "encoding/base64" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -37,16 +38,11 @@ type Database struct { } // NewDatabase creates a new device database -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { - var db *sql.DB - var err error - cs, err := sqlutil.ParseFileURI(dataSourceName) +func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName) (*Database, error) { + db, err := sqlutil.Open(dbProperties) if err != nil { return nil, err } - if db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { - return nil, err - } d := devicesStatements{} if err = d.prepare(db, serverName); err != nil { return nil, err diff --git a/userapi/storage/devices/storage.go b/userapi/storage/devices/storage.go index e094d202..1bd73a9f 100644 --- a/userapi/storage/devices/storage.go +++ b/userapi/storage/devices/storage.go @@ -17,9 +17,9 @@ package devices import ( - "net/url" + "fmt" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/userapi/storage/devices/postgres" "github.com/matrix-org/dendrite/userapi/storage/devices/sqlite3" "github.com/matrix-org/gomatrixserverlib" @@ -27,17 +27,13 @@ import ( // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets postgres connection parameters -func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgres.NewDatabase(dataSourceName, dbProperties, serverName) - } - switch uri.Scheme { - case "postgres": - return postgres.NewDatabase(dataSourceName, dbProperties, serverName) - case "file": - return sqlite3.NewDatabase(dataSourceName, serverName) +func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName) (Database, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(dbProperties, serverName) + case dbProperties.ConnectionString.IsPostgres(): + return postgres.NewDatabase(dbProperties, serverName) default: - return postgres.NewDatabase(dataSourceName, dbProperties, serverName) + return nil, fmt.Errorf("unexpected database type") } } diff --git a/userapi/storage/devices/storage_wasm.go b/userapi/storage/devices/storage_wasm.go index a5a515ef..e966c37f 100644 --- a/userapi/storage/devices/storage_wasm.go +++ b/userapi/storage/devices/storage_wasm.go @@ -16,28 +16,22 @@ package devices import ( "fmt" - "net/url" - "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/userapi/storage/devices/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) func NewDatabase( - dataSourceName string, - dbProperties sqlutil.DbProperties, // nolint:unparam + dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, ) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return nil, fmt.Errorf("Cannot use postgres implementation") - } - switch uri.Scheme { - case "postgres": - return nil, fmt.Errorf("Cannot use postgres implementation") - case "file": - return sqlite3.NewDatabase(dataSourceName, serverName) + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewDatabase(dbProperties, serverName) + case dbProperties.ConnectionString.IsPostgres(): + return nil, fmt.Errorf("can't use Postgres implementation") default: - return nil, fmt.Errorf("Cannot use postgres implementation") + return nil, fmt.Errorf("unexpected database type") } } diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index dab1ec71..548148f2 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/userapi" @@ -23,11 +24,15 @@ const ( ) func MustMakeInternalAPI(t *testing.T) (api.UserInternalAPI, accounts.Database, devices.Database) { - accountDB, err := accounts.NewDatabase("file::memory:", nil, serverName) + accountDB, err := accounts.NewDatabase(&config.DatabaseOptions{ + ConnectionString: "file::memory:", + }, serverName) if err != nil { t.Fatalf("failed to create account DB: %s", err) } - deviceDB, err := devices.NewDatabase("file::memory:", nil, serverName) + deviceDB, err := devices.NewDatabase(&config.DatabaseOptions{ + ConnectionString: "file::memory:", + }, serverName) if err != nil { t.Fatalf("failed to create device DB: %s", err) } |