diff options
| author | Neil Alexander <neilalexander@users.noreply.github.com> | 2020-08-10 14:18:04 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-08-10 14:18:04 +0100 |
| commit | 4b09f445c992fd0a389efc34d75aaa7e5bd50e9c (patch) | |
| tree | 18d6168718ac06e569eb271f25ed4dc064010b50 /internal | |
| parent | fdabba1851c489d801ea4029bce9dec7d415b2df (diff) | |
Configuration format v1 (#1230)
* Initial pass at refactoring config (not finished)
* Don't forget current state and EDU servers
* More shifting around
* Update server key API tests
* Fix roomserver test
* Fix more tests
* Further tweaks
* Fix current state server test (sort of)
* Maybe fix appservices
* Fix client API test
* Include database connection string in database options
* Fix sync API build
* Update config test
* Fix unit tests
* Fix federation sender build
* Fix gobind build
* Set Listen address for all services in HTTP monolith mode
* Validate config, reinstate appservice derived in directory, tweaks
* Tweak federation API test
* Set MaxOpenConnections/MaxIdleConnections to previous values
* Update generate-config
Diffstat (limited to 'internal')
24 files changed, 1030 insertions, 589 deletions
diff --git a/internal/config/config.go b/internal/config/config.go index 900d3b14..cf9168f7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -24,7 +24,6 @@ import ( "path/filepath" "regexp" "strings" - "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/gomatrixserverlib" @@ -38,7 +37,7 @@ import ( // Version is the current version of the config format. // This will change whenever we make breaking changes to the config format. -const Version = 0 +const Version = 1 // Dendrite contains all the config used by a dendrite process. // Relative paths are resolved relative to the current working directory @@ -51,217 +50,19 @@ type Dendrite struct { // been a breaking change to the config file format. Version int `yaml:"version"` - // The configuration required for a matrix server. - Matrix struct { - // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. - ServerName gomatrixserverlib.ServerName `yaml:"server_name"` - // Path to the private key which will be used to sign requests and events. - PrivateKeyPath Path `yaml:"private_key"` - // The private key which will be used to sign requests and events. - PrivateKey ed25519.PrivateKey `yaml:"-"` - // An arbitrary string used to uniquely identify the PrivateKey. Must start with the - // prefix "ed25519:". - KeyID gomatrixserverlib.KeyID `yaml:"-"` - // List of paths to X509 certificates used by the external federation listeners. - // These are used to calculate the TLS fingerprints to publish for this server. - // Other matrix servers talking to this server will expect the x509 certificate - // to match one of these certificates. - // The certificates should be in PEM format. - FederationCertificatePaths []Path `yaml:"federation_certificates"` - // A list of SHA256 TLS fingerprints for the X509 certificates used by the - // federation listener for this server. - TLSFingerPrints []gomatrixserverlib.TLSFingerprint `yaml:"-"` - // How long a remote server can cache our server key for before requesting it again. - // Increasing this number will reduce the number of requests made by remote servers - // for our key, but increases the period a compromised key will be considered valid - // by remote servers. - // Defaults to 24 hours. - KeyValidityPeriod time.Duration `yaml:"key_validity_period"` - // List of domains that the server will trust as identity servers to - // verify third-party identifiers. - // Defaults to an empty array. - TrustedIDServers []string `yaml:"trusted_third_party_id_servers"` - // If set, allows registration by anyone who also has the shared - // secret, even if registration is otherwise disabled. - RegistrationSharedSecret string `yaml:"registration_shared_secret"` - // This Home Server's ReCAPTCHA public key. - RecaptchaPublicKey string `yaml:"recaptcha_public_key"` - // This Home Server's ReCAPTCHA private key. - RecaptchaPrivateKey string `yaml:"recaptcha_private_key"` - // Boolean stating whether catpcha registration is enabled - // and required - RecaptchaEnabled bool `yaml:"enable_registration_captcha"` - // Secret used to bypass the captcha registration entirely - RecaptchaBypassSecret string `yaml:"captcha_bypass_secret"` - // HTTP API endpoint used to verify whether the captcha response - // was successful - RecaptchaSiteVerifyAPI string `yaml:"recaptcha_siteverify_api"` - // If set disables new users from registering (except via shared - // secrets) - RegistrationDisabled bool `yaml:"registration_disabled"` - // Perspective keyservers, to use as a backup when direct key fetch - // requests don't succeed - KeyPerspectives KeyPerspectives `yaml:"key_perspectives"` - // Federation failure threshold. How many consecutive failures that we should - // tolerate when sending federation requests to a specific server. The backoff - // is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. - // The default value is 16 if not specified, which is circa 18 hours. - FederationMaxRetries uint32 `yaml:"federation_max_retries"` - // FederationDisableTLSValidation disables the validation of X.509 TLS certs - // on remote federation endpoints. This is not recommended in production! - FederationDisableTLSValidation bool `yaml:"federation_disable_tls_validation"` - } `yaml:"matrix"` - - // The configuration specific to the media repostitory. - Media struct { - // The base path to where the media files will be stored. May be relative or absolute. - BasePath Path `yaml:"base_path"` - // The absolute base path to where media files will be stored. - AbsBasePath Path `yaml:"-"` - // The maximum file size in bytes that is allowed to be stored on this server. - // Note: if max_file_size_bytes is set to 0, the size is unlimited. - // Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) - MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"` - // Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated - DynamicThumbnails bool `yaml:"dynamic_thumbnails"` - // The maximum number of simultaneous thumbnail generators. default: 10 - MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"` - // A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content - ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"` - } `yaml:"media"` - - // The configuration to use for Prometheus metrics - Metrics struct { - // Whether or not the metrics are enabled - Enabled bool `yaml:"enabled"` - // Use BasicAuth for Authorization - BasicAuth struct { - // Authorization via Static Username & Password - // Hardcoded Username and Password - Username string `yaml:"username"` - Password string `yaml:"password"` - } `yaml:"basic_auth"` - } `yaml:"metrics"` - - // The configuration for talking to kafka. - Kafka struct { - // A list of kafka addresses to connect to. - Addresses []string `yaml:"addresses"` - // Whether to use naffka instead of kafka. - // Naffka can only be used when running dendrite as a single monolithic server. - // Kafka can be used both with a monolithic server and when running the - // components as separate servers. - UseNaffka bool `yaml:"use_naffka,omitempty"` - // The names of the topics to use when reading and writing from kafka. - Topics struct { - // Topic for roomserver/api.OutputRoomEvent events. - OutputRoomEvent Topic `yaml:"output_room_event"` - // Topic for sending account data from client API to sync API - OutputClientData Topic `yaml:"output_client_data"` - // Topic for eduserver/api.OutputTypingEvent events. - OutputTypingEvent Topic `yaml:"output_typing_event"` - // Topic for eduserver/api.OutputSendToDeviceEvent events. - OutputSendToDeviceEvent Topic `yaml:"output_send_to_device_event"` - // Topic for keyserver when new device keys are added. - OutputKeyChangeEvent Topic `yaml:"output_key_change_event"` - } - } `yaml:"kafka"` - - // Postgres Config - Database struct { - // The Account database stores the login details and account information - // for local users. It is accessed by the UserAPI. - Account DataSource `yaml:"account"` - // The CurrentState database stores the current state of all rooms. - // It is accessed by the CurrentStateServer. - CurrentState DataSource `yaml:"current_state"` - // The Device database stores session information for the devices of logged - // in local users. It is accessed by the UserAPI. - Device DataSource `yaml:"device"` - // The MediaAPI database stores information about files uploaded and downloaded - // by local users. It is only accessed by the MediaAPI. - MediaAPI DataSource `yaml:"media_api"` - // The ServerKey database caches the public keys of remote servers. - // It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI. - ServerKey DataSource `yaml:"server_key"` - // The E2EKey database stores one-time public keys for devices in addition to - // signed device keys. Used for E2E. - E2EKey DataSource `yaml:"e2e_key"` - // The SyncAPI stores information used by the SyncAPI server. - // It is only accessed by the SyncAPI server. - SyncAPI DataSource `yaml:"sync_api"` - // The RoomServer database stores information about matrix rooms. - // It is only accessed by the RoomServer. - RoomServer DataSource `yaml:"room_server"` - // The FederationSender database stores information used by the FederationSender - // It is only accessed by the FederationSender. - FederationSender DataSource `yaml:"federation_sender"` - // The AppServices database stores information used by the AppService component. - // It is only accessed by the AppService component. - AppService DataSource `yaml:"appservice"` - // The Naffka database is used internally by the naffka library, if used. - Naffka DataSource `yaml:"naffka,omitempty"` - // Maximum open connections to the DB (0 = use default, negative means unlimited) - MaxOpenConns int `yaml:"max_open_conns"` - // Maximum idle connections to the DB (0 = use default, negative means unlimited) - MaxIdleConns int `yaml:"max_idle_conns"` - // maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited) - ConnMaxLifetimeSec int `yaml:"conn_max_lifetime"` - } `yaml:"database"` - - // TURN Server Config - TURN struct { - // TODO Guest Support - // Whether or not guests can request TURN credentials - //AllowGuests bool `yaml:"turn_allow_guests"` - // How long the authorization should last - UserLifetime string `yaml:"turn_user_lifetime"` - // The list of TURN URIs to pass to clients - URIs []string `yaml:"turn_uris"` - - // Authorization via Shared Secret - // The shared secret from coturn - SharedSecret string `yaml:"turn_shared_secret"` - - // Authorization via Static Username & Password - // Hardcoded Username and Password - Username string `yaml:"turn_username"` - Password string `yaml:"turn_password"` - } `yaml:"turn"` - - // The internal addresses the components will listen on. - // These should not be exposed externally as they expose metrics and debugging APIs. - // Falls back to addresses listed in Listen if not specified - Bind struct { - MediaAPI Address `yaml:"media_api"` - ClientAPI Address `yaml:"client_api"` - CurrentState Address `yaml:"current_state_server"` - FederationAPI Address `yaml:"federation_api"` - ServerKeyAPI Address `yaml:"server_key_api"` - AppServiceAPI Address `yaml:"appservice_api"` - SyncAPI Address `yaml:"sync_api"` - UserAPI Address `yaml:"user_api"` - RoomServer Address `yaml:"room_server"` - FederationSender Address `yaml:"federation_sender"` - EDUServer Address `yaml:"edu_server"` - KeyServer Address `yaml:"key_server"` - } `yaml:"bind"` - - // The addresses for talking to other microservices. - Listen struct { - MediaAPI Address `yaml:"media_api"` - ClientAPI Address `yaml:"client_api"` - CurrentState Address `yaml:"current_state_server"` - FederationAPI Address `yaml:"federation_api"` - ServerKeyAPI Address `yaml:"server_key_api"` - AppServiceAPI Address `yaml:"appservice_api"` - SyncAPI Address `yaml:"sync_api"` - UserAPI Address `yaml:"user_api"` - RoomServer Address `yaml:"room_server"` - FederationSender Address `yaml:"federation_sender"` - EDUServer Address `yaml:"edu_server"` - KeyServer Address `yaml:"key_server"` - } `yaml:"listen"` + Global Global `yaml:"global"` + AppServiceAPI AppServiceAPI `yaml:"app_service_api"` + ClientAPI ClientAPI `yaml:"client_api"` + CurrentStateServer CurrentStateServer `yaml:"current_state_server"` + EDUServer EDUServer `yaml:"edu_server"` + FederationAPI FederationAPI `yaml:"federation_api"` + FederationSender FederationSender `yaml:"federation_sender"` + KeyServer KeyServer `yaml:"key_server"` + MediaAPI MediaAPI `yaml:"media_api"` + RoomServer RoomServer `yaml:"room_server"` + ServerKeyAPI ServerKeyAPI `yaml:"server_key_api"` + SyncAPI SyncAPI `yaml:"sync_api"` + UserAPI UserAPI `yaml:"user_api"` // The config for tracing the dendrite servers. Tracing struct { @@ -271,56 +72,42 @@ type Dendrite struct { Jaeger jaegerconfig.Configuration `yaml:"jaeger"` } `yaml:"tracing"` - // Application Services - // https://matrix.org/docs/spec/application_service/unstable.html - ApplicationServices struct { - // Configuration files for various application services - ConfigFiles []string `yaml:"config_files"` - } `yaml:"application_services"` - // The config for logging informations. Each hook will be added to logrus. Logging []LogrusHook `yaml:"logging"` - // The config for setting a proxy to use for server->server requests - Proxy *struct { - // The protocol for the proxy (http / https / socks5) - Protocol string `yaml:"protocol"` - // The host where the proxy is listening - Host string `yaml:"host"` - // The port on which the proxy is listening - Port uint16 `yaml:"port"` - } `yaml:"proxy"` - // Any information derived from the configuration options for later use. - Derived struct { - Registration struct { - // Flows is a slice of flows, which represent one possible way that the client can authenticate a request. - // http://matrix.org/docs/spec/HEAD/client_server/r0.3.0.html#user-interactive-authentication-api - // As long as the generated flows only rely on config file options, - // we can generate them on startup and store them until needed - Flows []authtypes.Flow `json:"flows"` - - // Params that need to be returned to the client during - // registration in order to complete registration stages. - Params map[string]interface{} `json:"params"` - } + Derived Derived `yaml:"-"` +} - // Application services parsed from their config files - // The paths of which were given above in the main config file - ApplicationServices []ApplicationService +// TODO: Kill Derived +type Derived struct { + Registration struct { + // Flows is a slice of flows, which represent one possible way that the client can authenticate a request. + // http://matrix.org/docs/spec/HEAD/client_server/r0.3.0.html#user-interactive-authentication-api + // As long as the generated flows only rely on config file options, + // we can generate them on startup and store them until needed + Flows []authtypes.Flow `json:"flows"` + + // Params that need to be returned to the client during + // registration in order to complete registration stages. + Params map[string]interface{} `json:"params"` + } - // Meta-regexes compiled from all exclusive application service - // Regexes. - // - // When a user registers, we check that their username does not match any - // exclusive application service namespaces - ExclusiveApplicationServicesUsernameRegexp *regexp.Regexp - // When a user creates a room alias, we check that it isn't already - // reserved by an application service - ExclusiveApplicationServicesAliasRegexp *regexp.Regexp - // Note: An Exclusive Regex for room ID isn't necessary as we aren't blocking - // servers from creating RoomIDs in exclusive application service namespaces - } `yaml:"-"` + // Application services parsed from their config files + // The paths of which were given above in the main config file + ApplicationServices []ApplicationService + + // Meta-regexes compiled from all exclusive application service + // Regexes. + // + // When a user registers, we check that their username does not match any + // exclusive application service namespaces + ExclusiveApplicationServicesUsernameRegexp *regexp.Regexp + // When a user creates a room alias, we check that it isn't already + // reserved by an application service + ExclusiveApplicationServicesAliasRegexp *regexp.Regexp + // Note: An Exclusive Regex for room ID isn't necessary as we aren't blocking + // servers from creating RoomIDs in exclusive application service namespaces } // KeyPerspectives are used to configure perspective key servers for @@ -344,6 +131,16 @@ type Path string // A DataSource for opening a postgresql database using lib/pq. type DataSource string +func (d DataSource) IsSQLite() bool { + return strings.HasPrefix(string(d), "file:") +} + +func (d DataSource) IsPostgres() bool { + // commented line may not always be true? + // return strings.HasPrefix(string(d), "postgres:") + return !d.IsSQLite() +} + // A Topic in kafka. type Topic string @@ -379,9 +176,9 @@ type LogrusHook struct { Params map[string]interface{} `yaml:"params"` } -// configErrors stores problems encountered when parsing a config file. +// ConfigErrors stores problems encountered when parsing a config file. // It implements the error interface. -type configErrors []string +type ConfigErrors []string // Load a yaml config file for a server run as multiple processes or as a monolith. // Checks the config to ensure that it is valid. @@ -405,29 +202,29 @@ func loadConfig( readFile func(string) ([]byte, error), monolithic bool, ) (*Dendrite, error) { - var config Dendrite + var c Dendrite + c.Defaults() + var err error - if err = yaml.Unmarshal(configData, &config); err != nil { + if err = yaml.Unmarshal(configData, &c); err != nil { return nil, err } - config.SetDefaults() - - if err = config.check(monolithic); err != nil { + if err = c.check(monolithic); err != nil { return nil, err } - privateKeyPath := absPath(basePath, config.Matrix.PrivateKeyPath) + privateKeyPath := absPath(basePath, c.Global.PrivateKeyPath) privateKeyData, err := readFile(privateKeyPath) if err != nil { return nil, err } - if config.Matrix.KeyID, config.Matrix.PrivateKey, err = readKeyPEM(privateKeyPath, privateKeyData); err != nil { + if c.Global.KeyID, c.Global.PrivateKey, err = readKeyPEM(privateKeyPath, privateKeyData); err != nil { return nil, err } - for _, certPath := range config.Matrix.FederationCertificatePaths { + for _, certPath := range c.FederationAPI.FederationCertificatePaths { absCertPath := absPath(basePath, certPath) var pemData []byte pemData, err = readFile(absCertPath) @@ -438,18 +235,19 @@ func loadConfig( if fingerprint == nil { return nil, fmt.Errorf("no certificate PEM data in %q", absCertPath) } - config.Matrix.TLSFingerPrints = append(config.Matrix.TLSFingerPrints, *fingerprint) + c.FederationAPI.TLSFingerPrints = append(c.FederationAPI.TLSFingerPrints, *fingerprint) } - config.Media.AbsBasePath = Path(absPath(basePath, config.Media.BasePath)) + c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath)) // Generate data from config options - err = config.Derive() + err = c.Derive() if err != nil { return nil, err } - return &config, nil + c.Wiring() + return &c, nil } // Derive generates data that is derived from various values provided in @@ -462,8 +260,8 @@ func (config *Dendrite) Derive() error { // TODO: Add email auth type // TODO: Add MSISDN auth type - if config.Matrix.RecaptchaEnabled { - config.Derived.Registration.Params[authtypes.LoginTypeRecaptcha] = map[string]string{"public_key": config.Matrix.RecaptchaPublicKey} + if config.ClientAPI.RecaptchaEnabled { + config.Derived.Registration.Params[authtypes.LoginTypeRecaptcha] = map[string]string{"public_key": config.ClientAPI.RecaptchaPublicKey} config.Derived.Registration.Flows = append(config.Derived.Registration.Flows, authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeRecaptcha}}) } else { @@ -472,7 +270,7 @@ func (config *Dendrite) Derive() error { } // Load application service configuration files - if err := loadAppServices(config); err != nil { + if err := loadAppServices(&config.AppServiceAPI, &config.Derived); err != nil { return err } @@ -480,41 +278,62 @@ func (config *Dendrite) Derive() error { } // SetDefaults sets default config values if they are not explicitly set. -func (config *Dendrite) SetDefaults() { - if config.Matrix.KeyValidityPeriod == 0 { - config.Matrix.KeyValidityPeriod = 24 * time.Hour - } - - if config.Matrix.TrustedIDServers == nil { - config.Matrix.TrustedIDServers = []string{} - } - - if config.Matrix.FederationMaxRetries == 0 { - config.Matrix.FederationMaxRetries = 16 - } - - if config.Media.MaxThumbnailGenerators == 0 { - config.Media.MaxThumbnailGenerators = 10 - } - - if config.Media.MaxFileSizeBytes == nil { - defaultMaxFileSizeBytes := FileSizeBytes(10485760) - config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes - } - - if config.Database.MaxIdleConns == 0 { - config.Database.MaxIdleConns = 2 - } - - if config.Database.MaxOpenConns == 0 { - config.Database.MaxOpenConns = 100 - } - +func (c *Dendrite) Defaults() { + c.Version = 1 + + c.Global.Defaults() + c.ClientAPI.Defaults() + c.CurrentStateServer.Defaults() + c.EDUServer.Defaults() + c.FederationAPI.Defaults() + c.FederationSender.Defaults() + c.KeyServer.Defaults() + c.MediaAPI.Defaults() + c.RoomServer.Defaults() + c.ServerKeyAPI.Defaults() + c.SyncAPI.Defaults() + c.UserAPI.Defaults() + c.AppServiceAPI.Defaults() + + c.Wiring() +} + +func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) { + type verifiable interface { + Verify(configErrs *ConfigErrors, isMonolith bool) + } + for _, c := range []verifiable{ + &c.Global, &c.ClientAPI, &c.CurrentStateServer, + &c.EDUServer, &c.FederationAPI, &c.FederationSender, + &c.KeyServer, &c.MediaAPI, &c.RoomServer, + &c.ServerKeyAPI, &c.SyncAPI, &c.UserAPI, + &c.AppServiceAPI, + } { + c.Verify(configErrs, isMonolith) + } +} + +func (c *Dendrite) Wiring() { + c.ClientAPI.Matrix = &c.Global + c.CurrentStateServer.Matrix = &c.Global + c.EDUServer.Matrix = &c.Global + c.FederationAPI.Matrix = &c.Global + c.FederationSender.Matrix = &c.Global + c.KeyServer.Matrix = &c.Global + c.MediaAPI.Matrix = &c.Global + c.RoomServer.Matrix = &c.Global + c.ServerKeyAPI.Matrix = &c.Global + c.SyncAPI.Matrix = &c.Global + c.UserAPI.Matrix = &c.Global + c.AppServiceAPI.Matrix = &c.Global + + c.ClientAPI.Derived = &c.Derived + c.AppServiceAPI.Derived = &c.Derived } // Error returns a string detailing how many errors were contained within a // configErrors type. -func (errs configErrors) Error() string { +func (errs ConfigErrors) Error() string { if len(errs) == 1 { return errs[0] } @@ -528,13 +347,13 @@ func (errs configErrors) Error() string { // the client code. // This method is safe to use with an uninitialized configErrors because // if it is nil, it will be properly allocated. -func (errs *configErrors) Add(str string) { +func (errs *ConfigErrors) Add(str string) { *errs = append(*errs, str) } // checkNotEmpty verifies the given value is not empty in the configuration. // If it is, adds an error to the list. -func checkNotEmpty(configErrs *configErrors, key, value string) { +func checkNotEmpty(configErrs *ConfigErrors, key, value string) { if value == "" { configErrs.Add(fmt.Sprintf("missing config key %q", key)) } @@ -542,7 +361,7 @@ func checkNotEmpty(configErrs *configErrors, key, value string) { // checkNotZero verifies the given value is not zero in the configuration. // If it is, adds an error to the list. -func checkNotZero(configErrs *configErrors, key string, value int64) { +func checkNotZero(configErrs *ConfigErrors, key string, value int64) { if value == 0 { configErrs.Add(fmt.Sprintf("missing config key %q", key)) } @@ -550,96 +369,14 @@ func checkNotZero(configErrs *configErrors, key string, value int64) { // checkPositive verifies the given value is positive (zero included) // in the configuration. If it is not, adds an error to the list. -func checkPositive(configErrs *configErrors, key string, value int64) { +func checkPositive(configErrs *ConfigErrors, key string, value int64) { if value < 0 { configErrs.Add(fmt.Sprintf("invalid value for config key %q: %d", key, value)) } } -// checkTurn verifies the parameters turn.* are valid. -func (config *Dendrite) checkTurn(configErrs *configErrors) { - value := config.TURN.UserLifetime - if value != "" { - if _, err := time.ParseDuration(value); err != nil { - configErrs.Add(fmt.Sprintf("invalid duration for config key %q: %s", "turn.turn_user_lifetime", value)) - } - } -} - -// checkMatrix verifies the parameters matrix.* are valid. -func (config *Dendrite) checkMatrix(configErrs *configErrors) { - checkNotEmpty(configErrs, "matrix.server_name", string(config.Matrix.ServerName)) - checkNotEmpty(configErrs, "matrix.private_key", string(config.Matrix.PrivateKeyPath)) - checkNotZero(configErrs, "matrix.federation_certificates", int64(len(config.Matrix.FederationCertificatePaths))) - if config.Matrix.RecaptchaEnabled { - checkNotEmpty(configErrs, "matrix.recaptcha_public_key", string(config.Matrix.RecaptchaPublicKey)) - checkNotEmpty(configErrs, "matrix.recaptcha_private_key", string(config.Matrix.RecaptchaPrivateKey)) - checkNotEmpty(configErrs, "matrix.recaptcha_siteverify_api", string(config.Matrix.RecaptchaSiteVerifyAPI)) - } -} - -// checkMedia verifies the parameters media.* are valid. -func (config *Dendrite) checkMedia(configErrs *configErrors) { - checkNotEmpty(configErrs, "media.base_path", string(config.Media.BasePath)) - checkPositive(configErrs, "media.max_file_size_bytes", int64(*config.Media.MaxFileSizeBytes)) - checkPositive(configErrs, "media.max_thumbnail_generators", int64(config.Media.MaxThumbnailGenerators)) - - for i, size := range config.Media.ThumbnailSizes { - checkPositive(configErrs, fmt.Sprintf("media.thumbnail_sizes[%d].width", i), int64(size.Width)) - checkPositive(configErrs, fmt.Sprintf("media.thumbnail_sizes[%d].height", i), int64(size.Height)) - } -} - -// checkKafka verifies the parameters kafka.* and the related -// database.naffka are valid. -func (config *Dendrite) checkKafka(configErrs *configErrors, monolithic bool) { - - if config.Kafka.UseNaffka { - if !monolithic { - configErrs.Add(fmt.Sprintf("naffka can only be used in a monolithic server")) - } - - checkNotEmpty(configErrs, "database.naffka", string(config.Database.Naffka)) - } else { - // If we aren't using naffka then we need to have at least one kafka - // server to talk to. - checkNotZero(configErrs, "kafka.addresses", int64(len(config.Kafka.Addresses))) - } - checkNotEmpty(configErrs, "kafka.topics.output_room_event", string(config.Kafka.Topics.OutputRoomEvent)) - checkNotEmpty(configErrs, "kafka.topics.output_client_data", string(config.Kafka.Topics.OutputClientData)) - checkNotEmpty(configErrs, "kafka.topics.output_typing_event", string(config.Kafka.Topics.OutputTypingEvent)) - checkNotEmpty(configErrs, "kafka.topics.output_send_to_device_event", string(config.Kafka.Topics.OutputSendToDeviceEvent)) - checkNotEmpty(configErrs, "kafka.topics.output_key_change_event", string(config.Kafka.Topics.OutputKeyChangeEvent)) -} - -// checkDatabase verifies the parameters database.* are valid. -func (config *Dendrite) checkDatabase(configErrs *configErrors) { - checkNotEmpty(configErrs, "database.account", string(config.Database.Account)) - checkNotEmpty(configErrs, "database.device", string(config.Database.Device)) - checkNotEmpty(configErrs, "database.server_key", string(config.Database.ServerKey)) - checkNotEmpty(configErrs, "database.media_api", string(config.Database.MediaAPI)) - checkNotEmpty(configErrs, "database.sync_api", string(config.Database.SyncAPI)) - checkNotEmpty(configErrs, "database.room_server", string(config.Database.RoomServer)) - checkNotEmpty(configErrs, "database.current_state", string(config.Database.CurrentState)) - checkNotEmpty(configErrs, "database.e2e_key", string(config.Database.E2EKey)) -} - -// checkListen verifies the parameters listen.* are valid. -func (config *Dendrite) checkListen(configErrs *configErrors) { - checkNotEmpty(configErrs, "listen.media_api", string(config.Listen.MediaAPI)) - checkNotEmpty(configErrs, "listen.client_api", string(config.Listen.ClientAPI)) - checkNotEmpty(configErrs, "listen.federation_api", string(config.Listen.FederationAPI)) - checkNotEmpty(configErrs, "listen.sync_api", string(config.Listen.SyncAPI)) - checkNotEmpty(configErrs, "listen.room_server", string(config.Listen.RoomServer)) - checkNotEmpty(configErrs, "listen.edu_server", string(config.Listen.EDUServer)) - checkNotEmpty(configErrs, "listen.server_key_api", string(config.Listen.EDUServer)) - checkNotEmpty(configErrs, "listen.user_api", string(config.Listen.UserAPI)) - checkNotEmpty(configErrs, "listen.current_state_server", string(config.Listen.CurrentState)) - checkNotEmpty(configErrs, "listen.key_server", string(config.Listen.KeyServer)) -} - // checkLogging verifies the parameters logging.* are valid. -func (config *Dendrite) checkLogging(configErrs *configErrors) { +func (config *Dendrite) checkLogging(configErrs *ConfigErrors) { for _, logrusHook := range config.Logging { checkNotEmpty(configErrs, "logging.type", string(logrusHook.Type)) checkNotEmpty(configErrs, "logging.level", string(logrusHook.Level)) @@ -648,8 +385,8 @@ func (config *Dendrite) checkLogging(configErrs *configErrors) { // check returns an error type containing all errors found within the config // file. -func (config *Dendrite) check(monolithic bool) error { - var configErrs configErrors +func (config *Dendrite) check(_ bool) error { // monolithic + var configErrs ConfigErrors if config.Version != Version { configErrs.Add(fmt.Sprintf( @@ -658,17 +395,8 @@ func (config *Dendrite) check(monolithic bool) error { return configErrs } - config.checkMatrix(&configErrs) - config.checkMedia(&configErrs) - config.checkTurn(&configErrs) - config.checkKafka(&configErrs, monolithic) - config.checkDatabase(&configErrs) config.checkLogging(&configErrs) - if !monolithic { - config.checkListen(&configErrs) - } - // Due to how Golang manages its interface types, this condition is not redundant. // In order to get the proper behaviour, it is necessary to return an explicit nil // and not a nil configErrors. @@ -737,7 +465,7 @@ func (config *Dendrite) AppServiceURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.AppServiceAPI) + return "http://" + string(config.AppServiceAPI.Listen) } // RoomServerURL returns an HTTP URL for where the roomserver is listening. @@ -746,7 +474,7 @@ func (config *Dendrite) RoomServerURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.RoomServer) + return "http://" + string(config.RoomServer.Listen) } // UserAPIURL returns an HTTP URL for where the userapi is listening. @@ -755,7 +483,7 @@ func (config *Dendrite) UserAPIURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.UserAPI) + return "http://" + string(config.UserAPI.Listen) } // CurrentStateAPIURL returns an HTTP URL for where the currentstateserver is listening. @@ -764,7 +492,7 @@ func (config *Dendrite) CurrentStateAPIURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.CurrentState) + return "http://" + string(config.CurrentStateServer.Listen) } // EDUServerURL returns an HTTP URL for where the EDU server is listening. @@ -773,7 +501,7 @@ func (config *Dendrite) EDUServerURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.EDUServer) + return "http://" + string(config.EDUServer.Listen) } // FederationSenderURL returns an HTTP URL for where the federation sender is listening. @@ -782,7 +510,7 @@ func (config *Dendrite) FederationSenderURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.FederationSender) + return "http://" + string(config.FederationSender.Listen) } // ServerKeyAPIURL returns an HTTP URL for where the server key API is listening. @@ -791,7 +519,7 @@ func (config *Dendrite) ServerKeyAPIURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.ServerKeyAPI) + return "http://" + string(config.ServerKeyAPI.Listen) } // KeyServerURL returns an HTTP URL for where the key server is listening. @@ -800,7 +528,7 @@ func (config *Dendrite) KeyServerURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.KeyServer) + return "http://" + string(config.KeyServer.Listen) } // SetupTracing configures the opentracing using the supplied configuration. @@ -815,33 +543,6 @@ func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err ) } -// MaxIdleConns returns maximum idle connections to the DB -func (config Dendrite) MaxIdleConns() int { - return config.Database.MaxIdleConns -} - -// MaxOpenConns returns maximum open connections to the DB -func (config Dendrite) MaxOpenConns() int { - return config.Database.MaxOpenConns -} - -// ConnMaxLifetime returns maximum amount of time a connection may be reused -func (config Dendrite) ConnMaxLifetime() time.Duration { - return time.Duration(config.Database.ConnMaxLifetimeSec) * time.Second -} - -// DbProperties functions return properties used by database/sql/DB -type DbProperties interface { - MaxIdleConns() int - MaxOpenConns() int - ConnMaxLifetime() time.Duration -} - -// DbProperties returns cfg as a DbProperties interface -func (config Dendrite) DbProperties() DbProperties { - return config -} - // logrusLogger is a small wrapper that implements jaeger.Logger using logrus. type logrusLogger struct { l *logrus.Logger diff --git a/internal/config/appservice.go b/internal/config/config_appservice.go index bf5f089b..b8962ded 100644 --- a/internal/config/appservice.go +++ b/internal/config/config_appservice.go @@ -25,6 +25,31 @@ import ( yaml "gopkg.in/yaml.v2" ) +type AppServiceAPI struct { + Matrix *Global `yaml:"-"` + Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + Database DatabaseOptions `yaml:"database"` + + ConfigFiles []string `yaml:"config_files"` +} + +func (c *AppServiceAPI) Defaults() { + c.Listen = "localhost:7777" + c.Bind = "localhost:7777" + c.Database.Defaults() + c.Database.ConnectionString = "file:appservice.db" +} + +func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "app_service_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "app_service_api.bind", string(c.Bind)) + checkNotEmpty(configErrs, "app_service_api.database.connection_string", string(c.Database.ConnectionString)) +} + // ApplicationServiceNamespace is the namespace that a specific application // service has management over. type ApplicationServiceNamespace struct { @@ -132,8 +157,8 @@ func (a *ApplicationService) IsInterestedInRoomAlias( // loadAppServices iterates through all application service config files // and loads their data into the config object for later access. -func loadAppServices(config *Dendrite) error { - for _, configPath := range config.ApplicationServices.ConfigFiles { +func loadAppServices(config *AppServiceAPI, derived *Derived) error { + for _, configPath := range config.ConfigFiles { // Create a new application service with default options appservice := ApplicationService{ RateLimited: true, @@ -157,26 +182,26 @@ func loadAppServices(config *Dendrite) error { } // Append the parsed application service to the global config - config.Derived.ApplicationServices = append( - config.Derived.ApplicationServices, appservice, + derived.ApplicationServices = append( + derived.ApplicationServices, appservice, ) } // Check for any errors in the loaded application services - return checkErrors(config) + return checkErrors(config, derived) } // setupRegexps will create regex objects for exclusive and non-exclusive // usernames, aliases and rooms of all application services, so that other // methods can quickly check if a particular string matches any of them. -func setupRegexps(cfg *Dendrite) (err error) { +func setupRegexps(_ *AppServiceAPI, derived *Derived) (err error) { // Combine all exclusive namespaces for later string checking var exclusiveUsernameStrings, exclusiveAliasStrings []string // If an application service's regex is marked as exclusive, add // its contents to the overall exlusive regex string. Room regex // not necessary as we aren't denying exclusive room ID creation - for _, appservice := range cfg.Derived.ApplicationServices { + for _, appservice := range derived.ApplicationServices { for key, namespaceSlice := range appservice.NamespaceMap { switch key { case "users": @@ -204,10 +229,10 @@ func setupRegexps(cfg *Dendrite) (err error) { } // Store compiled Regex - if cfg.Derived.ExclusiveApplicationServicesUsernameRegexp, err = regexp.Compile(exclusiveUsernames); err != nil { + if derived.ExclusiveApplicationServicesUsernameRegexp, err = regexp.Compile(exclusiveUsernames); err != nil { return err } - if cfg.Derived.ExclusiveApplicationServicesAliasRegexp, err = regexp.Compile(exclusiveAliases); err != nil { + if derived.ExclusiveApplicationServicesAliasRegexp, err = regexp.Compile(exclusiveAliases); err != nil { return err } @@ -234,7 +259,7 @@ func appendExclusiveNamespaceRegexs( // checkErrors checks for any configuration errors amongst the loaded // application services according to the application service spec. -func checkErrors(config *Dendrite) (err error) { +func checkErrors(config *AppServiceAPI, derived *Derived) (err error) { var idMap = make(map[string]bool) var tokenMap = make(map[string]bool) @@ -242,7 +267,7 @@ func checkErrors(config *Dendrite) (err error) { groupIDRegexp := regexp.MustCompile(`\+.*:.*`) // Check each application service for any config errors - for _, appservice := range config.Derived.ApplicationServices { + for _, appservice := range derived.ApplicationServices { // Namespace-related checks for key, namespaceSlice := range appservice.NamespaceMap { for _, namespace := range namespaceSlice { @@ -258,13 +283,13 @@ func checkErrors(config *Dendrite) (err error) { // Check if we've already seen this ID. No two application services // can have the same ID or token. if idMap[appservice.ID] { - return configErrors([]string{fmt.Sprintf( + return ConfigErrors([]string{fmt.Sprintf( "Application service ID %s must be unique", appservice.ID, )}) } // Check if we've already seen this token if tokenMap[appservice.ASToken] { - return configErrors([]string{fmt.Sprintf( + return ConfigErrors([]string{fmt.Sprintf( "Application service Token %s must be unique", appservice.ASToken, )}) } @@ -284,7 +309,7 @@ func checkErrors(config *Dendrite) (err error) { } } - return setupRegexps(config) + return setupRegexps(config, derived) } // validateNamespace returns nil or an error based on whether a given @@ -298,7 +323,7 @@ func validateNamespace( ) error { // Check that namespace(s) are valid regex if !IsValidRegex(namespace.Regex) { - return configErrors([]string{fmt.Sprintf( + return ConfigErrors([]string{fmt.Sprintf( "Invalid regex string for Application Service %s", appservice.ID, )}) } @@ -310,7 +335,7 @@ func validateNamespace( correctFormat := groupIDRegexp.MatchString(namespace.GroupID) if !correctFormat { - return configErrors([]string{fmt.Sprintf( + return ConfigErrors([]string{fmt.Sprintf( "Invalid user group_id field for application service %s.", appservice.ID, )}) diff --git a/internal/config/config_clientapi.go b/internal/config/config_clientapi.go new file mode 100644 index 00000000..c441a9c0 --- /dev/null +++ b/internal/config/config_clientapi.go @@ -0,0 +1,87 @@ +package config + +import ( + "fmt" + "time" +) + +type ClientAPI struct { + Matrix *Global `yaml:"-"` + Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // If set, allows registration by anyone who also has the shared + // secret, even if registration is otherwise disabled. + RegistrationSharedSecret string `yaml:"registration_shared_secret"` + // This Home Server's ReCAPTCHA public key. + RecaptchaPublicKey string `yaml:"recaptcha_public_key"` + // This Home Server's ReCAPTCHA private key. + RecaptchaPrivateKey string `yaml:"recaptcha_private_key"` + // Boolean stating whether catpcha registration is enabled + // and required + RecaptchaEnabled bool `yaml:"enable_registration_captcha"` + // Secret used to bypass the captcha registration entirely + RecaptchaBypassSecret string `yaml:"captcha_bypass_secret"` + // HTTP API endpoint used to verify whether the captcha response + // was successful + RecaptchaSiteVerifyAPI string `yaml:"recaptcha_siteverify_api"` + // If set disables new users from registering (except via shared + // secrets) + RegistrationDisabled bool `yaml:"registration_disabled"` + + // TURN options + TURN TURN `yaml:"turn"` +} + +func (c *ClientAPI) Defaults() { + c.Listen = "localhost:7771" + c.Bind = "localhost:7771" + c.RegistrationSharedSecret = "" + c.RecaptchaPublicKey = "" + c.RecaptchaPrivateKey = "" + c.RecaptchaEnabled = false + c.RecaptchaBypassSecret = "" + c.RecaptchaSiteVerifyAPI = "" + c.RegistrationDisabled = false +} + +func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "client_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "client_api.bind", string(c.Bind)) + if c.RecaptchaEnabled { + checkNotEmpty(configErrs, "client_api.recaptcha_public_key", string(c.RecaptchaPublicKey)) + checkNotEmpty(configErrs, "client_api.recaptcha_private_key", string(c.RecaptchaPrivateKey)) + checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", string(c.RecaptchaSiteVerifyAPI)) + } + c.TURN.Verify(configErrs) +} + +type TURN struct { + // TODO Guest Support + // Whether or not guests can request TURN credentials + // AllowGuests bool `yaml:"turn_allow_guests"` + // How long the authorization should last + UserLifetime string `yaml:"turn_user_lifetime"` + // The list of TURN URIs to pass to clients + URIs []string `yaml:"turn_uris"` + + // Authorization via Shared Secret + // The shared secret from coturn + SharedSecret string `yaml:"turn_shared_secret"` + + // Authorization via Static Username & Password + // Hardcoded Username and Password + Username string `yaml:"turn_username"` + Password string `yaml:"turn_password"` +} + +func (c *TURN) Verify(configErrs *ConfigErrors) { + value := c.UserLifetime + if value != "" { + if _, err := time.ParseDuration(value); err != nil { + configErrs.Add(fmt.Sprintf("invalid duration for config key %q: %s", "client_api.turn.turn_user_lifetime", value)) + } + } +} diff --git a/internal/config/config_currentstate.go b/internal/config/config_currentstate.go new file mode 100644 index 00000000..2687f7f5 --- /dev/null +++ b/internal/config/config_currentstate.go @@ -0,0 +1,25 @@ +package config + +type CurrentStateServer struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // The CurrentState database stores the current state of all rooms. + // It is accessed by the CurrentStateServer. + Database DatabaseOptions `yaml:"database"` +} + +func (c *CurrentStateServer) Defaults() { + c.Listen = "localhost:7782" + c.Bind = "localhost:7782" + c.Database.Defaults() + c.Database.ConnectionString = "file:currentstate.db" +} + +func (c *CurrentStateServer) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "current_state_server.listen", string(c.Listen)) + checkNotEmpty(configErrs, "current_state_server.bind", string(c.Bind)) + checkNotEmpty(configErrs, "current_state_server.database.connection_string", string(c.Database.ConnectionString)) +} diff --git a/internal/config/config_eduserver.go b/internal/config/config_eduserver.go new file mode 100644 index 00000000..02743041 --- /dev/null +++ b/internal/config/config_eduserver.go @@ -0,0 +1,18 @@ +package config + +type EDUServer struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` +} + +func (c *EDUServer) Defaults() { + c.Listen = "localhost:7778" + c.Bind = "localhost:7778" +} + +func (c *EDUServer) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "edu_server.listen", string(c.Listen)) + checkNotEmpty(configErrs, "edu_server.bind", string(c.Bind)) +} diff --git a/internal/config/config_federationapi.go b/internal/config/config_federationapi.go new file mode 100644 index 00000000..d155ef25 --- /dev/null +++ b/internal/config/config_federationapi.go @@ -0,0 +1,33 @@ +package config + +import "github.com/matrix-org/gomatrixserverlib" + +type FederationAPI struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // List of paths to X509 certificates used by the external federation listeners. + // These are used to calculate the TLS fingerprints to publish for this server. + // Other matrix servers talking to this server will expect the x509 certificate + // to match one of these certificates. + // The certificates should be in PEM format. + FederationCertificatePaths []Path `yaml:"federation_certificates"` + + // A list of SHA256 TLS fingerprints for the X509 certificates used by the + // federation listener for this server. + TLSFingerPrints []gomatrixserverlib.TLSFingerprint `yaml:"-"` +} + +func (c *FederationAPI) Defaults() { + c.Listen = "localhost:7772" + c.Bind = "localhost:7772" +} + +func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "federation_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "federation_api.bind", string(c.Bind)) + // TODO: not applicable always, e.g. in demos + //checkNotZero(configErrs, "federation_api.federation_certificates", int64(len(c.FederationCertificatePaths))) +} diff --git a/internal/config/config_federationsender.go b/internal/config/config_federationsender.go new file mode 100644 index 00000000..09d8287b --- /dev/null +++ b/internal/config/config_federationsender.go @@ -0,0 +1,64 @@ +package config + +type FederationSender struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // The FederationSender database stores information used by the FederationSender + // It is only accessed by the FederationSender. + Database DatabaseOptions `yaml:"database"` + + // Federation failure threshold. How many consecutive failures that we should + // tolerate when sending federation requests to a specific server. The backoff + // is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. + // The default value is 16 if not specified, which is circa 18 hours. + FederationMaxRetries uint32 `yaml:"send_max_retries"` + + // FederationDisableTLSValidation disables the validation of X.509 TLS certs + // on remote federation endpoints. This is not recommended in production! + DisableTLSValidation bool `yaml:"disable_tls_validation"` + + Proxy Proxy `yaml:"proxy_outbound"` +} + +func (c *FederationSender) Defaults() { + c.Listen = "localhost:7775" + c.Bind = "localhost:7775" + c.Database.Defaults() + c.Database.ConnectionString = "file:federationsender.db" + + c.FederationMaxRetries = 16 + c.DisableTLSValidation = false + + c.Proxy.Defaults() +} + +func (c *FederationSender) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "federation_sender.listen", string(c.Listen)) + checkNotEmpty(configErrs, "federation_sender.bind", string(c.Bind)) + checkNotEmpty(configErrs, "federation_sender.database.connection_string", string(c.Database.ConnectionString)) +} + +// The config for setting a proxy to use for server->server requests +type Proxy struct { + // Is the proxy enabled? + Enabled bool `yaml:"enabled"` + // The protocol for the proxy (http / https / socks5) + Protocol string `yaml:"protocol"` + // The host where the proxy is listening + Host string `yaml:"host"` + // The port on which the proxy is listening + Port uint16 `yaml:"port"` +} + +func (c *Proxy) Defaults() { + c.Enabled = false + c.Protocol = "http" + c.Host = "localhost" + c.Port = 8080 +} + +func (c *Proxy) Verify(configErrs *ConfigErrors) { +} diff --git a/internal/config/config_global.go b/internal/config/config_global.go new file mode 100644 index 00000000..9456dd3f --- /dev/null +++ b/internal/config/config_global.go @@ -0,0 +1,172 @@ +package config + +import ( + "math/rand" + "time" + + "github.com/matrix-org/gomatrixserverlib" + "golang.org/x/crypto/ed25519" +) + +type Global struct { + // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. + ServerName gomatrixserverlib.ServerName `yaml:"server_name"` + + // Path to the private key which will be used to sign requests and events. + PrivateKeyPath Path `yaml:"private_key"` + + // The private key which will be used to sign requests and events. + PrivateKey ed25519.PrivateKey `yaml:"-"` + + // An arbitrary string used to uniquely identify the PrivateKey. Must start with the + // prefix "ed25519:". + KeyID gomatrixserverlib.KeyID `yaml:"-"` + + // How long a remote server can cache our server key for before requesting it again. + // Increasing this number will reduce the number of requests made by remote servers + // for our key, but increases the period a compromised key will be considered valid + // by remote servers. + // Defaults to 24 hours. + KeyValidityPeriod time.Duration `yaml:"key_validity_period"` + + // List of domains that the server will trust as identity servers to + // verify third-party identifiers. + // Defaults to an empty array. + TrustedIDServers []string `yaml:"trusted_third_party_id_servers"` + + // Kafka/Naffka configuration + Kafka Kafka `yaml:"kafka"` + + // Metrics configuration + Metrics Metrics `yaml:"metrics"` +} + +func (c *Global) Defaults() { + c.ServerName = "localhost" + c.PrivateKeyPath = "matrix.pem" + _, c.PrivateKey, _ = ed25519.GenerateKey(rand.New(rand.NewSource(0))) + c.KeyID = "ed25519:auto" + c.KeyValidityPeriod = time.Hour * 24 * 7 + + c.Kafka.Defaults() + c.Metrics.Defaults() +} + +func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "global.server_name", string(c.ServerName)) + checkNotEmpty(configErrs, "global.private_key", string(c.PrivateKeyPath)) + + c.Kafka.Verify(configErrs, isMonolith) + c.Metrics.Verify(configErrs, isMonolith) +} + +type Kafka struct { + // A list of kafka addresses to connect to. + Addresses []string `yaml:"addresses"` + // Whether to use naffka instead of kafka. + // Naffka can only be used when running dendrite as a single monolithic server. + // Kafka can be used both with a monolithic server and when running the + // components as separate servers. + UseNaffka bool `yaml:"use_naffka"` + // The Naffka database is used internally by the naffka library, if used. + Database DatabaseOptions `yaml:"naffka_database"` + // The names of the topics to use when reading and writing from kafka. + Topics struct { + // Topic for roomserver/api.OutputRoomEvent events. + OutputRoomEvent Topic `yaml:"output_room_event"` + // Topic for sending account data from client API to sync API + OutputClientData Topic `yaml:"output_client_data"` + // Topic for eduserver/api.OutputTypingEvent events. + OutputTypingEvent Topic `yaml:"output_typing_event"` + // Topic for eduserver/api.OutputSendToDeviceEvent events. + OutputSendToDeviceEvent Topic `yaml:"output_send_to_device_event"` + // Topic for keyserver when new device keys are added. + OutputKeyChangeEvent Topic `yaml:"output_key_change_event"` + } +} + +func (c *Kafka) Defaults() { + c.UseNaffka = true + c.Database.Defaults() + c.Database.ConnectionString = DataSource("file:naffka.db") + c.Topics.OutputRoomEvent = "OutputRoomEventTopic" + c.Topics.OutputClientData = "OutputClientDataTopic" + c.Topics.OutputTypingEvent = "OutputTypingEventTopic" + c.Topics.OutputSendToDeviceEvent = "OutputSendToDeviceEventTopic" + c.Topics.OutputKeyChangeEvent = "OutputKeyChangeEventTopic" +} + +func (c *Kafka) Verify(configErrs *ConfigErrors, isMonolith bool) { + if c.UseNaffka { + if !isMonolith { + configErrs.Add("naffka can only be used in a monolithic server") + } + checkNotEmpty(configErrs, "global.kafka.database.connection_string", string(c.Database.ConnectionString)) + } else { + // If we aren't using naffka then we need to have at least one kafka + // server to talk to. + checkNotZero(configErrs, "global.kafka.addresses", int64(len(c.Addresses))) + } + checkNotEmpty(configErrs, "global.kafka.topics.output_room_event", string(c.Topics.OutputRoomEvent)) + checkNotEmpty(configErrs, "global.kafka.topics.output_client_data", string(c.Topics.OutputClientData)) + checkNotEmpty(configErrs, "global.kafka.topics.output_typing_event", string(c.Topics.OutputTypingEvent)) + checkNotEmpty(configErrs, "global.kafka.topics.output_send_to_device_event", string(c.Topics.OutputSendToDeviceEvent)) + checkNotEmpty(configErrs, "global.kafka.topics.output_key_change_event", string(c.Topics.OutputKeyChangeEvent)) +} + +// The configuration to use for Prometheus metrics +type Metrics struct { + // Whether or not the metrics are enabled + Enabled bool `yaml:"enabled"` + // Use BasicAuth for Authorization + BasicAuth struct { + // Authorization via Static Username & Password + // Hardcoded Username and Password + Username string `yaml:"username"` + Password string `yaml:"password"` + } `yaml:"basic_auth"` +} + +func (c *Metrics) Defaults() { + c.Enabled = false + c.BasicAuth.Username = "metrics" + c.BasicAuth.Password = "metrics" +} + +func (c *Metrics) Verify(configErrs *ConfigErrors, isMonolith bool) { +} + +type DatabaseOptions struct { + // The connection string, file:filename.db or postgres://server.... + ConnectionString DataSource `yaml:"connection_string"` + // Maximum open connections to the DB (0 = use default, negative means unlimited) + MaxOpenConnections int `yaml:"max_open_conns"` + // Maximum idle connections to the DB (0 = use default, negative means unlimited) + MaxIdleConnections int `yaml:"max_idle_conns"` + // maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited) + ConnMaxLifetimeSeconds int `yaml:"conn_max_lifetime"` +} + +func (c *DatabaseOptions) Defaults() { + c.MaxOpenConnections = 100 + c.MaxIdleConnections = 2 + c.ConnMaxLifetimeSeconds = -1 +} + +func (c *DatabaseOptions) Verify(configErrs *ConfigErrors, isMonolith bool) { +} + +// MaxIdleConns returns maximum idle connections to the DB +func (c DatabaseOptions) MaxIdleConns() int { + return c.MaxIdleConnections +} + +// MaxOpenConns returns maximum open connections to the DB +func (c DatabaseOptions) MaxOpenConns() int { + return c.MaxOpenConnections +} + +// ConnMaxLifetime returns maximum amount of time a connection may be reused +func (c DatabaseOptions) ConnMaxLifetime() time.Duration { + return time.Duration(c.ConnMaxLifetimeSeconds) * time.Second +} diff --git a/internal/config/config_keyserver.go b/internal/config/config_keyserver.go new file mode 100644 index 00000000..c0967a8a --- /dev/null +++ b/internal/config/config_keyserver.go @@ -0,0 +1,23 @@ +package config + +type KeyServer struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + Database DatabaseOptions `yaml:"database"` +} + +func (c *KeyServer) Defaults() { + c.Listen = "localhost:7779" + c.Bind = "localhost:7779" + c.Database.Defaults() + c.Database.ConnectionString = "file:keyserver.db" +} + +func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "key_server.listen", string(c.Listen)) + checkNotEmpty(configErrs, "key_server.bind", string(c.Bind)) + checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString)) +} diff --git a/internal/config/config_mediaapi.go b/internal/config/config_mediaapi.go new file mode 100644 index 00000000..9a4d7e0a --- /dev/null +++ b/internal/config/config_mediaapi.go @@ -0,0 +1,63 @@ +package config + +import ( + "fmt" +) + +type MediaAPI struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // The MediaAPI database stores information about files uploaded and downloaded + // by local users. It is only accessed by the MediaAPI. + Database DatabaseOptions `yaml:"database"` + + // The base path to where the media files will be stored. May be relative or absolute. + BasePath Path `yaml:"base_path"` + + // The absolute base path to where media files will be stored. + AbsBasePath Path `yaml:"-"` + + // The maximum file size in bytes that is allowed to be stored on this server. + // Note: if max_file_size_bytes is set to 0, the size is unlimited. + // Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) + MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"` + + // Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated + DynamicThumbnails bool `yaml:"dynamic_thumbnails"` + + // The maximum number of simultaneous thumbnail generators. default: 10 + MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"` + + // A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content + ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"` +} + +func (c *MediaAPI) Defaults() { + c.Listen = "localhost:7774" + c.Bind = "localhost:7774" + c.Database.Defaults() + c.Database.ConnectionString = "file:mediaapi.db" + + defaultMaxFileSizeBytes := FileSizeBytes(10485760) + c.MaxFileSizeBytes = &defaultMaxFileSizeBytes + c.MaxThumbnailGenerators = 10 + c.BasePath = "./media_store" +} + +func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "media_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "media_api.bind", string(c.Bind)) + checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString)) + + checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath)) + checkPositive(configErrs, "media_api.max_file_size_bytes", int64(*c.MaxFileSizeBytes)) + checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators)) + + for i, size := range c.ThumbnailSizes { + checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].width", i), int64(size.Width)) + checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].height", i), int64(size.Height)) + } +} diff --git a/internal/config/config_roomserver.go b/internal/config/config_roomserver.go new file mode 100644 index 00000000..1a16e2b1 --- /dev/null +++ b/internal/config/config_roomserver.go @@ -0,0 +1,23 @@ +package config + +type RoomServer struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + Database DatabaseOptions `yaml:"database"` +} + +func (c *RoomServer) Defaults() { + c.Listen = "localhost:7770" + c.Bind = "localhost:7770" + c.Database.Defaults() + c.Database.ConnectionString = "file:roomserver.db" +} + +func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "room_server.listen", string(c.Listen)) + checkNotEmpty(configErrs, "room_server.bind", string(c.Bind)) + checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString)) +} diff --git a/internal/config/config_serverkey.go b/internal/config/config_serverkey.go new file mode 100644 index 00000000..cf1f537a --- /dev/null +++ b/internal/config/config_serverkey.go @@ -0,0 +1,29 @@ +package config + +type ServerKeyAPI struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // The ServerKey database caches the public keys of remote servers. + // It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI. + Database DatabaseOptions `yaml:"database"` + + // Perspective keyservers, to use as a backup when direct key fetch + // requests don't succeed + KeyPerspectives KeyPerspectives `yaml:"key_perspectives"` +} + +func (c *ServerKeyAPI) Defaults() { + c.Listen = "localhost:7780" + c.Bind = "localhost:7780" + c.Database.Defaults() + c.Database.ConnectionString = "file:serverkeyapi.db" +} + +func (c *ServerKeyAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "server_key_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "server_key_api.bind", string(c.Bind)) + checkNotEmpty(configErrs, "server_key_api.database.connection_string", string(c.Database.ConnectionString)) +} diff --git a/internal/config/config_syncapi.go b/internal/config/config_syncapi.go new file mode 100644 index 00000000..488f6658 --- /dev/null +++ b/internal/config/config_syncapi.go @@ -0,0 +1,23 @@ +package config + +type SyncAPI struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + Database DatabaseOptions `yaml:"database"` +} + +func (c *SyncAPI) Defaults() { + c.Listen = "localhost:7773" + c.Bind = "localhost:7773" + c.Database.Defaults() + c.Database.ConnectionString = "file:syncapi.db" +} + +func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "sync_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "sync_api.bind", string(c.Bind)) + checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString)) +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 758d7552..4ff170e4 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -33,48 +33,157 @@ func TestLoadConfigRelative(t *testing.T) { } const testConfig = ` -version: 0 -matrix: +version: 1 +global: server_name: localhost private_key: matrix_key.pem - federation_certificates: [tls_cert.pem] -media: - base_path: media_store -kafka: - addresses: ["localhost:9092"] - topics: - output_room_event: output.room - output_client_data: output.client - output_typing_event: output.typing - output_send_to_device_event: output.std - output_key_change_event: output.key_change - user_updates: output.user -database: - media_api: "postgresql:///media_api" - account: "postgresql:///account" - device: "postgresql:///device" - server_key: "postgresql:///server_keys" - sync_api: "postgresql:///syn_api" - room_server: "postgresql:///room_server" - appservice: "postgresql:///appservice" - current_state: "postgresql:///current_state" - e2e_key: "postgresql:///e2e_key" -listen: - room_server: "localhost:7770" - client_api: "localhost:7771" - federation_api: "localhost:7772" - sync_api: "localhost:7773" - media_api: "localhost:7774" - appservice_api: "localhost:7777" - edu_server: "localhost:7778" - user_api: "localhost:7779" - current_state_server: "localhost:7775" - key_server: "localhost:7776" -logging: - - type: "file" - level: "info" - params: - path: "/my/log/dir" + key_validity_period: 168h0m0s + trusted_third_party_id_servers: [] + kafka: + addresses: [] + use_naffka: true + naffka_database: + connection_string: file:naffka.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 + topics: + output_room_event: OutputRoomEventTopic + output_client_data: OutputClientDataTopic + output_typing_event: OutputTypingEventTopic + output_send_to_device_event: OutputSendToDeviceEventTopic + output_key_change_event: OutputKeyChangeEventTopic + metrics: + enabled: false + basic_auth: + username: metrics + password: metrics +app_service_api: + listen: localhost:7777 + bind: localhost:7777 + database: + connection_string: file:appservice.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 + config_files: [] +client_api: + listen: localhost:7771 + bind: localhost:7771 + registration_shared_secret: "" + recaptcha_public_key: "" + recaptcha_private_key: "" + enable_registration_captcha: false + captcha_bypass_secret: "" + recaptcha_siteverify_api: "" + registration_disabled: false + turn: + turn_user_lifetime: "" + turn_uris: [] + turn_shared_secret: "" + turn_username: "" + turn_password: "" +current_state_server: + listen: localhost:7782 + bind: localhost:7782 + database: + connection_string: file:currentstate.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 +edu_server: + listen: localhost:7778 + bind: localhost:7778 +federation_api: + listen: localhost:7772 + bind: localhost:7772 + federation_certificates: [] +federation_sender: + listen: localhost:7775 + bind: localhost:7775 + database: + connection_string: file:federationsender.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 + federation_max_retries: 16 + proxy_outbound: + enabled: false + protocol: http + host: localhost + port: 8080 +key_server: + listen: localhost:7779 + bind: localhost:7779 + database: + connection_string: file:keyserver.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 +media_api: + listen: localhost:7774 + bind: localhost:7774 + database: + connection_string: file:mediaapi.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 + base_path: "" + max_file_size_bytes: 10485760 + dynamic_thumbnails: false + max_thumbnail_generators: 10 + thumbnail_sizes: [] +room_server: + listen: localhost:7770 + bind: localhost:7770 + database: + connection_string: file:roomserver.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 +server_key_api: + listen: localhost:7780 + bind: localhost:7780 + database: + connection_string: file:serverkeyapi.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 + key_perspectives: [] +sync_api: + listen: localhost:7773 + bind: localhost:7773 + database: + connection_string: file:syncapi.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 +user_api: + listen: localhost:7781 + bind: localhost:7781 + account_database: + connection_string: file:userapi_accounts.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 + device_database: + connection_string: file:userapi_devices.db + max_open_conns: 0 + max_idle_conns: 0 + conn_max_lifetime: -1 +tracing: + enabled: false + jaeger: + serviceName: "" + disabled: false + rpc_metrics: false + tags: [] + sampler: null + reporter: null + headers: null + baggage_restrictions: null + throttler: null +logging: [] ` type mockReadFile map[string]string diff --git a/internal/config/config_userapi.go b/internal/config/config_userapi.go new file mode 100644 index 00000000..f7da9e59 --- /dev/null +++ b/internal/config/config_userapi.go @@ -0,0 +1,31 @@ +package config + +type UserAPI struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // The Account database stores the login details and account information + // for local users. It is accessed by the UserAPI. + AccountDatabase DatabaseOptions `yaml:"account_database"` + // The Device database stores session information for the devices of logged + // in local users. It is accessed by the UserAPI. + DeviceDatabase DatabaseOptions `yaml:"device_database"` +} + +func (c *UserAPI) Defaults() { + c.Listen = "localhost:7781" + c.Bind = "localhost:7781" + c.AccountDatabase.Defaults() + c.DeviceDatabase.Defaults() + c.AccountDatabase.ConnectionString = "file:userapi_accounts.db" + c.DeviceDatabase.ConnectionString = "file:userapi_devices.db" +} + +func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "user_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "user_api.bind", string(c.Bind)) + checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString)) + checkNotEmpty(configErrs, "user_api.device_database.connection_string", string(c.DeviceDatabase.ConnectionString)) +} diff --git a/internal/eventutil/events.go b/internal/eventutil/events.go index 1e3afac8..35c7f33d 100644 --- a/internal/eventutil/events.go +++ b/internal/eventutil/events.go @@ -38,7 +38,7 @@ var ErrRoomNoExists = errors.New("Room does not exist") // Returns an error if something else went wrong func BuildEvent( ctx context.Context, - builder *gomatrixserverlib.EventBuilder, cfg *config.Dendrite, evTime time.Time, + builder *gomatrixserverlib.EventBuilder, cfg *config.Global, evTime time.Time, rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) (*gomatrixserverlib.HeaderedEvent, error) { if queryRes == nil { @@ -52,8 +52,8 @@ func BuildEvent( } event, err := builder.Build( - evTime, cfg.Matrix.ServerName, cfg.Matrix.KeyID, - cfg.Matrix.PrivateKey, queryRes.RoomVersion, + evTime, cfg.ServerName, cfg.KeyID, + cfg.PrivateKey, queryRes.RoomVersion, ) if err != nil { return nil, err diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index d371d172..8f7723ef 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -234,7 +234,7 @@ func (f *FederationWakeups) Wakeup(ctx context.Context, origin gomatrixserverlib } // SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics listener -func SetupHTTPAPI(servMux, publicApiMux, internalApiMux *mux.Router, cfg *config.Dendrite, enableHTTPAPIs bool) { +func SetupHTTPAPI(servMux, publicApiMux, internalApiMux *mux.Router, cfg *config.Global, enableHTTPAPIs bool) { if cfg.Metrics.Enabled { servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) } diff --git a/internal/setup/base.go b/internal/setup/base.go index 4fef0cbc..f59d136e 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -15,7 +15,6 @@ package setup import ( - "database/sql" "fmt" "io" "net/http" @@ -85,6 +84,15 @@ const HTTPClientTimeout = time.Second * 30 // The componentName is used for logging purposes, and should be a friendly name // of the compontent running, e.g. "SyncAPI" func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs bool) *BaseDendrite { + configErrors := &config.ConfigErrors{} + cfg.Verify(configErrors, componentName == "Monolith") // TODO: better way? + if len(*configErrors) > 0 { + for _, err := range *configErrors { + logrus.Errorf("Configuration error: %s", err) + } + logrus.Fatalf("Failed to start due to configuration errors") + } + internal.SetupStdLogging() internal.SetupHookLogging(cfg.Logging, componentName) internal.SetupPprof() @@ -96,7 +104,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo var kafkaConsumer sarama.Consumer var kafkaProducer sarama.SyncProducer - if cfg.Kafka.UseNaffka { + if cfg.Global.Kafka.UseNaffka { kafkaConsumer, kafkaProducer = setupNaffka(cfg) } else { kafkaConsumer, kafkaProducer = setupKafka(cfg) @@ -108,10 +116,10 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo } client := http.Client{Timeout: HTTPClientTimeout} - if cfg.Proxy != nil { + if cfg.FederationSender.Proxy.Enabled { client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{ - Scheme: cfg.Proxy.Protocol, - Host: fmt.Sprintf("%s:%d", cfg.Proxy.Host, cfg.Proxy.Port), + Scheme: cfg.FederationSender.Proxy.Protocol, + Host: fmt.Sprintf("%s:%d", cfg.FederationSender.Proxy.Host, cfg.FederationSender.Proxy.Port), })} } @@ -228,7 +236,7 @@ func (b *BaseDendrite) KeyServerHTTPClient() keyserverAPI.KeyInternalAPI { // CreateDeviceDB creates a new instance of the device database. Should only be // called once per component. func (b *BaseDendrite) CreateDeviceDB() devices.Database { - db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName) + db, err := devices.NewDatabase(&b.Cfg.UserAPI.DeviceDatabase, b.Cfg.Global.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to devices db") } @@ -239,7 +247,7 @@ func (b *BaseDendrite) CreateDeviceDB() devices.Database { // CreateAccountsDB creates a new instance of the accounts database. Should only // be called once per component. func (b *BaseDendrite) CreateAccountsDB() accounts.Database { - db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName) + db, err := accounts.NewDatabase(&b.Cfg.UserAPI.AccountDatabase, b.Cfg.Global.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to accounts db") } @@ -251,8 +259,8 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database { // once per component. func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationClient { return gomatrixserverlib.NewFederationClient( - b.Cfg.Matrix.ServerName, b.Cfg.Matrix.KeyID, b.Cfg.Matrix.PrivateKey, - b.Cfg.Matrix.FederationDisableTLSValidation, + b.Cfg.Global.ServerName, b.Cfg.Global.KeyID, b.Cfg.Global.PrivateKey, + b.Cfg.FederationSender.DisableTLSValidation, ) } @@ -277,7 +285,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { b.BaseMux, b.PublicAPIMux, b.InternalAPIMux, - b.Cfg, + &b.Cfg.Global, b.UseHTTPAPIs, ) serv.Handler = b.BaseMux @@ -293,12 +301,12 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { // setupKafka creates kafka consumer/producer pair from the config. func setupKafka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { - consumer, err := sarama.NewConsumer(cfg.Kafka.Addresses, nil) + consumer, err := sarama.NewConsumer(cfg.Global.Kafka.Addresses, nil) if err != nil { logrus.WithError(err).Panic("failed to start kafka consumer") } - producer, err := sarama.NewSyncProducer(cfg.Kafka.Addresses, nil) + producer, err := sarama.NewSyncProducer(cfg.Global.Kafka.Addresses, nil) if err != nil { logrus.WithError(err).Panic("failed to setup kafka producers") } @@ -308,36 +316,26 @@ func setupKafka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { // setupNaffka creates kafka consumer/producer pair from the config. func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { - var err error - var db *sql.DB var naffkaDB *naffka.DatabaseImpl - uri, err := url.Parse(string(cfg.Database.Naffka)) - if err != nil || uri.Scheme == "file" { - var cs string - cs, err = sqlutil.ParseFileURI(string(cfg.Database.Naffka)) - if err != nil { - logrus.WithError(err).Panic("Failed to parse naffka database file URI") - } - db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil) - if err != nil { - logrus.WithError(err).Panic("Failed to open naffka database") - } + db, err := sqlutil.Open(&cfg.Global.Kafka.Database) + if err != nil { + logrus.WithError(err).Panic("Failed to open naffka database") + } + switch { + case cfg.Global.Kafka.Database.ConnectionString.IsSQLite(): naffkaDB, err = naffka.NewSqliteDatabase(db) if err != nil { logrus.WithError(err).Panic("Failed to setup naffka database") } - } else { - db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka), nil) - if err != nil { - logrus.WithError(err).Panic("Failed to open naffka database") - } - + case cfg.Global.Kafka.Database.ConnectionString.IsPostgres(): naffkaDB, err = naffka.NewPostgresqlDatabase(db) if err != nil { logrus.WithError(err).Panic("Failed to setup naffka database") } + default: + panic("unknown naffka database type") } if naffkaDB == nil { diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index f33f97ee..92bbca72 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -65,19 +65,19 @@ type Monolith struct { // AddAllPublicRoutes attaches all public paths to the given router func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) { clientapi.AddPublicRoutes( - publicMux, m.Config, m.KafkaProducer, m.DeviceDB, m.AccountDB, + publicMux, &m.Config.ClientAPI, m.KafkaProducer, m.DeviceDB, m.AccountDB, m.FedClient, m.RoomserverAPI, m.EDUInternalAPI, m.AppserviceAPI, m.StateAPI, transactions.New(), m.FederationSenderAPI, m.UserAPI, m.KeyAPI, m.ExtPublicRoomsProvider, ) federationapi.AddPublicRoutes( - publicMux, m.Config, m.UserAPI, m.FedClient, + publicMux, &m.Config.FederationAPI, m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationSenderAPI, m.EDUInternalAPI, m.StateAPI, m.KeyAPI, ) - mediaapi.AddPublicRoutes(publicMux, m.Config, m.UserAPI, m.Client) + mediaapi.AddPublicRoutes(publicMux, &m.Config.MediaAPI, m.UserAPI, m.Client) syncapi.AddPublicRoutes( publicMux, m.KafkaConsumer, m.UserAPI, m.RoomserverAPI, - m.KeyAPI, m.StateAPI, m.FedClient, m.Config, + m.KeyAPI, m.StateAPI, m.FedClient, &m.Config.SyncAPI, ) } diff --git a/internal/sqlutil/sql.go b/internal/sqlutil/sql.go index 2ec6ce29..95467c63 100644 --- a/internal/sqlutil/sql.go +++ b/internal/sqlutil/sql.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "runtime" - "time" "go.uber.org/atomic" ) @@ -107,13 +106,6 @@ func SQLiteDriverName() string { return "sqlite3" } -// DbProperties functions return properties used by database/sql/DB -type DbProperties interface { - MaxIdleConns() int - MaxOpenConns() int - ConnMaxLifetime() time.Duration -} - // TransactionWriter allows queuing database writes so that you don't // contend on database locks in, e.g. SQLite. Only one task will run // at a time on a given TransactionWriter. diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go index f6644d59..fbd983be 100644 --- a/internal/sqlutil/trace.go +++ b/internal/sqlutil/trace.go @@ -25,6 +25,7 @@ import ( "strings" "time" + "github.com/matrix-org/dendrite/internal/config" "github.com/ngrok/sqlmw" "github.com/sirupsen/logrus" ) @@ -77,7 +78,22 @@ func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest [ // Open opens a database specified by its database driver name and a driver-specific data source name, // usually consisting of at least a database name and connection information. Includes tracing driver // if DENDRITE_TRACE_SQL=1 -func Open(driverName, dsn string, dbProperties DbProperties) (*sql.DB, error) { +func Open(dbProperties *config.DatabaseOptions) (*sql.DB, error) { + var err error + var driverName, dsn string + switch { + case dbProperties.ConnectionString.IsSQLite(): + driverName = SQLiteDriverName() + dsn, err = ParseFileURI(dbProperties.ConnectionString) + if err != nil { + return nil, fmt.Errorf("ParseFileURI: %w", err) + } + case dbProperties.ConnectionString.IsPostgres(): + driverName = "postgres" + dsn = string(dbProperties.ConnectionString) + default: + return nil, fmt.Errorf("invalid database connection string %q", dbProperties.ConnectionString) + } if tracingEnabled { // install the wrapped driver driverName += "-trace" @@ -86,11 +102,11 @@ func Open(driverName, dsn string, dbProperties DbProperties) (*sql.DB, error) { if err != nil { return nil, err } - if driverName != SQLiteDriverName() && dbProperties != nil { + if driverName != SQLiteDriverName() { logrus.WithFields(logrus.Fields{ - "MaxOpenConns": dbProperties.MaxOpenConns(), - "MaxIdleConns": dbProperties.MaxIdleConns(), - "ConnMaxLifetime": dbProperties.ConnMaxLifetime(), + "MaxOpenConns": dbProperties.MaxOpenConns, + "MaxIdleConns": dbProperties.MaxIdleConns, + "ConnMaxLifetime": dbProperties.ConnMaxLifetime, "dataSourceName": regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"), }).Debug("Setting DB connection limits") db.SetMaxOpenConns(dbProperties.MaxOpenConns()) diff --git a/internal/sqlutil/uri.go b/internal/sqlutil/uri.go index 703258e6..e2c825d9 100644 --- a/internal/sqlutil/uri.go +++ b/internal/sqlutil/uri.go @@ -15,14 +15,20 @@ package sqlutil import ( + "errors" "fmt" "net/url" + + "github.com/matrix-org/dendrite/internal/config" ) // ParseFileURI returns the filepath in the given file: URI. Specifically, this will handle // both relative (file:foo.db) and absolute (file:///path/to/foo) paths. -func ParseFileURI(dataSourceName string) (string, error) { - uri, err := url.Parse(dataSourceName) +func ParseFileURI(dataSourceName config.DataSource) (string, error) { + if !dataSourceName.IsSQLite() { + return "", errors.New("ParseFileURI expects SQLite connection string") + } + uri, err := url.Parse(string(dataSourceName)) if err != nil { return "", err } diff --git a/internal/test/config.go b/internal/test/config.go index bbcc9bed..43a5d1ff 100644 --- a/internal/test/config.go +++ b/internal/test/config.go @@ -49,6 +49,7 @@ const ( // Generates new matrix and TLS keys for the server. func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*config.Dendrite, int, error) { var cfg config.Dendrite + cfg.Defaults() port := startPort assignAddress := func() config.Address { @@ -72,51 +73,53 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Version = config.Version - cfg.Matrix.ServerName = gomatrixserverlib.ServerName(assignAddress()) - cfg.Matrix.PrivateKeyPath = config.Path(serverKeyPath) - cfg.Matrix.FederationCertificatePaths = []config.Path{config.Path(tlsCertPath)} + cfg.Global.ServerName = gomatrixserverlib.ServerName(assignAddress()) + cfg.Global.PrivateKeyPath = config.Path(serverKeyPath) - cfg.Media.BasePath = config.Path(mediaBasePath) + cfg.FederationAPI.FederationCertificatePaths = []config.Path{config.Path(tlsCertPath)} - cfg.Kafka.Addresses = []string{kafkaURI} - // TODO: Different servers should be using different topics. - // Make this configurable somehow? - cfg.Kafka.Topics.OutputRoomEvent = "test.room.output" - cfg.Kafka.Topics.OutputClientData = "test.clientapi.output" - cfg.Kafka.Topics.OutputTypingEvent = "test.typing.output" + cfg.MediaAPI.BasePath = config.Path(mediaBasePath) + + cfg.Global.Kafka.Addresses = []string{kafkaURI} // TODO: Use different databases for the different schemas. // Using the same database for every schema currently works because // the table names are globally unique. But we might not want to // rely on that in the future. - cfg.Database.Account = config.DataSource(database) - cfg.Database.AppService = config.DataSource(database) - cfg.Database.Device = config.DataSource(database) - cfg.Database.MediaAPI = config.DataSource(database) - cfg.Database.RoomServer = config.DataSource(database) - cfg.Database.ServerKey = config.DataSource(database) - cfg.Database.SyncAPI = config.DataSource(database) - cfg.Database.CurrentState = config.DataSource(database) - - cfg.Listen.ClientAPI = assignAddress() - cfg.Listen.AppServiceAPI = assignAddress() - cfg.Listen.FederationAPI = assignAddress() - cfg.Listen.MediaAPI = assignAddress() - cfg.Listen.RoomServer = assignAddress() - cfg.Listen.SyncAPI = assignAddress() - cfg.Listen.CurrentState = assignAddress() - cfg.Listen.EDUServer = assignAddress() - - // Bind to the same address as the listen address - // All microservices are run on the same host in testing - cfg.Bind.ClientAPI = cfg.Listen.ClientAPI - cfg.Bind.AppServiceAPI = cfg.Listen.AppServiceAPI - cfg.Bind.FederationAPI = cfg.Listen.FederationAPI - cfg.Bind.MediaAPI = cfg.Listen.MediaAPI - cfg.Bind.RoomServer = cfg.Listen.RoomServer - cfg.Bind.SyncAPI = cfg.Listen.SyncAPI - cfg.Bind.CurrentState = cfg.Listen.CurrentState - cfg.Bind.EDUServer = cfg.Listen.EDUServer + cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(database) + cfg.CurrentStateServer.Database.ConnectionString = config.DataSource(database) + cfg.FederationSender.Database.ConnectionString = config.DataSource(database) + cfg.KeyServer.Database.ConnectionString = config.DataSource(database) + cfg.MediaAPI.Database.ConnectionString = config.DataSource(database) + cfg.RoomServer.Database.ConnectionString = config.DataSource(database) + cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(database) + cfg.SyncAPI.Database.ConnectionString = config.DataSource(database) + cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(database) + cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(database) + + cfg.AppServiceAPI.Listen = assignAddress() + cfg.CurrentStateServer.Listen = assignAddress() + cfg.EDUServer.Listen = assignAddress() + cfg.FederationAPI.Listen = assignAddress() + cfg.FederationSender.Listen = assignAddress() + cfg.KeyServer.Listen = assignAddress() + cfg.MediaAPI.Listen = assignAddress() + cfg.RoomServer.Listen = assignAddress() + cfg.ServerKeyAPI.Listen = assignAddress() + cfg.SyncAPI.Listen = assignAddress() + cfg.UserAPI.Listen = assignAddress() + + cfg.AppServiceAPI.Bind = cfg.AppServiceAPI.Listen + cfg.CurrentStateServer.Bind = cfg.CurrentStateServer.Listen + cfg.EDUServer.Bind = cfg.EDUServer.Listen + cfg.FederationAPI.Bind = cfg.FederationAPI.Listen + cfg.FederationSender.Bind = cfg.FederationSender.Listen + cfg.KeyServer.Bind = cfg.KeyServer.Listen + cfg.MediaAPI.Bind = cfg.MediaAPI.Listen + cfg.RoomServer.Bind = cfg.RoomServer.Listen + cfg.ServerKeyAPI.Bind = cfg.ServerKeyAPI.Listen + cfg.SyncAPI.Bind = cfg.SyncAPI.Listen + cfg.UserAPI.Bind = cfg.UserAPI.Listen return &cfg, port, nil } diff --git a/internal/test/server.go b/internal/test/server.go index 2d4117f4..57df21db 100644 --- a/internal/test/server.go +++ b/internal/test/server.go @@ -96,9 +96,9 @@ func InitDatabase(postgresDatabase, postgresContainerName string, databases []st func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) { proxyArgs := []string{ "--bind-address", bindAddr, - "--sync-api-server-url", "http://" + string(cfg.Listen.SyncAPI), - "--client-api-server-url", "http://" + string(cfg.Listen.ClientAPI), - "--media-api-server-url", "http://" + string(cfg.Listen.MediaAPI), + "--sync-api-server-url", "http://" + string(cfg.SyncAPI.Listen), + "--client-api-server-url", "http://" + string(cfg.ClientAPI.Listen), + "--media-api-server-url", "http://" + string(cfg.MediaAPI.Listen), "--tls-cert", "server.crt", "--tls-key", "server.key", } |