From b5aa7ca3ab1c91397700637c91d60860a0535f1e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 2 Dec 2020 17:41:00 +0000 Subject: Top-level setup package (#1605) * Move config, setup, mscs into "setup" top-level folder * oops, forgot the EDU server * Add setup * goimports --- setup/config/config.go | 572 ++++++++++++++++++++++++++++++++ setup/config/config_appservice.go | 353 ++++++++++++++++++++ setup/config/config_clientapi.go | 123 +++++++ setup/config/config_eduserver.go | 17 + setup/config/config_federationapi.go | 31 ++ setup/config/config_federationsender.go | 63 ++++ setup/config/config_global.go | 142 ++++++++ setup/config/config_kafka.go | 61 ++++ setup/config/config_keyserver.go | 22 ++ setup/config/config_mediaapi.go | 67 ++++ setup/config/config_mscs.go | 19 ++ setup/config/config_roomserver.go | 22 ++ setup/config/config_signingkeyserver.go | 52 +++ setup/config/config_syncapi.go | 29 ++ setup/config/config_test.go | 285 ++++++++++++++++ setup/config/config_userapi.go | 30 ++ 16 files changed, 1888 insertions(+) create mode 100644 setup/config/config.go create mode 100644 setup/config/config_appservice.go create mode 100644 setup/config/config_clientapi.go create mode 100644 setup/config/config_eduserver.go create mode 100644 setup/config/config_federationapi.go create mode 100644 setup/config/config_federationsender.go create mode 100644 setup/config/config_global.go create mode 100644 setup/config/config_kafka.go create mode 100644 setup/config/config_keyserver.go create mode 100644 setup/config/config_mediaapi.go create mode 100644 setup/config/config_mscs.go create mode 100644 setup/config/config_roomserver.go create mode 100644 setup/config/config_signingkeyserver.go create mode 100644 setup/config/config_syncapi.go create mode 100644 setup/config/config_test.go create mode 100644 setup/config/config_userapi.go (limited to 'setup/config') diff --git a/setup/config/config.go b/setup/config/config.go new file mode 100644 index 00000000..b8b12d0c --- /dev/null +++ b/setup/config/config.go @@ -0,0 +1,572 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "bytes" + "encoding/pem" + "fmt" + "io" + "io/ioutil" + "net/url" + "path/filepath" + "regexp" + "strings" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" + "golang.org/x/crypto/ed25519" + yaml "gopkg.in/yaml.v2" + + jaegerconfig "github.com/uber/jaeger-client-go/config" + jaegermetrics "github.com/uber/jaeger-lib/metrics" +) + +// keyIDRegexp defines allowable characters in Key IDs. +var keyIDRegexp = regexp.MustCompile("^ed25519:[a-zA-Z0-9_]+$") + +// Version is the current version of the config format. +// This will change whenever we make breaking changes to the config format. +const Version = 1 + +// Dendrite contains all the config used by a dendrite process. +// Relative paths are resolved relative to the current working directory +type Dendrite struct { + // The version of the configuration file. + // If the version in a file doesn't match the current dendrite config + // version then we can give a clear error message telling the user + // to update their config file to the current version. + // The version of the file should only be different if there has + // been a breaking change to the config file format. + Version int `yaml:"version"` + + Global Global `yaml:"global"` + AppServiceAPI AppServiceAPI `yaml:"app_service_api"` + ClientAPI ClientAPI `yaml:"client_api"` + EDUServer EDUServer `yaml:"edu_server"` + FederationAPI FederationAPI `yaml:"federation_api"` + FederationSender FederationSender `yaml:"federation_sender"` + KeyServer KeyServer `yaml:"key_server"` + MediaAPI MediaAPI `yaml:"media_api"` + RoomServer RoomServer `yaml:"room_server"` + SigningKeyServer SigningKeyServer `yaml:"signing_key_server"` + SyncAPI SyncAPI `yaml:"sync_api"` + UserAPI UserAPI `yaml:"user_api"` + + MSCs MSCs `yaml:"mscs"` + + // The config for tracing the dendrite servers. + Tracing struct { + // Set to true to enable tracer hooks. If false, no tracing is set up. + Enabled bool `yaml:"enabled"` + // The config for the jaeger opentracing reporter. + Jaeger jaegerconfig.Configuration `yaml:"jaeger"` + } `yaml:"tracing"` + + // The config for logging informations. Each hook will be added to logrus. + Logging []LogrusHook `yaml:"logging"` + + // Any information derived from the configuration options for later use. + Derived Derived `yaml:"-"` +} + +// TODO: Kill Derived +type Derived struct { + Registration struct { + // Flows is a slice of flows, which represent one possible way that the client can authenticate a request. + // http://matrix.org/docs/spec/HEAD/client_server/r0.3.0.html#user-interactive-authentication-api + // As long as the generated flows only rely on config file options, + // we can generate them on startup and store them until needed + Flows []authtypes.Flow `json:"flows"` + + // Params that need to be returned to the client during + // registration in order to complete registration stages. + Params map[string]interface{} `json:"params"` + } + + // Application services parsed from their config files + // The paths of which were given above in the main config file + ApplicationServices []ApplicationService + + // Meta-regexes compiled from all exclusive application service + // Regexes. + // + // When a user registers, we check that their username does not match any + // exclusive application service namespaces + ExclusiveApplicationServicesUsernameRegexp *regexp.Regexp + // When a user creates a room alias, we check that it isn't already + // reserved by an application service + ExclusiveApplicationServicesAliasRegexp *regexp.Regexp + // Note: An Exclusive Regex for room ID isn't necessary as we aren't blocking + // servers from creating RoomIDs in exclusive application service namespaces +} + +type InternalAPIOptions struct { + Listen HTTPAddress `yaml:"listen"` + Connect HTTPAddress `yaml:"connect"` +} + +type ExternalAPIOptions struct { + Listen HTTPAddress `yaml:"listen"` +} + +// A Path on the filesystem. +type Path string + +// A DataSource for opening a postgresql database using lib/pq. +type DataSource string + +func (d DataSource) IsSQLite() bool { + return strings.HasPrefix(string(d), "file:") +} + +func (d DataSource) IsPostgres() bool { + // commented line may not always be true? + // return strings.HasPrefix(string(d), "postgres:") + return !d.IsSQLite() +} + +// A Topic in kafka. +type Topic string + +// An Address to listen on. +type Address string + +// An HTTPAddress to listen on, starting with either http:// or https://. +type HTTPAddress string + +func (h HTTPAddress) Address() (Address, error) { + url, err := url.Parse(string(h)) + if err != nil { + return "", err + } + return Address(url.Host), nil +} + +// FileSizeBytes is a file size in bytes +type FileSizeBytes int64 + +// ThumbnailSize contains a single thumbnail size configuration +type ThumbnailSize struct { + // Maximum width of the thumbnail image + Width int `yaml:"width"` + // Maximum height of the thumbnail image + Height int `yaml:"height"` + // ResizeMethod is one of crop or scale. + // crop scales to fill the requested dimensions and crops the excess. + // scale scales to fit the requested dimensions and one dimension may be smaller than requested. + ResizeMethod string `yaml:"method,omitempty"` +} + +// LogrusHook represents a single logrus hook. At this point, only parsing and +// verification of the proper values for type and level are done. +// Validity/integrity checks on the parameters are done when configuring logrus. +type LogrusHook struct { + // The type of hook, currently only "file" is supported. + Type string `yaml:"type"` + + // The level of the logs to produce. Will output only this level and above. + Level string `yaml:"level"` + + // The parameters for this hook. + Params map[string]interface{} `yaml:"params"` +} + +// ConfigErrors stores problems encountered when parsing a config file. +// It implements the error interface. +type ConfigErrors []string + +// Load a yaml config file for a server run as multiple processes or as a monolith. +// Checks the config to ensure that it is valid. +func Load(configPath string, monolith bool) (*Dendrite, error) { + configData, err := ioutil.ReadFile(configPath) + if err != nil { + return nil, err + } + basePath, err := filepath.Abs(".") + if err != nil { + return nil, err + } + // Pass the current working directory and ioutil.ReadFile so that they can + // be mocked in the tests + return loadConfig(basePath, configData, ioutil.ReadFile, monolith) +} + +func loadConfig( + basePath string, + configData []byte, + readFile func(string) ([]byte, error), + monolithic bool, +) (*Dendrite, error) { + var c Dendrite + c.Defaults() + + var err error + if err = yaml.Unmarshal(configData, &c); err != nil { + return nil, err + } + + if err = c.check(monolithic); err != nil { + return nil, err + } + + privateKeyPath := absPath(basePath, c.Global.PrivateKeyPath) + privateKeyData, err := readFile(privateKeyPath) + if err != nil { + return nil, err + } + + if c.Global.KeyID, c.Global.PrivateKey, err = readKeyPEM(privateKeyPath, privateKeyData, true); err != nil { + return nil, err + } + + for i, oldPrivateKey := range c.Global.OldVerifyKeys { + var oldPrivateKeyData []byte + + oldPrivateKeyPath := absPath(basePath, oldPrivateKey.PrivateKeyPath) + oldPrivateKeyData, err = readFile(oldPrivateKeyPath) + if err != nil { + return nil, err + } + + // NOTSPEC: Ordinarily we should enforce key ID formatting, but since there are + // a number of private keys out there with non-compatible symbols in them due + // to lack of validation in Synapse, we won't enforce that for old verify keys. + keyID, privateKey, perr := readKeyPEM(oldPrivateKeyPath, oldPrivateKeyData, false) + if perr != nil { + return nil, perr + } + + c.Global.OldVerifyKeys[i].KeyID, c.Global.OldVerifyKeys[i].PrivateKey = keyID, privateKey + } + + c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath)) + + // Generate data from config options + err = c.Derive() + if err != nil { + return nil, err + } + + c.Wiring() + return &c, nil +} + +// Derive generates data that is derived from various values provided in +// the config file. +func (config *Dendrite) Derive() error { + // Determine registrations flows based off config values + + config.Derived.Registration.Params = make(map[string]interface{}) + + // TODO: Add email auth type + // TODO: Add MSISDN auth type + + if config.ClientAPI.RecaptchaEnabled { + config.Derived.Registration.Params[authtypes.LoginTypeRecaptcha] = map[string]string{"public_key": config.ClientAPI.RecaptchaPublicKey} + config.Derived.Registration.Flows = append(config.Derived.Registration.Flows, + authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeRecaptcha}}) + } else { + config.Derived.Registration.Flows = append(config.Derived.Registration.Flows, + authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeDummy}}) + } + + // Load application service configuration files + if err := loadAppServices(&config.AppServiceAPI, &config.Derived); err != nil { + return err + } + + return nil +} + +// SetDefaults sets default config values if they are not explicitly set. +func (c *Dendrite) Defaults() { + c.Version = 1 + + c.Global.Defaults() + c.ClientAPI.Defaults() + c.EDUServer.Defaults() + c.FederationAPI.Defaults() + c.FederationSender.Defaults() + c.KeyServer.Defaults() + c.MediaAPI.Defaults() + c.RoomServer.Defaults() + c.SigningKeyServer.Defaults() + c.SyncAPI.Defaults() + c.UserAPI.Defaults() + c.AppServiceAPI.Defaults() + c.MSCs.Defaults() + + c.Wiring() +} + +func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) { + type verifiable interface { + Verify(configErrs *ConfigErrors, isMonolith bool) + } + for _, c := range []verifiable{ + &c.Global, &c.ClientAPI, + &c.EDUServer, &c.FederationAPI, &c.FederationSender, + &c.KeyServer, &c.MediaAPI, &c.RoomServer, + &c.SigningKeyServer, &c.SyncAPI, &c.UserAPI, + &c.AppServiceAPI, &c.MSCs, + } { + c.Verify(configErrs, isMonolith) + } +} + +func (c *Dendrite) Wiring() { + c.ClientAPI.Matrix = &c.Global + c.EDUServer.Matrix = &c.Global + c.FederationAPI.Matrix = &c.Global + c.FederationSender.Matrix = &c.Global + c.KeyServer.Matrix = &c.Global + c.MediaAPI.Matrix = &c.Global + c.RoomServer.Matrix = &c.Global + c.SigningKeyServer.Matrix = &c.Global + c.SyncAPI.Matrix = &c.Global + c.UserAPI.Matrix = &c.Global + c.AppServiceAPI.Matrix = &c.Global + c.MSCs.Matrix = &c.Global + + c.ClientAPI.Derived = &c.Derived + c.AppServiceAPI.Derived = &c.Derived +} + +// Error returns a string detailing how many errors were contained within a +// configErrors type. +func (errs ConfigErrors) Error() string { + if len(errs) == 1 { + return errs[0] + } + return fmt.Sprintf( + "%s (and %d other problems)", errs[0], len(errs)-1, + ) +} + +// Add appends an error to the list of errors in this configErrors. +// It is a wrapper to the builtin append and hides pointers from +// the client code. +// This method is safe to use with an uninitialized configErrors because +// if it is nil, it will be properly allocated. +func (errs *ConfigErrors) Add(str string) { + *errs = append(*errs, str) +} + +// checkNotEmpty verifies the given value is not empty in the configuration. +// If it is, adds an error to the list. +func checkNotEmpty(configErrs *ConfigErrors, key, value string) { + if value == "" { + configErrs.Add(fmt.Sprintf("missing config key %q", key)) + } +} + +// checkNotZero verifies the given value is not zero in the configuration. +// If it is, adds an error to the list. +func checkNotZero(configErrs *ConfigErrors, key string, value int64) { + if value == 0 { + configErrs.Add(fmt.Sprintf("missing config key %q", key)) + } +} + +// checkPositive verifies the given value is positive (zero included) +// in the configuration. If it is not, adds an error to the list. +func checkPositive(configErrs *ConfigErrors, key string, value int64) { + if value < 0 { + configErrs.Add(fmt.Sprintf("invalid value for config key %q: %d", key, value)) + } +} + +// checkURL verifies that the parameter is a valid URL +func checkURL(configErrs *ConfigErrors, key, value string) { + if value == "" { + configErrs.Add(fmt.Sprintf("missing config key %q", key)) + return + } + url, err := url.Parse(value) + if err != nil { + configErrs.Add(fmt.Sprintf("config key %q contains invalid URL (%s)", key, err.Error())) + return + } + switch url.Scheme { + case "http": + case "https": + default: + configErrs.Add(fmt.Sprintf("config key %q URL should be http:// or https://", key)) + return + } +} + +// checkLogging verifies the parameters logging.* are valid. +func (config *Dendrite) checkLogging(configErrs *ConfigErrors) { + for _, logrusHook := range config.Logging { + checkNotEmpty(configErrs, "logging.type", string(logrusHook.Type)) + checkNotEmpty(configErrs, "logging.level", string(logrusHook.Level)) + } +} + +// check returns an error type containing all errors found within the config +// file. +func (config *Dendrite) check(_ bool) error { // monolithic + var configErrs ConfigErrors + + if config.Version != Version { + configErrs.Add(fmt.Sprintf( + "unknown config version %q, expected %q", config.Version, Version, + )) + return configErrs + } + + config.checkLogging(&configErrs) + + // Due to how Golang manages its interface types, this condition is not redundant. + // In order to get the proper behaviour, it is necessary to return an explicit nil + // and not a nil configErrors. + // This is because the following equalities hold: + // error(nil) == nil + // error(configErrors(nil)) != nil + if configErrs != nil { + return configErrs + } + return nil +} + +// absPath returns the absolute path for a given relative or absolute path. +func absPath(dir string, path Path) string { + if filepath.IsAbs(string(path)) { + // filepath.Join cleans the path so we should clean the absolute paths as well for consistency. + return filepath.Clean(string(path)) + } + return filepath.Join(dir, string(path)) +} + +func readKeyPEM(path string, data []byte, enforceKeyIDFormat bool) (gomatrixserverlib.KeyID, ed25519.PrivateKey, error) { + for { + var keyBlock *pem.Block + keyBlock, data = pem.Decode(data) + if data == nil { + return "", nil, fmt.Errorf("no matrix private key PEM data in %q", path) + } + if keyBlock == nil { + return "", nil, fmt.Errorf("keyBlock is nil %q", path) + } + if keyBlock.Type == "MATRIX PRIVATE KEY" { + keyID := keyBlock.Headers["Key-ID"] + if keyID == "" { + return "", nil, fmt.Errorf("missing key ID in PEM data in %q", path) + } + if !strings.HasPrefix(keyID, "ed25519:") { + return "", nil, fmt.Errorf("key ID %q doesn't start with \"ed25519:\" in %q", keyID, path) + } + if enforceKeyIDFormat && !keyIDRegexp.MatchString(keyID) { + return "", nil, fmt.Errorf("key ID %q in %q contains illegal characters (use a-z, A-Z, 0-9 and _ only)", keyID, path) + } + _, privKey, err := ed25519.GenerateKey(bytes.NewReader(keyBlock.Bytes)) + if err != nil { + return "", nil, err + } + return gomatrixserverlib.KeyID(keyID), privKey, nil + } + } +} + +// AppServiceURL returns a HTTP URL for where the appservice component is listening. +func (config *Dendrite) AppServiceURL() string { + // Hard code the appservice server to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return string(config.AppServiceAPI.InternalAPI.Connect) +} + +// RoomServerURL returns an HTTP URL for where the roomserver is listening. +func (config *Dendrite) RoomServerURL() string { + // Hard code the roomserver to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return string(config.RoomServer.InternalAPI.Connect) +} + +// UserAPIURL returns an HTTP URL for where the userapi is listening. +func (config *Dendrite) UserAPIURL() string { + // Hard code the userapi to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return string(config.UserAPI.InternalAPI.Connect) +} + +// EDUServerURL returns an HTTP URL for where the EDU server is listening. +func (config *Dendrite) EDUServerURL() string { + // Hard code the EDU server to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return string(config.EDUServer.InternalAPI.Connect) +} + +// FederationSenderURL returns an HTTP URL for where the federation sender is listening. +func (config *Dendrite) FederationSenderURL() string { + // Hard code the federation sender server to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return string(config.FederationSender.InternalAPI.Connect) +} + +// SigningKeyServerURL returns an HTTP URL for where the signing key server is listening. +func (config *Dendrite) SigningKeyServerURL() string { + // Hard code the signing key server to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return string(config.SigningKeyServer.InternalAPI.Connect) +} + +// KeyServerURL returns an HTTP URL for where the key server is listening. +func (config *Dendrite) KeyServerURL() string { + // Hard code the key server to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return string(config.KeyServer.InternalAPI.Connect) +} + +// SetupTracing configures the opentracing using the supplied configuration. +func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err error) { + if !config.Tracing.Enabled { + return ioutil.NopCloser(bytes.NewReader([]byte{})), nil + } + return config.Tracing.Jaeger.InitGlobalTracer( + serviceName, + jaegerconfig.Logger(logrusLogger{logrus.StandardLogger()}), + jaegerconfig.Metrics(jaegermetrics.NullFactory), + ) +} + +// logrusLogger is a small wrapper that implements jaeger.Logger using logrus. +type logrusLogger struct { + l *logrus.Logger +} + +func (l logrusLogger) Error(msg string) { + l.l.Error(msg) +} + +func (l logrusLogger) Infof(msg string, args ...interface{}) { + l.l.Infof(msg, args...) +} diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go new file mode 100644 index 00000000..a042691d --- /dev/null +++ b/setup/config/config_appservice.go @@ -0,0 +1,353 @@ +// Copyright 2017 Andrew Morgan +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "regexp" + "strings" + + log "github.com/sirupsen/logrus" + yaml "gopkg.in/yaml.v2" +) + +type AppServiceAPI struct { + Matrix *Global `yaml:"-"` + Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit + + InternalAPI InternalAPIOptions `yaml:"internal_api"` + + Database DatabaseOptions `yaml:"database"` + + ConfigFiles []string `yaml:"config_files"` +} + +func (c *AppServiceAPI) Defaults() { + c.InternalAPI.Listen = "http://localhost:7777" + c.InternalAPI.Connect = "http://localhost:7777" + c.Database.Defaults() + c.Database.ConnectionString = "file:appservice.db" +} + +func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkURL(configErrs, "app_service_api.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "app_service_api.internal_api.bind", string(c.InternalAPI.Connect)) + checkNotEmpty(configErrs, "app_service_api.database.connection_string", string(c.Database.ConnectionString)) +} + +// ApplicationServiceNamespace is the namespace that a specific application +// service has management over. +type ApplicationServiceNamespace struct { + // Whether or not the namespace is managed solely by this application service + Exclusive bool `yaml:"exclusive"` + // A regex pattern that represents the namespace + Regex string `yaml:"regex"` + // The ID of an existing group that all users of this application service will + // be added to. This field is only relevant to the `users` namespace. + // Note that users who are joined to this group through an application service + // are not to be listed when querying for the group's members, however the + // group should be listed when querying an application service user's groups. + // This is to prevent making spamming all users of an application service + // trivial. + GroupID string `yaml:"group_id"` + // Regex object representing our pattern. Saves having to recompile every time + RegexpObject *regexp.Regexp +} + +// ApplicationService represents a Matrix application service. +// https://matrix.org/docs/spec/application_service/unstable.html +type ApplicationService struct { + // User-defined, unique, persistent ID of the application service + ID string `yaml:"id"` + // Base URL of the application service + URL string `yaml:"url"` + // Application service token provided in requests to a homeserver + ASToken string `yaml:"as_token"` + // Homeserver token provided in requests to an application service + HSToken string `yaml:"hs_token"` + // Localpart of application service user + SenderLocalpart string `yaml:"sender_localpart"` + // Information about an application service's namespaces. Key is either + // "users", "aliases" or "rooms" + NamespaceMap map[string][]ApplicationServiceNamespace `yaml:"namespaces"` + // Whether rate limiting is applied to each application service user + RateLimited bool `yaml:"rate_limited"` + // Any custom protocols that this application service provides (e.g. IRC) + Protocols []string `yaml:"protocols"` +} + +// IsInterestedInRoomID returns a bool on whether an application service's +// namespace includes the given room ID +func (a *ApplicationService) IsInterestedInRoomID( + roomID string, +) bool { + if namespaceSlice, ok := a.NamespaceMap["rooms"]; ok { + for _, namespace := range namespaceSlice { + if namespace.RegexpObject.MatchString(roomID) { + return true + } + } + } + + return false +} + +// IsInterestedInUserID returns a bool on whether an application service's +// namespace includes the given user ID +func (a *ApplicationService) IsInterestedInUserID( + userID string, +) bool { + if namespaceSlice, ok := a.NamespaceMap["users"]; ok { + for _, namespace := range namespaceSlice { + if namespace.RegexpObject.MatchString(userID) { + return true + } + } + } + + return false +} + +// OwnsNamespaceCoveringUserId returns a bool on whether an application service's +// namespace is exclusive and includes the given user ID +func (a *ApplicationService) OwnsNamespaceCoveringUserId( + userID string, +) bool { + if namespaceSlice, ok := a.NamespaceMap["users"]; ok { + for _, namespace := range namespaceSlice { + if namespace.Exclusive && namespace.RegexpObject.MatchString(userID) { + return true + } + } + } + + return false +} + +// IsInterestedInRoomAlias returns a bool on whether an application service's +// namespace includes the given room alias +func (a *ApplicationService) IsInterestedInRoomAlias( + roomAlias string, +) bool { + if namespaceSlice, ok := a.NamespaceMap["aliases"]; ok { + for _, namespace := range namespaceSlice { + if namespace.RegexpObject.MatchString(roomAlias) { + return true + } + } + } + + return false +} + +// loadAppServices iterates through all application service config files +// and loads their data into the config object for later access. +func loadAppServices(config *AppServiceAPI, derived *Derived) error { + for _, configPath := range config.ConfigFiles { + // Create a new application service with default options + appservice := ApplicationService{ + RateLimited: true, + } + + // Create an absolute path from a potentially relative path + absPath, err := filepath.Abs(configPath) + if err != nil { + return err + } + + // Read the application service's config file + configData, err := ioutil.ReadFile(absPath) + if err != nil { + return err + } + + // Load the config data into our struct + if err = yaml.UnmarshalStrict(configData, &appservice); err != nil { + return err + } + + // Append the parsed application service to the global config + derived.ApplicationServices = append( + derived.ApplicationServices, appservice, + ) + } + + // Check for any errors in the loaded application services + return checkErrors(config, derived) +} + +// setupRegexps will create regex objects for exclusive and non-exclusive +// usernames, aliases and rooms of all application services, so that other +// methods can quickly check if a particular string matches any of them. +func setupRegexps(_ *AppServiceAPI, derived *Derived) (err error) { + // Combine all exclusive namespaces for later string checking + var exclusiveUsernameStrings, exclusiveAliasStrings []string + + // If an application service's regex is marked as exclusive, add + // its contents to the overall exlusive regex string. Room regex + // not necessary as we aren't denying exclusive room ID creation + for _, appservice := range derived.ApplicationServices { + for key, namespaceSlice := range appservice.NamespaceMap { + switch key { + case "users": + appendExclusiveNamespaceRegexs(&exclusiveUsernameStrings, namespaceSlice) + case "aliases": + appendExclusiveNamespaceRegexs(&exclusiveAliasStrings, namespaceSlice) + } + } + } + + // Join the regexes together into one big regex. + // i.e. "app1.*", "app2.*" -> "(app1.*)|(app2.*)" + // Later we can check if a username or alias matches any exclusive regex and + // deny access if it isn't from an application service + exclusiveUsernames := strings.Join(exclusiveUsernameStrings, "|") + exclusiveAliases := strings.Join(exclusiveAliasStrings, "|") + + // If there are no exclusive regexes, compile string so that it will not match + // any valid usernames/aliases/roomIDs + if exclusiveUsernames == "" { + exclusiveUsernames = "^$" + } + if exclusiveAliases == "" { + exclusiveAliases = "^$" + } + + // Store compiled Regex + if derived.ExclusiveApplicationServicesUsernameRegexp, err = regexp.Compile(exclusiveUsernames); err != nil { + return err + } + if derived.ExclusiveApplicationServicesAliasRegexp, err = regexp.Compile(exclusiveAliases); err != nil { + return err + } + + return nil +} + +// appendExclusiveNamespaceRegexs takes a slice of strings and a slice of +// namespaces and will append the regexes of only the exclusive namespaces +// into the string slice +func appendExclusiveNamespaceRegexs( + exclusiveStrings *[]string, namespaces []ApplicationServiceNamespace, +) { + for index, namespace := range namespaces { + if namespace.Exclusive { + // We append parenthesis to later separate each regex when we compile + // i.e. "app1.*", "app2.*" -> "(app1.*)|(app2.*)" + *exclusiveStrings = append(*exclusiveStrings, "("+namespace.Regex+")") + } + + // Compile this regex into a Regexp object for later use + namespaces[index].RegexpObject, _ = regexp.Compile(namespace.Regex) + } +} + +// checkErrors checks for any configuration errors amongst the loaded +// application services according to the application service spec. +func checkErrors(config *AppServiceAPI, derived *Derived) (err error) { + var idMap = make(map[string]bool) + var tokenMap = make(map[string]bool) + + // Compile regexp object for checking groupIDs + groupIDRegexp := regexp.MustCompile(`\+.*:.*`) + + // Check each application service for any config errors + for _, appservice := range derived.ApplicationServices { + // Namespace-related checks + for key, namespaceSlice := range appservice.NamespaceMap { + for _, namespace := range namespaceSlice { + if err := validateNamespace(&appservice, key, &namespace, groupIDRegexp); err != nil { + return err + } + } + } + + // Check if the url has trailing /'s. If so, remove them + appservice.URL = strings.TrimRight(appservice.URL, "/") + + // Check if we've already seen this ID. No two application services + // can have the same ID or token. + if idMap[appservice.ID] { + return ConfigErrors([]string{fmt.Sprintf( + "Application service ID %s must be unique", appservice.ID, + )}) + } + // Check if we've already seen this token + if tokenMap[appservice.ASToken] { + return ConfigErrors([]string{fmt.Sprintf( + "Application service Token %s must be unique", appservice.ASToken, + )}) + } + + // Add the id/token to their respective maps if we haven't already + // seen them. + idMap[appservice.ID] = true + tokenMap[appservice.ASToken] = true + + // TODO: Remove once rate_limited is implemented + if appservice.RateLimited { + log.Warn("WARNING: Application service option rate_limited is currently unimplemented") + } + // TODO: Remove once protocols is implemented + if len(appservice.Protocols) > 0 { + log.Warn("WARNING: Application service option protocols is currently unimplemented") + } + } + + return setupRegexps(config, derived) +} + +// validateNamespace returns nil or an error based on whether a given +// application service namespace is valid. A namespace is valid if it has the +// required fields, and its regex is correct. +func validateNamespace( + appservice *ApplicationService, + key string, + namespace *ApplicationServiceNamespace, + groupIDRegexp *regexp.Regexp, +) error { + // Check that namespace(s) are valid regex + if !IsValidRegex(namespace.Regex) { + return ConfigErrors([]string{fmt.Sprintf( + "Invalid regex string for Application Service %s", appservice.ID, + )}) + } + + // Check if GroupID for the users namespace is in the correct format + if key == "users" && namespace.GroupID != "" { + // TODO: Remove once group_id is implemented + log.Warn("WARNING: Application service option group_id is currently unimplemented") + + correctFormat := groupIDRegexp.MatchString(namespace.GroupID) + if !correctFormat { + return ConfigErrors([]string{fmt.Sprintf( + "Invalid user group_id field for application service %s.", + appservice.ID, + )}) + } + } + + return nil +} + +// IsValidRegex returns true or false based on whether the +// given string is valid regex or not +func IsValidRegex(regexString string) bool { + _, err := regexp.Compile(regexString) + + return err == nil +} diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go new file mode 100644 index 00000000..52115491 --- /dev/null +++ b/setup/config/config_clientapi.go @@ -0,0 +1,123 @@ +package config + +import ( + "fmt" + "time" +) + +type ClientAPI struct { + Matrix *Global `yaml:"-"` + Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit + + InternalAPI InternalAPIOptions `yaml:"internal_api"` + ExternalAPI ExternalAPIOptions `yaml:"external_api"` + + // If set disables new users from registering (except via shared + // secrets) + RegistrationDisabled bool `yaml:"registration_disabled"` + // If set, allows registration by anyone who also has the shared + // secret, even if registration is otherwise disabled. + RegistrationSharedSecret string `yaml:"registration_shared_secret"` + + // Boolean stating whether catpcha registration is enabled + // and required + RecaptchaEnabled bool `yaml:"enable_registration_captcha"` + // This Home Server's ReCAPTCHA public key. + RecaptchaPublicKey string `yaml:"recaptcha_public_key"` + // This Home Server's ReCAPTCHA private key. + RecaptchaPrivateKey string `yaml:"recaptcha_private_key"` + // Secret used to bypass the captcha registration entirely + RecaptchaBypassSecret string `yaml:"recaptcha_bypass_secret"` + // HTTP API endpoint used to verify whether the captcha response + // was successful + RecaptchaSiteVerifyAPI string `yaml:"recaptcha_siteverify_api"` + + // TURN options + TURN TURN `yaml:"turn"` + + // Rate-limiting options + RateLimiting RateLimiting `yaml:"rate_limiting"` +} + +func (c *ClientAPI) Defaults() { + c.InternalAPI.Listen = "http://localhost:7771" + c.InternalAPI.Connect = "http://localhost:7771" + c.ExternalAPI.Listen = "http://[::]:8071" + c.RegistrationSharedSecret = "" + c.RecaptchaPublicKey = "" + c.RecaptchaPrivateKey = "" + c.RecaptchaEnabled = false + c.RecaptchaBypassSecret = "" + c.RecaptchaSiteVerifyAPI = "" + c.RegistrationDisabled = false + c.RateLimiting.Defaults() +} + +func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkURL(configErrs, "client_api.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "client_api.internal_api.connect", string(c.InternalAPI.Connect)) + if !isMonolith { + checkURL(configErrs, "client_api.external_api.listen", string(c.ExternalAPI.Listen)) + } + if c.RecaptchaEnabled { + checkNotEmpty(configErrs, "client_api.recaptcha_public_key", string(c.RecaptchaPublicKey)) + checkNotEmpty(configErrs, "client_api.recaptcha_private_key", string(c.RecaptchaPrivateKey)) + checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", string(c.RecaptchaSiteVerifyAPI)) + } + c.TURN.Verify(configErrs) + c.RateLimiting.Verify(configErrs) +} + +type TURN struct { + // TODO Guest Support + // Whether or not guests can request TURN credentials + // AllowGuests bool `yaml:"turn_allow_guests"` + // How long the authorization should last + UserLifetime string `yaml:"turn_user_lifetime"` + // The list of TURN URIs to pass to clients + URIs []string `yaml:"turn_uris"` + + // Authorization via Shared Secret + // The shared secret from coturn + SharedSecret string `yaml:"turn_shared_secret"` + + // Authorization via Static Username & Password + // Hardcoded Username and Password + Username string `yaml:"turn_username"` + Password string `yaml:"turn_password"` +} + +func (c *TURN) Verify(configErrs *ConfigErrors) { + value := c.UserLifetime + if value != "" { + if _, err := time.ParseDuration(value); err != nil { + configErrs.Add(fmt.Sprintf("invalid duration for config key %q: %s", "client_api.turn.turn_user_lifetime", value)) + } + } +} + +type RateLimiting struct { + // Is rate limiting enabled or disabled? + Enabled bool `yaml:"enabled"` + + // How many "slots" a user can occupy sending requests to a rate-limited + // endpoint before we apply rate-limiting + Threshold int64 `yaml:"threshold"` + + // The cooloff period in milliseconds after a request before the "slot" + // is freed again + CooloffMS int64 `yaml:"cooloff_ms"` +} + +func (r *RateLimiting) Verify(configErrs *ConfigErrors) { + if r.Enabled { + checkPositive(configErrs, "client_api.rate_limiting.threshold", r.Threshold) + checkPositive(configErrs, "client_api.rate_limiting.cooloff_ms", r.CooloffMS) + } +} + +func (r *RateLimiting) Defaults() { + r.Enabled = true + r.Threshold = 5 + r.CooloffMS = 500 +} diff --git a/setup/config/config_eduserver.go b/setup/config/config_eduserver.go new file mode 100644 index 00000000..a2ff3697 --- /dev/null +++ b/setup/config/config_eduserver.go @@ -0,0 +1,17 @@ +package config + +type EDUServer struct { + Matrix *Global `yaml:"-"` + + InternalAPI InternalAPIOptions `yaml:"internal_api"` +} + +func (c *EDUServer) Defaults() { + c.InternalAPI.Listen = "http://localhost:7778" + c.InternalAPI.Connect = "http://localhost:7778" +} + +func (c *EDUServer) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkURL(configErrs, "edu_server.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "edu_server.internal_api.connect", string(c.InternalAPI.Connect)) +} diff --git a/setup/config/config_federationapi.go b/setup/config/config_federationapi.go new file mode 100644 index 00000000..64803d95 --- /dev/null +++ b/setup/config/config_federationapi.go @@ -0,0 +1,31 @@ +package config + +type FederationAPI struct { + Matrix *Global `yaml:"-"` + + InternalAPI InternalAPIOptions `yaml:"internal_api"` + ExternalAPI ExternalAPIOptions `yaml:"external_api"` + + // List of paths to X509 certificates used by the external federation listeners. + // These are used to calculate the TLS fingerprints to publish for this server. + // Other matrix servers talking to this server will expect the x509 certificate + // to match one of these certificates. + // The certificates should be in PEM format. + FederationCertificatePaths []Path `yaml:"federation_certificates"` +} + +func (c *FederationAPI) Defaults() { + c.InternalAPI.Listen = "http://localhost:7772" + c.InternalAPI.Connect = "http://localhost:7772" + c.ExternalAPI.Listen = "http://[::]:8072" +} + +func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkURL(configErrs, "federation_api.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "federation_api.internal_api.connect", string(c.InternalAPI.Connect)) + if !isMonolith { + checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen)) + } + // TODO: not applicable always, e.g. in demos + //checkNotZero(configErrs, "federation_api.federation_certificates", int64(len(c.FederationCertificatePaths))) +} diff --git a/setup/config/config_federationsender.go b/setup/config/config_federationsender.go new file mode 100644 index 00000000..84f5b6f4 --- /dev/null +++ b/setup/config/config_federationsender.go @@ -0,0 +1,63 @@ +package config + +type FederationSender struct { + Matrix *Global `yaml:"-"` + + InternalAPI InternalAPIOptions `yaml:"internal_api"` + + // The FederationSender database stores information used by the FederationSender + // It is only accessed by the FederationSender. + Database DatabaseOptions `yaml:"database"` + + // Federation failure threshold. How many consecutive failures that we should + // tolerate when sending federation requests to a specific server. The backoff + // is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. + // The default value is 16 if not specified, which is circa 18 hours. + FederationMaxRetries uint32 `yaml:"send_max_retries"` + + // FederationDisableTLSValidation disables the validation of X.509 TLS certs + // on remote federation endpoints. This is not recommended in production! + DisableTLSValidation bool `yaml:"disable_tls_validation"` + + Proxy Proxy `yaml:"proxy_outbound"` +} + +func (c *FederationSender) Defaults() { + c.InternalAPI.Listen = "http://localhost:7775" + c.InternalAPI.Connect = "http://localhost:7775" + c.Database.Defaults() + c.Database.ConnectionString = "file:federationsender.db" + + c.FederationMaxRetries = 16 + c.DisableTLSValidation = false + + c.Proxy.Defaults() +} + +func (c *FederationSender) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkURL(configErrs, "federation_sender.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "federation_sender.internal_api.connect", string(c.InternalAPI.Connect)) + checkNotEmpty(configErrs, "federation_sender.database.connection_string", string(c.Database.ConnectionString)) +} + +// The config for setting a proxy to use for server->server requests +type Proxy struct { + // Is the proxy enabled? + Enabled bool `yaml:"enabled"` + // The protocol for the proxy (http / https / socks5) + Protocol string `yaml:"protocol"` + // The host where the proxy is listening + Host string `yaml:"host"` + // The port on which the proxy is listening + Port uint16 `yaml:"port"` +} + +func (c *Proxy) Defaults() { + c.Enabled = false + c.Protocol = "http" + c.Host = "localhost" + c.Port = 8080 +} + +func (c *Proxy) Verify(configErrs *ConfigErrors) { +} diff --git a/setup/config/config_global.go b/setup/config/config_global.go new file mode 100644 index 00000000..95652217 --- /dev/null +++ b/setup/config/config_global.go @@ -0,0 +1,142 @@ +package config + +import ( + "math/rand" + "time" + + "github.com/matrix-org/gomatrixserverlib" + "golang.org/x/crypto/ed25519" +) + +type Global struct { + // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. + ServerName gomatrixserverlib.ServerName `yaml:"server_name"` + + // Path to the private key which will be used to sign requests and events. + PrivateKeyPath Path `yaml:"private_key"` + + // The private key which will be used to sign requests and events. + PrivateKey ed25519.PrivateKey `yaml:"-"` + + // An arbitrary string used to uniquely identify the PrivateKey. Must start with the + // prefix "ed25519:". + KeyID gomatrixserverlib.KeyID `yaml:"-"` + + // Information about old private keys that used to be used to sign requests and + // events on this domain. They will not be used but will be advertised to other + // servers that ask for them to help verify old events. + OldVerifyKeys []OldVerifyKeys `yaml:"old_private_keys"` + + // How long a remote server can cache our server key for before requesting it again. + // Increasing this number will reduce the number of requests made by remote servers + // for our key, but increases the period a compromised key will be considered valid + // by remote servers. + // Defaults to 24 hours. + KeyValidityPeriod time.Duration `yaml:"key_validity_period"` + + // Disables federation. Dendrite will not be able to make any outbound HTTP requests + // to other servers and the federation API will not be exposed. + DisableFederation bool `yaml:"disable_federation"` + + // List of domains that the server will trust as identity servers to + // verify third-party identifiers. + // Defaults to an empty array. + TrustedIDServers []string `yaml:"trusted_third_party_id_servers"` + + // Kafka/Naffka configuration + Kafka Kafka `yaml:"kafka"` + + // Metrics configuration + Metrics Metrics `yaml:"metrics"` +} + +func (c *Global) Defaults() { + c.ServerName = "localhost" + c.PrivateKeyPath = "matrix_key.pem" + _, c.PrivateKey, _ = ed25519.GenerateKey(rand.New(rand.NewSource(0))) + c.KeyID = "ed25519:auto" + c.KeyValidityPeriod = time.Hour * 24 * 7 + + c.Kafka.Defaults() + c.Metrics.Defaults() +} + +func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "global.server_name", string(c.ServerName)) + checkNotEmpty(configErrs, "global.private_key", string(c.PrivateKeyPath)) + + c.Kafka.Verify(configErrs, isMonolith) + c.Metrics.Verify(configErrs, isMonolith) +} + +type OldVerifyKeys struct { + // Path to the private key. + PrivateKeyPath Path `yaml:"private_key"` + + // The private key itself. + PrivateKey ed25519.PrivateKey `yaml:"-"` + + // The key ID of the private key. + KeyID gomatrixserverlib.KeyID `yaml:"-"` + + // When the private key was designed as "expired", as a UNIX timestamp + // in millisecond precision. + ExpiredAt gomatrixserverlib.Timestamp `yaml:"expired_at"` +} + +// The configuration to use for Prometheus metrics +type Metrics struct { + // Whether or not the metrics are enabled + Enabled bool `yaml:"enabled"` + // Use BasicAuth for Authorization + BasicAuth struct { + // Authorization via Static Username & Password + // Hardcoded Username and Password + Username string `yaml:"username"` + Password string `yaml:"password"` + } `yaml:"basic_auth"` +} + +func (c *Metrics) Defaults() { + c.Enabled = false + c.BasicAuth.Username = "metrics" + c.BasicAuth.Password = "metrics" +} + +func (c *Metrics) Verify(configErrs *ConfigErrors, isMonolith bool) { +} + +type DatabaseOptions struct { + // The connection string, file:filename.db or postgres://server.... + ConnectionString DataSource `yaml:"connection_string"` + // Maximum open connections to the DB (0 = use default, negative means unlimited) + MaxOpenConnections int `yaml:"max_open_conns"` + // Maximum idle connections to the DB (0 = use default, negative means unlimited) + MaxIdleConnections int `yaml:"max_idle_conns"` + // maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited) + ConnMaxLifetimeSeconds int `yaml:"conn_max_lifetime"` +} + +func (c *DatabaseOptions) Defaults() { + c.MaxOpenConnections = 100 + c.MaxIdleConnections = 2 + c.ConnMaxLifetimeSeconds = -1 +} + +func (c *DatabaseOptions) Verify(configErrs *ConfigErrors, isMonolith bool) { +} + +// MaxIdleConns returns maximum idle connections to the DB +func (c DatabaseOptions) MaxIdleConns() int { + return c.MaxIdleConnections +} + +// MaxOpenConns returns maximum open connections to the DB +func (c DatabaseOptions) MaxOpenConns() int { + return c.MaxOpenConnections +} + +// ConnMaxLifetime returns maximum amount of time a connection may be reused +func (c DatabaseOptions) ConnMaxLifetime() time.Duration { + return time.Duration(c.ConnMaxLifetimeSeconds) * time.Second +} diff --git a/setup/config/config_kafka.go b/setup/config/config_kafka.go new file mode 100644 index 00000000..aa91e558 --- /dev/null +++ b/setup/config/config_kafka.go @@ -0,0 +1,61 @@ +package config + +import "fmt" + +// Defined Kafka topics. +const ( + TopicOutputTypingEvent = "OutputTypingEvent" + TopicOutputSendToDeviceEvent = "OutputSendToDeviceEvent" + TopicOutputKeyChangeEvent = "OutputKeyChangeEvent" + TopicOutputRoomEvent = "OutputRoomEvent" + TopicOutputClientData = "OutputClientData" + TopicOutputReceiptEvent = "OutputReceiptEvent" +) + +type Kafka struct { + // A list of kafka addresses to connect to. + Addresses []string `yaml:"addresses"` + // The prefix to use for Kafka topic names for this homeserver - really only + // useful if running more than one Dendrite on the same Kafka deployment. + TopicPrefix string `yaml:"topic_prefix"` + // Whether to use naffka instead of kafka. + // Naffka can only be used when running dendrite as a single monolithic server. + // Kafka can be used both with a monolithic server and when running the + // components as separate servers. + UseNaffka bool `yaml:"use_naffka"` + // The Naffka database is used internally by the naffka library, if used. + Database DatabaseOptions `yaml:"naffka_database"` + // The max size a Kafka message passed between consumer/producer can have + // Equals roughly max.message.bytes / fetch.message.max.bytes in Kafka + MaxMessageBytes *int `yaml:"max_message_bytes"` +} + +func (k *Kafka) TopicFor(name string) string { + return fmt.Sprintf("%s%s", k.TopicPrefix, name) +} + +func (c *Kafka) Defaults() { + c.UseNaffka = true + c.Database.Defaults() + c.Addresses = []string{"localhost:2181"} + c.Database.ConnectionString = DataSource("file:naffka.db") + c.TopicPrefix = "Dendrite" + + maxBytes := 1024 * 1024 * 8 // about 8MB + c.MaxMessageBytes = &maxBytes +} + +func (c *Kafka) Verify(configErrs *ConfigErrors, isMonolith bool) { + if c.UseNaffka { + if !isMonolith { + configErrs.Add("naffka can only be used in a monolithic server") + } + checkNotEmpty(configErrs, "global.kafka.database.connection_string", string(c.Database.ConnectionString)) + } else { + // If we aren't using naffka then we need to have at least one kafka + // server to talk to. + checkNotZero(configErrs, "global.kafka.addresses", int64(len(c.Addresses))) + } + checkNotEmpty(configErrs, "global.kafka.topic_prefix", string(c.TopicPrefix)) + checkPositive(configErrs, "global.kafka.max_message_bytes", int64(*c.MaxMessageBytes)) +} diff --git a/setup/config/config_keyserver.go b/setup/config/config_keyserver.go new file mode 100644 index 00000000..89162300 --- /dev/null +++ b/setup/config/config_keyserver.go @@ -0,0 +1,22 @@ +package config + +type KeyServer struct { + Matrix *Global `yaml:"-"` + + InternalAPI InternalAPIOptions `yaml:"internal_api"` + + Database DatabaseOptions `yaml:"database"` +} + +func (c *KeyServer) Defaults() { + c.InternalAPI.Listen = "http://localhost:7779" + c.InternalAPI.Connect = "http://localhost:7779" + c.Database.Defaults() + c.Database.ConnectionString = "file:keyserver.db" +} + +func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "key_server.internal_api.bind", string(c.InternalAPI.Connect)) + checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString)) +} diff --git a/setup/config/config_mediaapi.go b/setup/config/config_mediaapi.go new file mode 100644 index 00000000..a9425b7b --- /dev/null +++ b/setup/config/config_mediaapi.go @@ -0,0 +1,67 @@ +package config + +import ( + "fmt" +) + +type MediaAPI struct { + Matrix *Global `yaml:"-"` + + InternalAPI InternalAPIOptions `yaml:"internal_api"` + ExternalAPI ExternalAPIOptions `yaml:"external_api"` + + // The MediaAPI database stores information about files uploaded and downloaded + // by local users. It is only accessed by the MediaAPI. + Database DatabaseOptions `yaml:"database"` + + // The base path to where the media files will be stored. May be relative or absolute. + BasePath Path `yaml:"base_path"` + + // The absolute base path to where media files will be stored. + AbsBasePath Path `yaml:"-"` + + // The maximum file size in bytes that is allowed to be stored on this server. + // Note: if max_file_size_bytes is set to 0, the size is unlimited. + // Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) + MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"` + + // Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated + DynamicThumbnails bool `yaml:"dynamic_thumbnails"` + + // The maximum number of simultaneous thumbnail generators. default: 10 + MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"` + + // A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content + ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"` +} + +func (c *MediaAPI) Defaults() { + c.InternalAPI.Listen = "http://localhost:7774" + c.InternalAPI.Connect = "http://localhost:7774" + c.ExternalAPI.Listen = "http://[::]:8074" + c.Database.Defaults() + c.Database.ConnectionString = "file:mediaapi.db" + + defaultMaxFileSizeBytes := FileSizeBytes(10485760) + c.MaxFileSizeBytes = &defaultMaxFileSizeBytes + c.MaxThumbnailGenerators = 10 + c.BasePath = "./media_store" +} + +func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkURL(configErrs, "media_api.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "media_api.internal_api.connect", string(c.InternalAPI.Connect)) + if !isMonolith { + checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen)) + } + checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString)) + + checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath)) + checkPositive(configErrs, "media_api.max_file_size_bytes", int64(*c.MaxFileSizeBytes)) + checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators)) + + for i, size := range c.ThumbnailSizes { + checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].width", i), int64(size.Width)) + checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].height", i), int64(size.Height)) + } +} diff --git a/setup/config/config_mscs.go b/setup/config/config_mscs.go new file mode 100644 index 00000000..776d0b64 --- /dev/null +++ b/setup/config/config_mscs.go @@ -0,0 +1,19 @@ +package config + +type MSCs struct { + Matrix *Global `yaml:"-"` + + // The MSCs to enable, currently only `msc2836` is supported. + MSCs []string `yaml:"mscs"` + + Database DatabaseOptions `yaml:"database"` +} + +func (c *MSCs) Defaults() { + c.Database.Defaults() + c.Database.ConnectionString = "file:mscs.db" +} + +func (c *MSCs) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkNotEmpty(configErrs, "mscs.database.connection_string", string(c.Database.ConnectionString)) +} diff --git a/setup/config/config_roomserver.go b/setup/config/config_roomserver.go new file mode 100644 index 00000000..2a1fc38b --- /dev/null +++ b/setup/config/config_roomserver.go @@ -0,0 +1,22 @@ +package config + +type RoomServer struct { + Matrix *Global `yaml:"-"` + + InternalAPI InternalAPIOptions `yaml:"internal_api"` + + Database DatabaseOptions `yaml:"database"` +} + +func (c *RoomServer) Defaults() { + c.InternalAPI.Listen = "http://localhost:7770" + c.InternalAPI.Connect = "http://localhost:7770" + c.Database.Defaults() + c.Database.ConnectionString = "file:roomserver.db" +} + +func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "room_server.internal_ap.bind", string(c.InternalAPI.Connect)) + checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString)) +} diff --git a/setup/config/config_signingkeyserver.go b/setup/config/config_signingkeyserver.go new file mode 100644 index 00000000..51aca38b --- /dev/null +++ b/setup/config/config_signingkeyserver.go @@ -0,0 +1,52 @@ +package config + +import "github.com/matrix-org/gomatrixserverlib" + +type SigningKeyServer struct { + Matrix *Global `yaml:"-"` + + InternalAPI InternalAPIOptions `yaml:"internal_api"` + + // The SigningKeyServer database caches the public keys of remote servers. + // It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI. + Database DatabaseOptions `yaml:"database"` + + // Perspective keyservers, to use as a backup when direct key fetch + // requests don't succeed + KeyPerspectives KeyPerspectives `yaml:"key_perspectives"` + + // Should we prefer direct key fetches over perspective ones? + PreferDirectFetch bool `yaml:"prefer_direct_fetch"` +} + +func (c *SigningKeyServer) Defaults() { + c.InternalAPI.Listen = "http://localhost:7780" + c.InternalAPI.Connect = "http://localhost:7780" + c.Database.Defaults() + c.Database.ConnectionString = "file:signingkeyserver.db" +} + +func (c *SigningKeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkURL(configErrs, "signing_key_server.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "signing_key_server.internal_api.bind", string(c.InternalAPI.Connect)) + checkNotEmpty(configErrs, "signing_key_server.database.connection_string", string(c.Database.ConnectionString)) +} + +// KeyPerspectives are used to configure perspective key servers for +// retrieving server keys. +type KeyPerspectives []KeyPerspective + +type KeyPerspective struct { + // The server name of the perspective key server + ServerName gomatrixserverlib.ServerName `yaml:"server_name"` + // Server keys for the perspective user, used to verify the + // keys have been signed by the perspective server + Keys []KeyPerspectiveTrustKey `yaml:"keys"` +} + +type KeyPerspectiveTrustKey struct { + // The key ID, e.g. ed25519:auto + KeyID gomatrixserverlib.KeyID `yaml:"key_id"` + // The public key in base64 unpadded format + PublicKey string `yaml:"public_key"` +} diff --git a/setup/config/config_syncapi.go b/setup/config/config_syncapi.go new file mode 100644 index 00000000..fc08f738 --- /dev/null +++ b/setup/config/config_syncapi.go @@ -0,0 +1,29 @@ +package config + +type SyncAPI struct { + Matrix *Global `yaml:"-"` + + InternalAPI InternalAPIOptions `yaml:"internal_api"` + ExternalAPI ExternalAPIOptions `yaml:"external_api"` + + Database DatabaseOptions `yaml:"database"` + + RealIPHeader string `yaml:"real_ip_header"` +} + +func (c *SyncAPI) Defaults() { + c.InternalAPI.Listen = "http://localhost:7773" + c.InternalAPI.Connect = "http://localhost:7773" + c.ExternalAPI.Listen = "http://localhost:8073" + c.Database.Defaults() + c.Database.ConnectionString = "file:syncapi.db" +} + +func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "sync_api.internal_api.bind", string(c.InternalAPI.Connect)) + if !isMonolith { + checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen)) + } + checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString)) +} diff --git a/setup/config/config_test.go b/setup/config/config_test.go new file mode 100644 index 00000000..4107b684 --- /dev/null +++ b/setup/config/config_test.go @@ -0,0 +1,285 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "testing" +) + +func TestLoadConfigRelative(t *testing.T) { + _, err := loadConfig("/my/config/dir", []byte(testConfig), + mockReadFile{ + "/my/config/dir/matrix_key.pem": testKey, + "/my/config/dir/tls_cert.pem": testCert, + }.readFile, + false, + ) + if err != nil { + t.Error("failed to load config:", err) + } +} + +const testConfig = ` +version: 1 +global: + server_name: localhost + private_key: matrix_key.pem + key_id: ed25519:auto + key_validity_period: 168h0m0s + trusted_third_party_id_servers: + - matrix.org + - vector.im + kafka: + addresses: + - localhost:2181 + topic_prefix: Dendrite + use_naffka: true + naffka_database: + connection_string: file:naffka.db + max_open_conns: 100 + max_idle_conns: 2 + conn_max_lifetime: -1 + metrics: + enabled: false + basic_auth: + username: metrics + password: metrics +app_service_api: + internal_api: + listen: http://localhost:7777 + connect: http://localhost:7777 + database: + connection_string: file:appservice.db + max_open_conns: 100 + max_idle_conns: 2 + conn_max_lifetime: -1 + config_files: [] +client_api: + internal_api: + listen: http://localhost:7771 + connect: http://localhost:7771 + external_api: + listen: http://[::]:8071 + registration_disabled: false + registration_shared_secret: "" + enable_registration_captcha: false + recaptcha_public_key: "" + recaptcha_private_key: "" + recaptcha_bypass_secret: "" + recaptcha_siteverify_api: "" + turn: + turn_user_lifetime: "" + turn_uris: [] + turn_shared_secret: "" + turn_username: "" + turn_password: "" +current_state_server: + internal_api: + listen: http://localhost:7782 + connect: http://localhost:7782 + database: + connection_string: file:currentstate.db + max_open_conns: 100 + max_idle_conns: 2 + conn_max_lifetime: -1 +edu_server: + internal_api: + listen: http://localhost:7778 + connect: http://localhost:7778 +federation_api: + internal_api: + listen: http://localhost:7772 + connect: http://localhost:7772 + external_api: + listen: http://[::]:8072 + federation_certificates: [] +federation_sender: + internal_api: + listen: http://localhost:7775 + connect: http://localhost:7775 + database: + connection_string: file:federationsender.db + max_open_conns: 100 + max_idle_conns: 2 + conn_max_lifetime: -1 + send_max_retries: 16 + disable_tls_validation: false + proxy_outbound: + enabled: false + protocol: http + host: localhost + port: 8080 +key_server: + internal_api: + listen: http://localhost:7779 + connect: http://localhost:7779 + database: + connection_string: file:keyserver.db + max_open_conns: 100 + max_idle_conns: 2 + conn_max_lifetime: -1 +media_api: + internal_api: + listen: http://localhost:7774 + connect: http://localhost:7774 + external_api: + listen: http://[::]:8074 + database: + connection_string: file:mediaapi.db + max_open_conns: 100 + max_idle_conns: 2 + conn_max_lifetime: -1 + base_path: ./media_store + max_file_size_bytes: 10485760 + dynamic_thumbnails: false + max_thumbnail_generators: 10 + thumbnail_sizes: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 640 + height: 480 + method: scale +room_server: + internal_api: + listen: http://localhost:7770 + connect: http://localhost:7770 + database: + connection_string: file:roomserver.db + max_open_conns: 100 + max_idle_conns: 2 + conn_max_lifetime: -1 +server_key_api: + internal_api: + listen: http://localhost:7780 + connect: http://localhost:7780 + database: + connection_string: file:serverkeyapi.db + max_open_conns: 100 + max_idle_conns: 2 + conn_max_lifetime: -1 + key_perspectives: + - server_name: matrix.org + keys: + - key_id: ed25519:auto + public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw + - key_id: ed25519:a_RXGa + public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ +sync_api: + internal_api: + listen: http://localhost:7773 + connect: http://localhost:7773 + database: + connection_string: file:syncapi.db + max_open_conns: 100 + max_idle_conns: 2 + conn_max_lifetime: -1 +user_api: + internal_api: + listen: http://localhost:7781 + connect: http://localhost:7781 + account_database: + connection_string: file:userapi_accounts.db + max_open_conns: 100 + max_idle_conns: 2 + conn_max_lifetime: -1 + device_database: + connection_string: file:userapi_devices.db + max_open_conns: 100 + max_idle_conns: 2 + conn_max_lifetime: -1 +tracing: + enabled: false + jaeger: + serviceName: "" + disabled: false + rpc_metrics: false + tags: [] + sampler: null + reporter: null + headers: null + baggage_restrictions: null + throttler: null +logging: +- type: file + level: info + params: + path: /var/log/dendrite +` + +type mockReadFile map[string]string + +func (m mockReadFile) readFile(path string) ([]byte, error) { + data, ok := m[path] + if !ok { + return nil, fmt.Errorf("no such file %q", path) + } + return []byte(data), nil +} + +func TestReadKey(t *testing.T) { + keyID, _, err := readKeyPEM("path/to/key", []byte(testKey), true) + if err != nil { + t.Error("failed to load private key:", err) + } + wantKeyID := testKeyID + if wantKeyID != string(keyID) { + t.Errorf("wanted key ID to be %q, got %q", wantKeyID, keyID) + } +} + +const testKeyID = "ed25519:c8NsuQ" + +const testKey = ` +-----BEGIN MATRIX PRIVATE KEY----- +Key-ID: ` + testKeyID + ` +7KRZiZ2sTyRR8uqqUjRwczuwRXXkUMYIUHq4Mc3t4bE= +-----END MATRIX PRIVATE KEY----- +` + +const testCert = ` +-----BEGIN CERTIFICATE----- +MIIE0zCCArugAwIBAgIJAPype3u24LJeMA0GCSqGSIb3DQEBCwUAMAAwHhcNMTcw +NjEzMTQyODU4WhcNMTgwNjEzMTQyODU4WjAAMIICIjANBgkqhkiG9w0BAQEFAAOC +Ag8AMIICCgKCAgEA3vNSr7lCh/alxPFqairp/PYohwdsqPvOD7zf7dJCNhy0gbdC +9/APwIbPAPL9nU+o9ud1ACNCKBCQin/9LnI5vd5pa/Ne+mmRADDLB/BBBoywSJWG +NSfKJ9n3XY1bjgtqi53uUh+RDdQ7sXudDqCUxiiJZmS7oqK/mp88XXAgCbuXUY29 +GmzbbDz37vntuSxDgUOnJ8uPSvRp5YPKogA3JwW1SyrlLt4Z30CQ6nH3Y2Q5SVfJ +NIQyMrnwyjA9bCdXezv1cLXoTYn7U9BRyzXTZeXs3y3ldnRfISXN35CU04Az1F8j +lfj7nXMEqI/qAj/qhxZ8nVBB+rpNOZy9RJko3O+G5Qa/EvzkQYV1rW4TM2Yme88A +QyJspoV/0bXk6gG987PonK2Uk5djxSULhnGVIqswydyH0Nzb+slRp2bSoWbaNlee ++6TIeiyTQYc055pCHOp22gtLrC5LQGchksi02St2ZzRHdnlfqCJ8S9sS7x3trzds +cYueg1sGI+O8szpQ3eUM7OhJOBrx6OlR7+QYnQg1wr/V+JAz1qcyTC1URcwfeqtg +QjxFdBD9LfCtfK+AO51H9ugtsPJqOh33PmvfvUBEM05OHCA0lNaWJHROGpm4T4cc +YQI9JQk/0lB7itF1qK5RG74qgKdjkBkfZxi0OqkUgHk6YHtJlKfET8zfrtcCAwEA +AaNQME4wHQYDVR0OBBYEFGwb0NgH0Zr7Ga23njEJ85Ozf8M9MB8GA1UdIwQYMBaA +FGwb0NgH0Zr7Ga23njEJ85Ozf8M9MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggIBAKU3RHXggbq/pLhGinU5q/9QT0TB/0bBnF1wNFkKQC0FrNJ+ZnBNmusy +oqOn7DEohBCCDxT0kgOC05gLEsGLkSXlVyqCsPFfycCFhtu1QzSRtQNRxB3pW3Wq +4/RFVYv0PGBjVBKxImQlEmXJWEDwemGKqDQZPtqR/FTHTbJcaT0xQr5+1oG6lawt +I/2cW6GQ0kYW/Szps8FgNdSNgVqCjjNIzBYbWhRWMx/63qD1ReUbY7/Yw9KKT8nK +zXERpbTM9k+Pnm0g9Gep+9HJ1dBFJeuTPugKeSeyqg2OJbENw1hxGs/HjBXw7580 +ioiMn/kMj6Tg/f3HCfKrdHHBFQw0/fJW6o17QImYIpPOPzc5RjXBrCJWb34kxqEd +NQdKgejWiV/LlVsguIF8hVZH2kRzvoyypkVUtSUYGmjvA5UXoORQZfJ+b41llq1B +GcSF6iaVbAFKnsUyyr1i9uHz/6Muqflphv/SfZxGheIn5u3PnhXrzDagvItjw0NS +n0Xq64k7fc42HXJpF8CGBkSaIhtlzcruO+vqR80B9r62+D0V7VmHOnP135MT6noU +8F0JQfEtP+I8NII5jHSF/khzSgP5g80LS9tEc2ILnIHK1StkInAoRQQ+/HsQsgbz +ANAf5kxmMsM0zlN2hkxl0H6o7wKlBSw3RI3cjfilXiMWRPJrzlc4 +-----END CERTIFICATE----- +` diff --git a/setup/config/config_userapi.go b/setup/config/config_userapi.go new file mode 100644 index 00000000..2cbd8a45 --- /dev/null +++ b/setup/config/config_userapi.go @@ -0,0 +1,30 @@ +package config + +type UserAPI struct { + Matrix *Global `yaml:"-"` + + InternalAPI InternalAPIOptions `yaml:"internal_api"` + + // The Account database stores the login details and account information + // for local users. It is accessed by the UserAPI. + AccountDatabase DatabaseOptions `yaml:"account_database"` + // The Device database stores session information for the devices of logged + // in local users. It is accessed by the UserAPI. + DeviceDatabase DatabaseOptions `yaml:"device_database"` +} + +func (c *UserAPI) Defaults() { + c.InternalAPI.Listen = "http://localhost:7781" + c.InternalAPI.Connect = "http://localhost:7781" + c.AccountDatabase.Defaults() + c.DeviceDatabase.Defaults() + c.AccountDatabase.ConnectionString = "file:userapi_accounts.db" + c.DeviceDatabase.ConnectionString = "file:userapi_devices.db" +} + +func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect)) + checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString)) + checkNotEmpty(configErrs, "user_api.device_database.connection_string", string(c.DeviceDatabase.ConnectionString)) +} -- cgit v1.2.3