diff options
author | Kegsay <kegan@matrix.org> | 2020-05-21 14:40:13 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-05-21 14:40:13 +0100 |
commit | 24d8df664c21fa8bd68d80b5585a496e264c410a (patch) | |
tree | 0a176d6dfd7f81522c5739b53313366b552b0ce1 /internal/config | |
parent | 3fdb045116c9cd2f2a3badfebec0645d0381bacb (diff) |
Fix #897 and shuffle directory around (#1054)
* Fix #897 and shuffle directory around
* Update find-lint
* goimports
Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
Diffstat (limited to 'internal/config')
-rw-r--r-- | internal/config/appservice.go | 329 | ||||
-rw-r--r-- | internal/config/config.go | 804 | ||||
-rw-r--r-- | internal/config/config_test.go | 145 |
3 files changed, 1278 insertions, 0 deletions
diff --git a/internal/config/appservice.go b/internal/config/appservice.go new file mode 100644 index 00000000..bf5f089b --- /dev/null +++ b/internal/config/appservice.go @@ -0,0 +1,329 @@ +// Copyright 2017 Andrew Morgan <andrew@amorgan.xyz> +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "regexp" + "strings" + + log "github.com/sirupsen/logrus" + yaml "gopkg.in/yaml.v2" +) + +// ApplicationServiceNamespace is the namespace that a specific application +// service has management over. +type ApplicationServiceNamespace struct { + // Whether or not the namespace is managed solely by this application service + Exclusive bool `yaml:"exclusive"` + // A regex pattern that represents the namespace + Regex string `yaml:"regex"` + // The ID of an existing group that all users of this application service will + // be added to. This field is only relevant to the `users` namespace. + // Note that users who are joined to this group through an application service + // are not to be listed when querying for the group's members, however the + // group should be listed when querying an application service user's groups. + // This is to prevent making spamming all users of an application service + // trivial. + GroupID string `yaml:"group_id"` + // Regex object representing our pattern. Saves having to recompile every time + RegexpObject *regexp.Regexp +} + +// ApplicationService represents a Matrix application service. +// https://matrix.org/docs/spec/application_service/unstable.html +type ApplicationService struct { + // User-defined, unique, persistent ID of the application service + ID string `yaml:"id"` + // Base URL of the application service + URL string `yaml:"url"` + // Application service token provided in requests to a homeserver + ASToken string `yaml:"as_token"` + // Homeserver token provided in requests to an application service + HSToken string `yaml:"hs_token"` + // Localpart of application service user + SenderLocalpart string `yaml:"sender_localpart"` + // Information about an application service's namespaces. Key is either + // "users", "aliases" or "rooms" + NamespaceMap map[string][]ApplicationServiceNamespace `yaml:"namespaces"` + // Whether rate limiting is applied to each application service user + RateLimited bool `yaml:"rate_limited"` + // Any custom protocols that this application service provides (e.g. IRC) + Protocols []string `yaml:"protocols"` +} + +// IsInterestedInRoomID returns a bool on whether an application service's +// namespace includes the given room ID +func (a *ApplicationService) IsInterestedInRoomID( + roomID string, +) bool { + if namespaceSlice, ok := a.NamespaceMap["rooms"]; ok { + for _, namespace := range namespaceSlice { + if namespace.RegexpObject.MatchString(roomID) { + return true + } + } + } + + return false +} + +// IsInterestedInUserID returns a bool on whether an application service's +// namespace includes the given user ID +func (a *ApplicationService) IsInterestedInUserID( + userID string, +) bool { + if namespaceSlice, ok := a.NamespaceMap["users"]; ok { + for _, namespace := range namespaceSlice { + if namespace.RegexpObject.MatchString(userID) { + return true + } + } + } + + return false +} + +// OwnsNamespaceCoveringUserId returns a bool on whether an application service's +// namespace is exclusive and includes the given user ID +func (a *ApplicationService) OwnsNamespaceCoveringUserId( + userID string, +) bool { + if namespaceSlice, ok := a.NamespaceMap["users"]; ok { + for _, namespace := range namespaceSlice { + if namespace.Exclusive && namespace.RegexpObject.MatchString(userID) { + return true + } + } + } + + return false +} + +// IsInterestedInRoomAlias returns a bool on whether an application service's +// namespace includes the given room alias +func (a *ApplicationService) IsInterestedInRoomAlias( + roomAlias string, +) bool { + if namespaceSlice, ok := a.NamespaceMap["aliases"]; ok { + for _, namespace := range namespaceSlice { + if namespace.RegexpObject.MatchString(roomAlias) { + return true + } + } + } + + return false +} + +// loadAppServices iterates through all application service config files +// and loads their data into the config object for later access. +func loadAppServices(config *Dendrite) error { + for _, configPath := range config.ApplicationServices.ConfigFiles { + // Create a new application service with default options + appservice := ApplicationService{ + RateLimited: true, + } + + // Create an absolute path from a potentially relative path + absPath, err := filepath.Abs(configPath) + if err != nil { + return err + } + + // Read the application service's config file + configData, err := ioutil.ReadFile(absPath) + if err != nil { + return err + } + + // Load the config data into our struct + if err = yaml.UnmarshalStrict(configData, &appservice); err != nil { + return err + } + + // Append the parsed application service to the global config + config.Derived.ApplicationServices = append( + config.Derived.ApplicationServices, appservice, + ) + } + + // Check for any errors in the loaded application services + return checkErrors(config) +} + +// setupRegexps will create regex objects for exclusive and non-exclusive +// usernames, aliases and rooms of all application services, so that other +// methods can quickly check if a particular string matches any of them. +func setupRegexps(cfg *Dendrite) (err error) { + // Combine all exclusive namespaces for later string checking + var exclusiveUsernameStrings, exclusiveAliasStrings []string + + // If an application service's regex is marked as exclusive, add + // its contents to the overall exlusive regex string. Room regex + // not necessary as we aren't denying exclusive room ID creation + for _, appservice := range cfg.Derived.ApplicationServices { + for key, namespaceSlice := range appservice.NamespaceMap { + switch key { + case "users": + appendExclusiveNamespaceRegexs(&exclusiveUsernameStrings, namespaceSlice) + case "aliases": + appendExclusiveNamespaceRegexs(&exclusiveAliasStrings, namespaceSlice) + } + } + } + + // Join the regexes together into one big regex. + // i.e. "app1.*", "app2.*" -> "(app1.*)|(app2.*)" + // Later we can check if a username or alias matches any exclusive regex and + // deny access if it isn't from an application service + exclusiveUsernames := strings.Join(exclusiveUsernameStrings, "|") + exclusiveAliases := strings.Join(exclusiveAliasStrings, "|") + + // If there are no exclusive regexes, compile string so that it will not match + // any valid usernames/aliases/roomIDs + if exclusiveUsernames == "" { + exclusiveUsernames = "^$" + } + if exclusiveAliases == "" { + exclusiveAliases = "^$" + } + + // Store compiled Regex + if cfg.Derived.ExclusiveApplicationServicesUsernameRegexp, err = regexp.Compile(exclusiveUsernames); err != nil { + return err + } + if cfg.Derived.ExclusiveApplicationServicesAliasRegexp, err = regexp.Compile(exclusiveAliases); err != nil { + return err + } + + return nil +} + +// appendExclusiveNamespaceRegexs takes a slice of strings and a slice of +// namespaces and will append the regexes of only the exclusive namespaces +// into the string slice +func appendExclusiveNamespaceRegexs( + exclusiveStrings *[]string, namespaces []ApplicationServiceNamespace, +) { + for index, namespace := range namespaces { + if namespace.Exclusive { + // We append parenthesis to later separate each regex when we compile + // i.e. "app1.*", "app2.*" -> "(app1.*)|(app2.*)" + *exclusiveStrings = append(*exclusiveStrings, "("+namespace.Regex+")") + } + + // Compile this regex into a Regexp object for later use + namespaces[index].RegexpObject, _ = regexp.Compile(namespace.Regex) + } +} + +// checkErrors checks for any configuration errors amongst the loaded +// application services according to the application service spec. +func checkErrors(config *Dendrite) (err error) { + var idMap = make(map[string]bool) + var tokenMap = make(map[string]bool) + + // Compile regexp object for checking groupIDs + groupIDRegexp := regexp.MustCompile(`\+.*:.*`) + + // Check each application service for any config errors + for _, appservice := range config.Derived.ApplicationServices { + // Namespace-related checks + for key, namespaceSlice := range appservice.NamespaceMap { + for _, namespace := range namespaceSlice { + if err := validateNamespace(&appservice, key, &namespace, groupIDRegexp); err != nil { + return err + } + } + } + + // Check if the url has trailing /'s. If so, remove them + appservice.URL = strings.TrimRight(appservice.URL, "/") + + // Check if we've already seen this ID. No two application services + // can have the same ID or token. + if idMap[appservice.ID] { + return configErrors([]string{fmt.Sprintf( + "Application service ID %s must be unique", appservice.ID, + )}) + } + // Check if we've already seen this token + if tokenMap[appservice.ASToken] { + return configErrors([]string{fmt.Sprintf( + "Application service Token %s must be unique", appservice.ASToken, + )}) + } + + // Add the id/token to their respective maps if we haven't already + // seen them. + idMap[appservice.ID] = true + tokenMap[appservice.ASToken] = true + + // TODO: Remove once rate_limited is implemented + if appservice.RateLimited { + log.Warn("WARNING: Application service option rate_limited is currently unimplemented") + } + // TODO: Remove once protocols is implemented + if len(appservice.Protocols) > 0 { + log.Warn("WARNING: Application service option protocols is currently unimplemented") + } + } + + return setupRegexps(config) +} + +// validateNamespace returns nil or an error based on whether a given +// application service namespace is valid. A namespace is valid if it has the +// required fields, and its regex is correct. +func validateNamespace( + appservice *ApplicationService, + key string, + namespace *ApplicationServiceNamespace, + groupIDRegexp *regexp.Regexp, +) error { + // Check that namespace(s) are valid regex + if !IsValidRegex(namespace.Regex) { + return configErrors([]string{fmt.Sprintf( + "Invalid regex string for Application Service %s", appservice.ID, + )}) + } + + // Check if GroupID for the users namespace is in the correct format + if key == "users" && namespace.GroupID != "" { + // TODO: Remove once group_id is implemented + log.Warn("WARNING: Application service option group_id is currently unimplemented") + + correctFormat := groupIDRegexp.MatchString(namespace.GroupID) + if !correctFormat { + return configErrors([]string{fmt.Sprintf( + "Invalid user group_id field for application service %s.", + appservice.ID, + )}) + } + } + + return nil +} + +// IsValidRegex returns true or false based on whether the +// given string is valid regex or not +func IsValidRegex(regexString string) bool { + _, err := regexp.Compile(regexString) + + return err == nil +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 00000000..e1e96f9d --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,804 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "bytes" + "crypto/sha256" + "encoding/pem" + "fmt" + "io" + "io/ioutil" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" + "golang.org/x/crypto/ed25519" + yaml "gopkg.in/yaml.v2" + + jaegerconfig "github.com/uber/jaeger-client-go/config" + jaegermetrics "github.com/uber/jaeger-lib/metrics" +) + +// Version is the current version of the config format. +// This will change whenever we make breaking changes to the config format. +const Version = 0 + +// Dendrite contains all the config used by a dendrite process. +// Relative paths are resolved relative to the current working directory +type Dendrite struct { + // The version of the configuration file. + // If the version in a file doesn't match the current dendrite config + // version then we can give a clear error message telling the user + // to update their config file to the current version. + // The version of the file should only be different if there has + // been a breaking change to the config file format. + Version int `yaml:"version"` + + // The configuration required for a matrix server. + Matrix struct { + // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. + ServerName gomatrixserverlib.ServerName `yaml:"server_name"` + // Path to the private key which will be used to sign requests and events. + PrivateKeyPath Path `yaml:"private_key"` + // The private key which will be used to sign requests and events. + PrivateKey ed25519.PrivateKey `yaml:"-"` + // An arbitrary string used to uniquely identify the PrivateKey. Must start with the + // prefix "ed25519:". + KeyID gomatrixserverlib.KeyID `yaml:"-"` + // List of paths to X509 certificates used by the external federation listeners. + // These are used to calculate the TLS fingerprints to publish for this server. + // Other matrix servers talking to this server will expect the x509 certificate + // to match one of these certificates. + // The certificates should be in PEM format. + FederationCertificatePaths []Path `yaml:"federation_certificates"` + // A list of SHA256 TLS fingerprints for the X509 certificates used by the + // federation listener for this server. + TLSFingerPrints []gomatrixserverlib.TLSFingerprint `yaml:"-"` + // How long a remote server can cache our server key for before requesting it again. + // Increasing this number will reduce the number of requests made by remote servers + // for our key, but increases the period a compromised key will be considered valid + // by remote servers. + // Defaults to 24 hours. + KeyValidityPeriod time.Duration `yaml:"key_validity_period"` + // List of domains that the server will trust as identity servers to + // verify third-party identifiers. + // Defaults to an empty array. + TrustedIDServers []string `yaml:"trusted_third_party_id_servers"` + // If set, allows registration by anyone who also has the shared + // secret, even if registration is otherwise disabled. + RegistrationSharedSecret string `yaml:"registration_shared_secret"` + // This Home Server's ReCAPTCHA public key. + RecaptchaPublicKey string `yaml:"recaptcha_public_key"` + // This Home Server's ReCAPTCHA private key. + RecaptchaPrivateKey string `yaml:"recaptcha_private_key"` + // Boolean stating whether catpcha registration is enabled + // and required + RecaptchaEnabled bool `yaml:"enable_registration_captcha"` + // Secret used to bypass the captcha registration entirely + RecaptchaBypassSecret string `yaml:"captcha_bypass_secret"` + // HTTP API endpoint used to verify whether the captcha response + // was successful + RecaptchaSiteVerifyAPI string `yaml:"recaptcha_siteverify_api"` + // If set disables new users from registering (except via shared + // secrets) + RegistrationDisabled bool `yaml:"registration_disabled"` + // Perspective keyservers, to use as a backup when direct key fetch + // requests don't succeed + KeyPerspectives KeyPerspectives `yaml:"key_perspectives"` + } `yaml:"matrix"` + + // The configuration specific to the media repostitory. + Media struct { + // The base path to where the media files will be stored. May be relative or absolute. + BasePath Path `yaml:"base_path"` + // The absolute base path to where media files will be stored. + AbsBasePath Path `yaml:"-"` + // The maximum file size in bytes that is allowed to be stored on this server. + // Note: if max_file_size_bytes is set to 0, the size is unlimited. + // Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) + MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"` + // Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated + DynamicThumbnails bool `yaml:"dynamic_thumbnails"` + // The maximum number of simultaneous thumbnail generators. default: 10 + MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"` + // A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content + ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"` + } `yaml:"media"` + + // The configuration to use for Prometheus metrics + Metrics struct { + // Whether or not the metrics are enabled + Enabled bool `yaml:"enabled"` + // Use BasicAuth for Authorization + BasicAuth struct { + // Authorization via Static Username & Password + // Hardcoded Username and Password + Username string `yaml:"username"` + Password string `yaml:"password"` + } `yaml:"basic_auth"` + } `yaml:"metrics"` + + // The configuration for talking to kafka. + Kafka struct { + // A list of kafka addresses to connect to. + Addresses []string `yaml:"addresses"` + // Whether to use naffka instead of kafka. + // Naffka can only be used when running dendrite as a single monolithic server. + // Kafka can be used both with a monolithic server and when running the + // components as separate servers. + UseNaffka bool `yaml:"use_naffka,omitempty"` + // The names of the topics to use when reading and writing from kafka. + Topics struct { + // Topic for roomserver/api.OutputRoomEvent events. + OutputRoomEvent Topic `yaml:"output_room_event"` + // Topic for sending account data from client API to sync API + OutputClientData Topic `yaml:"output_client_data"` + // Topic for eduserver/api.OutputTypingEvent events. + OutputTypingEvent Topic `yaml:"output_typing_event"` + // Topic for user updates (profile, presence) + UserUpdates Topic `yaml:"user_updates"` + } + } `yaml:"kafka"` + + // Postgres Config + Database struct { + // The Account database stores the login details and account information + // for local users. It is accessed by the ClientAPI. + Account DataSource `yaml:"account"` + // The Device database stores session information for the devices of logged + // in local users. It is accessed by the ClientAPI, the MediaAPI and the SyncAPI. + Device DataSource `yaml:"device"` + // The MediaAPI database stores information about files uploaded and downloaded + // by local users. It is only accessed by the MediaAPI. + MediaAPI DataSource `yaml:"media_api"` + // The ServerKey database caches the public keys of remote servers. + // It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI. + ServerKey DataSource `yaml:"server_key"` + // The SyncAPI stores information used by the SyncAPI server. + // It is only accessed by the SyncAPI server. + SyncAPI DataSource `yaml:"sync_api"` + // The RoomServer database stores information about matrix rooms. + // It is only accessed by the RoomServer. + RoomServer DataSource `yaml:"room_server"` + // The FederationSender database stores information used by the FederationSender + // It is only accessed by the FederationSender. + FederationSender DataSource `yaml:"federation_sender"` + // The AppServices database stores information used by the AppService component. + // It is only accessed by the AppService component. + AppService DataSource `yaml:"appservice"` + // The PublicRoomsAPI database stores information used to compute the public + // room directory. It is only accessed by the PublicRoomsAPI server. + PublicRoomsAPI DataSource `yaml:"public_rooms_api"` + // The Naffka database is used internally by the naffka library, if used. + Naffka DataSource `yaml:"naffka,omitempty"` + // Maximum open connections to the DB (0 = use default, negative means unlimited) + MaxOpenConns int `yaml:"max_open_conns"` + // Maximum idle connections to the DB (0 = use default, negative means unlimited) + MaxIdleConns int `yaml:"max_idle_conns"` + // maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited) + ConnMaxLifetimeSec int `yaml:"conn_max_lifetime"` + } `yaml:"database"` + + // TURN Server Config + TURN struct { + // TODO Guest Support + // Whether or not guests can request TURN credentials + //AllowGuests bool `yaml:"turn_allow_guests"` + // How long the authorization should last + UserLifetime string `yaml:"turn_user_lifetime"` + // The list of TURN URIs to pass to clients + URIs []string `yaml:"turn_uris"` + + // Authorization via Shared Secret + // The shared secret from coturn + SharedSecret string `yaml:"turn_shared_secret"` + + // Authorization via Static Username & Password + // Hardcoded Username and Password + Username string `yaml:"turn_username"` + Password string `yaml:"turn_password"` + } `yaml:"turn"` + + // The internal addresses the components will listen on. + // These should not be exposed externally as they expose metrics and debugging APIs. + // Falls back to addresses listed in Listen if not specified + Bind struct { + MediaAPI Address `yaml:"media_api"` + ClientAPI Address `yaml:"client_api"` + FederationAPI Address `yaml:"federation_api"` + AppServiceAPI Address `yaml:"appservice_api"` + SyncAPI Address `yaml:"sync_api"` + RoomServer Address `yaml:"room_server"` + FederationSender Address `yaml:"federation_sender"` + PublicRoomsAPI Address `yaml:"public_rooms_api"` + EDUServer Address `yaml:"edu_server"` + KeyServer Address `yaml:"key_server"` + } `yaml:"bind"` + + // The addresses for talking to other microservices. + Listen struct { + MediaAPI Address `yaml:"media_api"` + ClientAPI Address `yaml:"client_api"` + FederationAPI Address `yaml:"federation_api"` + AppServiceAPI Address `yaml:"appservice_api"` + SyncAPI Address `yaml:"sync_api"` + RoomServer Address `yaml:"room_server"` + FederationSender Address `yaml:"federation_sender"` + PublicRoomsAPI Address `yaml:"public_rooms_api"` + EDUServer Address `yaml:"edu_server"` + KeyServer Address `yaml:"key_server"` + } `yaml:"listen"` + + // The config for tracing the dendrite servers. + Tracing struct { + // Set to true to enable tracer hooks. If false, no tracing is set up. + Enabled bool `yaml:"enabled"` + // The config for the jaeger opentracing reporter. + Jaeger jaegerconfig.Configuration `yaml:"jaeger"` + } `yaml:"tracing"` + + // Application Services + // https://matrix.org/docs/spec/application_service/unstable.html + ApplicationServices struct { + // Configuration files for various application services + ConfigFiles []string `yaml:"config_files"` + } `yaml:"application_services"` + + // The config for logging informations. Each hook will be added to logrus. + Logging []LogrusHook `yaml:"logging"` + + // Any information derived from the configuration options for later use. + Derived struct { + Registration struct { + // Flows is a slice of flows, which represent one possible way that the client can authenticate a request. + // http://matrix.org/docs/spec/HEAD/client_server/r0.3.0.html#user-interactive-authentication-api + // As long as the generated flows only rely on config file options, + // we can generate them on startup and store them until needed + Flows []authtypes.Flow `json:"flows"` + + // Params that need to be returned to the client during + // registration in order to complete registration stages. + Params map[string]interface{} `json:"params"` + } + + // Application services parsed from their config files + // The paths of which were given above in the main config file + ApplicationServices []ApplicationService + + // Meta-regexes compiled from all exclusive application service + // Regexes. + // + // When a user registers, we check that their username does not match any + // exclusive application service namespaces + ExclusiveApplicationServicesUsernameRegexp *regexp.Regexp + // When a user creates a room alias, we check that it isn't already + // reserved by an application service + ExclusiveApplicationServicesAliasRegexp *regexp.Regexp + // Note: An Exclusive Regex for room ID isn't necessary as we aren't blocking + // servers from creating RoomIDs in exclusive application service namespaces + } `yaml:"-"` +} + +// KeyPerspectives are used to configure perspective key servers for +// retrieving server keys. +type KeyPerspectives []struct { + // The server name of the perspective key server + ServerName gomatrixserverlib.ServerName `yaml:"server_name"` + // Server keys for the perspective user, used to verify the + // keys have been signed by the perspective server + Keys []struct { + // The key ID, e.g. ed25519:auto + KeyID gomatrixserverlib.KeyID `yaml:"key_id"` + // The public key in base64 unpadded format + PublicKey string `yaml:"public_key"` + } `yaml:"keys"` +} + +// A Path on the filesystem. +type Path string + +// A DataSource for opening a postgresql database using lib/pq. +type DataSource string + +// A Topic in kafka. +type Topic string + +// An Address to listen on. +type Address string + +// FileSizeBytes is a file size in bytes +type FileSizeBytes int64 + +// ThumbnailSize contains a single thumbnail size configuration +type ThumbnailSize struct { + // Maximum width of the thumbnail image + Width int `yaml:"width"` + // Maximum height of the thumbnail image + Height int `yaml:"height"` + // ResizeMethod is one of crop or scale. + // crop scales to fill the requested dimensions and crops the excess. + // scale scales to fit the requested dimensions and one dimension may be smaller than requested. + ResizeMethod string `yaml:"method,omitempty"` +} + +// LogrusHook represents a single logrus hook. At this point, only parsing and +// verification of the proper values for type and level are done. +// Validity/integrity checks on the parameters are done when configuring logrus. +type LogrusHook struct { + // The type of hook, currently only "file" is supported. + Type string `yaml:"type"` + + // The level of the logs to produce. Will output only this level and above. + Level string `yaml:"level"` + + // The parameters for this hook. + Params map[string]interface{} `yaml:"params"` +} + +// configErrors stores problems encountered when parsing a config file. +// It implements the error interface. +type configErrors []string + +// Load a yaml config file for a server run as multiple processes. +// Checks the config to ensure that it is valid. +// The checks are different if the server is run as a monolithic process instead +// of being split into multiple components +func Load(configPath string) (*Dendrite, error) { + configData, err := ioutil.ReadFile(configPath) + if err != nil { + return nil, err + } + basePath, err := filepath.Abs(".") + if err != nil { + return nil, err + } + // Pass the current working directory and ioutil.ReadFile so that they can + // be mocked in the tests + monolithic := false + return loadConfig(basePath, configData, ioutil.ReadFile, monolithic) +} + +// LoadMonolithic loads a yaml config file for a server run as a single monolith. +// Checks the config to ensure that it is valid. +// The checks are different if the server is run as a monolithic process instead +// of being split into multiple components +func LoadMonolithic(configPath string) (*Dendrite, error) { + configData, err := ioutil.ReadFile(configPath) + if err != nil { + return nil, err + } + basePath, err := filepath.Abs(".") + if err != nil { + return nil, err + } + // Pass the current working directory and ioutil.ReadFile so that they can + // be mocked in the tests + monolithic := true + return loadConfig(basePath, configData, ioutil.ReadFile, monolithic) +} + +func loadConfig( + basePath string, + configData []byte, + readFile func(string) ([]byte, error), + monolithic bool, +) (*Dendrite, error) { + var config Dendrite + var err error + if err = yaml.Unmarshal(configData, &config); err != nil { + return nil, err + } + + config.SetDefaults() + + if err = config.check(monolithic); err != nil { + return nil, err + } + + privateKeyPath := absPath(basePath, config.Matrix.PrivateKeyPath) + privateKeyData, err := readFile(privateKeyPath) + if err != nil { + return nil, err + } + + if config.Matrix.KeyID, config.Matrix.PrivateKey, err = readKeyPEM(privateKeyPath, privateKeyData); err != nil { + return nil, err + } + + for _, certPath := range config.Matrix.FederationCertificatePaths { + absCertPath := absPath(basePath, certPath) + var pemData []byte + pemData, err = readFile(absCertPath) + if err != nil { + return nil, err + } + fingerprint := fingerprintPEM(pemData) + if fingerprint == nil { + return nil, fmt.Errorf("no certificate PEM data in %q", absCertPath) + } + config.Matrix.TLSFingerPrints = append(config.Matrix.TLSFingerPrints, *fingerprint) + } + + config.Media.AbsBasePath = Path(absPath(basePath, config.Media.BasePath)) + + // Generate data from config options + err = config.Derive() + if err != nil { + return nil, err + } + + return &config, nil +} + +// Derive generates data that is derived from various values provided in +// the config file. +func (config *Dendrite) Derive() error { + // Determine registrations flows based off config values + + config.Derived.Registration.Params = make(map[string]interface{}) + + // TODO: Add email auth type + // TODO: Add MSISDN auth type + + if config.Matrix.RecaptchaEnabled { + config.Derived.Registration.Params[authtypes.LoginTypeRecaptcha] = map[string]string{"public_key": config.Matrix.RecaptchaPublicKey} + config.Derived.Registration.Flows = append(config.Derived.Registration.Flows, + authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeRecaptcha}}) + } else { + config.Derived.Registration.Flows = append(config.Derived.Registration.Flows, + authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeDummy}}) + } + + // Load application service configuration files + if err := loadAppServices(config); err != nil { + return err + } + + return nil +} + +// SetDefaults sets default config values if they are not explicitly set. +func (config *Dendrite) SetDefaults() { + if config.Matrix.KeyValidityPeriod == 0 { + config.Matrix.KeyValidityPeriod = 24 * time.Hour + } + + if config.Matrix.TrustedIDServers == nil { + config.Matrix.TrustedIDServers = []string{} + } + + if config.Media.MaxThumbnailGenerators == 0 { + config.Media.MaxThumbnailGenerators = 10 + } + + if config.Media.MaxFileSizeBytes == nil { + defaultMaxFileSizeBytes := FileSizeBytes(10485760) + config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes + } + + if config.Database.MaxIdleConns == 0 { + config.Database.MaxIdleConns = 2 + } + + if config.Database.MaxOpenConns == 0 { + config.Database.MaxOpenConns = 100 + } + +} + +// Error returns a string detailing how many errors were contained within a +// configErrors type. +func (errs configErrors) Error() string { + if len(errs) == 1 { + return errs[0] + } + return fmt.Sprintf( + "%s (and %d other problems)", errs[0], len(errs)-1, + ) +} + +// Add appends an error to the list of errors in this configErrors. +// It is a wrapper to the builtin append and hides pointers from +// the client code. +// This method is safe to use with an uninitialized configErrors because +// if it is nil, it will be properly allocated. +func (errs *configErrors) Add(str string) { + *errs = append(*errs, str) +} + +// checkNotEmpty verifies the given value is not empty in the configuration. +// If it is, adds an error to the list. +func checkNotEmpty(configErrs *configErrors, key, value string) { + if value == "" { + configErrs.Add(fmt.Sprintf("missing config key %q", key)) + } +} + +// checkNotZero verifies the given value is not zero in the configuration. +// If it is, adds an error to the list. +func checkNotZero(configErrs *configErrors, key string, value int64) { + if value == 0 { + configErrs.Add(fmt.Sprintf("missing config key %q", key)) + } +} + +// checkPositive verifies the given value is positive (zero included) +// in the configuration. If it is not, adds an error to the list. +func checkPositive(configErrs *configErrors, key string, value int64) { + if value < 0 { + configErrs.Add(fmt.Sprintf("invalid value for config key %q: %d", key, value)) + } +} + +// checkTurn verifies the parameters turn.* are valid. +func (config *Dendrite) checkTurn(configErrs *configErrors) { + value := config.TURN.UserLifetime + if value != "" { + if _, err := time.ParseDuration(value); err != nil { + configErrs.Add(fmt.Sprintf("invalid duration for config key %q: %s", "turn.turn_user_lifetime", value)) + } + } +} + +// checkMatrix verifies the parameters matrix.* are valid. +func (config *Dendrite) checkMatrix(configErrs *configErrors) { + checkNotEmpty(configErrs, "matrix.server_name", string(config.Matrix.ServerName)) + checkNotEmpty(configErrs, "matrix.private_key", string(config.Matrix.PrivateKeyPath)) + checkNotZero(configErrs, "matrix.federation_certificates", int64(len(config.Matrix.FederationCertificatePaths))) + if config.Matrix.RecaptchaEnabled { + checkNotEmpty(configErrs, "matrix.recaptcha_public_key", string(config.Matrix.RecaptchaPublicKey)) + checkNotEmpty(configErrs, "matrix.recaptcha_private_key", string(config.Matrix.RecaptchaPrivateKey)) + checkNotEmpty(configErrs, "matrix.recaptcha_siteverify_api", string(config.Matrix.RecaptchaSiteVerifyAPI)) + } +} + +// checkMedia verifies the parameters media.* are valid. +func (config *Dendrite) checkMedia(configErrs *configErrors) { + checkNotEmpty(configErrs, "media.base_path", string(config.Media.BasePath)) + checkPositive(configErrs, "media.max_file_size_bytes", int64(*config.Media.MaxFileSizeBytes)) + checkPositive(configErrs, "media.max_thumbnail_generators", int64(config.Media.MaxThumbnailGenerators)) + + for i, size := range config.Media.ThumbnailSizes { + checkPositive(configErrs, fmt.Sprintf("media.thumbnail_sizes[%d].width", i), int64(size.Width)) + checkPositive(configErrs, fmt.Sprintf("media.thumbnail_sizes[%d].height", i), int64(size.Height)) + } +} + +// checkKafka verifies the parameters kafka.* and the related +// database.naffka are valid. +func (config *Dendrite) checkKafka(configErrs *configErrors, monolithic bool) { + + if config.Kafka.UseNaffka { + if !monolithic { + configErrs.Add(fmt.Sprintf("naffka can only be used in a monolithic server")) + } + + checkNotEmpty(configErrs, "database.naffka", string(config.Database.Naffka)) + } else { + // If we aren't using naffka then we need to have at least one kafka + // server to talk to. + checkNotZero(configErrs, "kafka.addresses", int64(len(config.Kafka.Addresses))) + } + checkNotEmpty(configErrs, "kafka.topics.output_room_event", string(config.Kafka.Topics.OutputRoomEvent)) + checkNotEmpty(configErrs, "kafka.topics.output_client_data", string(config.Kafka.Topics.OutputClientData)) + checkNotEmpty(configErrs, "kafka.topics.output_typing_event", string(config.Kafka.Topics.OutputTypingEvent)) + checkNotEmpty(configErrs, "kafka.topics.user_updates", string(config.Kafka.Topics.UserUpdates)) +} + +// checkDatabase verifies the parameters database.* are valid. +func (config *Dendrite) checkDatabase(configErrs *configErrors) { + checkNotEmpty(configErrs, "database.account", string(config.Database.Account)) + checkNotEmpty(configErrs, "database.device", string(config.Database.Device)) + checkNotEmpty(configErrs, "database.server_key", string(config.Database.ServerKey)) + checkNotEmpty(configErrs, "database.media_api", string(config.Database.MediaAPI)) + checkNotEmpty(configErrs, "database.sync_api", string(config.Database.SyncAPI)) + checkNotEmpty(configErrs, "database.room_server", string(config.Database.RoomServer)) +} + +// checkListen verifies the parameters listen.* are valid. +func (config *Dendrite) checkListen(configErrs *configErrors) { + checkNotEmpty(configErrs, "listen.media_api", string(config.Listen.MediaAPI)) + checkNotEmpty(configErrs, "listen.client_api", string(config.Listen.ClientAPI)) + checkNotEmpty(configErrs, "listen.federation_api", string(config.Listen.FederationAPI)) + checkNotEmpty(configErrs, "listen.sync_api", string(config.Listen.SyncAPI)) + checkNotEmpty(configErrs, "listen.room_server", string(config.Listen.RoomServer)) + checkNotEmpty(configErrs, "listen.edu_server", string(config.Listen.EDUServer)) +} + +// checkLogging verifies the parameters logging.* are valid. +func (config *Dendrite) checkLogging(configErrs *configErrors) { + for _, logrusHook := range config.Logging { + checkNotEmpty(configErrs, "logging.type", string(logrusHook.Type)) + checkNotEmpty(configErrs, "logging.level", string(logrusHook.Level)) + } +} + +// check returns an error type containing all errors found within the config +// file. +func (config *Dendrite) check(monolithic bool) error { + var configErrs configErrors + + if config.Version != Version { + configErrs.Add(fmt.Sprintf( + "unknown config version %q, expected %q", config.Version, Version, + )) + return configErrs + } + + config.checkMatrix(&configErrs) + config.checkMedia(&configErrs) + config.checkTurn(&configErrs) + config.checkKafka(&configErrs, monolithic) + config.checkDatabase(&configErrs) + config.checkLogging(&configErrs) + + if !monolithic { + config.checkListen(&configErrs) + } + + // Due to how Golang manages its interface types, this condition is not redundant. + // In order to get the proper behaviour, it is necessary to return an explicit nil + // and not a nil configErrors. + // This is because the following equalities hold: + // error(nil) == nil + // error(configErrors(nil)) != nil + if configErrs != nil { + return configErrs + } + return nil +} + +// absPath returns the absolute path for a given relative or absolute path. +func absPath(dir string, path Path) string { + if filepath.IsAbs(string(path)) { + // filepath.Join cleans the path so we should clean the absolute paths as well for consistency. + return filepath.Clean(string(path)) + } + return filepath.Join(dir, string(path)) +} + +func readKeyPEM(path string, data []byte) (gomatrixserverlib.KeyID, ed25519.PrivateKey, error) { + for { + var keyBlock *pem.Block + keyBlock, data = pem.Decode(data) + if data == nil { + return "", nil, fmt.Errorf("no matrix private key PEM data in %q", path) + } + if keyBlock == nil { + return "", nil, fmt.Errorf("keyBlock is nil %q", path) + } + if keyBlock.Type == "MATRIX PRIVATE KEY" { + keyID := keyBlock.Headers["Key-ID"] + if keyID == "" { + return "", nil, fmt.Errorf("missing key ID in PEM data in %q", path) + } + if !strings.HasPrefix(keyID, "ed25519:") { + return "", nil, fmt.Errorf("key ID %q doesn't start with \"ed25519:\" in %q", keyID, path) + } + _, privKey, err := ed25519.GenerateKey(bytes.NewReader(keyBlock.Bytes)) + if err != nil { + return "", nil, err + } + return gomatrixserverlib.KeyID(keyID), privKey, nil + } + } +} + +func fingerprintPEM(data []byte) *gomatrixserverlib.TLSFingerprint { + for { + var certDERBlock *pem.Block + certDERBlock, data = pem.Decode(data) + if data == nil { + return nil + } + if certDERBlock.Type == "CERTIFICATE" { + digest := sha256.Sum256(certDERBlock.Bytes) + return &gomatrixserverlib.TLSFingerprint{SHA256: digest[:]} + } + } +} + +// AppServiceURL returns a HTTP URL for where the appservice component is listening. +func (config *Dendrite) AppServiceURL() string { + // Hard code the appservice server to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return "http://" + string(config.Listen.AppServiceAPI) +} + +// RoomServerURL returns an HTTP URL for where the roomserver is listening. +func (config *Dendrite) RoomServerURL() string { + // Hard code the roomserver to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return "http://" + string(config.Listen.RoomServer) +} + +// EDUServerURL returns an HTTP URL for where the EDU server is listening. +func (config *Dendrite) EDUServerURL() string { + // Hard code the EDU server to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return "http://" + string(config.Listen.EDUServer) +} + +// FederationSenderURL returns an HTTP URL for where the federation sender is listening. +func (config *Dendrite) FederationSenderURL() string { + // Hard code the federation sender server to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return "http://" + string(config.Listen.FederationSender) +} + +// SetupTracing configures the opentracing using the supplied configuration. +func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err error) { + if !config.Tracing.Enabled { + return ioutil.NopCloser(bytes.NewReader([]byte{})), nil + } + return config.Tracing.Jaeger.InitGlobalTracer( + serviceName, + jaegerconfig.Logger(logrusLogger{logrus.StandardLogger()}), + jaegerconfig.Metrics(jaegermetrics.NullFactory), + ) +} + +// MaxIdleConns returns maximum idle connections to the DB +func (config Dendrite) MaxIdleConns() int { + return config.Database.MaxIdleConns +} + +// MaxOpenConns returns maximum open connections to the DB +func (config Dendrite) MaxOpenConns() int { + return config.Database.MaxOpenConns +} + +// ConnMaxLifetime returns maximum amount of time a connection may be reused +func (config Dendrite) ConnMaxLifetime() time.Duration { + return time.Duration(config.Database.ConnMaxLifetimeSec) * time.Second +} + +// DbProperties functions return properties used by database/sql/DB +type DbProperties interface { + MaxIdleConns() int + MaxOpenConns() int + ConnMaxLifetime() time.Duration +} + +// DbProperties returns cfg as a DbProperties interface +func (config Dendrite) DbProperties() DbProperties { + return config +} + +// logrusLogger is a small wrapper that implements jaeger.Logger using logrus. +type logrusLogger struct { + l *logrus.Logger +} + +func (l logrusLogger) Error(msg string) { + l.l.Error(msg) +} + +func (l logrusLogger) Infof(msg string, args ...interface{}) { + l.l.Infof(msg, args...) +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 00000000..b72f5fad --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,145 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "testing" +) + +func TestLoadConfigRelative(t *testing.T) { + _, err := loadConfig("/my/config/dir", []byte(testConfig), + mockReadFile{ + "/my/config/dir/matrix_key.pem": testKey, + "/my/config/dir/tls_cert.pem": testCert, + }.readFile, + false, + ) + if err != nil { + t.Error("failed to load config:", err) + } +} + +const testConfig = ` +version: 0 +matrix: + server_name: localhost + private_key: matrix_key.pem + federation_certificates: [tls_cert.pem] +media: + base_path: media_store +kafka: + addresses: ["localhost:9092"] + topics: + output_room_event: output.room + output_client_data: output.client + output_typing_event: output.typing + user_updates: output.user +database: + media_api: "postgresql:///media_api" + account: "postgresql:///account" + device: "postgresql:///device" + server_key: "postgresql:///server_keys" + sync_api: "postgresql:///syn_api" + room_server: "postgresql:///room_server" + appservice: "postgresql:///appservice" +listen: + room_server: "localhost:7770" + client_api: "localhost:7771" + federation_api: "localhost:7772" + sync_api: "localhost:7773" + media_api: "localhost:7774" + appservice_api: "localhost:7777" + edu_server: "localhost:7778" +logging: + - type: "file" + level: "info" + params: + path: "/my/log/dir" +` + +type mockReadFile map[string]string + +func (m mockReadFile) readFile(path string) ([]byte, error) { + data, ok := m[path] + if !ok { + return nil, fmt.Errorf("no such file %q", path) + } + return []byte(data), nil +} + +func TestReadKey(t *testing.T) { + keyID, _, err := readKeyPEM("path/to/key", []byte(testKey)) + if err != nil { + t.Error("failed to load private key:", err) + } + wantKeyID := testKeyID + if wantKeyID != string(keyID) { + t.Errorf("wanted key ID to be %q, got %q", wantKeyID, keyID) + } +} + +const testKeyID = "ed25519:c8NsuQ" + +const testKey = ` +-----BEGIN MATRIX PRIVATE KEY----- +Key-ID: ` + testKeyID + ` +7KRZiZ2sTyRR8uqqUjRwczuwRXXkUMYIUHq4Mc3t4bE= +-----END MATRIX PRIVATE KEY----- +` + +func TestFingerprintPEM(t *testing.T) { + got := fingerprintPEM([]byte(testCert)) + if got == nil { + t.Error("failed to calculate fingerprint") + } + if string(got.SHA256) != testCertFingerprint { + t.Errorf("bad fingerprint: wanted %q got %q", got, testCertFingerprint) + } + +} + +const testCertFingerprint = "56.\\SPQxE\xd4\x95\xfb\xf6\xd5\x04\x91\xcb/\x07\xb1^\x88\x08\xe3\xc1p\xdfY\x04\x19w\xcb" + +const testCert = ` +-----BEGIN CERTIFICATE----- +MIIE0zCCArugAwIBAgIJAPype3u24LJeMA0GCSqGSIb3DQEBCwUAMAAwHhcNMTcw +NjEzMTQyODU4WhcNMTgwNjEzMTQyODU4WjAAMIICIjANBgkqhkiG9w0BAQEFAAOC +Ag8AMIICCgKCAgEA3vNSr7lCh/alxPFqairp/PYohwdsqPvOD7zf7dJCNhy0gbdC +9/APwIbPAPL9nU+o9ud1ACNCKBCQin/9LnI5vd5pa/Ne+mmRADDLB/BBBoywSJWG +NSfKJ9n3XY1bjgtqi53uUh+RDdQ7sXudDqCUxiiJZmS7oqK/mp88XXAgCbuXUY29 +GmzbbDz37vntuSxDgUOnJ8uPSvRp5YPKogA3JwW1SyrlLt4Z30CQ6nH3Y2Q5SVfJ +NIQyMrnwyjA9bCdXezv1cLXoTYn7U9BRyzXTZeXs3y3ldnRfISXN35CU04Az1F8j +lfj7nXMEqI/qAj/qhxZ8nVBB+rpNOZy9RJko3O+G5Qa/EvzkQYV1rW4TM2Yme88A +QyJspoV/0bXk6gG987PonK2Uk5djxSULhnGVIqswydyH0Nzb+slRp2bSoWbaNlee ++6TIeiyTQYc055pCHOp22gtLrC5LQGchksi02St2ZzRHdnlfqCJ8S9sS7x3trzds +cYueg1sGI+O8szpQ3eUM7OhJOBrx6OlR7+QYnQg1wr/V+JAz1qcyTC1URcwfeqtg +QjxFdBD9LfCtfK+AO51H9ugtsPJqOh33PmvfvUBEM05OHCA0lNaWJHROGpm4T4cc +YQI9JQk/0lB7itF1qK5RG74qgKdjkBkfZxi0OqkUgHk6YHtJlKfET8zfrtcCAwEA +AaNQME4wHQYDVR0OBBYEFGwb0NgH0Zr7Ga23njEJ85Ozf8M9MB8GA1UdIwQYMBaA +FGwb0NgH0Zr7Ga23njEJ85Ozf8M9MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggIBAKU3RHXggbq/pLhGinU5q/9QT0TB/0bBnF1wNFkKQC0FrNJ+ZnBNmusy +oqOn7DEohBCCDxT0kgOC05gLEsGLkSXlVyqCsPFfycCFhtu1QzSRtQNRxB3pW3Wq +4/RFVYv0PGBjVBKxImQlEmXJWEDwemGKqDQZPtqR/FTHTbJcaT0xQr5+1oG6lawt +I/2cW6GQ0kYW/Szps8FgNdSNgVqCjjNIzBYbWhRWMx/63qD1ReUbY7/Yw9KKT8nK +zXERpbTM9k+Pnm0g9Gep+9HJ1dBFJeuTPugKeSeyqg2OJbENw1hxGs/HjBXw7580 +ioiMn/kMj6Tg/f3HCfKrdHHBFQw0/fJW6o17QImYIpPOPzc5RjXBrCJWb34kxqEd +NQdKgejWiV/LlVsguIF8hVZH2kRzvoyypkVUtSUYGmjvA5UXoORQZfJ+b41llq1B +GcSF6iaVbAFKnsUyyr1i9uHz/6Muqflphv/SfZxGheIn5u3PnhXrzDagvItjw0NS +n0Xq64k7fc42HXJpF8CGBkSaIhtlzcruO+vqR80B9r62+D0V7VmHOnP135MT6noU +8F0JQfEtP+I8NII5jHSF/khzSgP5g80LS9tEc2ILnIHK1StkInAoRQQ+/HsQsgbz +ANAf5kxmMsM0zlN2hkxl0H6o7wKlBSw3RI3cjfilXiMWRPJrzlc4 +-----END CERTIFICATE----- +` |