diff options
author | Till <2353100+S7evinK@users.noreply.github.com> | 2023-02-14 12:47:47 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-02-14 12:47:47 +0100 |
commit | 11d9b9db0e96c51c1430d451d23cf5ae9f36e4ee (patch) | |
tree | a0837bfa69051295b76140e3940a45fc61854cab /cmd | |
parent | cc59879faa57cac043cf5f1585773b301994bebf (diff) |
Remove polylith/API mode (#2967)
This removes most of the code used for polylith/API mode.
This removes the `/api` internal endpoints entirely.
Binary size change roughly 5%:
```
51437560 Feb 13 10:15 dendrite-monolith-server # old
48759008 Feb 13 10:15 dendrite-monolith-server # new
```
Diffstat (limited to 'cmd')
18 files changed, 187 insertions, 618 deletions
diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go index 6f1c69a7..fe19593c 100644 --- a/cmd/dendrite-demo-pinecone/monolith/monolith.go +++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go @@ -80,8 +80,8 @@ type P2PMonolith struct { func GenerateDefaultConfig(sk ed25519.PrivateKey, storageDir string, cacheDir string, dbPrefix string) *config.Dendrite { cfg := config.Dendrite{} cfg.Defaults(config.DefaultOpts{ - Generate: true, - Monolithic: true, + Generate: true, + SingleDatabase: true, }) cfg.Global.PrivateKey = sk cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", filepath.Join(cacheDir, dbPrefix))) @@ -121,9 +121,9 @@ func (p *P2PMonolith) SetupPinecone(sk ed25519.PrivateKey) { func (p *P2PMonolith) SetupDendrite(cfg *config.Dendrite, port int, enableRelaying bool, enableMetrics bool, enableWebsockets bool) { if enableMetrics { - p.BaseDendrite = base.NewBaseDendrite(cfg, "Monolith") + p.BaseDendrite = base.NewBaseDendrite(cfg) } else { - p.BaseDendrite = base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics) + p.BaseDendrite = base.NewBaseDendrite(cfg, base.DisableMetrics) } p.port = port p.BaseDendrite.ConfigureAdminEndpoints() @@ -236,7 +236,6 @@ func (p *P2PMonolith) Addr() string { func (p *P2PMonolith) setupHttpServers(userProvider *users.PineconeUserProvider, enableWebsockets bool) { p.httpMux = mux.NewRouter().SkipClean(true).UseEncodedPath() - p.httpMux.PathPrefix(httputil.InternalPathPrefix).Handler(p.BaseDendrite.InternalAPIMux) p.httpMux.PathPrefix(httputil.PublicClientPathPrefix).Handler(p.BaseDendrite.PublicClientAPIMux) p.httpMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(p.BaseDendrite.PublicMediaAPIMux) p.httpMux.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(p.BaseDendrite.DendriteAdminMux) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 3ea4a08b..842682b4 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -117,8 +117,8 @@ func main() { cfg = setup.ParseFlags(true) } else { cfg.Defaults(config.DefaultOpts{ - Generate: true, - Monolithic: true, + Generate: true, + SingleDatabase: true, }) cfg.Global.PrivateKey = sk cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(*instanceDir, *instanceName)) @@ -143,7 +143,7 @@ func main() { cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk)) cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) - base := base.NewBaseDendrite(cfg, "Monolith") + base := base.NewBaseDendrite(cfg) base.ConfigureAdminEndpoints() defer base.Close() // nolint: errcheck @@ -157,13 +157,11 @@ func main() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - rsComponent := roomserver.NewInternalAPI( + rsAPI := roomserver.NewInternalAPI( base, ) - keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsComponent) - - rsAPI := rsComponent + keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsAPI) userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient()) keyAPI.SetUserAPI(userAPI) @@ -174,7 +172,7 @@ func main() { base, federation, rsAPI, base.Caches, keyRing, true, ) - rsComponent.SetFederationAPI(fsAPI, keyRing) + rsAPI.SetFederationAPI(fsAPI, keyRing) monolith := setup.Monolith{ Config: base.Cfg, @@ -197,7 +195,6 @@ func main() { } httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() - httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux) httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux) httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go deleted file mode 100644 index 6836b642..00000000 --- a/cmd/dendrite-monolith-server/main.go +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "os" - - "github.com/sirupsen/logrus" - - "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/federationapi" - "github.com/matrix-org/dendrite/keyserver" - "github.com/matrix-org/dendrite/roomserver" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/setup/mscs" - "github.com/matrix-org/dendrite/userapi" - uapi "github.com/matrix-org/dendrite/userapi/api" -) - -var ( - httpBindAddr = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server") - httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server") - apiBindAddr = flag.String("api-bind-address", "localhost:18008", "The HTTP listening port for the internal HTTP APIs (if -api is enabled)") - certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") - keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") - enableHTTPAPIs = flag.Bool("api", false, "Use HTTP APIs instead of short-circuiting (warning: exposes API endpoints!)") - traceInternal = os.Getenv("DENDRITE_TRACE_INTERNAL") == "1" -) - -func main() { - cfg := setup.ParseFlags(true) - httpAddr := config.HTTPAddress("http://" + *httpBindAddr) - httpsAddr := config.HTTPAddress("https://" + *httpsBindAddr) - httpAPIAddr := httpAddr - options := []basepkg.BaseDendriteOptions{} - if *enableHTTPAPIs { - logrus.Warnf("DANGER! The -api option is enabled, exposing internal APIs on %q!", *apiBindAddr) - httpAPIAddr = config.HTTPAddress("http://" + *apiBindAddr) - // If the HTTP APIs are enabled then we need to update the Listen - // statements in the configuration so that we know where to find - // the API endpoints. They'll listen on the same port as the monolith - // itself. - cfg.AppServiceAPI.InternalAPI.Connect = httpAPIAddr - cfg.ClientAPI.InternalAPI.Connect = httpAPIAddr - cfg.FederationAPI.InternalAPI.Connect = httpAPIAddr - cfg.KeyServer.InternalAPI.Connect = httpAPIAddr - cfg.MediaAPI.InternalAPI.Connect = httpAPIAddr - cfg.RoomServer.InternalAPI.Connect = httpAPIAddr - cfg.SyncAPI.InternalAPI.Connect = httpAPIAddr - cfg.UserAPI.InternalAPI.Connect = httpAPIAddr - options = append(options, basepkg.UseHTTPAPIs) - } - - base := basepkg.NewBaseDendrite(cfg, "Monolith", options...) - defer base.Close() // nolint: errcheck - - federation := base.CreateFederationClient() - - rsImpl := roomserver.NewInternalAPI(base) - // call functions directly on the impl unless running in HTTP mode - rsAPI := rsImpl - if base.UseHTTPAPIs { - roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl, base.EnableMetrics) - rsAPI = base.RoomserverHTTPClient() - } - if traceInternal { - rsAPI = &api.RoomserverInternalAPITrace{ - Impl: rsAPI, - } - } - - fsAPI := federationapi.NewInternalAPI( - base, federation, rsAPI, base.Caches, nil, false, - ) - fsImplAPI := fsAPI - if base.UseHTTPAPIs { - federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics) - fsAPI = base.FederationAPIHTTPClient() - } - keyRing := fsAPI.KeyRing() - - keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI) - keyAPI := keyImpl - if base.UseHTTPAPIs { - keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI, base.EnableMetrics) - keyAPI = base.KeyServerHTTPClient() - } - - pgClient := base.PushGatewayHTTPClient() - userImpl := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, pgClient) - userAPI := userImpl - if base.UseHTTPAPIs { - userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics) - userAPI = base.UserAPIClient() - } - if traceInternal { - userAPI = &uapi.UserInternalAPITrace{ - Impl: userAPI, - } - } - - // TODO: This should use userAPI, not userImpl, but the appservice setup races with - // the listeners and panics at startup if it tries to create appservice accounts - // before the listeners are up. - asAPI := appservice.NewInternalAPI(base, userImpl, rsAPI) - if base.UseHTTPAPIs { - appservice.AddInternalRoutes(base.InternalAPIMux, asAPI, base.EnableMetrics) - asAPI = base.AppserviceHTTPClient() - } - - // The underlying roomserver implementation needs to be able to call the fedsender. - // This is different to rsAPI which can be the http client which doesn't need this - // dependency. Other components also need updating after their dependencies are up. - rsImpl.SetFederationAPI(fsAPI, keyRing) - rsImpl.SetAppserviceAPI(asAPI) - rsImpl.SetUserAPI(userAPI) - keyImpl.SetUserAPI(userAPI) - - monolith := setup.Monolith{ - Config: base.Cfg, - Client: base.CreateClient(), - FedClient: federation, - KeyRing: keyRing, - - AppserviceAPI: asAPI, - // always use the concrete impl here even in -http mode because adding public routes - // must be done on the concrete impl not an HTTP client else fedapi will call itself - FederationAPI: fsImplAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, - } - monolith.AddAllPublicRoutes(base) - - if len(base.Cfg.MSCs.MSCs) > 0 { - if err := mscs.Enable(base, &monolith); err != nil { - logrus.WithError(err).Fatalf("Failed to enable MSCs") - } - } - - // Expose the matrix APIs directly rather than putting them under a /api path. - go func() { - base.SetupAndServeHTTP( - httpAPIAddr, // internal API - httpAddr, // external API - nil, nil, // TLS settings - ) - }() - // Handle HTTPS if certificate and key are provided - if *certFile != "" && *keyFile != "" { - go func() { - base.SetupAndServeHTTP( - basepkg.NoListener, // internal API - httpsAddr, // external API - certFile, keyFile, // TLS settings - ) - }() - } - - // We want to block forever to let the HTTP and HTTPS handler serve the APIs - base.WaitForShutdown() -} diff --git a/cmd/dendrite-polylith-multi/main.go b/cmd/dendrite-polylith-multi/main.go deleted file mode 100644 index c6a560b1..00000000 --- a/cmd/dendrite-polylith-multi/main.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "os" - "strings" - - "github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities" - "github.com/matrix-org/dendrite/setup" - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" - "github.com/sirupsen/logrus" -) - -type entrypoint func(base *base.BaseDendrite, cfg *config.Dendrite) - -func main() { - cfg := setup.ParseFlags(false) - - component := "" - if flag.NFlag() > 0 { - component = flag.Arg(0) // ./dendrite-polylith-multi --config=... clientapi - } else if len(os.Args) > 1 { - component = os.Args[1] // ./dendrite-polylith-multi clientapi - } - - components := map[string]entrypoint{ - "appservice": personalities.Appservice, - "clientapi": personalities.ClientAPI, - "federationapi": personalities.FederationAPI, - "keyserver": personalities.KeyServer, - "mediaapi": personalities.MediaAPI, - "roomserver": personalities.RoomServer, - "syncapi": personalities.SyncAPI, - "userapi": personalities.UserAPI, - } - - start, ok := components[component] - if !ok { - if component == "" { - logrus.Errorf("No component specified") - logrus.Info("The first argument on the command line must be the name of the component to run") - } else { - logrus.Errorf("Unknown component %q specified", component) - } - - var list []string - for c := range components { - list = append(list, c) - } - logrus.Infof("Valid components: %s", strings.Join(list, ", ")) - - os.Exit(1) - } - - logrus.Infof("Starting %q component", component) - - base := base.NewBaseDendrite(cfg, component, base.PolylithMode) // TODO - defer base.Close() // nolint: errcheck - - go start(base, cfg) - base.WaitForShutdown() -} diff --git a/cmd/dendrite-polylith-multi/personalities/appservice.go b/cmd/dendrite-polylith-multi/personalities/appservice.go deleted file mode 100644 index 0547d57f..00000000 --- a/cmd/dendrite-polylith-multi/personalities/appservice.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/setup/base" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func Appservice(base *base.BaseDendrite, cfg *config.Dendrite) { - userAPI := base.UserAPIClient() - rsAPI := base.RoomserverHTTPClient() - - intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) - appservice.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics) - - base.SetupAndServeHTTP( - base.Cfg.AppServiceAPI.InternalAPI.Listen, // internal listener - basepkg.NoListener, // external listener - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/clientapi.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go deleted file mode 100644 index a5d69d07..00000000 --- a/cmd/dendrite-polylith-multi/personalities/clientapi.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - "github.com/matrix-org/dendrite/clientapi" - "github.com/matrix-org/dendrite/internal/transactions" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func ClientAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - federation := base.CreateFederationClient() - - asQuery := base.AppserviceHTTPClient() - rsAPI := base.RoomserverHTTPClient() - fsAPI := base.FederationAPIHTTPClient() - userAPI := base.UserAPIClient() - keyAPI := base.KeyServerHTTPClient() - - clientapi.AddPublicRoutes( - base, federation, rsAPI, asQuery, - transactions.New(), fsAPI, userAPI, userAPI, - keyAPI, nil, - ) - - base.SetupAndServeHTTP( - base.Cfg.ClientAPI.InternalAPI.Listen, - base.Cfg.ClientAPI.ExternalAPI.Listen, - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/federationapi.go b/cmd/dendrite-polylith-multi/personalities/federationapi.go deleted file mode 100644 index 48da42fb..00000000 --- a/cmd/dendrite-polylith-multi/personalities/federationapi.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - "github.com/matrix-org/dendrite/federationapi" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - userAPI := base.UserAPIClient() - federation := base.CreateFederationClient() - rsAPI := base.RoomserverHTTPClient() - keyAPI := base.KeyServerHTTPClient() - fsAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, nil, true) - keyRing := fsAPI.KeyRing() - - federationapi.AddPublicRoutes( - base, - userAPI, federation, keyRing, - rsAPI, fsAPI, keyAPI, nil, - ) - - federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics) - - base.SetupAndServeHTTP( - base.Cfg.FederationAPI.InternalAPI.Listen, - base.Cfg.FederationAPI.ExternalAPI.Listen, - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/keyserver.go b/cmd/dendrite-polylith-multi/personalities/keyserver.go deleted file mode 100644 index ad0bd0e5..00000000 --- a/cmd/dendrite-polylith-multi/personalities/keyserver.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - "github.com/matrix-org/dendrite/keyserver" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func KeyServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - fsAPI := base.FederationAPIHTTPClient() - rsAPI := base.RoomserverHTTPClient() - intAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI) - intAPI.SetUserAPI(base.UserAPIClient()) - - keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics) - - base.SetupAndServeHTTP( - base.Cfg.KeyServer.InternalAPI.Listen, // internal listener - basepkg.NoListener, // external listener - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/mediaapi.go b/cmd/dendrite-polylith-multi/personalities/mediaapi.go deleted file mode 100644 index 69d5fd5a..00000000 --- a/cmd/dendrite-polylith-multi/personalities/mediaapi.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - "github.com/matrix-org/dendrite/mediaapi" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func MediaAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - userAPI := base.UserAPIClient() - client := base.CreateClient() - - mediaapi.AddPublicRoutes( - base, userAPI, client, - ) - - base.SetupAndServeHTTP( - base.Cfg.MediaAPI.InternalAPI.Listen, - base.Cfg.MediaAPI.ExternalAPI.Listen, - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/roomserver.go b/cmd/dendrite-polylith-multi/personalities/roomserver.go deleted file mode 100644 index 974559bd..00000000 --- a/cmd/dendrite-polylith-multi/personalities/roomserver.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - "github.com/matrix-org/dendrite/roomserver" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func RoomServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - asAPI := base.AppserviceHTTPClient() - fsAPI := base.FederationAPIHTTPClient() - rsAPI := roomserver.NewInternalAPI(base) - rsAPI.SetFederationAPI(fsAPI, fsAPI.KeyRing()) - rsAPI.SetAppserviceAPI(asAPI) - roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI, base.EnableMetrics) - - base.SetupAndServeHTTP( - base.Cfg.RoomServer.InternalAPI.Listen, // internal listener - basepkg.NoListener, // external listener - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/syncapi.go b/cmd/dendrite-polylith-multi/personalities/syncapi.go deleted file mode 100644 index 41637fe1..00000000 --- a/cmd/dendrite-polylith-multi/personalities/syncapi.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/syncapi" -) - -func SyncAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - userAPI := base.UserAPIClient() - - rsAPI := base.RoomserverHTTPClient() - - syncapi.AddPublicRoutes( - base, - userAPI, rsAPI, - base.KeyServerHTTPClient(), - ) - - base.SetupAndServeHTTP( - base.Cfg.SyncAPI.InternalAPI.Listen, - base.Cfg.SyncAPI.ExternalAPI.Listen, - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/userapi.go b/cmd/dendrite-polylith-multi/personalities/userapi.go deleted file mode 100644 index 1bc88cb5..00000000 --- a/cmd/dendrite-polylith-multi/personalities/userapi.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/userapi" -) - -func UserAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - userAPI := userapi.NewInternalAPI( - base, &cfg.UserAPI, cfg.Derived.ApplicationServices, - base.KeyServerHTTPClient(), base.RoomserverHTTPClient(), - base.PushGatewayHTTPClient(), - ) - - userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics) - - base.SetupAndServeHTTP( - base.Cfg.UserAPI.InternalAPI.Listen, // internal listener - basepkg.NoListener, // external listener - nil, nil, - ) -} diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go index 39b9320c..174a80a3 100644 --- a/cmd/dendrite-upgrade-tests/main.go +++ b/cmd/dendrite-upgrade-tests/main.go @@ -45,6 +45,10 @@ var ( const HEAD = "HEAD" +// The binary was renamed after v0.11.1, so everything after that should use the new name +var binaryChangeVersion, _ = semver.NewVersion("v0.11.1") +var latest, _ = semver.NewVersion("v6.6.6") // Dummy version, used as "HEAD" + // Embed the Dockerfile to use when building dendrite versions. // We cannot use the dockerfile associated with the repo with each version sadly due to changes in // Docker versions. Specifically, earlier Dendrite versions are incompatible with newer Docker clients @@ -54,12 +58,13 @@ const HEAD = "HEAD" const DockerfilePostgreSQL = `FROM golang:1.18-stretch as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build +ARG BINARY # Copy the build context to the repo as this is the right dendrite code. This is different to the # Complement Dockerfile which wgets a branch. COPY . . -RUN go build ./cmd/dendrite-monolith-server +RUN go build ./cmd/${BINARY} RUN go build ./cmd/generate-keys RUN go build ./cmd/generate-config RUN go build ./cmd/create-account @@ -88,22 +93,24 @@ done \n\ \n\ sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\ PARAMS="--tls-cert server.crt --tls-key server.key --config dendrite.yaml" \n\ -./dendrite-monolith-server --really-enable-open-registration ${PARAMS} || ./dendrite-monolith-server ${PARAMS} \n\ +./${BINARY} --really-enable-open-registration ${PARAMS} || ./${BINARY} ${PARAMS} \n\ ' > run_dendrite.sh && chmod +x run_dendrite.sh ENV SERVER_NAME=localhost +ENV BINARY=dendrite EXPOSE 8008 8448 -CMD /build/run_dendrite.sh ` +CMD /build/run_dendrite.sh` const DockerfileSQLite = `FROM golang:1.18-stretch as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build +ARG BINARY # Copy the build context to the repo as this is the right dendrite code. This is different to the # Complement Dockerfile which wgets a branch. COPY . . -RUN go build ./cmd/dendrite-monolith-server +RUN go build ./cmd/${BINARY} RUN go build ./cmd/generate-keys RUN go build ./cmd/generate-config RUN go build ./cmd/create-account @@ -118,10 +125,11 @@ RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postg RUN echo '\ sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\ PARAMS="--tls-cert server.crt --tls-key server.key --config dendrite.yaml" \n\ -./dendrite-monolith-server --really-enable-open-registration ${PARAMS} || ./dendrite-monolith-server ${PARAMS} \n\ +./${BINARY} --really-enable-open-registration ${PARAMS} || ./${BINARY} ${PARAMS} \n\ ' > run_dendrite.sh && chmod +x run_dendrite.sh ENV SERVER_NAME=localhost +ENV BINARY=dendrite EXPOSE 8008 8448 CMD /build/run_dendrite.sh ` @@ -182,7 +190,7 @@ func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []b } // buildDendrite builds Dendrite on the branchOrTagName given. Returns the image ID or an error -func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir, branchOrTagName string) (string, error) { +func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir string, branchOrTagName, binary string) (string, error) { var tarball *bytes.Buffer var err error // If a custom HEAD location is given, use that, else pull from github. Mostly useful for CI @@ -216,6 +224,9 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir, log.Printf("%s: Building version %s\n", branchOrTagName, branchOrTagName) res, err := dockerClient.ImageBuild(context.Background(), tarball, types.ImageBuildOptions{ Tags: []string{"dendrite-upgrade"}, + BuildArgs: map[string]*string{ + "BINARY": &binary, + }, }) if err != nil { return "", fmt.Errorf("failed to start building image: %s", err) @@ -272,7 +283,7 @@ func getAndSortVersionsFromGithub(httpClient *http.Client) (semVers []*semver.Ve return semVers, nil } -func calculateVersions(cli *http.Client, from, to string, direct bool) []string { +func calculateVersions(cli *http.Client, from, to string, direct bool) []*semver.Version { semvers, err := getAndSortVersionsFromGithub(cli) if err != nil { log.Fatalf("failed to collect semvers from github: %s", err) @@ -320,28 +331,25 @@ func calculateVersions(cli *http.Client, from, to string, direct bool) []string } semvers = semvers[:i+1] } - var versions []string - for _, sv := range semvers { - versions = append(versions, sv.Original()) - } + if to == HEAD { - versions = append(versions, HEAD) + semvers = append(semvers, latest) } if direct { - versions = []string{versions[0], versions[len(versions)-1]} + semvers = []*semver.Version{semvers[0], semvers[len(semvers)-1]} } - return versions + return semvers } -func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, baseTempDir string, concurrency int, branchOrTagNames []string) map[string]string { +func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, baseTempDir string, concurrency int, versions []*semver.Version) map[string]string { // concurrently build all versions, this can be done in any order. The mutex protects the map branchToImageID := make(map[string]string) var mu sync.Mutex var wg sync.WaitGroup wg.Add(concurrency) - ch := make(chan string, len(branchOrTagNames)) - for _, branchName := range branchOrTagNames { + ch := make(chan *semver.Version, len(versions)) + for _, branchName := range versions { ch <- branchName } close(ch) @@ -349,11 +357,13 @@ func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, b for i := 0; i < concurrency; i++ { go func() { defer wg.Done() - for branchName := range ch { + for version := range ch { + branchName, binary := versionToBranchAndBinary(version) + log.Printf("Building version %s with binary %s", branchName, binary) tmpDir := baseTempDir + alphaNumerics.ReplaceAllString(branchName, "") - imgID, err := buildDendrite(httpClient, dockerClient, tmpDir, branchName) + imgID, err := buildDendrite(httpClient, dockerClient, tmpDir, branchName, binary) if err != nil { - log.Fatalf("%s: failed to build dendrite image: %s", branchName, err) + log.Fatalf("%s: failed to build dendrite image: %s", version, err) } mu.Lock() branchToImageID[branchName] = imgID @@ -365,13 +375,14 @@ func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, b return branchToImageID } -func runImage(dockerClient *client.Client, volumeName, version, imageID string) (csAPIURL, containerID string, err error) { - log.Printf("%s: running image %s\n", version, imageID) +func runImage(dockerClient *client.Client, volumeName string, branchNameToImageID map[string]string, version *semver.Version) (csAPIURL, containerID string, err error) { + branchName, binary := versionToBranchAndBinary(version) + imageID := branchNameToImageID[branchName] ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) defer cancel() body, err := dockerClient.ContainerCreate(ctx, &container.Config{ Image: imageID, - Env: []string{"SERVER_NAME=hs1"}, + Env: []string{"SERVER_NAME=hs1", fmt.Sprintf("BINARY=%s", binary)}, Labels: map[string]string{ dendriteUpgradeTestLabel: "yes", }, @@ -384,7 +395,7 @@ func runImage(dockerClient *client.Client, volumeName, version, imageID string) Target: "/var/lib/postgresql/9.6/main", }, }, - }, nil, nil, "dendrite_upgrade_test_"+version) + }, nil, nil, "dendrite_upgrade_test_"+branchName) if err != nil { return "", "", fmt.Errorf("failed to ContainerCreate: %s", err) } @@ -451,8 +462,8 @@ func destroyContainer(dockerClient *client.Client, containerID string) { } } -func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchToImageID map[string]string) error { - csAPIURL, containerID, err := runImage(dockerClient, volumeName, v, branchToImageID[v]) +func loadAndRunTests(dockerClient *client.Client, volumeName string, v *semver.Version, branchToImageID map[string]string) error { + csAPIURL, containerID, err := runImage(dockerClient, volumeName, branchToImageID, v) if err != nil { return fmt.Errorf("failed to run container for branch %v: %v", v, err) } @@ -470,9 +481,10 @@ func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchTo } // test that create-account is working -func testCreateAccount(dockerClient *client.Client, v string, containerID string) error { - createUser := strings.ToLower("createaccountuser-" + v) - log.Printf("%s: Creating account %s with create-account\n", v, createUser) +func testCreateAccount(dockerClient *client.Client, version *semver.Version, containerID string) error { + branchName, _ := versionToBranchAndBinary(version) + createUser := strings.ToLower("createaccountuser-" + branchName) + log.Printf("%s: Creating account %s with create-account\n", branchName, createUser) respID, err := dockerClient.ContainerExecCreate(context.Background(), containerID, types.ExecConfig{ AttachStderr: true, @@ -504,9 +516,21 @@ func testCreateAccount(dockerClient *client.Client, v string, containerID string return nil } -func verifyTests(dockerClient *client.Client, volumeName string, versions []string, branchToImageID map[string]string) error { +func versionToBranchAndBinary(version *semver.Version) (branchName, binary string) { + binary = "dendrite-monolith-server" + branchName = version.Original() + if version.GreaterThan(binaryChangeVersion) { + binary = "dendrite" + if version.Equal(latest) { + branchName = HEAD + } + } + return +} + +func verifyTests(dockerClient *client.Client, volumeName string, versions []*semver.Version, branchToImageID map[string]string) error { lastVer := versions[len(versions)-1] - csAPIURL, containerID, err := runImage(dockerClient, volumeName, lastVer, branchToImageID[lastVer]) + csAPIURL, containerID, err := runImage(dockerClient, volumeName, branchToImageID, lastVer) if err != nil { return fmt.Errorf("failed to run container for branch %v: %v", lastVer, err) } diff --git a/cmd/dendrite-upgrade-tests/tests.go b/cmd/dendrite-upgrade-tests/tests.go index 5c9589df..03438bd4 100644 --- a/cmd/dendrite-upgrade-tests/tests.go +++ b/cmd/dendrite-upgrade-tests/tests.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/Masterminds/semver/v3" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" ) @@ -22,7 +23,8 @@ type user struct { // - register alice and bob with branch name muxed into the localpart // - create a DM room for the 2 users and exchange messages // - create/join a public #global room and exchange messages -func runTests(baseURL, branchName string) error { +func runTests(baseURL string, v *semver.Version) error { + branchName, _ := versionToBranchAndBinary(v) // register 2 users users := []user{ { @@ -164,15 +166,16 @@ func runTests(baseURL, branchName string) error { } // verifyTestsRan checks that the HS has the right rooms/messages -func verifyTestsRan(baseURL string, branchNames []string) error { +func verifyTestsRan(baseURL string, versions []*semver.Version) error { log.Println("Verifying tests....") // check we can login as all users var resp *gomatrix.RespLogin - for _, branchName := range branchNames { + for _, version := range versions { client, err := gomatrix.NewClient(baseURL, "", "") if err != nil { return err } + branchName, _ := versionToBranchAndBinary(version) userLocalparts := []string{ "alice" + branchName, "bob" + branchName, @@ -224,7 +227,7 @@ func verifyTestsRan(baseURL string, branchNames []string) error { msgCount += 1 } } - wantMsgCount := len(branchNames) * 4 + wantMsgCount := len(versions) * 4 if msgCount != wantMsgCount { return fmt.Errorf("got %d messages in global room, want %d", msgCount, wantMsgCount) } diff --git a/cmd/dendrite/main.go b/cmd/dendrite/main.go new file mode 100644 index 00000000..35bfc1d6 --- /dev/null +++ b/cmd/dendrite/main.go @@ -0,0 +1,109 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + + "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/keyserver" + "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/setup" + basepkg "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/mscs" + "github.com/matrix-org/dendrite/userapi" +) + +var ( + httpBindAddr = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server") + httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server") + certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") + keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") +) + +func main() { + cfg := setup.ParseFlags(true) + httpAddr := config.HTTPAddress("http://" + *httpBindAddr) + httpsAddr := config.HTTPAddress("https://" + *httpsBindAddr) + options := []basepkg.BaseDendriteOptions{} + + base := basepkg.NewBaseDendrite(cfg, options...) + defer base.Close() // nolint: errcheck + + federation := base.CreateFederationClient() + + rsAPI := roomserver.NewInternalAPI(base) + + fsAPI := federationapi.NewInternalAPI( + base, federation, rsAPI, base.Caches, nil, false, + ) + + keyRing := fsAPI.KeyRing() + + keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI) + + pgClient := base.PushGatewayHTTPClient() + userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, pgClient) + + asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) + + // The underlying roomserver implementation needs to be able to call the fedsender. + // This is different to rsAPI which can be the http client which doesn't need this + // dependency. Other components also need updating after their dependencies are up. + rsAPI.SetFederationAPI(fsAPI, keyRing) + rsAPI.SetAppserviceAPI(asAPI) + rsAPI.SetUserAPI(userAPI) + keyAPI.SetUserAPI(userAPI) + + monolith := setup.Monolith{ + Config: base.Cfg, + Client: base.CreateClient(), + FedClient: federation, + KeyRing: keyRing, + + AppserviceAPI: asAPI, + // always use the concrete impl here even in -http mode because adding public routes + // must be done on the concrete impl not an HTTP client else fedapi will call itself + FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + KeyAPI: keyAPI, + } + monolith.AddAllPublicRoutes(base) + + if len(base.Cfg.MSCs.MSCs) > 0 { + if err := mscs.Enable(base, &monolith); err != nil { + logrus.WithError(err).Fatalf("Failed to enable MSCs") + } + } + + // Expose the matrix APIs directly rather than putting them under a /api path. + go func() { + base.SetupAndServeHTTP(httpAddr, nil, nil) + }() + // Handle HTTPS if certificate and key are provided + if *certFile != "" && *keyFile != "" { + go func() { + base.SetupAndServeHTTP(httpsAddr, certFile, keyFile) + }() + } + + // We want to block forever to let the HTTP and HTTPS handler serve the APIs + base.WaitForShutdown() +} diff --git a/cmd/dendrite-monolith-server/main_test.go b/cmd/dendrite/main_test.go index efa1a926..efa1a926 100644 --- a/cmd/dendrite-monolith-server/main_test.go +++ b/cmd/dendrite/main_test.go diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index 56a14565..b0707be1 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -18,7 +18,6 @@ func main() { dbURI := flag.String("db", "", "The DB URI to use for all components (PostgreSQL only)") dirPath := flag.String("dir", "./", "The folder to use for paths (like SQLite databases, media storage)") normalise := flag.String("normalise", "", "Normalise an existing configuration file by adding new/missing options and defaults") - polylith := flag.Bool("polylith", false, "Generate a config that makes sense for polylith deployments") flag.Parse() var cfg *config.Dendrite @@ -27,14 +26,14 @@ func main() { Version: config.Version, } cfg.Defaults(config.DefaultOpts{ - Generate: true, - Monolithic: !*polylith, + Generate: true, + SingleDatabase: true, }) if *serverName != "" { cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName) } uri := config.DataSource(*dbURI) - if *polylith || uri.IsSQLite() || uri == "" { + if uri.IsSQLite() || uri == "" { for name, db := range map[string]*config.DatabaseOptions{ "federationapi": &cfg.FederationAPI.Database, "keyserver": &cfg.KeyServer.Database, @@ -43,6 +42,7 @@ func main() { "roomserver": &cfg.RoomServer.Database, "syncapi": &cfg.SyncAPI.Database, "userapi": &cfg.UserAPI.AccountDatabase, + "relayapi": &cfg.RelayAPI.Database, } { if uri == "" { path := filepath.Join(*dirPath, fmt.Sprintf("dendrite_%s.db", name)) @@ -96,7 +96,7 @@ func main() { } } else { var err error - if cfg, err = config.Load(*normalise, !*polylith); err != nil { + if cfg, err = config.Load(*normalise); err != nil { panic(err) } } diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go index f8bb130c..59c66aed 100644 --- a/cmd/resolve-state/main.go +++ b/cmd/resolve-state/main.go @@ -40,7 +40,7 @@ func main() { Level: "error", }) cfg.ClientAPI.RegistrationDisabled = true - base := base.NewBaseDendrite(cfg, "ResolveState", base.DisableMetrics) + base := base.NewBaseDendrite(cfg, base.DisableMetrics) args := flag.Args() fmt.Println("Room version", *roomVersion) |