p2p: Implement published rooms (#923)

* Create and glue ExternalPublicRoomsProvider into the public rooms component

This is how we will link p2p stuff to dendrite proper.

* Use gmsl structs rather than our own

* Implement federated public rooms

- Make thirdparty endpoint r0 so riot-web loads the public room list

* Typo

* Missing callsites

16 files changed, 247 insertions, 65 deletions

diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go
index 47b7b267..22ff12b0 100644
--- a/clientapi/routing/routing.go
+++ b/clientapi/routing/routing.go
@@ -396,7 +396,7 @@ func Setup(
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
-	unstableMux.Handle("/thirdparty/protocols",
+	r0mux.Handle("/thirdparty/protocols",
 		common.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse {
 			// TODO: Return the third party protcols
 			return util.JSONResponse{
diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go
index b3de9add..27c3054b 100644
--- a/cmd/dendrite-monolith-server/main.go
+++ b/cmd/dendrite-monolith-server/main.go
@@ -69,7 +69,7 @@ func main() {
 	)
 	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI)
 	mediaapi.SetupMediaAPIComponent(base, deviceDB)
-	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query)
+	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, federation, nil)
 	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg)
 
 	httpHandler := common.WrapHandlerInCORS(base.APIMux)
diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go
index f8bd8b06..6b7eac7d 100644
--- a/cmd/dendrite-public-rooms-api-server/main.go
+++ b/cmd/dendrite-public-rooms-api-server/main.go
@@ -28,7 +28,7 @@ func main() {
 
 	_, _, query := base.CreateHTTPRoomserverAPIs()
 
-	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query)
+	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, nil, nil)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI))
 
diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go
index 25492b16..7c852671 100644
--- a/cmd/dendritejs/main.go
+++ b/cmd/dendritejs/main.go
@@ -119,6 +119,7 @@ func main() {
 		},
 		KeyDatabase: keyDB,
 	}
+	p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node)
 
 	alias, input, query := roomserver.SetupRoomServerComponent(base)
 	typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache())
@@ -134,7 +135,7 @@ func main() {
 	)
 	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI)
 	mediaapi.SetupMediaAPIComponent(base, deviceDB)
-	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query)
+	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, federation, p2pPublicRoomProvider)
 	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg)
 
 	httpHandler := common.WrapHandlerInCORS(base.APIMux)
diff --git a/cmd/dendritejs/publicrooms.go b/cmd/dendritejs/publicrooms.go
new file mode 100644
index 00000000..17822e7a
--- /dev/null
+++ b/cmd/dendritejs/publicrooms.go
@@ -0,0 +1,46 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build wasm
+
+package main
+
+import (
+	"github.com/matrix-org/go-http-js-libp2p/go_http_js_libp2p"
+)
+
+type libp2pPublicRoomsProvider struct {
+	node      *go_http_js_libp2p.P2pLocalNode
+	providers []go_http_js_libp2p.PeerInfo
+}
+
+func NewLibP2PPublicRoomsProvider(node *go_http_js_libp2p.P2pLocalNode) *libp2pPublicRoomsProvider {
+	p := &libp2pPublicRoomsProvider{
+		node: node,
+	}
+	node.RegisterFoundProviders(p.foundProviders)
+	return p
+}
+
+func (p *libp2pPublicRoomsProvider) foundProviders(peerInfos []go_http_js_libp2p.PeerInfo) {
+	p.providers = peerInfos
+}
+
+func (p *libp2pPublicRoomsProvider) Homeservers() []string {
+	result := make([]string, len(p.providers))
+	for i := range p.providers {
+		result[i] = p.providers[i].Id
+	}
+	return result
+}
diff --git a/go.mod b/go.mod
index 25debd74..556a6b8f 100644
--- a/go.mod
+++ b/go.mod
@@ -8,15 +8,13 @@ require (
 	github.com/lib/pq v1.2.0
 	github.com/libp2p/go-libp2p-core v0.5.0
 	github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5
-	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c
+	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd
 	github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1
 	github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
 	github.com/mattn/go-sqlite3 v2.0.2+incompatible
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pkg/errors v0.8.1
diff --git a/go.sum b/go.sum
index 3c7cf281..2c51915b 100644
--- a/go.sum
+++ b/go.sum
@@ -230,6 +230,8 @@ github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437 h1:zc
 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437/go.mod h1:/giSXVd8D6DZGSfTmhQrLEoZZwsfkC14kSqP9MiLqIY=
 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c h1:jj/LIZKMO7GK6O0UarpRwse9L3ZyzozpyMtdPA7ddSk=
 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk=
+github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f h1:5TOte9uk/epk8L+Pbp6qwaV8YsKYXKjyECPHUhJTWQc=
+github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk=
 github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074 h1:UWz6vfhmQVshBuE67X1BCsdMhEDtd+uOz8CJ48Fc0F4=
 github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db h1:ERuFJq4DI8fakfBZlvXHltHZ0ix3K5YsLG0tQfQn6TI=
diff --git a/publicroomsapi/directory/public_rooms.go b/publicroomsapi/directory/public_rooms.go
index fd327942..7bd6740e 100644
--- a/publicroomsapi/directory/public_rooms.go
+++ b/publicroomsapi/directory/public_rooms.go
@@ -15,17 +15,21 @@
 package directory
 
 import (
+	"context"
 	"net/http"
 	"strconv"
+	"sync"
+	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/publicroomsapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
-type publicRoomReq struct {
+type PublicRoomReq struct {
 	Since  string `json:"since,omitempty"`
 	Limit  int16  `json:"limit,omitempty"`
 	Filter filter `json:"filter,omitempty"`
@@ -35,65 +39,182 @@ type filter struct {
 	SearchTerms string `json:"generic_search_term,omitempty"`
 }
 
-type publicRoomRes struct {
-	Chunk     []types.PublicRoom `json:"chunk"`
-	NextBatch string             `json:"next_batch,omitempty"`
-	PrevBatch string             `json:"prev_batch,omitempty"`
-	Estimate  int64              `json:"total_room_count_estimate,omitempty"`
-}
-
 // GetPostPublicRooms implements GET and POST /publicRooms
 func GetPostPublicRooms(
 	req *http.Request, publicRoomDatabase storage.Database,
 ) util.JSONResponse {
-	var limit int16
-	var offset int64
-	var request publicRoomReq
-	var response publicRoomRes
+	var request PublicRoomReq
+	if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
+		return *fillErr
+	}
+	response, err := publicRooms(req.Context(), request, publicRoomDatabase)
+	if err != nil {
+		return jsonerror.InternalServerError()
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: response,
+	}
+}
 
+// GetPostPublicRoomsWithExternal is the same as GetPostPublicRooms but also mixes in public rooms from the provider supplied.
+func GetPostPublicRoomsWithExternal(
+	req *http.Request, publicRoomDatabase storage.Database, fedClient *gomatrixserverlib.FederationClient,
+	extRoomsProvider types.ExternalPublicRoomsProvider,
+) util.JSONResponse {
+	var request PublicRoomReq
 	if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
 		return *fillErr
 	}
+	response, err := publicRooms(req.Context(), request, publicRoomDatabase)
+	if err != nil {
+		return jsonerror.InternalServerError()
+	}
+
+	if request.Since != "" {
+		// TODO: handle pagination tokens sensibly rather than ignoring them.
+		// ignore paginated requests since we don't handle them yet over federation.
+		// Only the initial request will contain federated rooms.
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: response,
+		}
+	}
+
+	// If we have already hit the limit on the number of rooms, bail.
+	var limit int
+	if request.Limit > 0 {
+		limit = int(request.Limit) - len(response.Chunk)
+		if limit <= 0 {
+			return util.JSONResponse{
+				Code: http.StatusOK,
+				JSON: response,
+			}
+		}
+	}
 
+	// downcasting `limit` is safe as we know it isn't bigger than request.Limit which is int16
+	fedRooms := bulkFetchPublicRoomsFromServers(req.Context(), fedClient, extRoomsProvider.Homeservers(), int16(limit))
+	response.Chunk = append(response.Chunk, fedRooms...)
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: response,
+	}
+}
+
+// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers.
+// Returns a list of public rooms up to the limit specified.
+func bulkFetchPublicRoomsFromServers(
+	ctx context.Context, fedClient *gomatrixserverlib.FederationClient, homeservers []string, limit int16,
+) (publicRooms []gomatrixserverlib.PublicRoom) {
+	// follow pipeline semantics, see https://blog.golang.org/pipelines for more info.
+	// goroutines send rooms to this channel
+	roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit))
+	// signalling channel to tell goroutines to stop sending rooms and quit
+	done := make(chan bool)
+	// signalling to say when we can close the room channel
+	var wg sync.WaitGroup
+	wg.Add(len(homeservers))
+	// concurrently query for public rooms
+	for _, hs := range homeservers {
+		go func(homeserverDomain string) {
+			defer wg.Done()
+			util.GetLogger(ctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")
+			fres, err := fedClient.GetPublicRooms(ctx, gomatrixserverlib.ServerName(homeserverDomain), int(limit), "", false, "")
+			if err != nil {
+				util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Warn(
+					"bulkFetchPublicRoomsFromServers: failed to query hs",
+				)
+				return
+			}
+			for _, room := range fres.Chunk {
+				// atomically send a room or stop
+				select {
+				case roomCh <- room:
+				case <-done:
+					util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending rooms")
+					return
+				}
+			}
+		}(hs)
+	}
+
+	// Close the room channel when the goroutines have quit so we don't leak, but don't let it stop the in-flight request.
+	// This also allows the request to fail fast if all HSes experience errors as it will cause the room channel to be
+	// closed.
+	go func() {
+		wg.Wait()
+		util.GetLogger(ctx).Info("Cleaning up resources")
+		close(roomCh)
+	}()
+
+	// fan-in results with timeout. We stop when we reach the limit.
+FanIn:
+	for len(publicRooms) < int(limit) || limit == 0 {
+		// add a room or timeout
+		select {
+		case room, ok := <-roomCh:
+			if !ok {
+				util.GetLogger(ctx).Info("All homeservers have been queried, returning results.")
+				break FanIn
+			}
+			publicRooms = append(publicRooms, room)
+		case <-time.After(15 * time.Second): // we've waited long enough, let's tell the client what we got.
+			util.GetLogger(ctx).Info("Waited 15s for federated public rooms, returning early")
+			break FanIn
+		case <-ctx.Done(): // the client hung up on us, let's stop.
+			util.GetLogger(ctx).Info("Client hung up, returning early")
+			break FanIn
+		}
+	}
+	// tell goroutines to stop
+	close(done)
+
+	return publicRooms
+}
+
+func publicRooms(ctx context.Context, request PublicRoomReq, publicRoomDatabase storage.Database) (*gomatrixserverlib.RespPublicRooms, error) {
+	var response gomatrixserverlib.RespPublicRooms
+	var limit int16
+	var offset int64
 	limit = request.Limit
 	offset, err := strconv.ParseInt(request.Since, 10, 64)
 	// ParseInt returns 0 and an error when trying to parse an empty string
 	// In that case, we want to assign 0 so we ignore the error
 	if err != nil && len(request.Since) > 0 {
-		util.GetLogger(req.Context()).WithError(err).Error("strconv.ParseInt failed")
-		return jsonerror.InternalServerError()
+		util.GetLogger(ctx).WithError(err).Error("strconv.ParseInt failed")
+		return nil, err
 	}
 
-	if response.Estimate, err = publicRoomDatabase.CountPublicRooms(req.Context()); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("publicRoomDatabase.CountPublicRooms failed")
-		return jsonerror.InternalServerError()
+	est, err := publicRoomDatabase.CountPublicRooms(ctx)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.CountPublicRooms failed")
+		return nil, err
 	}
+	response.TotalRoomCountEstimate = int(est)
 
 	if offset > 0 {
 		response.PrevBatch = strconv.Itoa(int(offset) - 1)
 	}
 	nextIndex := int(offset) + int(limit)
-	if response.Estimate > int64(nextIndex) {
+	if response.TotalRoomCountEstimate > nextIndex {
 		response.NextBatch = strconv.Itoa(nextIndex)
 	}
 
 	if response.Chunk, err = publicRoomDatabase.GetPublicRooms(
-		req.Context(), offset, limit, request.Filter.SearchTerms,
+		ctx, offset, limit, request.Filter.SearchTerms,
 	); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("publicRoomDatabase.GetPublicRooms failed")
-		return jsonerror.InternalServerError()
+		util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.GetPublicRooms failed")
+		return nil, err
 	}
 
-	return util.JSONResponse{
-		Code: http.StatusOK,
-		JSON: response,
-	}
+	return &response, nil
 }
 
 // fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request
 // on /publicRooms by parsing the incoming HTTP request
 // Filter is only filled for POST requests
-func fillPublicRoomsReq(httpReq *http.Request, request *publicRoomReq) *util.JSONResponse {
+func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSONResponse {
 	if httpReq.Method == http.MethodGet {
 		limit, err := strconv.Atoi(httpReq.FormValue("limit"))
 		// Atoi returns 0 and an error when trying to parse an empty string
diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go
index 1e2a3f9b..399c0cc5 100644
--- a/publicroomsapi/publicroomsapi.go
+++ b/publicroomsapi/publicroomsapi.go
@@ -20,7 +20,9 @@ import (
 	"github.com/matrix-org/dendrite/publicroomsapi/consumers"
 	"github.com/matrix-org/dendrite/publicroomsapi/routing"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage"
+	"github.com/matrix-org/dendrite/publicroomsapi/types"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
 )
 
@@ -30,6 +32,8 @@ func SetupPublicRoomsAPIComponent(
 	base *basecomponent.BaseDendrite,
 	deviceDB devices.Database,
 	rsQueryAPI roomserverAPI.RoomserverQueryAPI,
+	fedClient *gomatrixserverlib.FederationClient,
+	extRoomsProvider types.ExternalPublicRoomsProvider,
 ) {
 	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI))
 	if err != nil {
@@ -43,5 +47,5 @@ func SetupPublicRoomsAPIComponent(
 		logrus.WithError(err).Panic("failed to start public rooms server consumer")
 	}
 
-	routing.Setup(base.APIMux, deviceDB, publicRoomsDB)
+	routing.Setup(base.APIMux, deviceDB, publicRoomsDB, fedClient, extRoomsProvider)
 }
diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go
index 1953e04f..321b61b8 100644
--- a/publicroomsapi/routing/routing.go
+++ b/publicroomsapi/routing/routing.go
@@ -24,6 +24,8 @@ import (
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/publicroomsapi/directory"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage"
+	"github.com/matrix-org/dendrite/publicroomsapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
 
@@ -34,7 +36,10 @@ const pathPrefixR0 = "/_matrix/client/r0"
 // Due to Setup being used to call many other functions, a gocyclo nolint is
 // applied:
 // nolint: gocyclo
-func Setup(apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database) {
+func Setup(
+	apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database,
+	fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider,
+) {
 	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
 
 	authData := auth.Data{
@@ -64,7 +69,17 @@ func Setup(apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.
 	).Methods(http.MethodPut, http.MethodOptions)
 	r0mux.Handle("/publicRooms",
 		common.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse {
+			if extRoomsProvider != nil {
+				return directory.GetPostPublicRoomsWithExternal(req, publicRoomsDB, fedClient, extRoomsProvider)
+			}
 			return directory.GetPostPublicRooms(req, publicRoomsDB)
 		}),
 	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
+
+	// Federation - TODO: should this live here or in federation API? It's sure easier if it's here so here it is.
+	apiMux.Handle("/_matrix/federation/v1/publicRooms",
+		common.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse {
+			return directory.GetPostPublicRooms(req, publicRoomsDB)
+		}),
+	).Methods(http.MethodGet)
 }
diff --git a/publicroomsapi/storage/interface.go b/publicroomsapi/storage/interface.go
index 3f7e6589..0feca0e2 100644
--- a/publicroomsapi/storage/interface.go
+++ b/publicroomsapi/storage/interface.go
@@ -18,7 +18,6 @@ import (
 	"context"
 
 	"github.com/matrix-org/dendrite/common"
-	"github.com/matrix-org/dendrite/publicroomsapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -27,7 +26,7 @@ type Database interface {
 	GetRoomVisibility(ctx context.Context, roomID string) (bool, error)
 	SetRoomVisibility(ctx context.Context, visible bool, roomID string) error
 	CountPublicRooms(ctx context.Context) (int64, error)
-	GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]types.PublicRoom, error)
+	GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error)
 	UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error
 	UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error
 }
diff --git a/publicroomsapi/storage/postgres/public_rooms_table.go b/publicroomsapi/storage/postgres/public_rooms_table.go
index 7e606e93..7e31afd2 100644
--- a/publicroomsapi/storage/postgres/public_rooms_table.go
+++ b/publicroomsapi/storage/postgres/public_rooms_table.go
@@ -22,9 +22,9 @@ import (
 	"fmt"
 
 	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/lib/pq"
-	"github.com/matrix-org/dendrite/publicroomsapi/types"
 )
 
 var editableAttributes = []string{
@@ -177,7 +177,7 @@ func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64,
 
 func (s *publicRoomsStatements) selectPublicRooms(
 	ctx context.Context, offset int64, limit int16, filter string,
-) ([]types.PublicRoom, error) {
+) ([]gomatrixserverlib.PublicRoom, error) {
 	var rows *sql.Rows
 	var err error
 
@@ -203,17 +203,17 @@ func (s *publicRoomsStatements) selectPublicRooms(
 	}
 
 	if err != nil {
-		return []types.PublicRoom{}, nil
+		return []gomatrixserverlib.PublicRoom{}, nil
 	}
 	defer common.CloseAndLogIfError(ctx, rows, "selectPublicRooms: rows.close() failed")
 
-	rooms := []types.PublicRoom{}
+	rooms := []gomatrixserverlib.PublicRoom{}
 	for rows.Next() {
-		var r types.PublicRoom
+		var r gomatrixserverlib.PublicRoom
 		var aliases pq.StringArray
 
 		err = rows.Scan(
-			&r.RoomID, &r.NumJoinedMembers, &aliases, &r.CanonicalAlias,
+			&r.RoomID, &r.JoinedMembersCount, &aliases, &r.CanonicalAlias,
 			&r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL,
 		)
 		if err != nil {
diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go
index 5365c766..5a4bc8f9 100644
--- a/publicroomsapi/storage/postgres/storage.go
+++ b/publicroomsapi/storage/postgres/storage.go
@@ -21,7 +21,6 @@ import (
 	"encoding/json"
 
 	"github.com/matrix-org/dendrite/common"
-	"github.com/matrix-org/dendrite/publicroomsapi/types"
 
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -85,7 +84,7 @@ func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64
 // Returns an error if the retrieval failed.
 func (d *PublicRoomsServerDatabase) GetPublicRooms(
 	ctx context.Context, offset int64, limit int16, filter string,
-) ([]types.PublicRoom, error) {
+) ([]gomatrixserverlib.PublicRoom, error) {
 	return d.statements.selectPublicRooms(ctx, offset, limit, filter)
 }
 
diff --git a/publicroomsapi/storage/sqlite3/public_rooms_table.go b/publicroomsapi/storage/sqlite3/public_rooms_table.go
index beba0d69..44679837 100644
--- a/publicroomsapi/storage/sqlite3/public_rooms_table.go
+++ b/publicroomsapi/storage/sqlite3/public_rooms_table.go
@@ -22,7 +22,8 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/matrix-org/dendrite/publicroomsapi/types"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 var editableAttributes = []string{
@@ -66,7 +67,7 @@ const selectPublicRoomsWithLimitSQL = "" +
 	"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
 	" FROM publicroomsapi_public_rooms WHERE visibility = true" +
 	" ORDER BY joined_members DESC" +
-	" LIMIT $2 OFFSET $1"
+	" LIMIT $1 OFFSET $2"
 
 const selectPublicRoomsWithFilterSQL = "" +
 	"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
@@ -164,7 +165,7 @@ func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64,
 
 func (s *publicRoomsStatements) selectPublicRooms(
 	ctx context.Context, offset int64, limit int16, filter string,
-) ([]types.PublicRoom, error) {
+) ([]gomatrixserverlib.PublicRoom, error) {
 	var rows *sql.Rows
 	var err error
 
@@ -190,16 +191,17 @@ func (s *publicRoomsStatements) selectPublicRooms(
 	}
 
 	if err != nil {
-		return []types.PublicRoom{}, nil
+		return []gomatrixserverlib.PublicRoom{}, nil
 	}
+	defer common.CloseAndLogIfError(ctx, rows, "selectPublicRooms failed to close rows")
 
-	rooms := []types.PublicRoom{}
+	rooms := []gomatrixserverlib.PublicRoom{}
 	for rows.Next() {
-		var r types.PublicRoom
+		var r gomatrixserverlib.PublicRoom
 		var aliasesJSON string
 
 		err = rows.Scan(
-			&r.RoomID, &r.NumJoinedMembers, &aliasesJSON, &r.CanonicalAlias,
+			&r.RoomID, &r.JoinedMembersCount, &aliasesJSON, &r.CanonicalAlias,
 			&r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL,
 		)
 		if err != nil {
diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go
index f8ba71a8..80c04cab 100644
--- a/publicroomsapi/storage/sqlite3/storage.go
+++ b/publicroomsapi/storage/sqlite3/storage.go
@@ -23,7 +23,6 @@ import (
 	_ "github.com/mattn/go-sqlite3"
 
 	"github.com/matrix-org/dendrite/common"
-	"github.com/matrix-org/dendrite/publicroomsapi/types"
 
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -87,7 +86,7 @@ func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64
 // Returns an error if the retrieval failed.
 func (d *PublicRoomsServerDatabase) GetPublicRooms(
 	ctx context.Context, offset int64, limit int16, filter string,
-) ([]types.PublicRoom, error) {
+) ([]gomatrixserverlib.PublicRoom, error) {
 	return d.statements.selectPublicRooms(ctx, offset, limit, filter)
 }
 
diff --git a/publicroomsapi/types/types.go b/publicroomsapi/types/types.go
index c284bcca..11cb0d20 100644
--- a/publicroomsapi/types/types.go
+++ b/publicroomsapi/types/types.go
@@ -14,15 +14,11 @@
 
 package types
 
-// PublicRoom represents a local public room
-type PublicRoom struct {
-	RoomID           string   `json:"room_id"`
-	Aliases          []string `json:"aliases,omitempty"`
-	CanonicalAlias   string   `json:"canonical_alias,omitempty"`
-	Name             string   `json:"name,omitempty"`
-	Topic            string   `json:"topic,omitempty"`
-	AvatarURL        string   `json:"avatar_url,omitempty"`
-	NumJoinedMembers int64    `json:"num_joined_members"`
-	WorldReadable    bool     `json:"world_readable"`
-	GuestCanJoin     bool     `json:"guest_can_join"`
+// ExternalPublicRoomsProvider provides a list of homeservers who should be queried
+// periodically for a list of public rooms on their server.
+type ExternalPublicRoomsProvider interface {
+	// The list of homeserver domains to query. These servers will receive a request
+	// via this API: https://matrix.org/docs/spec/server_server/latest#public-room-directory
+	// This will be called -on demand- by clients, so cache appropriately!
+	Homeservers() []string
 }