Replace publicroomsapi with a combination of clientapi/roomserver/currentstateserver (#1174)

* Use content_value instead of membership

* Fix build

* Replace publicroomsapi with a combination of clientapi/roomserver/currentstateserver

- All public rooms paths are now handled by clientapi
- Requests to (un)publish rooms are sent to the roomserver via `PerformPublish`
  which are stored in a new `published_table.go`
- Requests for public rooms are handled in clientapi by:
    * Fetch all room IDs which are published using `QueryPublishedRooms` on the roomserver.
    * Apply pagination parameters to the slice.
    * Do a `QueryBulkStateContent` request to the currentstateserver to pull out
      required state event *content* (not entire events).
    * Aggregate and return the chunk.

Mostly but not fully implemented (DB queries on currentstateserver are missing)

* Fix pq query

* Make postgres work

* Make sqlite work

* Fix tests

* Unbreak pagination tests

* Linting

15 files changed, 370 insertions, 0 deletions

diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index 26ec8ca1..0a5845dd 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -36,6 +36,18 @@ type RoomserverInternalAPI interface {
 		res *PerformLeaveResponse,
 	) error
 
+	PerformPublish(
+		ctx context.Context,
+		req *PerformPublishRequest,
+		res *PerformPublishResponse,
+	)
+
+	QueryPublishedRooms(
+		ctx context.Context,
+		req *QueryPublishedRoomsRequest,
+		res *QueryPublishedRoomsResponse,
+	) error
+
 	// Query the latest events and state for a room from the room server.
 	QueryLatestEventsAndState(
 		ctx context.Context,
diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go
index 8645b6f2..bdebc57b 100644
--- a/roomserver/api/api_trace.go
+++ b/roomserver/api/api_trace.go
@@ -57,6 +57,25 @@ func (t *RoomserverInternalAPITrace) PerformLeave(
 	return err
 }
 
+func (t *RoomserverInternalAPITrace) PerformPublish(
+	ctx context.Context,
+	req *PerformPublishRequest,
+	res *PerformPublishResponse,
+) {
+	t.Impl.PerformPublish(ctx, req, res)
+	util.GetLogger(ctx).Infof("PerformPublish req=%+v res=%+v", js(req), js(res))
+}
+
+func (t *RoomserverInternalAPITrace) QueryPublishedRooms(
+	ctx context.Context,
+	req *QueryPublishedRoomsRequest,
+	res *QueryPublishedRoomsResponse,
+) error {
+	err := t.Impl.QueryPublishedRooms(ctx, req, res)
+	util.GetLogger(ctx).WithError(err).Infof("QueryPublishedRooms req=%+v res=%+v", js(req), js(res))
+	return err
+}
+
 func (t *RoomserverInternalAPITrace) QueryLatestEventsAndState(
 	ctx context.Context,
 	req *QueryLatestEventsAndStateRequest,
diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go
index 5d8d88a5..9e844733 100644
--- a/roomserver/api/perform.go
+++ b/roomserver/api/perform.go
@@ -136,3 +136,13 @@ type PerformBackfillResponse struct {
 	// Missing events, arbritrary order.
 	Events []gomatrixserverlib.HeaderedEvent `json:"events"`
 }
+
+type PerformPublishRequest struct {
+	RoomID     string
+	Visibility string
+}
+
+type PerformPublishResponse struct {
+	// If non-nil, the publish request failed. Contains more information why it failed.
+	Error *PerformError
+}
diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index f0cb9374..4e1d09c3 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -215,3 +215,13 @@ type QueryRoomVersionForRoomRequest struct {
 type QueryRoomVersionForRoomResponse struct {
 	RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"`
 }
+
+type QueryPublishedRoomsRequest struct {
+	// Optional. If specified, returns whether this room is published or not.
+	RoomID string
+}
+
+type QueryPublishedRoomsResponse struct {
+	// The list of published rooms.
+	RoomIDs []string
+}
diff --git a/roomserver/internal/perform_publish.go b/roomserver/internal/perform_publish.go
new file mode 100644
index 00000000..d7863620
--- /dev/null
+++ b/roomserver/internal/perform_publish.go
@@ -0,0 +1,20 @@
+package internal
+
+import (
+	"context"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+)
+
+func (r *RoomserverInternalAPI) PerformPublish(
+	ctx context.Context,
+	req *api.PerformPublishRequest,
+	res *api.PerformPublishResponse,
+) {
+	err := r.DB.PublishRoom(ctx, req.RoomID, req.Visibility == "public")
+	if err != nil {
+		res.Error = &api.PerformError{
+			Msg: err.Error(),
+		}
+	}
+}
diff --git a/roomserver/internal/query.go b/roomserver/internal/query.go
index 19236bfb..7fa3247a 100644
--- a/roomserver/internal/query.go
+++ b/roomserver/internal/query.go
@@ -930,3 +930,16 @@ func (r *RoomserverInternalAPI) QueryRoomVersionForRoom(
 	r.Cache.StoreRoomVersion(request.RoomID, response.RoomVersion)
 	return nil
 }
+
+func (r *RoomserverInternalAPI) QueryPublishedRooms(
+	ctx context.Context,
+	req *api.QueryPublishedRoomsRequest,
+	res *api.QueryPublishedRoomsResponse,
+) error {
+	rooms, err := r.DB.GetPublishedRooms(ctx)
+	if err != nil {
+		return err
+	}
+	res.RoomIDs = rooms
+	return nil
+}
diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go
index 8a2b1204..ad24af4a 100644
--- a/roomserver/inthttp/client.go
+++ b/roomserver/inthttp/client.go
@@ -29,6 +29,7 @@ const (
 	RoomserverPerformJoinPath     = "/roomserver/performJoin"
 	RoomserverPerformLeavePath    = "/roomserver/performLeave"
 	RoomserverPerformBackfillPath = "/roomserver/performBackfill"
+	RoomserverPerformPublishPath  = "/roomserver/performPublish"
 
 	// Query operations
 	RoomserverQueryLatestEventsAndStatePath    = "/roomserver/queryLatestEventsAndState"
@@ -41,6 +42,7 @@ const (
 	RoomserverQueryStateAndAuthChainPath       = "/roomserver/queryStateAndAuthChain"
 	RoomserverQueryRoomVersionCapabilitiesPath = "/roomserver/queryRoomVersionCapabilities"
 	RoomserverQueryRoomVersionForRoomPath      = "/roomserver/queryRoomVersionForRoom"
+	RoomserverQueryPublishedRoomsPath          = "/roomserver/queryPublishedRooms"
 )
 
 type httpRoomserverInternalAPI struct {
@@ -194,6 +196,23 @@ func (h *httpRoomserverInternalAPI) PerformLeave(
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
+func (h *httpRoomserverInternalAPI) PerformPublish(
+	ctx context.Context,
+	req *api.PerformPublishRequest,
+	res *api.PerformPublishResponse,
+) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformPublish")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverPerformPublishPath
+	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err),
+		}
+	}
+}
+
 // QueryLatestEventsAndState implements RoomserverQueryAPI
 func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState(
 	ctx context.Context,
@@ -233,6 +252,18 @@ func (h *httpRoomserverInternalAPI) QueryEventsByID(
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
+func (h *httpRoomserverInternalAPI) QueryPublishedRooms(
+	ctx context.Context,
+	request *api.QueryPublishedRoomsRequest,
+	response *api.QueryPublishedRoomsResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPublishedRooms")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverQueryPublishedRoomsPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
+
 // QueryMembershipForUser implements RoomserverQueryAPI
 func (h *httpRoomserverInternalAPI) QueryMembershipForUser(
 	ctx context.Context,
diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go
index 1c47e87e..bb54abf9 100644
--- a/roomserver/inthttp/server.go
+++ b/roomserver/inthttp/server.go
@@ -61,6 +61,31 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(RoomserverPerformPublishPath,
+		httputil.MakeInternalAPI("performPublish", func(req *http.Request) util.JSONResponse {
+			var request api.PerformPublishRequest
+			var response api.PerformPublishResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			r.PerformPublish(req.Context(), &request, &response)
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	internalAPIMux.Handle(
+		RoomserverQueryPublishedRoomsPath,
+		httputil.MakeInternalAPI("queryPublishedRooms", func(req *http.Request) util.JSONResponse {
+			var request api.QueryPublishedRoomsRequest
+			var response api.QueryPublishedRoomsResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryPublishedRooms(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 	internalAPIMux.Handle(
 		RoomserverQueryLatestEventsAndStatePath,
 		httputil.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse {
diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go
index 0c4e2e0b..5c916f29 100644
--- a/roomserver/storage/interface.go
+++ b/roomserver/storage/interface.go
@@ -139,4 +139,8 @@ type Database interface {
 	EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error)
 	// Look up the room version for a given room.
 	GetRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error)
+	// Publish or unpublish a room from the room directory.
+	PublishRoom(ctx context.Context, roomID string, publish bool) error
+	// Returns a list of room IDs for rooms which are published.
+	GetPublishedRooms(ctx context.Context) ([]string, error)
 }
diff --git a/roomserver/storage/postgres/published_table.go b/roomserver/storage/postgres/published_table.go
new file mode 100644
index 00000000..23a9b067
--- /dev/null
+++ b/roomserver/storage/postgres/published_table.go
@@ -0,0 +1,101 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/roomserver/storage/shared"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+)
+
+const publishedSchema = `
+-- Stores which rooms are published in the room directory
+CREATE TABLE IF NOT EXISTS roomserver_published (
+    -- The room ID of the room
+    room_id TEXT NOT NULL PRIMARY KEY,
+    -- Whether it is published or not
+    published BOOLEAN NOT NULL DEFAULT false
+);
+`
+
+const upsertPublishedSQL = "" +
+	"INSERT INTO roomserver_published (room_id, published) VALUES ($1, $2) " +
+	"ON CONFLICT (room_id) DO UPDATE SET published=$2"
+
+const selectAllPublishedSQL = "" +
+	"SELECT room_id FROM roomserver_published WHERE published = $1 ORDER BY room_id ASC"
+
+const selectPublishedSQL = "" +
+	"SELECT published FROM roomserver_published WHERE room_id = $1"
+
+type publishedStatements struct {
+	upsertPublishedStmt    *sql.Stmt
+	selectAllPublishedStmt *sql.Stmt
+	selectPublishedStmt    *sql.Stmt
+}
+
+func NewPostgresPublishedTable(db *sql.DB) (tables.Published, error) {
+	s := &publishedStatements{}
+	_, err := db.Exec(publishedSchema)
+	if err != nil {
+		return nil, err
+	}
+	return s, shared.StatementList{
+		{&s.upsertPublishedStmt, upsertPublishedSQL},
+		{&s.selectAllPublishedStmt, selectAllPublishedSQL},
+		{&s.selectPublishedStmt, selectPublishedSQL},
+	}.Prepare(db)
+}
+
+func (s *publishedStatements) UpsertRoomPublished(
+	ctx context.Context, roomID string, published bool,
+) (err error) {
+	_, err = s.upsertPublishedStmt.ExecContext(ctx, roomID, published)
+	return
+}
+
+func (s *publishedStatements) SelectPublishedFromRoomID(
+	ctx context.Context, roomID string,
+) (published bool, err error) {
+	err = s.selectPublishedStmt.QueryRowContext(ctx, roomID).Scan(&published)
+	if err == sql.ErrNoRows {
+		return false, nil
+	}
+	return
+}
+
+func (s *publishedStatements) SelectAllPublishedRooms(
+	ctx context.Context, published bool,
+) ([]string, error) {
+	rows, err := s.selectAllPublishedStmt.QueryContext(ctx, published)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectAllPublishedStmt: rows.close() failed")
+
+	var roomIDs []string
+	for rows.Next() {
+		var roomID string
+		if err = rows.Scan(&roomID); err != nil {
+			return nil, err
+		}
+
+		roomIDs = append(roomIDs, roomID)
+	}
+	return roomIDs, rows.Err()
+}
diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go
index d76ee0a9..23d078e4 100644
--- a/roomserver/storage/postgres/storage.go
+++ b/roomserver/storage/postgres/storage.go
@@ -87,6 +87,10 @@ func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database,
 	if err != nil {
 		return nil, err
 	}
+	published, err := NewPostgresPublishedTable(db)
+	if err != nil {
+		return nil, err
+	}
 	d.Database = shared.Database{
 		DB:                  db,
 		EventTypesTable:     eventTypes,
@@ -101,6 +105,7 @@ func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database,
 		RoomAliasesTable:    roomAliases,
 		InvitesTable:        invites,
 		MembershipTable:     membership,
+		PublishedTable:      published,
 	}
 	return &d, nil
 }
diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go
index e6d0e34e..166822d0 100644
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@@ -26,6 +26,7 @@ type Database struct {
 	PrevEventsTable     tables.PreviousEvents
 	InvitesTable        tables.Invites
 	MembershipTable     tables.Membership
+	PublishedTable      tables.Published
 }
 
 func (d *Database) EventTypeNIDs(
@@ -420,6 +421,14 @@ func (d *Database) StoreEvent(
 	}, nil
 }
 
+func (d *Database) PublishRoom(ctx context.Context, roomID string, publish bool) error {
+	return d.PublishedTable.UpsertRoomPublished(ctx, roomID, publish)
+}
+
+func (d *Database) GetPublishedRooms(ctx context.Context) ([]string, error) {
+	return d.PublishedTable.SelectAllPublishedRooms(ctx, true)
+}
+
 func (d *Database) assignRoomNID(
 	ctx context.Context, txn *sql.Tx,
 	roomID string, roomVersion gomatrixserverlib.RoomVersion,
diff --git a/roomserver/storage/sqlite3/published_table.go b/roomserver/storage/sqlite3/published_table.go
new file mode 100644
index 00000000..9995fff6
--- /dev/null
+++ b/roomserver/storage/sqlite3/published_table.go
@@ -0,0 +1,100 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/roomserver/storage/shared"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+)
+
+const publishedSchema = `
+-- Stores which rooms are published in the room directory
+CREATE TABLE IF NOT EXISTS roomserver_published (
+    -- The room ID of the room
+    room_id TEXT NOT NULL PRIMARY KEY,
+    -- Whether it is published or not
+    published BOOLEAN NOT NULL DEFAULT false
+);
+`
+
+const upsertPublishedSQL = "" +
+	"INSERT OR REPLACE INTO roomserver_published (room_id, published) VALUES ($1, $2)"
+
+const selectAllPublishedSQL = "" +
+	"SELECT room_id FROM roomserver_published WHERE published = $1 ORDER BY room_id ASC"
+
+const selectPublishedSQL = "" +
+	"SELECT published FROM roomserver_published WHERE room_id = $1"
+
+type publishedStatements struct {
+	upsertPublishedStmt    *sql.Stmt
+	selectAllPublishedStmt *sql.Stmt
+	selectPublishedStmt    *sql.Stmt
+}
+
+func NewSqlitePublishedTable(db *sql.DB) (tables.Published, error) {
+	s := &publishedStatements{}
+	_, err := db.Exec(publishedSchema)
+	if err != nil {
+		return nil, err
+	}
+	return s, shared.StatementList{
+		{&s.upsertPublishedStmt, upsertPublishedSQL},
+		{&s.selectAllPublishedStmt, selectAllPublishedSQL},
+		{&s.selectPublishedStmt, selectPublishedSQL},
+	}.Prepare(db)
+}
+
+func (s *publishedStatements) UpsertRoomPublished(
+	ctx context.Context, roomID string, published bool,
+) (err error) {
+	_, err = s.upsertPublishedStmt.ExecContext(ctx, roomID, published)
+	return
+}
+
+func (s *publishedStatements) SelectPublishedFromRoomID(
+	ctx context.Context, roomID string,
+) (published bool, err error) {
+	err = s.selectPublishedStmt.QueryRowContext(ctx, roomID).Scan(&published)
+	if err == sql.ErrNoRows {
+		return false, nil
+	}
+	return
+}
+
+func (s *publishedStatements) SelectAllPublishedRooms(
+	ctx context.Context, published bool,
+) ([]string, error) {
+	rows, err := s.selectAllPublishedStmt.QueryContext(ctx, published)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectAllPublishedStmt: rows.close() failed")
+
+	var roomIDs []string
+	for rows.Next() {
+		var roomID string
+		if err = rows.Scan(&roomID); err != nil {
+			return nil, err
+		}
+
+		roomIDs = append(roomIDs, roomID)
+	}
+	return roomIDs, rows.Err()
+}
diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go
index 8e935219..767b13ce 100644
--- a/roomserver/storage/sqlite3/storage.go
+++ b/roomserver/storage/sqlite3/storage.go
@@ -110,6 +110,10 @@ func Open(dataSourceName string) (*Database, error) {
 	if err != nil {
 		return nil, err
 	}
+	published, err := NewSqlitePublishedTable(d.db)
+	if err != nil {
+		return nil, err
+	}
 	d.Database = shared.Database{
 		DB:                  d.db,
 		EventsTable:         d.events,
@@ -124,6 +128,7 @@ func Open(dataSourceName string) (*Database, error) {
 		RoomAliasesTable:    roomAliases,
 		InvitesTable:        d.invites,
 		MembershipTable:     d.membership,
+		PublishedTable:      published,
 	}
 	return &d, nil
 }
diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go
index 3aa8c538..7499089c 100644
--- a/roomserver/storage/tables/interface.go
+++ b/roomserver/storage/tables/interface.go
@@ -120,3 +120,9 @@ type Membership interface {
 	SelectMembershipsFromRoomAndMembership(ctx context.Context, roomNID types.RoomNID, membership MembershipState, localOnly bool) (eventNIDs []types.EventNID, err error)
 	UpdateMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, senderUserNID types.EventStateKeyNID, membership MembershipState, eventNID types.EventNID) error
 }
+
+type Published interface {
+	UpsertRoomPublished(ctx context.Context, roomID string, published bool) (err error)
+	SelectPublishedFromRoomID(ctx context.Context, roomID string) (published bool, err error)
+	SelectAllPublishedRooms(ctx context.Context, published bool) ([]string, error)
+}