Set historyVisibility for backfilled events over federation (#2656)

This should hopefully deflake Backfill works correctly with history visibility set to joined as we were using the default shared visibility, even if the events are set to joined (or something else)

3 files changed, 31 insertions, 13 deletions

diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go
index d9ea9dd1..20931f80 100644
--- a/roomserver/api/perform.go
+++ b/roomserver/api/perform.go
@@ -5,9 +5,10 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 )
 
 type PerformErrorCode int
@@ -161,7 +162,8 @@ func (r *PerformBackfillRequest) PrevEventIDs() []string {
 // PerformBackfillResponse is a response to PerformBackfill.
 type PerformBackfillResponse struct {
 	// Missing events, arbritrary order.
-	Events []*gomatrixserverlib.HeaderedEvent `json:"events"`
+	Events            []*gomatrixserverlib.HeaderedEvent  `json:"events"`
+	HistoryVisibility gomatrixserverlib.HistoryVisibility `json:"history_visibility"`
 }
 
 type PerformPublishRequest struct {
diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go
index aecff8b8..de76b641 100644
--- a/roomserver/internal/perform/perform_backfill.go
+++ b/roomserver/internal/perform/perform_backfill.go
@@ -164,6 +164,7 @@ func (r *Backfiller) backfillViaFederation(ctx context.Context, req *api.Perform
 	// TODO: update backwards extremities, as that should be moved from syncapi to roomserver at some point.
 
 	res.Events = events
+	res.HistoryVisibility = requester.historyVisiblity
 	return nil
 }
 
@@ -248,6 +249,7 @@ type backfillRequester struct {
 	servers                 []gomatrixserverlib.ServerName
 	eventIDToBeforeStateIDs map[string][]string
 	eventIDMap              map[string]*gomatrixserverlib.Event
+	historyVisiblity        gomatrixserverlib.HistoryVisibility
 }
 
 func newBackfillRequester(
@@ -266,6 +268,7 @@ func newBackfillRequester(
 		eventIDMap:              make(map[string]*gomatrixserverlib.Event),
 		bwExtrems:               bwExtrems,
 		preferServer:            preferServer,
+		historyVisiblity:        gomatrixserverlib.HistoryVisibilityShared,
 	}
 }
 
@@ -447,7 +450,8 @@ FindSuccessor:
 	}
 
 	// possibly return all joined servers depending on history visiblity
-	memberEventsFromVis, err := joinEventsFromHistoryVisibility(ctx, b.db, roomID, stateEntries, b.thisServer)
+	memberEventsFromVis, visibility, err := joinEventsFromHistoryVisibility(ctx, b.db, roomID, stateEntries, b.thisServer)
+	b.historyVisiblity = visibility
 	if err != nil {
 		logrus.WithError(err).Error("ServersAtEvent: failed calculate servers from history visibility rules")
 		return nil
@@ -528,7 +532,7 @@ func (b *backfillRequester) ProvideEvents(roomVer gomatrixserverlib.RoomVersion,
 // pull all events and then filter by that table.
 func joinEventsFromHistoryVisibility(
 	ctx context.Context, db storage.Database, roomID string, stateEntries []types.StateEntry,
-	thisServer gomatrixserverlib.ServerName) ([]types.Event, error) {
+	thisServer gomatrixserverlib.ServerName) ([]types.Event, gomatrixserverlib.HistoryVisibility, error) {
 
 	var eventNIDs []types.EventNID
 	for _, entry := range stateEntries {
@@ -542,7 +546,9 @@ func joinEventsFromHistoryVisibility(
 	// Get all of the events in this state
 	stateEvents, err := db.Events(ctx, eventNIDs)
 	if err != nil {
-		return nil, err
+		// even though the default should be shared, restricting the visibility to joined
+		// feels more secure here.
+		return nil, gomatrixserverlib.HistoryVisibilityJoined, err
 	}
 	events := make([]*gomatrixserverlib.Event, len(stateEvents))
 	for i := range stateEvents {
@@ -551,20 +557,22 @@ func joinEventsFromHistoryVisibility(
 
 	// Can we see events in the room?
 	canSeeEvents := auth.IsServerAllowed(thisServer, true, events)
+	visibility := gomatrixserverlib.HistoryVisibility(auth.HistoryVisibilityForRoom(events))
 	if !canSeeEvents {
-		logrus.Infof("ServersAtEvent history not visible to us: %s", auth.HistoryVisibilityForRoom(events))
-		return nil, nil
+		logrus.Infof("ServersAtEvent history not visible to us: %s", visibility)
+		return nil, visibility, nil
 	}
 	// get joined members
 	info, err := db.RoomInfo(ctx, roomID)
 	if err != nil {
-		return nil, err
+		return nil, visibility, nil
 	}
 	joinEventNIDs, err := db.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, true, false)
 	if err != nil {
-		return nil, err
+		return nil, visibility, err
 	}
-	return db.Events(ctx, joinEventNIDs)
+	evs, err := db.Events(ctx, joinEventNIDs)
+	return evs, visibility, err
 }
 
 func persistEvents(ctx context.Context, db storage.Database, events []*gomatrixserverlib.HeaderedEvent) (types.RoomNID, map[string]types.Event) {
diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go
index 9db3d8e1..03614302 100644
--- a/syncapi/routing/messages.go
+++ b/syncapi/routing/messages.go
@@ -350,8 +350,10 @@ func (r *messagesReq) retrieveEvents() (
 	startTime := time.Now()
 	filteredEvents, err := internal.ApplyHistoryVisibilityFilter(r.ctx, r.db, r.rsAPI, events, nil, r.device.UserID, "messages")
 	logrus.WithFields(logrus.Fields{
-		"duration": time.Since(startTime),
-		"room_id":  r.roomID,
+		"duration":      time.Since(startTime),
+		"room_id":       r.roomID,
+		"events_before": len(events),
+		"events_after":  len(filteredEvents),
 	}).Debug("applied history visibility (messages)")
 	return gomatrixserverlib.HeaderedToClientEvents(filteredEvents, gomatrixserverlib.FormatAll), start, end, err
 }
@@ -513,6 +515,9 @@ func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][]
 
 	// Store the events in the database, while marking them as unfit to show
 	// up in responses to sync requests.
+	if res.HistoryVisibility == "" {
+		res.HistoryVisibility = gomatrixserverlib.HistoryVisibilityShared
+	}
 	for i := range res.Events {
 		_, err = r.db.WriteEvent(
 			context.Background(),
@@ -521,7 +526,7 @@ func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][]
 			[]string{},
 			[]string{},
 			nil, true,
-			gomatrixserverlib.HistoryVisibilityShared,
+			res.HistoryVisibility,
 		)
 		if err != nil {
 			return nil, err
@@ -534,6 +539,9 @@ func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][]
 		// last `limit` events
 		events = events[len(events)-limit:]
 	}
+	for _, ev := range events {
+		ev.Visibility = res.HistoryVisibility
+	}
 
 	return events, nil
 }