Handle state before, send history visibility in output (#2532)

* Check state before event

* Tweaks

* Refactor a bit, include in output events

* Don't waste time if soft failed either

* Tweak control flow, comments, use GMSL history visibility type

7 files changed, 142 insertions, 22 deletions

diff --git a/clientapi/routing/aliases.go b/clientapi/routing/aliases.go
index 504d6026..68d0f419 100644
--- a/clientapi/routing/aliases.go
+++ b/clientapi/routing/aliases.go
@@ -44,7 +44,7 @@ func GetAliases(
 		return util.ErrorResponse(fmt.Errorf("rsAPI.QueryCurrentState: %w", err))
 	}
 
-	visibility := "invite"
+	visibility := gomatrixserverlib.HistoryVisibilityInvited
 	if historyVisEvent, ok := stateRes.StateEvents[stateTuple]; ok {
 		var err error
 		visibility, err = historyVisEvent.HistoryVisibility()
diff --git a/go.mod b/go.mod
index ea6e8cae..b2a09675 100644
--- a/go.mod
+++ b/go.mod
@@ -34,7 +34,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220607143425-e55d796fd0b3
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a
 	github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.13
diff --git a/go.sum b/go.sum
index e21794f4..3a35c47d 100644
--- a/go.sum
+++ b/go.sum
@@ -418,8 +418,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220607143425-e55d796fd0b3 h1:2eYcBt8Kg+nW/xIJY5x8Uo2dQLjUF+oxLap00uFC5l8=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220607143425-e55d796fd0b3/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a h1:jOkrb6twViAGTHHadA51sQwdloHT0Vx1MCptk9InTHo=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
diff --git a/roomserver/api/output.go b/roomserver/api/output.go
index a82bf870..36d0625c 100644
--- a/roomserver/api/output.go
+++ b/roomserver/api/output.go
@@ -161,6 +161,8 @@ type OutputNewRoomEvent struct {
 	// The transaction ID of the send request if sent by a local user and one
 	// was specified
 	TransactionID *TransactionID `json:"transaction_id,omitempty"`
+	// The history visibility of the event.
+	HistoryVisibility gomatrixserverlib.HistoryVisibility `json:"history_visibility"`
 }
 
 func (o *OutputNewRoomEvent) NeededStateEventIDs() ([]*gomatrixserverlib.HeaderedEvent, []string) {
@@ -187,7 +189,8 @@ func (o *OutputNewRoomEvent) NeededStateEventIDs() ([]*gomatrixserverlib.Headere
 // should build their current room state up from OutputNewRoomEvents only.
 type OutputOldRoomEvent struct {
 	// The Event.
-	Event *gomatrixserverlib.HeaderedEvent `json:"event"`
+	Event             *gomatrixserverlib.HeaderedEvent    `json:"event"`
+	HistoryVisibility gomatrixserverlib.HistoryVisibility `json:"history_visibility"`
 }
 
 // An OutputNewInviteEvent is written whenever an invite becomes active.
diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go
index deb88ea8..ff05f798 100644
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@@ -295,6 +295,22 @@ func (r *Inputer) processRoomEvent(
 		}
 	}
 
+	// Get the state before the event so that we can work out if the event was
+	// allowed at the time, and also to get the history visibility. We won't
+	// bother doing this if the event was already rejected as it just ends up
+	// burning CPU time.
+	historyVisibility := gomatrixserverlib.HistoryVisibilityJoined // Default to restrictive.
+	if rejectionErr == nil && !isRejected && !softfail {
+		var err error
+		historyVisibility, rejectionErr, err = r.processStateBefore(ctx, input, missingPrev)
+		if err != nil {
+			return fmt.Errorf("r.processStateBefore: %w", err)
+		}
+		if rejectionErr != nil {
+			isRejected = true
+		}
+	}
+
 	// Store the event.
 	_, _, stateAtEvent, redactionEvent, redactedEventID, err := r.DB.StoreEvent(ctx, event, authEventNIDs, isRejected || softfail)
 	if err != nil {
@@ -360,6 +376,7 @@ func (r *Inputer) processRoomEvent(
 			input.SendAsServer,  // send as server
 			input.TransactionID, // transaction ID
 			input.HasState,      // rewrites state?
+			historyVisibility,   // the history visibility before the event
 		); err != nil {
 			return fmt.Errorf("r.updateLatestEvents: %w", err)
 		}
@@ -368,7 +385,8 @@ func (r *Inputer) processRoomEvent(
 			{
 				Type: api.OutputTypeOldRoomEvent,
 				OldRoomEvent: &api.OutputOldRoomEvent{
-					Event: headered,
+					Event:             headered,
+					HistoryVisibility: historyVisibility,
 				},
 			},
 		})
@@ -402,6 +420,100 @@ func (r *Inputer) processRoomEvent(
 	return nil
 }
 
+// processStateBefore works out what the state is before the event and
+// then checks the event auths against the state at the time. It also
+// tries to determine what the history visibility was of the event at
+// the time, so that it can be sent in the output event to downstream
+// components.
+// nolint:nakedret
+func (r *Inputer) processStateBefore(
+	ctx context.Context,
+	input *api.InputRoomEvent,
+	missingPrev bool,
+) (historyVisibility gomatrixserverlib.HistoryVisibility, rejectionErr error, err error) {
+	historyVisibility = gomatrixserverlib.HistoryVisibilityJoined // Default to restrictive.
+	event := input.Event.Unwrap()
+	isCreateEvent := event.Type() == gomatrixserverlib.MRoomCreate && event.StateKeyEquals("")
+	var stateBeforeEvent []*gomatrixserverlib.Event
+	switch {
+	case isCreateEvent:
+		// There's no state before a create event so there is nothing
+		// else to do.
+		return
+	case input.HasState:
+		// If we're overriding the state then we need to go and retrieve
+		// them from the database. It's a hard error if they are missing.
+		stateEvents, err := r.DB.EventsFromIDs(ctx, input.StateEventIDs)
+		if err != nil {
+			return "", nil, fmt.Errorf("r.DB.EventsFromIDs: %w", err)
+		}
+		stateBeforeEvent = make([]*gomatrixserverlib.Event, 0, len(stateEvents))
+		for _, entry := range stateEvents {
+			stateBeforeEvent = append(stateBeforeEvent, entry.Event)
+		}
+	case missingPrev:
+		// We don't know all of the prev events, so we can't work out
+		// the state before the event. Reject it in that case.
+		rejectionErr = fmt.Errorf("event %q has missing prev events", event.EventID())
+		return
+	case len(event.PrevEventIDs()) == 0:
+		// There should be prev events since it's not a create event.
+		// A non-create event that claims to have no prev events is
+		// invalid, so reject it.
+		rejectionErr = fmt.Errorf("event %q must have prev events", event.EventID())
+		return
+	default:
+		// For all non-create events, there must be prev events, so we'll
+		// ask the query API for the relevant tuples needed for auth. We
+		// will include the history visibility here even though we don't
+		// actually need it for auth, because we want to send it in the
+		// output events.
+		tuplesNeeded := gomatrixserverlib.StateNeededForAuth([]*gomatrixserverlib.Event{event}).Tuples()
+		tuplesNeeded = append(tuplesNeeded, gomatrixserverlib.StateKeyTuple{
+			EventType: gomatrixserverlib.MRoomHistoryVisibility,
+			StateKey:  "",
+		})
+		stateBeforeReq := &api.QueryStateAfterEventsRequest{
+			RoomID:       event.RoomID(),
+			PrevEventIDs: event.PrevEventIDs(),
+			StateToFetch: tuplesNeeded,
+		}
+		stateBeforeRes := &api.QueryStateAfterEventsResponse{}
+		if err := r.Queryer.QueryStateAfterEvents(ctx, stateBeforeReq, stateBeforeRes); err != nil {
+			return "", nil, fmt.Errorf("r.Queryer.QueryStateAfterEvents: %w", err)
+		}
+		switch {
+		case !stateBeforeRes.RoomExists:
+			rejectionErr = fmt.Errorf("room %q does not exist", event.RoomID())
+			return
+		case !stateBeforeRes.PrevEventsExist:
+			rejectionErr = fmt.Errorf("prev events of %q are not known", event.EventID())
+			return
+		default:
+			stateBeforeEvent = gomatrixserverlib.UnwrapEventHeaders(stateBeforeRes.StateEvents)
+		}
+	}
+	// At this point, stateBeforeEvent should be populated either by
+	// the supplied state in the input request, or from the prev events.
+	// Check whether the event is allowed or not.
+	stateBeforeAuth := gomatrixserverlib.NewAuthEvents(stateBeforeEvent)
+	if rejectionErr = gomatrixserverlib.Allowed(event, &stateBeforeAuth); rejectionErr != nil {
+		return
+	}
+	// Work out what the history visibility was at the time of the
+	// event.
+	for _, event := range stateBeforeEvent {
+		if event.Type() != gomatrixserverlib.MRoomHistoryVisibility || !event.StateKeyEquals("") {
+			continue
+		}
+		if hisVis, err := event.HistoryVisibility(); err == nil {
+			historyVisibility = hisVis
+			break
+		}
+	}
+	return
+}
+
 // fetchAuthEvents will check to see if any of the
 // auth events specified by the given event are unknown. If they are
 // then we will go off and request them from the federation and then
diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go
index 9738ed4e..e76f4ba8 100644
--- a/roomserver/internal/input/input_latest_events.go
+++ b/roomserver/internal/input/input_latest_events.go
@@ -56,6 +56,7 @@ func (r *Inputer) updateLatestEvents(
 	sendAsServer string,
 	transactionID *api.TransactionID,
 	rewritesState bool,
+	historyVisibility gomatrixserverlib.HistoryVisibility,
 ) (err error) {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "updateLatestEvents")
 	defer span.Finish()
@@ -69,15 +70,16 @@ func (r *Inputer) updateLatestEvents(
 	defer sqlutil.EndTransactionWithCheck(updater, &succeeded, &err)
 
 	u := latestEventsUpdater{
-		ctx:           ctx,
-		api:           r,
-		updater:       updater,
-		roomInfo:      roomInfo,
-		stateAtEvent:  stateAtEvent,
-		event:         event,
-		sendAsServer:  sendAsServer,
-		transactionID: transactionID,
-		rewritesState: rewritesState,
+		ctx:               ctx,
+		api:               r,
+		updater:           updater,
+		roomInfo:          roomInfo,
+		stateAtEvent:      stateAtEvent,
+		event:             event,
+		sendAsServer:      sendAsServer,
+		transactionID:     transactionID,
+		rewritesState:     rewritesState,
+		historyVisibility: historyVisibility,
 	}
 
 	if err = u.doUpdateLatestEvents(); err != nil {
@@ -119,6 +121,8 @@ type latestEventsUpdater struct {
 	// The snapshots of current state before and after processing this event
 	oldStateNID types.StateSnapshotNID
 	newStateNID types.StateSnapshotNID
+	// The history visibility of the event itself (from the state before the event).
+	historyVisibility gomatrixserverlib.HistoryVisibility
 }
 
 func (u *latestEventsUpdater) doUpdateLatestEvents() error {
@@ -365,12 +369,13 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error)
 	}
 
 	ore := api.OutputNewRoomEvent{
-		Event:           u.event.Headered(u.roomInfo.RoomVersion),
-		RewritesState:   u.rewritesState,
-		LastSentEventID: u.lastEventIDSent,
-		LatestEventIDs:  latestEventIDs,
-		TransactionID:   u.transactionID,
-		SendAsServer:    u.sendAsServer,
+		Event:             u.event.Headered(u.roomInfo.RoomVersion),
+		RewritesState:     u.rewritesState,
+		LastSentEventID:   u.lastEventIDSent,
+		LatestEventIDs:    latestEventIDs,
+		TransactionID:     u.transactionID,
+		SendAsServer:      u.sendAsServer,
+		HistoryVisibility: u.historyVisibility,
 	}
 
 	eventIDMap, err := u.stateEventMap()
diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go
index 96438e18..d021d365 100644
--- a/syncapi/routing/context.go
+++ b/syncapi/routing/context.go
@@ -97,7 +97,7 @@ func Context(
 	state, _ := syncDB.CurrentState(ctx, roomID, &stateFilter, nil)
 	// verify the user is allowed to see the context for this room/event
 	for _, x := range state {
-		var hisVis string
+		var hisVis gomatrixserverlib.HistoryVisibility
 		hisVis, err = x.HistoryVisibility()
 		if err != nil {
 			continue