MSC2836 threading: part 2 (#1596)

* Update GMSL

* Add MSC2836EventRelationships to fedsender

* Call MSC2836EventRelationships in reqCtx

* auth remote servers

* Extract room ID and servers from previous events; refactor a bit

* initial cut of federated threading

* Use the right client/fed struct in the response

* Add QueryAuthChain for use with MSC2836

* Add auth chain to federated response

* Fix pointers

* under CI: more logging and enable mscs, nil fix

* Handle direction: up

* Actually send message events to the roomserver..

* Add children and children_hash to unsigned, with tests

* Add logic for exploring threads and tracking children; missing storage functions

* Implement storage functions for children

* Add fetchUnknownEvent

* Do federated hits for include_children if we have unexplored children

* Use /ev_rel rather than /event as the former includes child metadata

* Remove cross-room threading impl

* Enable MSC2836 in the p2p demo

* Namespace mscs db

* Enable msc2836 for ygg

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>

7 files changed, 66 insertions, 2 deletions

diff --git a/roomserver/api/api.go b/roomserver/api/api.go
index bef2bb3f..ebc068ac 100644
--- a/roomserver/api/api.go
+++ b/roomserver/api/api.go
@@ -132,6 +132,15 @@ type RoomserverInternalAPI interface {
 		response *QueryStateAndAuthChainResponse,
 	) error
 
+	// QueryAuthChain returns the entire auth chain for the event IDs given.
+	// The response includes the events in the request.
+	// Omits without error for any missing auth events. There will be no duplicates.
+	QueryAuthChain(
+		ctx context.Context,
+		request *QueryAuthChainRequest,
+		response *QueryAuthChainResponse,
+	) error
+
 	// QueryCurrentState retrieves the requested state events. If state events are not found, they will be missing from
 	// the response.
 	QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error
diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go
index eb2b2e1d..c279807e 100644
--- a/roomserver/api/api_trace.go
+++ b/roomserver/api/api_trace.go
@@ -324,6 +324,16 @@ func (t *RoomserverInternalAPITrace) QueryServerBannedFromRoom(ctx context.Conte
 	return err
 }
 
+func (t *RoomserverInternalAPITrace) QueryAuthChain(
+	ctx context.Context,
+	request *QueryAuthChainRequest,
+	response *QueryAuthChainResponse,
+) error {
+	err := t.Impl.QueryAuthChain(ctx, request, response)
+	util.GetLogger(ctx).WithError(err).Infof("QueryAuthChain req=%+v res=%+v", js(request), js(response))
+	return err
+}
+
 func js(thing interface{}) string {
 	b, err := json.Marshal(thing)
 	if err != nil {
diff --git a/roomserver/api/query.go b/roomserver/api/query.go
index 55922b7f..43e562a9 100644
--- a/roomserver/api/query.go
+++ b/roomserver/api/query.go
@@ -275,6 +275,14 @@ type QueryPublishedRoomsResponse struct {
 	RoomIDs []string
 }
 
+type QueryAuthChainRequest struct {
+	EventIDs []string
+}
+
+type QueryAuthChainResponse struct {
+	AuthChain []*gomatrixserverlib.HeaderedEvent
+}
+
 type QuerySharedUsersRequest struct {
 	UserID         string
 	ExcludeRoomIDs []string
diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go
index 79dc2fe1..2bed0c7f 100644
--- a/roomserver/internal/input/input.go
+++ b/roomserver/internal/input/input.go
@@ -62,10 +62,10 @@ func (w *inputWorker) start() {
 	for {
 		select {
 		case task := <-w.input:
-			hooks.Run(hooks.KindNewEventReceived, &task.event.Event)
+			hooks.Run(hooks.KindNewEventReceived, task.event.Event)
 			_, task.err = w.r.processRoomEvent(task.ctx, task.event)
 			if task.err == nil {
-				hooks.Run(hooks.KindNewEventPersisted, &task.event.Event)
+				hooks.Run(hooks.KindNewEventPersisted, task.event.Event)
 			}
 			task.wg.Done()
 		case <-time.After(time.Second * 5):
diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go
index c9940c3c..7346c7a7 100644
--- a/roomserver/internal/query/query.go
+++ b/roomserver/internal/query/query.go
@@ -716,3 +716,16 @@ func (r *Queryer) QueryServerBannedFromRoom(ctx context.Context, req *api.QueryS
 	res.Banned = r.ServerACLs.IsServerBannedFromRoom(req.ServerName, req.RoomID)
 	return nil
 }
+
+func (r *Queryer) QueryAuthChain(ctx context.Context, req *api.QueryAuthChainRequest, res *api.QueryAuthChainResponse) error {
+	chain, err := getAuthChain(ctx, r.DB.EventsFromIDs, req.EventIDs)
+	if err != nil {
+		return err
+	}
+	hchain := make([]*gomatrixserverlib.HeaderedEvent, len(chain))
+	for i := range chain {
+		hchain[i] = chain[i].Headered(chain[i].Version())
+	}
+	res.AuthChain = hchain
+	return nil
+}
diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go
index e496b81e..8a1c91d2 100644
--- a/roomserver/inthttp/client.go
+++ b/roomserver/inthttp/client.go
@@ -54,6 +54,7 @@ const (
 	RoomserverQuerySharedUsersPath             = "/roomserver/querySharedUsers"
 	RoomserverQueryKnownUsersPath              = "/roomserver/queryKnownUsers"
 	RoomserverQueryServerBannedFromRoomPath    = "/roomserver/queryServerBannedFromRoom"
+	RoomserverQueryAuthChainPath               = "/roomserver/queryAuthChain"
 )
 
 type httpRoomserverInternalAPI struct {
@@ -502,6 +503,16 @@ func (h *httpRoomserverInternalAPI) QueryKnownUsers(
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
 }
 
+func (h *httpRoomserverInternalAPI) QueryAuthChain(
+	ctx context.Context, req *api.QueryAuthChainRequest, res *api.QueryAuthChainResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryAuthChain")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverQueryAuthChainPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+}
+
 func (h *httpRoomserverInternalAPI) QueryServerBannedFromRoom(
 	ctx context.Context, req *api.QueryServerBannedFromRoomRequest, res *api.QueryServerBannedFromRoomResponse,
 ) error {
diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go
index ac1fc25b..f9c8ef9f 100644
--- a/roomserver/inthttp/server.go
+++ b/roomserver/inthttp/server.go
@@ -452,4 +452,17 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(RoomserverQueryAuthChainPath,
+		httputil.MakeInternalAPI("queryAuthChain", func(req *http.Request) util.JSONResponse {
+			request := api.QueryAuthChainRequest{}
+			response := api.QueryAuthChainResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := r.QueryAuthChain(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 }