From f8d3a762c49a1dafe4e484a2440ade2bb6ba32ac Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 30 Mar 2021 10:01:32 +0100 Subject: Add a per-room mutex to federationapi when processing transactions (#1810) * Add a per-room mutex to federationapi when processing transactions This has numerous benefits: - Prevents us doing lots of state resolutions in busy rooms. Previously, room forks would always result in a state resolution being performed immediately, without checking if we were already doing this in a different transaction. Now they will queue up, resulting in fewer calls to `/state_ids`, `/g_m_e`, etc. - Prevents memory usage from growing too large as a result and potentially OOMing. And costs: - High traffic rooms will be slightly slower due to head-of-line blocking from other servers, though this has always been an issue as roomserver has a per-room mutex already. * Fix unit tests * Correct mutex lock ordering --- federationapi/routing/send.go | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'federationapi/routing/send.go') diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 231a1686..b48d6c0b 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -26,6 +26,7 @@ import ( "github.com/getsentry/sentry-go" "github.com/matrix-org/dendrite/clientapi/jsonerror" eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" + "github.com/matrix-org/dendrite/internal" keyapi "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" @@ -98,6 +99,7 @@ func Send( keyAPI keyapi.KeyInternalAPI, keys gomatrixserverlib.JSONVerifier, federation *gomatrixserverlib.FederationClient, + mu *internal.MutexByRoom, ) util.JSONResponse { t := txnReq{ rsAPI: rsAPI, @@ -107,6 +109,7 @@ func Send( haveEvents: make(map[string]*gomatrixserverlib.HeaderedEvent), newEvents: make(map[string]bool), keyAPI: keyAPI, + roomsMu: mu, } var txnEvents struct { @@ -163,6 +166,7 @@ type txnReq struct { federation txnFederationClient servers []gomatrixserverlib.ServerName serversMutex sync.RWMutex + roomsMu *internal.MutexByRoom // local cache of events for auth checks, etc - this may include events // which the roomserver is unaware of. haveEvents map[string]*gomatrixserverlib.HeaderedEvent @@ -494,6 +498,8 @@ func (t *txnReq) getServers(ctx context.Context, roomID string) []gomatrixserver } func (t *txnReq) processEvent(ctx context.Context, e *gomatrixserverlib.Event) error { + t.roomsMu.Lock(e.RoomID()) + defer t.roomsMu.Unlock(e.RoomID()) logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID()) t.work = "" // reset from previous event -- cgit v1.2.3