Implement /sync `limited` and read timeline limit from stored filters (#1168)

* Move filter table to syncapi where it is used

* Implement /sync `limited` and read timeline limit from stored filters

We now fully handle `room.timeline.limit` filters (in-line + stored) and
return the right value for `limited` syncs.

* Update whitelist

* Default to the default timeline limit if it's unset, also strip the extra event correctly

* Update whitelist

5 files changed, 2 insertions, 312 deletions

diff --git a/userapi/storage/accounts/interface.go b/userapi/storage/accounts/interface.go
index c6692879..9ed33e1b 100644
--- a/userapi/storage/accounts/interface.go
+++ b/userapi/storage/accounts/interface.go
@@ -52,8 +52,6 @@ type Database interface {
 	RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error)
 	GetLocalpartForThreePID(ctx context.Context, threepid string, medium string) (localpart string, err error)
 	GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error)
-	GetFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error)
-	PutFilter(ctx context.Context, localpart string, filter *gomatrixserverlib.Filter) (string, error)
 	CheckAccountAvailability(ctx context.Context, localpart string) (bool, error)
 	GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error)
 }
diff --git a/userapi/storage/accounts/postgres/filter_table.go b/userapi/storage/accounts/postgres/filter_table.go
deleted file mode 100644
index c54e4bc4..00000000
--- a/userapi/storage/accounts/postgres/filter_table.go
+++ /dev/null
@@ -1,127 +0,0 @@
-// Copyright 2017 Jan Christian Grünhage
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package postgres
-
-import (
-	"context"
-	"database/sql"
-	"encoding/json"
-
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-const filterSchema = `
--- Stores data about filters
-CREATE TABLE IF NOT EXISTS account_filter (
-	-- The filter
-	filter TEXT NOT NULL,
-	-- The ID
-	id SERIAL UNIQUE,
-	-- The localpart of the Matrix user ID associated to this filter
-	localpart TEXT NOT NULL,
-
-	PRIMARY KEY(id, localpart)
-);
-
-CREATE INDEX IF NOT EXISTS account_filter_localpart ON account_filter(localpart);
-`
-
-const selectFilterSQL = "" +
-	"SELECT filter FROM account_filter WHERE localpart = $1 AND id = $2"
-
-const selectFilterIDByContentSQL = "" +
-	"SELECT id FROM account_filter WHERE localpart = $1 AND filter = $2"
-
-const insertFilterSQL = "" +
-	"INSERT INTO account_filter (filter, id, localpart) VALUES ($1, DEFAULT, $2) RETURNING id"
-
-type filterStatements struct {
-	selectFilterStmt            *sql.Stmt
-	selectFilterIDByContentStmt *sql.Stmt
-	insertFilterStmt            *sql.Stmt
-}
-
-func (s *filterStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(filterSchema)
-	if err != nil {
-		return
-	}
-	if s.selectFilterStmt, err = db.Prepare(selectFilterSQL); err != nil {
-		return
-	}
-	if s.selectFilterIDByContentStmt, err = db.Prepare(selectFilterIDByContentSQL); err != nil {
-		return
-	}
-	if s.insertFilterStmt, err = db.Prepare(insertFilterSQL); err != nil {
-		return
-	}
-	return
-}
-
-func (s *filterStatements) selectFilter(
-	ctx context.Context, localpart string, filterID string,
-) (*gomatrixserverlib.Filter, error) {
-	// Retrieve filter from database (stored as canonical JSON)
-	var filterData []byte
-	err := s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filterData)
-	if err != nil {
-		return nil, err
-	}
-
-	// Unmarshal JSON into Filter struct
-	var filter gomatrixserverlib.Filter
-	if err = json.Unmarshal(filterData, &filter); err != nil {
-		return nil, err
-	}
-	return &filter, nil
-}
-
-func (s *filterStatements) insertFilter(
-	ctx context.Context, filter *gomatrixserverlib.Filter, localpart string,
-) (filterID string, err error) {
-	var existingFilterID string
-
-	// Serialise json
-	filterJSON, err := json.Marshal(filter)
-	if err != nil {
-		return "", err
-	}
-	// Remove whitespaces and sort JSON data
-	// needed to prevent from inserting the same filter multiple times
-	filterJSON, err = gomatrixserverlib.CanonicalJSON(filterJSON)
-	if err != nil {
-		return "", err
-	}
-
-	// Check if filter already exists in the database using its localpart and content
-	//
-	// This can result in a race condition when two clients try to insert the
-	// same filter and localpart at the same time, however this is not a
-	// problem as both calls will result in the same filterID
-	err = s.selectFilterIDByContentStmt.QueryRowContext(ctx,
-		localpart, filterJSON).Scan(&existingFilterID)
-	if err != nil && err != sql.ErrNoRows {
-		return "", err
-	}
-	// If it does, return the existing ID
-	if existingFilterID != "" {
-		return existingFilterID, err
-	}
-
-	// Otherwise insert the filter and return the new ID
-	err = s.insertFilterStmt.QueryRowContext(ctx, filterJSON, localpart).
-		Scan(&filterID)
-	return
-}
diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go
index e5509980..f0b11bfd 100644
--- a/userapi/storage/accounts/postgres/storage.go
+++ b/userapi/storage/accounts/postgres/storage.go
@@ -40,7 +40,6 @@ type Database struct {
 	memberships  membershipStatements
 	accountDatas accountDataStatements
 	threepids    threepidStatements
-	filter       filterStatements
 	serverName   gomatrixserverlib.ServerName
 }
 
@@ -75,11 +74,7 @@ func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serve
 	if err = t.prepare(db); err != nil {
 		return nil, err
 	}
-	f := filterStatements{}
-	if err = f.prepare(db); err != nil {
-		return nil, err
-	}
-	return &Database{db, partitions, a, p, m, ac, t, f, serverName}, nil
+	return &Database{db, partitions, a, p, m, ac, t, serverName}, nil
 }
 
 // GetAccountByPassword returns the account associated with the given localpart and password.
@@ -396,24 +391,6 @@ func (d *Database) GetThreePIDsForLocalpart(
 	return d.threepids.selectThreePIDsForLocalpart(ctx, localpart)
 }
 
-// GetFilter looks up the filter associated with a given local user and filter ID.
-// Returns a filter structure. Otherwise returns an error if no such filter exists
-// or if there was an error talking to the database.
-func (d *Database) GetFilter(
-	ctx context.Context, localpart string, filterID string,
-) (*gomatrixserverlib.Filter, error) {
-	return d.filter.selectFilter(ctx, localpart, filterID)
-}
-
-// PutFilter puts the passed filter into the database.
-// Returns the filterID as a string. Otherwise returns an error if something
-// goes wrong.
-func (d *Database) PutFilter(
-	ctx context.Context, localpart string, filter *gomatrixserverlib.Filter,
-) (string, error) {
-	return d.filter.insertFilter(ctx, filter, localpart)
-}
-
 // CheckAccountAvailability checks if the username/localpart is already present
 // in the database.
 // If the DB returns sql.ErrNoRows the Localpart isn't taken.
diff --git a/userapi/storage/accounts/sqlite3/filter_table.go b/userapi/storage/accounts/sqlite3/filter_table.go
deleted file mode 100644
index 7f1a0c24..00000000
--- a/userapi/storage/accounts/sqlite3/filter_table.go
+++ /dev/null
@@ -1,135 +0,0 @@
-// Copyright 2017 Jan Christian Grünhage
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sqlite3
-
-import (
-	"context"
-	"database/sql"
-	"encoding/json"
-	"fmt"
-
-	"github.com/matrix-org/gomatrixserverlib"
-)
-
-const filterSchema = `
--- Stores data about filters
-CREATE TABLE IF NOT EXISTS account_filter (
-	-- The filter
-	filter TEXT NOT NULL,
-	-- The ID
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	-- The localpart of the Matrix user ID associated to this filter
-	localpart TEXT NOT NULL,
-
-	UNIQUE (id, localpart)
-);
-
-CREATE INDEX IF NOT EXISTS account_filter_localpart ON account_filter(localpart);
-`
-
-const selectFilterSQL = "" +
-	"SELECT filter FROM account_filter WHERE localpart = $1 AND id = $2"
-
-const selectFilterIDByContentSQL = "" +
-	"SELECT id FROM account_filter WHERE localpart = $1 AND filter = $2"
-
-const insertFilterSQL = "" +
-	"INSERT INTO account_filter (filter, localpart) VALUES ($1, $2)"
-
-type filterStatements struct {
-	selectFilterStmt            *sql.Stmt
-	selectFilterIDByContentStmt *sql.Stmt
-	insertFilterStmt            *sql.Stmt
-}
-
-func (s *filterStatements) prepare(db *sql.DB) (err error) {
-	_, err = db.Exec(filterSchema)
-	if err != nil {
-		return
-	}
-	if s.selectFilterStmt, err = db.Prepare(selectFilterSQL); err != nil {
-		return
-	}
-	if s.selectFilterIDByContentStmt, err = db.Prepare(selectFilterIDByContentSQL); err != nil {
-		return
-	}
-	if s.insertFilterStmt, err = db.Prepare(insertFilterSQL); err != nil {
-		return
-	}
-	return
-}
-
-func (s *filterStatements) selectFilter(
-	ctx context.Context, localpart string, filterID string,
-) (*gomatrixserverlib.Filter, error) {
-	// Retrieve filter from database (stored as canonical JSON)
-	var filterData []byte
-	err := s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filterData)
-	if err != nil {
-		return nil, err
-	}
-
-	// Unmarshal JSON into Filter struct
-	var filter gomatrixserverlib.Filter
-	if err = json.Unmarshal(filterData, &filter); err != nil {
-		return nil, err
-	}
-	return &filter, nil
-}
-
-func (s *filterStatements) insertFilter(
-	ctx context.Context, filter *gomatrixserverlib.Filter, localpart string,
-) (filterID string, err error) {
-	var existingFilterID string
-
-	// Serialise json
-	filterJSON, err := json.Marshal(filter)
-	if err != nil {
-		return "", err
-	}
-	// Remove whitespaces and sort JSON data
-	// needed to prevent from inserting the same filter multiple times
-	filterJSON, err = gomatrixserverlib.CanonicalJSON(filterJSON)
-	if err != nil {
-		return "", err
-	}
-
-	// Check if filter already exists in the database using its localpart and content
-	//
-	// This can result in a race condition when two clients try to insert the
-	// same filter and localpart at the same time, however this is not a
-	// problem as both calls will result in the same filterID
-	err = s.selectFilterIDByContentStmt.QueryRowContext(ctx,
-		localpart, filterJSON).Scan(&existingFilterID)
-	if err != nil && err != sql.ErrNoRows {
-		return "", err
-	}
-	// If it does, return the existing ID
-	if existingFilterID != "" {
-		return existingFilterID, err
-	}
-
-	// Otherwise insert the filter and return the new ID
-	res, err := s.insertFilterStmt.ExecContext(ctx, filterJSON, localpart)
-	if err != nil {
-		return "", err
-	}
-	rowid, err := res.LastInsertId()
-	if err != nil {
-		return "", err
-	}
-	filterID = fmt.Sprintf("%d", rowid)
-	return
-}
diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go
index d84f25b1..e965df4f 100644
--- a/userapi/storage/accounts/sqlite3/storage.go
+++ b/userapi/storage/accounts/sqlite3/storage.go
@@ -39,7 +39,6 @@ type Database struct {
 	memberships  membershipStatements
 	accountDatas accountDataStatements
 	threepids    threepidStatements
-	filter       filterStatements
 	serverName   gomatrixserverlib.ServerName
 
 	createAccountMu sync.Mutex
@@ -80,11 +79,7 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName)
 	if err = t.prepare(db); err != nil {
 		return nil, err
 	}
-	f := filterStatements{}
-	if err = f.prepare(db); err != nil {
-		return nil, err
-	}
-	return &Database{db, partitions, a, p, m, ac, t, f, serverName, sync.Mutex{}}, nil
+	return &Database{db, partitions, a, p, m, ac, t, serverName, sync.Mutex{}}, nil
 }
 
 // GetAccountByPassword returns the account associated with the given localpart and password.
@@ -410,24 +405,6 @@ func (d *Database) GetThreePIDsForLocalpart(
 	return d.threepids.selectThreePIDsForLocalpart(ctx, localpart)
 }
 
-// GetFilter looks up the filter associated with a given local user and filter ID.
-// Returns a filter structure. Otherwise returns an error if no such filter exists
-// or if there was an error talking to the database.
-func (d *Database) GetFilter(
-	ctx context.Context, localpart string, filterID string,
-) (*gomatrixserverlib.Filter, error) {
-	return d.filter.selectFilter(ctx, localpart, filterID)
-}
-
-// PutFilter puts the passed filter into the database.
-// Returns the filterID as a string. Otherwise returns an error if something
-// goes wrong.
-func (d *Database) PutFilter(
-	ctx context.Context, localpart string, filter *gomatrixserverlib.Filter,
-) (string, error) {
-	return d.filter.insertFilter(ctx, filter, localpart)
-}
-
 // CheckAccountAvailability checks if the username/localpart is already present
 // in the database.
 // If the DB returns sql.ErrNoRows the Localpart isn't taken.