diff options
33 files changed, 127 insertions, 451 deletions
diff --git a/docs/development/sytest.md b/docs/development/sytest.md index 3cfb99e6..4fae2ea3 100644 --- a/docs/development/sytest.md +++ b/docs/development/sytest.md @@ -61,7 +61,6 @@ When debugging, the following Docker `run` options may also be useful: * `-e "DENDRITE_TRACE_HTTP=1"`: Adds HTTP tracing to server logs. * `-e "DENDRITE_TRACE_INTERNAL=1"`: Adds roomserver internal API tracing to server logs. -* `-e "DENDRITE_TRACE_SQL=1"`: Adds tracing to all SQL statements to server logs. The docker command also supports a single positional argument for the test file to run, so you can run a single `.pl` file rather than the whole test suite. For example: diff --git a/federationapi/storage/postgres/blacklist_table.go b/federationapi/storage/postgres/blacklist_table.go index eef37318..1d931daa 100644 --- a/federationapi/storage/postgres/blacklist_table.go +++ b/federationapi/storage/postgres/blacklist_table.go @@ -60,19 +60,12 @@ func NewPostgresBlacklistTable(db *sql.DB) (s *blacklistStatements, err error) { return } - if s.insertBlacklistStmt, err = db.Prepare(insertBlacklistSQL); err != nil { - return - } - if s.selectBlacklistStmt, err = db.Prepare(selectBlacklistSQL); err != nil { - return - } - if s.deleteBlacklistStmt, err = db.Prepare(deleteBlacklistSQL); err != nil { - return - } - if s.deleteAllBlacklistStmt, err = db.Prepare(deleteAllBlacklistSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertBlacklistStmt, insertBlacklistSQL}, + {&s.selectBlacklistStmt, selectBlacklistSQL}, + {&s.deleteBlacklistStmt, deleteBlacklistSQL}, + {&s.deleteAllBlacklistStmt, deleteAllBlacklistSQL}, + }.Prepare(db) } func (s *blacklistStatements) InsertBlacklist( diff --git a/federationapi/storage/postgres/joined_hosts_table.go b/federationapi/storage/postgres/joined_hosts_table.go index 9a397756..8806db55 100644 --- a/federationapi/storage/postgres/joined_hosts_table.go +++ b/federationapi/storage/postgres/joined_hosts_table.go @@ -90,28 +90,15 @@ func NewPostgresJoinedHostsTable(db *sql.DB) (s *joinedHostsStatements, err erro if err != nil { return } - if s.insertJoinedHostsStmt, err = s.db.Prepare(insertJoinedHostsSQL); err != nil { - return - } - if s.deleteJoinedHostsStmt, err = s.db.Prepare(deleteJoinedHostsSQL); err != nil { - return - } - if s.deleteJoinedHostsForRoomStmt, err = s.db.Prepare(deleteJoinedHostsForRoomSQL); err != nil { - return - } - if s.selectJoinedHostsStmt, err = s.db.Prepare(selectJoinedHostsSQL); err != nil { - return - } - if s.selectAllJoinedHostsStmt, err = s.db.Prepare(selectAllJoinedHostsSQL); err != nil { - return - } - if s.selectJoinedHostsForRoomsStmt, err = s.db.Prepare(selectJoinedHostsForRoomsSQL); err != nil { - return - } - if s.selectJoinedHostsForRoomsExcludingBlacklistedStmt, err = s.db.Prepare(selectJoinedHostsForRoomsExcludingBlacklistedSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertJoinedHostsStmt, insertJoinedHostsSQL}, + {&s.deleteJoinedHostsStmt, deleteJoinedHostsSQL}, + {&s.deleteJoinedHostsForRoomStmt, deleteJoinedHostsForRoomSQL}, + {&s.selectJoinedHostsStmt, selectJoinedHostsSQL}, + {&s.selectAllJoinedHostsStmt, selectAllJoinedHostsSQL}, + {&s.selectJoinedHostsForRoomsStmt, selectJoinedHostsForRoomsSQL}, + {&s.selectJoinedHostsForRoomsExcludingBlacklistedStmt, selectJoinedHostsForRoomsExcludingBlacklistedSQL}, + }.Prepare(db) } func (s *joinedHostsStatements) InsertJoinedHosts( diff --git a/federationapi/storage/postgres/notary_server_keys_json_table.go b/federationapi/storage/postgres/notary_server_keys_json_table.go index 65221c08..9fc93a61 100644 --- a/federationapi/storage/postgres/notary_server_keys_json_table.go +++ b/federationapi/storage/postgres/notary_server_keys_json_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/federationapi/storage/tables" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -50,10 +51,9 @@ func NewPostgresNotaryServerKeysTable(db *sql.DB) (s *notaryServerKeysStatements return } - if s.insertServerKeysJSONStmt, err = db.Prepare(insertServerKeysJSONSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertServerKeysJSONStmt, insertServerKeysJSONSQL}, + }.Prepare(db) } func (s *notaryServerKeysStatements) InsertJSONResponse( diff --git a/federationapi/storage/postgres/notary_server_keys_metadata_table.go b/federationapi/storage/postgres/notary_server_keys_metadata_table.go index 72faf480..6d38ccab 100644 --- a/federationapi/storage/postgres/notary_server_keys_metadata_table.go +++ b/federationapi/storage/postgres/notary_server_keys_metadata_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/federationapi/storage/tables" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -91,22 +92,13 @@ func NewPostgresNotaryServerKeysMetadataTable(db *sql.DB) (s *notaryServerKeysMe return } - if s.upsertServerKeysStmt, err = db.Prepare(upsertServerKeysSQL); err != nil { - return - } - if s.selectNotaryKeyResponsesStmt, err = db.Prepare(selectNotaryKeyResponsesSQL); err != nil { - return - } - if s.selectNotaryKeyResponsesWithKeyIDsStmt, err = db.Prepare(selectNotaryKeyResponsesWithKeyIDsSQL); err != nil { - return - } - if s.selectNotaryKeyMetadataStmt, err = db.Prepare(selectNotaryKeyMetadataSQL); err != nil { - return - } - if s.deleteUnusedServerKeysJSONStmt, err = db.Prepare(deleteUnusedServerKeysJSONSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.upsertServerKeysStmt, upsertServerKeysSQL}, + {&s.selectNotaryKeyResponsesStmt, selectNotaryKeyResponsesSQL}, + {&s.selectNotaryKeyResponsesWithKeyIDsStmt, selectNotaryKeyResponsesWithKeyIDsSQL}, + {&s.selectNotaryKeyMetadataStmt, selectNotaryKeyMetadataSQL}, + {&s.deleteUnusedServerKeysJSONStmt, deleteUnusedServerKeysJSONSQL}, + }.Prepare(db) } func (s *notaryServerKeysMetadataStatements) UpsertKey( diff --git a/federationapi/storage/postgres/queue_json_table.go b/federationapi/storage/postgres/queue_json_table.go index e3307418..563738dd 100644 --- a/federationapi/storage/postgres/queue_json_table.go +++ b/federationapi/storage/postgres/queue_json_table.go @@ -65,16 +65,11 @@ func NewPostgresQueueJSONTable(db *sql.DB) (s *queueJSONStatements, err error) { if err != nil { return } - if s.insertJSONStmt, err = s.db.Prepare(insertJSONSQL); err != nil { - return - } - if s.deleteJSONStmt, err = s.db.Prepare(deleteJSONSQL); err != nil { - return - } - if s.selectJSONStmt, err = s.db.Prepare(selectJSONSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertJSONStmt, insertJSONSQL}, + {&s.deleteJSONStmt, deleteJSONSQL}, + {&s.selectJSONStmt, selectJSONSQL}, + }.Prepare(db) } func (s *queueJSONStatements) InsertQueueJSON( diff --git a/federationapi/storage/postgres/queue_pdus_table.go b/federationapi/storage/postgres/queue_pdus_table.go index 3b0bef9a..b97be482 100644 --- a/federationapi/storage/postgres/queue_pdus_table.go +++ b/federationapi/storage/postgres/queue_pdus_table.go @@ -78,22 +78,13 @@ func NewPostgresQueuePDUsTable(db *sql.DB) (s *queuePDUsStatements, err error) { if err != nil { return } - if s.insertQueuePDUStmt, err = s.db.Prepare(insertQueuePDUSQL); err != nil { - return - } - if s.deleteQueuePDUsStmt, err = s.db.Prepare(deleteQueuePDUSQL); err != nil { - return - } - if s.selectQueuePDUsStmt, err = s.db.Prepare(selectQueuePDUsSQL); err != nil { - return - } - if s.selectQueuePDUReferenceJSONCountStmt, err = s.db.Prepare(selectQueuePDUReferenceJSONCountSQL); err != nil { - return - } - if s.selectQueuePDUServerNamesStmt, err = s.db.Prepare(selectQueuePDUServerNamesSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertQueuePDUStmt, insertQueuePDUSQL}, + {&s.deleteQueuePDUsStmt, deleteQueuePDUSQL}, + {&s.selectQueuePDUsStmt, selectQueuePDUsSQL}, + {&s.selectQueuePDUReferenceJSONCountStmt, selectQueuePDUReferenceJSONCountSQL}, + {&s.selectQueuePDUServerNamesStmt, selectQueuePDUServerNamesSQL}, + }.Prepare(db) } func (s *queuePDUsStatements) InsertQueuePDU( diff --git a/federationapi/storage/postgres/server_key_table.go b/federationapi/storage/postgres/server_key_table.go index 16e294e0..f393351b 100644 --- a/federationapi/storage/postgres/server_key_table.go +++ b/federationapi/storage/postgres/server_key_table.go @@ -72,13 +72,10 @@ func NewPostgresServerSigningKeysTable(db *sql.DB) (s *serverSigningKeyStatement if err != nil { return } - if s.bulkSelectServerKeysStmt, err = db.Prepare(bulkSelectServerSigningKeysSQL); err != nil { - return - } - if s.upsertServerKeysStmt, err = db.Prepare(upsertServerSigningKeysSQL); err != nil { - return - } - return s, nil + return s, sqlutil.StatementList{ + {&s.bulkSelectServerKeysStmt, bulkSelectServerSigningKeysSQL}, + {&s.upsertServerKeysStmt, upsertServerSigningKeysSQL}, + }.Prepare(db) } func (s *serverSigningKeyStatements) BulkSelectServerKeys( diff --git a/federationapi/storage/sqlite3/blacklist_table.go b/federationapi/storage/sqlite3/blacklist_table.go index 2694e630..5122bff1 100644 --- a/federationapi/storage/sqlite3/blacklist_table.go +++ b/federationapi/storage/sqlite3/blacklist_table.go @@ -60,19 +60,12 @@ func NewSQLiteBlacklistTable(db *sql.DB) (s *blacklistStatements, err error) { return } - if s.insertBlacklistStmt, err = db.Prepare(insertBlacklistSQL); err != nil { - return - } - if s.selectBlacklistStmt, err = db.Prepare(selectBlacklistSQL); err != nil { - return - } - if s.deleteBlacklistStmt, err = db.Prepare(deleteBlacklistSQL); err != nil { - return - } - if s.deleteAllBlacklistStmt, err = db.Prepare(deleteAllBlacklistSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertBlacklistStmt, insertBlacklistSQL}, + {&s.selectBlacklistStmt, selectBlacklistSQL}, + {&s.deleteBlacklistStmt, deleteBlacklistSQL}, + {&s.deleteAllBlacklistStmt, deleteAllBlacklistSQL}, + }.Prepare(db) } func (s *blacklistStatements) InsertBlacklist( diff --git a/federationapi/storage/sqlite3/joined_hosts_table.go b/federationapi/storage/sqlite3/joined_hosts_table.go index 83112c15..2f076382 100644 --- a/federationapi/storage/sqlite3/joined_hosts_table.go +++ b/federationapi/storage/sqlite3/joined_hosts_table.go @@ -90,22 +90,14 @@ func NewSQLiteJoinedHostsTable(db *sql.DB) (s *joinedHostsStatements, err error) if err != nil { return } - if s.insertJoinedHostsStmt, err = db.Prepare(insertJoinedHostsSQL); err != nil { - return - } - if s.deleteJoinedHostsStmt, err = db.Prepare(deleteJoinedHostsSQL); err != nil { - return - } - if s.deleteJoinedHostsForRoomStmt, err = s.db.Prepare(deleteJoinedHostsForRoomSQL); err != nil { - return - } - if s.selectJoinedHostsStmt, err = db.Prepare(selectJoinedHostsSQL); err != nil { - return - } - if s.selectAllJoinedHostsStmt, err = db.Prepare(selectAllJoinedHostsSQL); err != nil { - return - } - return + + return s, sqlutil.StatementList{ + {&s.insertJoinedHostsStmt, insertJoinedHostsSQL}, + {&s.deleteJoinedHostsStmt, deleteJoinedHostsSQL}, + {&s.deleteJoinedHostsForRoomStmt, deleteJoinedHostsForRoomSQL}, + {&s.selectJoinedHostsStmt, selectJoinedHostsSQL}, + {&s.selectAllJoinedHostsStmt, selectAllJoinedHostsSQL}, + }.Prepare(db) } func (s *joinedHostsStatements) InsertJoinedHosts( diff --git a/federationapi/storage/sqlite3/notary_server_keys_json_table.go b/federationapi/storage/sqlite3/notary_server_keys_json_table.go index 4a028fa2..24875569 100644 --- a/federationapi/storage/sqlite3/notary_server_keys_json_table.go +++ b/federationapi/storage/sqlite3/notary_server_keys_json_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/federationapi/storage/tables" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -49,10 +50,9 @@ func NewSQLiteNotaryServerKeysTable(db *sql.DB) (s *notaryServerKeysStatements, return } - if s.insertServerKeysJSONStmt, err = db.Prepare(insertServerKeysJSONSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertServerKeysJSONStmt, insertServerKeysJSONSQL}, + }.Prepare(db) } func (s *notaryServerKeysStatements) InsertJSONResponse( diff --git a/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go index 55709a96..7179eb8d 100644 --- a/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go +++ b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go @@ -92,19 +92,12 @@ func NewSQLiteNotaryServerKeysMetadataTable(db *sql.DB) (s *notaryServerKeysMeta return } - if s.upsertServerKeysStmt, err = db.Prepare(upsertServerKeysSQL); err != nil { - return - } - if s.selectNotaryKeyResponsesStmt, err = db.Prepare(selectNotaryKeyResponsesSQL); err != nil { - return - } - if s.selectNotaryKeyMetadataStmt, err = db.Prepare(selectNotaryKeyMetadataSQL); err != nil { - return - } - if s.deleteUnusedServerKeysJSONStmt, err = db.Prepare(deleteUnusedServerKeysJSONSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.upsertServerKeysStmt, upsertServerKeysSQL}, + {&s.selectNotaryKeyResponsesStmt, selectNotaryKeyResponsesSQL}, + {&s.selectNotaryKeyMetadataStmt, selectNotaryKeyMetadataSQL}, + {&s.deleteUnusedServerKeysJSONStmt, deleteUnusedServerKeysJSONSQL}, + }.Prepare(db) } func (s *notaryServerKeysMetadataStatements) UpsertKey( diff --git a/federationapi/storage/sqlite3/queue_json_table.go b/federationapi/storage/sqlite3/queue_json_table.go index fe5896c7..0e2806d5 100644 --- a/federationapi/storage/sqlite3/queue_json_table.go +++ b/federationapi/storage/sqlite3/queue_json_table.go @@ -66,10 +66,10 @@ func NewSQLiteQueueJSONTable(db *sql.DB) (s *queueJSONStatements, err error) { if err != nil { return } - if s.insertJSONStmt, err = db.Prepare(insertJSONSQL); err != nil { - return - } - return + + return s, sqlutil.StatementList{ + {&s.insertJSONStmt, insertJSONSQL}, + }.Prepare(db) } func (s *queueJSONStatements) InsertQueueJSON( diff --git a/federationapi/storage/sqlite3/queue_pdus_table.go b/federationapi/storage/sqlite3/queue_pdus_table.go index aee8b03d..d8d99f0c 100644 --- a/federationapi/storage/sqlite3/queue_pdus_table.go +++ b/federationapi/storage/sqlite3/queue_pdus_table.go @@ -87,25 +87,13 @@ func NewSQLiteQueuePDUsTable(db *sql.DB) (s *queuePDUsStatements, err error) { if err != nil { return } - if s.insertQueuePDUStmt, err = db.Prepare(insertQueuePDUSQL); err != nil { - return - } - //if s.deleteQueuePDUsStmt, err = db.Prepare(deleteQueuePDUsSQL); err != nil { - // return - //} - if s.selectQueueNextTransactionIDStmt, err = db.Prepare(selectQueueNextTransactionIDSQL); err != nil { - return - } - if s.selectQueuePDUsStmt, err = db.Prepare(selectQueuePDUsSQL); err != nil { - return - } - if s.selectQueueReferenceJSONCountStmt, err = db.Prepare(selectQueuePDUsReferenceJSONCountSQL); err != nil { - return - } - if s.selectQueueServerNamesStmt, err = db.Prepare(selectQueuePDUsServerNamesSQL); err != nil { - return - } - return + return s, sqlutil.StatementList{ + {&s.insertQueuePDUStmt, insertQueuePDUSQL}, + {&s.selectQueueNextTransactionIDStmt, selectQueueNextTransactionIDSQL}, + {&s.selectQueuePDUsStmt, selectQueuePDUsSQL}, + {&s.selectQueueReferenceJSONCountStmt, selectQueuePDUsReferenceJSONCountSQL}, + {&s.selectQueueServerNamesStmt, selectQueuePDUsServerNamesSQL}, + }.Prepare(db) } func (s *queuePDUsStatements) InsertQueuePDU( diff --git a/federationapi/storage/sqlite3/server_key_table.go b/federationapi/storage/sqlite3/server_key_table.go index 9b89649f..b32ff092 100644 --- a/federationapi/storage/sqlite3/server_key_table.go +++ b/federationapi/storage/sqlite3/server_key_table.go @@ -74,13 +74,10 @@ func NewSQLiteServerSigningKeysTable(db *sql.DB) (s *serverSigningKeyStatements, if err != nil { return } - if s.bulkSelectServerKeysStmt, err = db.Prepare(bulkSelectServerSigningKeysSQL); err != nil { - return - } - if s.upsertServerKeysStmt, err = db.Prepare(upsertServerSigningKeysSQL); err != nil { - return - } - return s, nil + return s, sqlutil.StatementList{ + {&s.bulkSelectServerKeysStmt, bulkSelectServerSigningKeysSQL}, + {&s.upsertServerKeysStmt, upsertServerSigningKeysSQL}, + }.Prepare(db) } func (s *serverSigningKeyStatements) BulkSelectServerKeys( @@ -30,7 +30,6 @@ require ( github.com/nats-io/nats.go v1.24.0 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 - github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79 github.com/opentracing/opentracing-go v1.2.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 @@ -368,8 +368,6 @@ github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQE github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= -github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79 h1:Dmx8g2747UTVPzSkmohk84S3g/uWqd6+f4SSLPhLcfA= -github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79/go.mod h1:E26fwEtRNigBfFfHDWsklmo0T7Ixbg0XXgck+Hq4O9k= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.3.0 h1:kUMoxMoQG3ogk/QWyKh3zibV7BKZ+xBpWil1cTylVqc= github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= diff --git a/internal/sqlutil/sqlite_cgo.go b/internal/sqlutil/sqlite_cgo.go index efb743fc..2fe2396f 100644 --- a/internal/sqlutil/sqlite_cgo.go +++ b/internal/sqlutil/sqlite_cgo.go @@ -4,7 +4,6 @@ package sqlutil import ( - "github.com/mattn/go-sqlite3" _ "github.com/mattn/go-sqlite3" ) @@ -13,7 +12,3 @@ const SQLITE_DRIVER_NAME = "sqlite3" func sqliteDSNExtension(dsn string) string { return dsn } - -func sqliteDriver() *sqlite3.SQLiteDriver { - return &sqlite3.SQLiteDriver{} -} diff --git a/internal/sqlutil/sqlite_native.go b/internal/sqlutil/sqlite_native.go index ed500afc..3c545b65 100644 --- a/internal/sqlutil/sqlite_native.go +++ b/internal/sqlutil/sqlite_native.go @@ -4,7 +4,6 @@ package sqlutil import ( - "modernc.org/sqlite" "strings" ) @@ -23,7 +22,3 @@ func sqliteDSNExtension(dsn string) string { dsn += "_pragma=busy_timeout%3d10000" return dsn } - -func sqliteDriver() *sqlite.Driver { - return &sqlite.Driver{} -} diff --git a/internal/sqlutil/sqlutil.go b/internal/sqlutil/sqlutil.go index 39a067e5..b2d0d218 100644 --- a/internal/sqlutil/sqlutil.go +++ b/internal/sqlutil/sqlutil.go @@ -13,8 +13,7 @@ import ( var skipSanityChecks = flag.Bool("skip-db-sanity", false, "Ignore sanity checks on the database connections (NOT RECOMMENDED!)") // Open opens a database specified by its database driver name and a driver-specific data source name, -// usually consisting of at least a database name and connection information. Includes tracing driver -// if DENDRITE_TRACE_SQL=1 +// usually consisting of at least a database name and connection information. func Open(dbProperties *config.DatabaseOptions, writer Writer) (*sql.DB, error) { var err error var driverName, dsn string @@ -32,10 +31,6 @@ func Open(dbProperties *config.DatabaseOptions, writer Writer) (*sql.DB, error) default: return nil, fmt.Errorf("invalid database connection string %q", dbProperties.ConnectionString) } - if tracingEnabled { - // install the wrapped driver - driverName += "-trace" - } db, err := sql.Open(driverName, dsn) if err != nil { return nil, err diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go deleted file mode 100644 index 7b637106..00000000 --- a/internal/sqlutil/trace.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlutil - -import ( - "context" - "database/sql/driver" - "fmt" - "io" - "os" - "runtime" - "strconv" - "strings" - "sync" - "time" - - "github.com/ngrok/sqlmw" - "github.com/sirupsen/logrus" -) - -var tracingEnabled = os.Getenv("DENDRITE_TRACE_SQL") == "1" -var goidToWriter sync.Map - -type traceInterceptor struct { - sqlmw.NullInterceptor -} - -func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.StmtQueryContext, query string, args []driver.NamedValue) (context.Context, driver.Rows, error) { - startedAt := time.Now() - rows, err := stmt.QueryContext(ctx, args) - - trackGoID(query) - - logrus.WithField("duration", time.Since(startedAt)).WithField(logrus.ErrorKey, err).Debug("executed sql query ", query, " args: ", args) - - return ctx, rows, err -} - -func (in *traceInterceptor) StmtExecContext(ctx context.Context, stmt driver.StmtExecContext, query string, args []driver.NamedValue) (driver.Result, error) { - startedAt := time.Now() - result, err := stmt.ExecContext(ctx, args) - - trackGoID(query) - - logrus.WithField("duration", time.Since(startedAt)).WithField(logrus.ErrorKey, err).Debug("executed sql query ", query, " args: ", args) - - return result, err -} - -func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest []driver.Value) error { - err := rows.Next(dest) - if err == io.EOF { - // For all cases, we call Next() n+1 times, the first to populate the initial dest, then eventually - // it will io.EOF. If we log on each Next() call we log the last element twice, so don't. - return err - } - cols := rows.Columns() - logrus.Debug(strings.Join(cols, " | ")) - - b := strings.Builder{} - for i, val := range dest { - b.WriteString(fmt.Sprintf("%q", val)) - if i+1 <= len(dest)-1 { - b.WriteString(" | ") - } - } - logrus.Debug(b.String()) - return err -} - -func trackGoID(query string) { - thisGoID := goid() - if _, ok := goidToWriter.Load(thisGoID); ok { - return // we're on a writer goroutine - } - - q := strings.TrimSpace(query) - if strings.HasPrefix(q, "SELECT") { - return // SELECTs can go on other goroutines - } - logrus.Warnf("unsafe goid %d: SQL executed not on an ExclusiveWriter: %s", thisGoID, q) -} - -func init() { - registerDrivers() -} - -func goid() int { - var buf [64]byte - n := runtime.Stack(buf[:], false) - idField := strings.Fields(strings.TrimPrefix(string(buf[:n]), "goroutine "))[0] - id, err := strconv.Atoi(idField) - if err != nil { - panic(fmt.Sprintf("cannot get goroutine id: %v", err)) - } - return id -} diff --git a/internal/sqlutil/trace_driver.go b/internal/sqlutil/trace_driver.go deleted file mode 100644 index a2e0d12e..00000000 --- a/internal/sqlutil/trace_driver.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !wasm -// +build !wasm - -package sqlutil - -import ( - "database/sql" - - "github.com/lib/pq" - "github.com/ngrok/sqlmw" -) - -func registerDrivers() { - if !tracingEnabled { - return - } - // install the wrapped drivers - sql.Register("postgres-trace", sqlmw.Driver(&pq.Driver{}, new(traceInterceptor))) - sql.Register("sqlite3-trace", sqlmw.Driver(sqliteDriver(), new(traceInterceptor))) - -} diff --git a/internal/sqlutil/trace_driver_wasm.go b/internal/sqlutil/trace_driver_wasm.go deleted file mode 100644 index 51b60c3c..00000000 --- a/internal/sqlutil/trace_driver_wasm.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build wasm -// +build wasm - -package sqlutil - -import ( - "database/sql" - - sqlitejs "github.com/matrix-org/go-sqlite3-js" - "github.com/ngrok/sqlmw" -) - -func registerDrivers() { - if !tracingEnabled { - return - } - // install the wrapped drivers - sql.Register("sqlite3_js-trace", sqlmw.Driver(&sqlitejs.SqliteJsDriver{}, new(traceInterceptor))) - -} diff --git a/internal/sqlutil/writer_exclusive.go b/internal/sqlutil/writer_exclusive.go index 8eff3ce5..c6a271c1 100644 --- a/internal/sqlutil/writer_exclusive.go +++ b/internal/sqlutil/writer_exclusive.go @@ -60,11 +60,6 @@ func (w *ExclusiveWriter) run() { if !w.running.CompareAndSwap(false, true) { return } - if tracingEnabled { - gid := goid() - goidToWriter.Store(gid, w) - defer goidToWriter.Delete(gid) - } defer w.running.Store(false) for task := range w.todo { diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index aa54cb08..8147b486 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -78,16 +78,11 @@ func NewPostgresAccountDataTable(db *sql.DB) (tables.AccountData, error) { if err != nil { return nil, err } - if s.insertAccountDataStmt, err = db.Prepare(insertAccountDataSQL); err != nil { - return nil, err - } - if s.selectAccountDataInRangeStmt, err = db.Prepare(selectAccountDataInRangeSQL); err != nil { - return nil, err - } - if s.selectMaxAccountDataIDStmt, err = db.Prepare(selectMaxAccountDataIDSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.insertAccountDataStmt, insertAccountDataSQL}, + {&s.selectAccountDataInRangeStmt, selectAccountDataInRangeSQL}, + {&s.selectMaxAccountDataIDStmt, selectMaxAccountDataIDSQL}, + }.Prepare(db) } func (s *accountDataStatements) InsertAccountData( diff --git a/syncapi/storage/postgres/filter_table.go b/syncapi/storage/postgres/filter_table.go index 86cec362..44c3de72 100644 --- a/syncapi/storage/postgres/filter_table.go +++ b/syncapi/storage/postgres/filter_table.go @@ -61,16 +61,11 @@ func NewPostgresFilterTable(db *sql.DB) (tables.Filter, error) { return nil, err } s := &filterStatements{} - if s.selectFilterStmt, err = db.Prepare(selectFilterSQL); err != nil { - return nil, err - } - if s.selectFilterIDByContentStmt, err = db.Prepare(selectFilterIDByContentSQL); err != nil { - return nil, err - } - if s.insertFilterStmt, err = db.Prepare(insertFilterSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.selectFilterStmt, selectFilterSQL}, + {&s.selectFilterIDByContentStmt, selectFilterIDByContentSQL}, + {&s.insertFilterStmt, insertFilterSQL}, + }.Prepare(db) } func (s *filterStatements) SelectFilter( diff --git a/syncapi/storage/postgres/ignores_table.go b/syncapi/storage/postgres/ignores_table.go index 97660725..a511c747 100644 --- a/syncapi/storage/postgres/ignores_table.go +++ b/syncapi/storage/postgres/ignores_table.go @@ -52,13 +52,11 @@ func NewPostgresIgnoresTable(db *sql.DB) (tables.Ignores, error) { return nil, err } s := &ignoresStatements{} - if s.selectIgnoresStmt, err = db.Prepare(selectIgnoresSQL); err != nil { - return nil, err - } - if s.upsertIgnoresStmt, err = db.Prepare(upsertIgnoresSQL); err != nil { - return nil, err - } - return s, nil + + return s, sqlutil.StatementList{ + {&s.selectIgnoresStmt, selectIgnoresSQL}, + {&s.upsertIgnoresStmt, upsertIgnoresSQL}, + }.Prepare(db) } func (s *ignoresStatements) SelectIgnores( diff --git a/syncapi/storage/postgres/send_to_device_table.go b/syncapi/storage/postgres/send_to_device_table.go index 6ab1f0f4..88b86ef7 100644 --- a/syncapi/storage/postgres/send_to_device_table.go +++ b/syncapi/storage/postgres/send_to_device_table.go @@ -88,19 +88,12 @@ func NewPostgresSendToDeviceTable(db *sql.DB) (tables.SendToDevice, error) { if err != nil { return nil, err } - if s.insertSendToDeviceMessageStmt, err = db.Prepare(insertSendToDeviceMessageSQL); err != nil { - return nil, err - } - if s.selectSendToDeviceMessagesStmt, err = db.Prepare(selectSendToDeviceMessagesSQL); err != nil { - return nil, err - } - if s.deleteSendToDeviceMessagesStmt, err = db.Prepare(deleteSendToDeviceMessagesSQL); err != nil { - return nil, err - } - if s.selectMaxSendToDeviceIDStmt, err = db.Prepare(selectMaxSendToDeviceIDSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.insertSendToDeviceMessageStmt, insertSendToDeviceMessageSQL}, + {&s.selectSendToDeviceMessagesStmt, selectSendToDeviceMessagesSQL}, + {&s.deleteSendToDeviceMessagesStmt, deleteSendToDeviceMessagesSQL}, + {&s.selectMaxSendToDeviceIDStmt, selectMaxSendToDeviceIDSQL}, + }.Prepare(db) } func (s *sendToDeviceStatements) InsertSendToDeviceMessage( diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index de0e72db..eaf52234 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -66,16 +66,11 @@ func NewSqliteAccountDataTable(db *sql.DB, streamID *StreamIDStatements) (tables if err != nil { return nil, err } - if s.insertAccountDataStmt, err = db.Prepare(insertAccountDataSQL); err != nil { - return nil, err - } - if s.selectMaxAccountDataIDStmt, err = db.Prepare(selectMaxAccountDataIDSQL); err != nil { - return nil, err - } - if s.selectAccountDataInRangeStmt, err = db.Prepare(selectAccountDataInRangeSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.insertAccountDataStmt, insertAccountDataSQL}, + {&s.selectMaxAccountDataIDStmt, selectMaxAccountDataIDSQL}, + {&s.selectAccountDataInRangeStmt, selectAccountDataInRangeSQL}, + }.Prepare(db) } func (s *accountDataStatements) InsertAccountData( diff --git a/syncapi/storage/sqlite3/filter_table.go b/syncapi/storage/sqlite3/filter_table.go index 5f1e980e..0132a766 100644 --- a/syncapi/storage/sqlite3/filter_table.go +++ b/syncapi/storage/sqlite3/filter_table.go @@ -65,16 +65,11 @@ func NewSqliteFilterTable(db *sql.DB) (tables.Filter, error) { s := &filterStatements{ db: db, } - if s.selectFilterStmt, err = db.Prepare(selectFilterSQL); err != nil { - return nil, err - } - if s.selectFilterIDByContentStmt, err = db.Prepare(selectFilterIDByContentSQL); err != nil { - return nil, err - } - if s.insertFilterStmt, err = db.Prepare(insertFilterSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.selectFilterStmt, selectFilterSQL}, + {&s.selectFilterIDByContentStmt, selectFilterIDByContentSQL}, + {&s.insertFilterStmt, insertFilterSQL}, + }.Prepare(db) } func (s *filterStatements) SelectFilter( diff --git a/syncapi/storage/sqlite3/ignores_table.go b/syncapi/storage/sqlite3/ignores_table.go index 5ee1a9fa..bff5780b 100644 --- a/syncapi/storage/sqlite3/ignores_table.go +++ b/syncapi/storage/sqlite3/ignores_table.go @@ -52,13 +52,10 @@ func NewSqliteIgnoresTable(db *sql.DB) (tables.Ignores, error) { return nil, err } s := &ignoresStatements{} - if s.selectIgnoresStmt, err = db.Prepare(selectIgnoresSQL); err != nil { - return nil, err - } - if s.upsertIgnoresStmt, err = db.Prepare(upsertIgnoresSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.selectIgnoresStmt, selectIgnoresSQL}, + {&s.upsertIgnoresStmt, upsertIgnoresSQL}, + }.Prepare(db) } func (s *ignoresStatements) SelectIgnores( diff --git a/syncapi/storage/sqlite3/send_to_device_table.go b/syncapi/storage/sqlite3/send_to_device_table.go index 0da06506..998a063c 100644 --- a/syncapi/storage/sqlite3/send_to_device_table.go +++ b/syncapi/storage/sqlite3/send_to_device_table.go @@ -88,19 +88,12 @@ func NewSqliteSendToDeviceTable(db *sql.DB) (tables.SendToDevice, error) { if err != nil { return nil, err } - if s.insertSendToDeviceMessageStmt, err = db.Prepare(insertSendToDeviceMessageSQL); err != nil { - return nil, err - } - if s.selectSendToDeviceMessagesStmt, err = db.Prepare(selectSendToDeviceMessagesSQL); err != nil { - return nil, err - } - if s.deleteSendToDeviceMessagesStmt, err = db.Prepare(deleteSendToDeviceMessagesSQL); err != nil { - return nil, err - } - if s.selectMaxSendToDeviceIDStmt, err = db.Prepare(selectMaxSendToDeviceIDSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.insertSendToDeviceMessageStmt, insertSendToDeviceMessageSQL}, + {&s.selectSendToDeviceMessagesStmt, selectSendToDeviceMessagesSQL}, + {&s.deleteSendToDeviceMessagesStmt, deleteSendToDeviceMessagesSQL}, + {&s.selectMaxSendToDeviceIDStmt, selectMaxSendToDeviceIDSQL}, + }.Prepare(db) } func (s *sendToDeviceStatements) InsertSendToDeviceMessage( diff --git a/syncapi/storage/sqlite3/stream_id_table.go b/syncapi/storage/sqlite3/stream_id_table.go index a4bba508..b51eccf5 100644 --- a/syncapi/storage/sqlite3/stream_id_table.go +++ b/syncapi/storage/sqlite3/stream_id_table.go @@ -47,10 +47,9 @@ func (s *StreamIDStatements) Prepare(db *sql.DB) (err error) { if err != nil { return } - if s.increaseStreamIDStmt, err = db.Prepare(increaseStreamIDStmt); err != nil { - return - } - return + return sqlutil.StatementList{ + {&s.increaseStreamIDStmt, increaseStreamIDStmt}, + }.Prepare(db) } func (s *StreamIDStatements) nextPDUID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { |