Rehuffle where things are in the internal package (#1122)

renamed:    internal/eventcontent.go -> internal/eventutil/eventcontent.go
	renamed:    internal/events.go -> internal/eventutil/events.go
	renamed:    internal/types.go -> internal/eventutil/types.go
	renamed:    internal/http/http.go -> internal/httputil/http.go
	renamed:    internal/httpapi.go -> internal/httputil/httpapi.go
	renamed:    internal/httpapi_test.go -> internal/httputil/httpapi_test.go
	renamed:    internal/httpapis/paths.go -> internal/httputil/paths.go
	renamed:    internal/routing.go -> internal/httputil/routing.go
	renamed:    internal/basecomponent/base.go -> internal/setup/base.go
	renamed:    internal/basecomponent/flags.go -> internal/setup/flags.go
	renamed:    internal/partition_offset_table.go -> internal/sqlutil/partition_offset_table.go
	renamed:    internal/postgres.go -> internal/sqlutil/postgres.go
	renamed:    internal/postgres_wasm.go -> internal/sqlutil/postgres_wasm.go
	renamed:    internal/sql.go -> internal/sqlutil/sql.go

5 files changed, 533 insertions, 0 deletions

diff --git a/internal/httputil/http.go b/internal/httputil/http.go
new file mode 100644
index 00000000..9197371a
--- /dev/null
+++ b/internal/httputil/http.go
@@ -0,0 +1,81 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httputil
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+
+	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
+)
+
+// PostJSON performs a POST request with JSON on an internal HTTP API
+func PostJSON(
+	ctx context.Context, span opentracing.Span, httpClient *http.Client,
+	apiURL string, request, response interface{},
+) error {
+	jsonBytes, err := json.Marshal(request)
+	if err != nil {
+		return err
+	}
+
+	parsedAPIURL, err := url.Parse(apiURL)
+	if err != nil {
+		return err
+	}
+
+	parsedAPIURL.Path = InternalPathPrefix + strings.TrimLeft(parsedAPIURL.Path, "/")
+	apiURL = parsedAPIURL.String()
+
+	req, err := http.NewRequest(http.MethodPost, apiURL, bytes.NewReader(jsonBytes))
+	if err != nil {
+		return err
+	}
+
+	// Mark the span as being an RPC client.
+	ext.SpanKindRPCClient.Set(span)
+	carrier := opentracing.HTTPHeadersCarrier(req.Header)
+	tracer := opentracing.GlobalTracer()
+
+	if err = tracer.Inject(span.Context(), opentracing.HTTPHeaders, carrier); err != nil {
+		return err
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	res, err := httpClient.Do(req.WithContext(ctx))
+	if res != nil {
+		defer (func() { err = res.Body.Close() })()
+	}
+	if err != nil {
+		return err
+	}
+	if res.StatusCode != http.StatusOK {
+		var errorBody struct {
+			Message string `json:"message"`
+		}
+		if msgerr := json.NewDecoder(res.Body).Decode(&errorBody); msgerr == nil {
+			return fmt.Errorf("Internal API: %d from %s: %s", res.StatusCode, apiURL, errorBody.Message)
+		}
+		return fmt.Errorf("Internal API: %d from %s", res.StatusCode, apiURL)
+	}
+	return json.NewDecoder(res.Body).Decode(response)
+}
diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go
new file mode 100644
index 00000000..0a37f06c
--- /dev/null
+++ b/internal/httputil/httpapi.go
@@ -0,0 +1,288 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httputil
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/clientapi/auth"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	federationsenderAPI "github.com/matrix-org/dendrite/federationsender/api"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/sirupsen/logrus"
+)
+
+// BasicAuth is used for authorization on /metrics handlers
+type BasicAuth struct {
+	Username string `yaml:"username"`
+	Password string `yaml:"password"`
+}
+
+// MakeAuthAPI turns a util.JSONRequestHandler function into an http.Handler which authenticates the request.
+func MakeAuthAPI(
+	metricsName string, data auth.Data,
+	f func(*http.Request, *authtypes.Device) util.JSONResponse,
+) http.Handler {
+	h := func(req *http.Request) util.JSONResponse {
+		device, err := auth.VerifyUserFromRequest(req, data)
+		if err != nil {
+			return *err
+		}
+		// add the user ID to the logger
+		logger := util.GetLogger((req.Context()))
+		logger = logger.WithField("user_id", device.UserID)
+		req = req.WithContext(util.ContextWithLogger(req.Context(), logger))
+
+		return f(req, device)
+	}
+	return MakeExternalAPI(metricsName, h)
+}
+
+// MakeExternalAPI turns a util.JSONRequestHandler function into an http.Handler.
+// This is used for APIs that are called from the internet.
+func MakeExternalAPI(metricsName string, f func(*http.Request) util.JSONResponse) http.Handler {
+	// TODO: We shouldn't be directly reading env vars here, inject it in instead.
+	// Refactor this when we split out config structs.
+	verbose := false
+	if os.Getenv("DENDRITE_TRACE_HTTP") == "1" {
+		verbose = true
+	}
+	h := util.MakeJSONAPI(util.NewJSONRequestHandler(f))
+	withSpan := func(w http.ResponseWriter, req *http.Request) {
+		nextWriter := w
+		if verbose {
+			logger := logrus.NewEntry(logrus.StandardLogger())
+			// Log outgoing response
+			rec := httptest.NewRecorder()
+			nextWriter = rec
+			defer func() {
+				resp := rec.Result()
+				dump, err := httputil.DumpResponse(resp, true)
+				if err != nil {
+					logger.Debugf("Failed to dump outgoing response: %s", err)
+				} else {
+					strSlice := strings.Split(string(dump), "\n")
+					for _, s := range strSlice {
+						logger.Debug(s)
+					}
+				}
+				// copy the response to the client
+				for hdr, vals := range resp.Header {
+					for _, val := range vals {
+						w.Header().Add(hdr, val)
+					}
+				}
+				w.WriteHeader(resp.StatusCode)
+				// discard errors as this is for debugging
+				_, _ = io.Copy(w, resp.Body)
+				_ = resp.Body.Close()
+			}()
+
+			// Log incoming request
+			dump, err := httputil.DumpRequest(req, true)
+			if err != nil {
+				logger.Debugf("Failed to dump incoming request: %s", err)
+			} else {
+				strSlice := strings.Split(string(dump), "\n")
+				for _, s := range strSlice {
+					logger.Debug(s)
+				}
+			}
+		}
+
+		span := opentracing.StartSpan(metricsName)
+		defer span.Finish()
+		req = req.WithContext(opentracing.ContextWithSpan(req.Context(), span))
+		h.ServeHTTP(nextWriter, req)
+
+	}
+
+	return http.HandlerFunc(withSpan)
+}
+
+// MakeHTMLAPI adds Span metrics to the HTML Handler function
+// This is used to serve HTML alongside JSON error messages
+func MakeHTMLAPI(metricsName string, f func(http.ResponseWriter, *http.Request) *util.JSONResponse) http.Handler {
+	withSpan := func(w http.ResponseWriter, req *http.Request) {
+		span := opentracing.StartSpan(metricsName)
+		defer span.Finish()
+		req = req.WithContext(opentracing.ContextWithSpan(req.Context(), span))
+		if err := f(w, req); err != nil {
+			h := util.MakeJSONAPI(util.NewJSONRequestHandler(func(req *http.Request) util.JSONResponse {
+				return *err
+			}))
+			h.ServeHTTP(w, req)
+		}
+	}
+
+	return promhttp.InstrumentHandlerCounter(
+		promauto.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: metricsName,
+				Help: "Total number of http requests for HTML resources",
+			},
+			[]string{"code"},
+		),
+		http.HandlerFunc(withSpan),
+	)
+}
+
+// MakeInternalAPI turns a util.JSONRequestHandler function into an http.Handler.
+// This is used for APIs that are internal to dendrite.
+// If we are passed a tracing context in the request headers then we use that
+// as the parent of any tracing spans we create.
+func MakeInternalAPI(metricsName string, f func(*http.Request) util.JSONResponse) http.Handler {
+	h := util.MakeJSONAPI(util.NewJSONRequestHandler(f))
+	withSpan := func(w http.ResponseWriter, req *http.Request) {
+		carrier := opentracing.HTTPHeadersCarrier(req.Header)
+		tracer := opentracing.GlobalTracer()
+		clientContext, err := tracer.Extract(opentracing.HTTPHeaders, carrier)
+		var span opentracing.Span
+		if err == nil {
+			// Default to a span without RPC context.
+			span = tracer.StartSpan(metricsName)
+		} else {
+			// Set the RPC context.
+			span = tracer.StartSpan(metricsName, ext.RPCServerOption(clientContext))
+		}
+		defer span.Finish()
+		req = req.WithContext(opentracing.ContextWithSpan(req.Context(), span))
+		h.ServeHTTP(w, req)
+	}
+
+	return http.HandlerFunc(withSpan)
+}
+
+// MakeFedAPI makes an http.Handler that checks matrix federation authentication.
+func MakeFedAPI(
+	metricsName string,
+	serverName gomatrixserverlib.ServerName,
+	keyRing gomatrixserverlib.KeyRing,
+	wakeup *FederationWakeups,
+	f func(*http.Request, *gomatrixserverlib.FederationRequest, map[string]string) util.JSONResponse,
+) http.Handler {
+	h := func(req *http.Request) util.JSONResponse {
+		fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
+			req, time.Now(), serverName, keyRing,
+		)
+		if fedReq == nil {
+			return errResp
+		}
+		go wakeup.Wakeup(req.Context(), fedReq.Origin())
+		vars, err := URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
+
+		return f(req, fedReq, vars)
+	}
+	return MakeExternalAPI(metricsName, h)
+}
+
+type FederationWakeups struct {
+	FsAPI   federationsenderAPI.FederationSenderInternalAPI
+	origins sync.Map
+}
+
+func (f *FederationWakeups) Wakeup(ctx context.Context, origin gomatrixserverlib.ServerName) {
+	key, keyok := f.origins.Load(origin)
+	if keyok {
+		lastTime, ok := key.(time.Time)
+		if ok && time.Since(lastTime) < time.Minute {
+			return
+		}
+	}
+	aliveReq := federationsenderAPI.PerformServersAliveRequest{
+		Servers: []gomatrixserverlib.ServerName{origin},
+	}
+	aliveRes := federationsenderAPI.PerformServersAliveResponse{}
+	if err := f.FsAPI.PerformServersAlive(ctx, &aliveReq, &aliveRes); err != nil {
+		util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{
+			"origin": origin,
+		}).Warn("incoming federation request failed to notify server alive")
+	} else {
+		f.origins.Store(origin, time.Now())
+	}
+}
+
+// SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics
+// listener.
+func SetupHTTPAPI(servMux *http.ServeMux, publicApiMux *mux.Router, internalApiMux *mux.Router, cfg *config.Dendrite, enableHTTPAPIs bool) {
+	if cfg.Metrics.Enabled {
+		servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth))
+	}
+	if enableHTTPAPIs {
+		servMux.Handle(InternalPathPrefix, internalApiMux)
+	}
+	servMux.Handle(PublicPathPrefix, WrapHandlerInCORS(publicApiMux))
+}
+
+// WrapHandlerInBasicAuth adds basic auth to a handler. Only used for /metrics
+func WrapHandlerInBasicAuth(h http.Handler, b BasicAuth) http.HandlerFunc {
+	if b.Username == "" || b.Password == "" {
+		logrus.Warn("Metrics are exposed without protection. Make sure you set up protection at proxy level.")
+	}
+	return func(w http.ResponseWriter, r *http.Request) {
+		// Serve without authorization if either Username or Password is unset
+		if b.Username == "" || b.Password == "" {
+			h.ServeHTTP(w, r)
+			return
+		}
+		user, pass, ok := r.BasicAuth()
+
+		if !ok || user != b.Username || pass != b.Password {
+			http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
+			return
+		}
+		h.ServeHTTP(w, r)
+	}
+}
+
+// WrapHandlerInCORS adds CORS headers to all responses, including all error
+// responses.
+// Handles OPTIONS requests directly.
+func WrapHandlerInCORS(h http.Handler) http.HandlerFunc {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+		w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
+		w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Authorization")
+
+		if r.Method == http.MethodOptions && r.Header.Get("Access-Control-Request-Method") != "" {
+			// Its easiest just to always return a 200 OK for everything. Whether
+			// this is technically correct or not is a question, but in the end this
+			// is what a lot of other people do (including synapse) and the clients
+			// are perfectly happy with it.
+			w.WriteHeader(http.StatusOK)
+		} else {
+			h.ServeHTTP(w, r)
+		}
+	})
+}
diff --git a/internal/httputil/httpapi_test.go b/internal/httputil/httpapi_test.go
new file mode 100644
index 00000000..de6ccf9b
--- /dev/null
+++ b/internal/httputil/httpapi_test.go
@@ -0,0 +1,109 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httputil
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestWrapHandlerInBasicAuth(t *testing.T) {
+	type args struct {
+		h http.Handler
+		b BasicAuth
+	}
+
+	dummyHandler := http.HandlerFunc(func(h http.ResponseWriter, r *http.Request) {
+		h.WriteHeader(http.StatusOK)
+	})
+
+	tests := []struct {
+		name    string
+		args    args
+		want    int
+		reqAuth bool
+	}{
+		{
+			name:    "no user or password setup",
+			args:    args{h: dummyHandler},
+			want:    http.StatusOK,
+			reqAuth: false,
+		},
+		{
+			name: "only user set",
+			args: args{
+				h: dummyHandler,
+				b: BasicAuth{Username: "test"}, // no basic auth
+			},
+			want:    http.StatusOK,
+			reqAuth: false,
+		},
+		{
+			name: "only pass set",
+			args: args{
+				h: dummyHandler,
+				b: BasicAuth{Password: "test"}, // no basic auth
+			},
+			want:    http.StatusOK,
+			reqAuth: false,
+		},
+		{
+			name: "credentials correct",
+			args: args{
+				h: dummyHandler,
+				b: BasicAuth{Username: "test", Password: "test"}, // basic auth enabled
+			},
+			want:    http.StatusOK,
+			reqAuth: true,
+		},
+		{
+			name: "credentials wrong",
+			args: args{
+				h: dummyHandler,
+				b: BasicAuth{Username: "test1", Password: "test"}, // basic auth enabled
+			},
+			want:    http.StatusForbidden,
+			reqAuth: true,
+		},
+		{
+			name: "no basic auth in request",
+			args: args{
+				h: dummyHandler,
+				b: BasicAuth{Username: "test", Password: "test"}, // basic auth enabled
+			},
+			want:    http.StatusForbidden,
+			reqAuth: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			baHandler := WrapHandlerInBasicAuth(tt.args.h, tt.args.b)
+
+			req := httptest.NewRequest("GET", "http://localhost/metrics", nil)
+			if tt.reqAuth {
+				req.SetBasicAuth("test", "test")
+			}
+
+			w := httptest.NewRecorder()
+			baHandler(w, req)
+			resp := w.Result()
+
+			if resp.StatusCode != tt.want {
+				t.Errorf("Expected status code %d, got %d", resp.StatusCode, tt.want)
+			}
+		})
+	}
+}
diff --git a/internal/httputil/paths.go b/internal/httputil/paths.go
new file mode 100644
index 00000000..728b5a87
--- /dev/null
+++ b/internal/httputil/paths.go
@@ -0,0 +1,20 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httputil
+
+const (
+	PublicPathPrefix   = "/_matrix/"
+	InternalPathPrefix = "/api/"
+)
diff --git a/internal/httputil/routing.go b/internal/httputil/routing.go
new file mode 100644
index 00000000..0bd3655e
--- /dev/null
+++ b/internal/httputil/routing.go
@@ -0,0 +1,35 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httputil
+
+import (
+	"net/url"
+)
+
+// URLDecodeMapValues is a function that iterates through each of the items in a
+// map, URL decodes the value, and returns a new map with the decoded values
+// under the same key names
+func URLDecodeMapValues(vmap map[string]string) (map[string]string, error) {
+	decoded := make(map[string]string, len(vmap))
+	for key, value := range vmap {
+		decodedVal, err := url.PathUnescape(value)
+		if err != nil {
+			return make(map[string]string), err
+		}
+		decoded[key] = decodedVal
+	}
+
+	return decoded, nil
+}