dendritejs-pinecone

4 files changed, 393 insertions, 13 deletions

diff --git a/cmd/dendrite-demo-pinecone/conn/client.go b/cmd/dendrite-demo-pinecone/conn/client.go
index 66f9ebfa..0889e781 100644
--- a/cmd/dendrite-demo-pinecone/conn/client.go
+++ b/cmd/dendrite-demo-pinecone/conn/client.go
@@ -1,14 +1,15 @@
 package conn
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/http"
 	"strings"
 
-	"github.com/gorilla/websocket"
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/gomatrixserverlib"
+	"nhooyr.io/websocket"
 
 	pineconeRouter "github.com/matrix-org/pinecone/router"
 	pineconeSessions "github.com/matrix-org/pinecone/sessions"
@@ -17,11 +18,12 @@ import (
 func ConnectToPeer(pRouter *pineconeRouter.Router, peer string) error {
 	var parent net.Conn
 	if strings.HasPrefix(peer, "ws://") || strings.HasPrefix(peer, "wss://") {
-		c, _, err := websocket.DefaultDialer.Dial(peer, nil)
+		ctx := context.Background()
+		c, _, err := websocket.Dial(ctx, peer, nil)
 		if err != nil {
-			return fmt.Errorf("websocket.DefaultDialer.Dial: %w", err)
+			return fmt.Errorf("websocket.Dial: %w", err)
 		}
-		parent = WrapWebSocketConn(c)
+		parent = websocket.NetConn(ctx, c, websocket.MessageBinary)
 	} else {
 		var err error
 		parent, err = net.Dial("tcp", peer)
@@ -41,21 +43,21 @@ type RoundTripper struct {
 }
 
 func (y *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
-	req.URL.Scheme = "http"
+	req.URL.Scheme = "https"
 	return y.inner.RoundTrip(req)
 }
 
 func createTransport(s *pineconeSessions.Sessions) *http.Transport {
-	tr := &http.Transport{}
+	tr := &http.Transport{
+		DisableKeepAlives: false,
+		Dial:              s.Dial,
+		DialContext:       s.DialContext,
+		DialTLS:           s.DialTLS,
+		DialTLSContext:    s.DialTLSContext,
+	}
 	tr.RegisterProtocol(
 		"matrix", &RoundTripper{
-			inner: &http.Transport{
-				DisableKeepAlives: false,
-				Dial:              s.Dial,
-				DialContext:       s.DialContext,
-				DialTLS:           s.DialTLS,
-				DialTLSContext:    s.DialTLSContext,
-			},
+			inner: tr,
 		},
 	)
 	return tr
diff --git a/cmd/dendritejs-pinecone/jsServer.go b/cmd/dendritejs-pinecone/jsServer.go
new file mode 100644
index 00000000..074d20cb
--- /dev/null
+++ b/cmd/dendritejs-pinecone/jsServer.go
@@ -0,0 +1,100 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build wasm
+
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"syscall/js"
+)
+
+// JSServer exposes an HTTP-like server interface which allows JS to 'send' requests to it.
+type JSServer struct {
+	// The router which will service requests
+	Mux http.Handler
+}
+
+// OnRequestFromJS is the function that JS will invoke when there is a new request.
+// The JS function signature is:
+//   function(reqString: string): Promise<{result: string, error: string}>
+// Usage is like:
+//   const res = await global._go_js_server.fetch(reqString);
+//   if (res.error) {
+//     // handle error: this is a 'network' error, not a non-2xx error.
+//   }
+//   const rawHttpResponse = res.result;
+func (h *JSServer) OnRequestFromJS(this js.Value, args []js.Value) interface{} {
+	// we HAVE to spawn a new goroutine and return immediately or else Go will deadlock
+	// if this request blocks at all e.g for /sync calls
+	httpStr := args[0].String()
+	promise := js.Global().Get("Promise").New(js.FuncOf(func(pthis js.Value, pargs []js.Value) interface{} {
+		// The initial callback code for new Promise() is also called on the critical path, which is why
+		// we need to put this in an immediately invoked goroutine.
+		go func() {
+			resolve := pargs[0]
+			resStr, err := h.handle(httpStr)
+			errStr := ""
+			if err != nil {
+				errStr = err.Error()
+			}
+			resolve.Invoke(map[string]interface{}{
+				"result": resStr,
+				"error":  errStr,
+			})
+		}()
+		return nil
+	}))
+	return promise
+}
+
+// handle invokes the http.ServeMux for this request and returns the raw HTTP response.
+func (h *JSServer) handle(httpStr string) (resStr string, err error) {
+	req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(httpStr)))
+	if err != nil {
+		return
+	}
+	w := httptest.NewRecorder()
+
+	h.Mux.ServeHTTP(w, req)
+
+	res := w.Result()
+	var resBuffer strings.Builder
+	err = res.Write(&resBuffer)
+	return resBuffer.String(), err
+}
+
+// ListenAndServe registers a variable in JS-land with the given namespace. This variable is
+// a function which JS-land can call to 'send' HTTP requests. The function is attached to
+// a global object called "_go_js_server". See OnRequestFromJS for more info.
+func (h *JSServer) ListenAndServe(namespace string) {
+	globalName := "_go_js_server"
+	// register a hook in JS-land for it to invoke stuff
+	server := js.Global().Get(globalName)
+	if !server.Truthy() {
+		server = js.Global().Get("Object").New()
+		js.Global().Set(globalName, server)
+	}
+
+	server.Set(namespace, js.FuncOf(h.OnRequestFromJS))
+
+	fmt.Printf("Listening for requests from JS on function %s.%s\n", globalName, namespace)
+	// Block forever to mimic http.ListenAndServe
+	select {}
+}
diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go
new file mode 100644
index 00000000..f4c8a982
--- /dev/null
+++ b/cmd/dendritejs-pinecone/main.go
@@ -0,0 +1,255 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build wasm
+
+package main
+
+import (
+	"crypto/ed25519"
+	"fmt"
+	"log"
+	"os"
+	"syscall/js"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
+	"github.com/matrix-org/dendrite/eduserver"
+	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/keyserver"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/userapi"
+
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/sirupsen/logrus"
+
+	_ "github.com/matrix-org/go-sqlite3-js"
+
+	pineconeRouter "github.com/matrix-org/pinecone/router"
+	pineconeSessions "github.com/matrix-org/pinecone/sessions"
+)
+
+var GitCommit string
+
+func init() {
+	fmt.Printf("[%s] dendrite.js starting...\n", GitCommit)
+}
+
+const publicPeer = "wss://pinecone.matrix.org/public"
+const keyNameEd25519 = "_go_ed25519_key"
+
+func readKeyFromLocalStorage() (key ed25519.PrivateKey, err error) {
+	localforage := js.Global().Get("localforage")
+	if !localforage.Truthy() {
+		err = fmt.Errorf("readKeyFromLocalStorage: no localforage")
+		return
+	}
+	// https://localforage.github.io/localForage/
+	item, ok := await(localforage.Call("getItem", keyNameEd25519))
+	if !ok || !item.Truthy() {
+		err = fmt.Errorf("readKeyFromLocalStorage: no key in localforage")
+		return
+	}
+	fmt.Println("Found key in localforage")
+	// extract []byte and make an ed25519 key
+	seed := make([]byte, 32, 32)
+	js.CopyBytesToGo(seed, item)
+
+	return ed25519.NewKeyFromSeed(seed), nil
+}
+
+func writeKeyToLocalStorage(key ed25519.PrivateKey) error {
+	localforage := js.Global().Get("localforage")
+	if !localforage.Truthy() {
+		return fmt.Errorf("writeKeyToLocalStorage: no localforage")
+	}
+
+	// make a Uint8Array from the key's seed
+	seed := key.Seed()
+	jsSeed := js.Global().Get("Uint8Array").New(len(seed))
+	js.CopyBytesToJS(jsSeed, seed)
+	// write it
+	localforage.Call("setItem", keyNameEd25519, jsSeed)
+	return nil
+}
+
+// taken from https://go-review.googlesource.com/c/go/+/150917
+
+// await waits until the promise v has been resolved or rejected and returns the promise's result value.
+// The boolean value ok is true if the promise has been resolved, false if it has been rejected.
+// If v is not a promise, v itself is returned as the value and ok is true.
+func await(v js.Value) (result js.Value, ok bool) {
+	if v.Type() != js.TypeObject || v.Get("then").Type() != js.TypeFunction {
+		return v, true
+	}
+	done := make(chan struct{})
+	onResolve := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
+		result = args[0]
+		ok = true
+		close(done)
+		return nil
+	})
+	defer onResolve.Release()
+	onReject := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
+		result = args[0]
+		ok = false
+		close(done)
+		return nil
+	})
+	defer onReject.Release()
+	v.Call("then", onResolve, onReject)
+	<-done
+	return
+}
+
+func generateKey() ed25519.PrivateKey {
+	// attempt to look for a seed in JS-land and if it exists use it.
+	priv, err := readKeyFromLocalStorage()
+	if err == nil {
+		fmt.Println("Read key from localStorage")
+		return priv
+	}
+	// generate a new key
+	fmt.Println(err, " : Generating new ed25519 key")
+	_, priv, err = ed25519.GenerateKey(nil)
+	if err != nil {
+		logrus.Fatalf("Failed to generate ed25519 key: %s", err)
+	}
+	if err := writeKeyToLocalStorage(priv); err != nil {
+		fmt.Println("failed to write key to localStorage: ", err)
+		// non-fatal, we'll just have amnesia for a while
+	}
+	return priv
+}
+
+func main() {
+	sk := generateKey()
+	pk := sk.Public().(ed25519.PublicKey)
+
+	logger := log.New(os.Stdout, "", 0)
+	pRouter := pineconeRouter.NewRouter(logger, "dendrite", sk, pk, nil)
+	pSessions := pineconeSessions.NewSessions(logger, pRouter)
+
+	cfg := &config.Dendrite{}
+	cfg.Defaults()
+	cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db"
+	cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db"
+	cfg.UserAPI.DeviceDatabase.ConnectionString = "file:/idb/dendritejs_device.db"
+	cfg.FederationSender.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db"
+	cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db"
+	cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db"
+	cfg.SigningKeyServer.Database.ConnectionString = "file:/idb/dendritejs_signingkeyserver.db"
+	cfg.SyncAPI.Database.ConnectionString = "file:/idb/dendritejs_syncapi.db"
+	cfg.KeyServer.Database.ConnectionString = "file:/idb/dendritejs_e2ekey.db"
+	cfg.Global.Kafka.UseNaffka = true
+	cfg.Global.Kafka.Database.ConnectionString = "file:/idb/dendritejs_naffka.db"
+	cfg.Global.TrustedIDServers = []string{}
+	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
+	cfg.Global.PrivateKey = sk
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(pk)
+
+	if err := cfg.Derive(); err != nil {
+		logrus.Fatalf("Failed to derive values from config: %s", err)
+	}
+	base := setup.NewBaseDendrite(cfg, "Monolith", false)
+	defer base.Close() // nolint: errcheck
+
+	accountDB := base.CreateAccountsDB()
+	federation := conn.CreateFederationClient(base, pSessions)
+	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation)
+	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI)
+	keyAPI.SetUserAPI(userAPI)
+
+	serverKeyAPI := &signing.YggdrasilKeys{}
+	keyRing := serverKeyAPI.KeyRing()
+
+	rsAPI := roomserver.NewInternalAPI(base, keyRing)
+	eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI)
+	asQuery := appservice.NewInternalAPI(
+		base, userAPI, rsAPI,
+	)
+	rsAPI.SetAppserviceAPI(asQuery)
+	fedSenderAPI := federationsender.NewInternalAPI(base, federation, rsAPI, keyRing, true)
+	rsAPI.SetFederationSenderAPI(fedSenderAPI)
+
+	monolith := setup.Monolith{
+		Config:    base.Cfg,
+		AccountDB: accountDB,
+		Client:    conn.CreateClient(base, pSessions),
+		FedClient: federation,
+		KeyRing:   keyRing,
+
+		AppserviceAPI:       asQuery,
+		EDUInternalAPI:      eduInputAPI,
+		FederationSenderAPI: fedSenderAPI,
+		RoomserverAPI:       rsAPI,
+		UserAPI:             userAPI,
+		KeyAPI:              keyAPI,
+		//ServerKeyAPI:        serverKeyAPI,
+		ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pSessions, fedSenderAPI, federation),
+	}
+	monolith.AddAllPublicRoutes(
+		base.ProcessContext,
+		base.PublicClientAPIMux,
+		base.PublicFederationAPIMux,
+		base.PublicKeyAPIMux,
+		base.PublicMediaAPIMux,
+	)
+
+	httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
+	httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
+	httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
+	httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
+
+	p2pRouter := pSessions.HTTP().Mux()
+	p2pRouter.Handle(httputil.PublicFederationPathPrefix, base.PublicFederationAPIMux)
+	p2pRouter.Handle(httputil.PublicMediaPathPrefix, base.PublicMediaAPIMux)
+
+	// Expose the matrix APIs via fetch - for local traffic
+	go func() {
+		logrus.Info("Listening for service-worker fetch traffic")
+		s := JSServer{
+			Mux: httpRouter,
+		}
+		s.ListenAndServe("fetch")
+	}()
+
+	// Connect to the static peer
+	go func() {
+		for {
+			if pRouter.PeerCount(pineconeRouter.PeerTypeRemote) == 0 {
+				if err := conn.ConnectToPeer(pRouter, publicPeer); err != nil {
+					logrus.WithError(err).Error("Failed to connect to static peer")
+				}
+			}
+			select {
+			case <-base.ProcessContext.Context().Done():
+				return
+			case <-time.After(time.Second * 5):
+			}
+		}
+	}()
+
+	// We want to block forever to let the fetch and libp2p handler serve the APIs
+	select {}
+}
diff --git a/cmd/dendritejs-pinecone/main_noop.go b/cmd/dendritejs-pinecone/main_noop.go
new file mode 100644
index 00000000..dcea032f
--- /dev/null
+++ b/cmd/dendritejs-pinecone/main_noop.go
@@ -0,0 +1,23 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !wasm
+
+package main
+
+import "fmt"
+
+func main() {
+	fmt.Println("dendritejs: no-op when not compiling for WebAssembly")
+}