Port to Golangv0.0.1-dev

9 files changed, 652 insertions, 0 deletions

diff --git a/internal/service/config.go b/internal/service/config.go
new file mode 100644
index 0000000..bcb9efc
--- /dev/null
+++ b/internal/service/config.go
@@ -0,0 +1,73 @@
+package service
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"time"
+
+	"dario.cat/mergo"
+)
+
+type Duration struct {
+	time.Duration
+}
+
+func (s Duration) MarshalJSON() ([]byte, error) {
+	return json.Marshal(s.Duration.String())
+}
+
+func (s *Duration) UnmarshalJSON(data []byte) error {
+	var str string
+	err := json.Unmarshal(data, &str)
+	if err != nil {
+		return err
+	}
+
+	v, err := time.ParseDuration(str)
+	if err != nil {
+		return err
+	}
+	*s = Duration{v}
+
+	return nil
+}
+
+type Config struct {
+	MaxInterval Duration  `json:"max-interval,omitempty"`
+	MinInterval Duration  `json:"min-interval,omitempty"`
+	Mirrors     []*Mirror `json:"mirrors,omitempty"`
+}
+
+var DefaultConfig = Config{
+	MaxInterval: Duration{24 * time.Hour},
+	MinInterval: Duration{time.Hour},
+}
+
+func (c *Config) Apply(arg Config) {
+	err := mergo.Merge(c, &arg)
+	if err != nil {
+		panic(err)
+	}
+}
+
+// ApplyFileConfig loads the configuration described by the given yaml file.
+func ApplyFileConfig(cfg *Config, filePath string) error {
+	var ret Config
+
+	f, err := os.Open(filePath)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	err = json.NewDecoder(f).Decode(&ret)
+	if err != nil {
+		return fmt.Errorf("loading configuration file: %w", err)
+	}
+
+	cfg.Apply(ret)
+	return nil
+}
diff --git a/internal/service/config_test.go b/internal/service/config_test.go
new file mode 100644
index 0000000..c9af1e7
--- /dev/null
+++ b/internal/service/config_test.go
@@ -0,0 +1,54 @@
+package service
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDurationMarshal(t *testing.T) {
+	tests := []struct {
+		arg Duration
+		exp string
+	}{
+		{
+			Duration{time.Second},
+			"\"1s\"",
+		},
+		{
+			Duration{time.Minute},
+			"\"1m0s\"",
+		},
+	}
+
+	for _, test := range tests {
+		v, err := json.Marshal(test.arg)
+		require.NoError(t, err)
+		require.Equal(t, test.exp, string(v))
+	}
+}
+
+func TestDurationUnmarshal(t *testing.T) {
+	tests := []struct {
+		arg string
+		exp Duration
+	}{
+		{
+			"\"1s\"",
+			Duration{time.Second},
+		},
+		{
+			"\"1m0s\"",
+			Duration{time.Minute},
+		},
+	}
+
+	for _, test := range tests {
+		var v Duration
+		err := json.Unmarshal([]byte(test.arg), &v)
+		require.NoError(t, err)
+		require.Equal(t, test.exp.Duration, v.Duration)
+	}
+}
diff --git a/internal/service/git.go b/internal/service/git.go
new file mode 100644
index 0000000..ad3a653
--- /dev/null
+++ b/internal/service/git.go
@@ -0,0 +1,134 @@
+package service
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+)
+
+const gitDescriptionFile = "description"
+
+func mapExecError(err error) error {
+	if ee, ok := err.(*exec.ExitError); ok {
+		return errors.New(string(ee.Stderr))
+	}
+	return err
+}
+
+// Set the description  for the projects repository.
+func setDescription(repo string, desc string) error {
+	descPath := path.Join(repo, gitDescriptionFile)
+
+	var curDesc string
+	buf, err := os.ReadFile(descPath)
+	if os.IsNotExist(err) {
+		// empty
+	} else if err != nil {
+		return err
+	} else {
+		curDesc = string(buf)
+	}
+
+	if curDesc != desc {
+		err = os.WriteFile(descPath, []byte(desc), 0750)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Set the remote origin's URL  for the projects repository.
+func setRemoteOrigin(repo string, origin *url.URL) error {
+	cmd := exec.Command("git", "remote", "get-url", "origin")
+	cmd.Dir = repo
+	buf, err := cmd.Output()
+	if err != nil {
+		return fmt.Errorf("getting current project remote origin: %w", err)
+	}
+	currentOrigin := strings.TrimSpace(string(buf))
+
+	if currentOrigin != origin.String() {
+		cmd = exec.Command("git", "remote", "set-url", "origin", origin.String())
+		cmd.Dir = repo
+		err = cmd.Run()
+		if err != nil {
+			err = mapExecError(err)
+			return fmt.Errorf("setting project remote origin: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func getRemoteHeadReference(repo string) (string, error) {
+	cmd := exec.Command("git", "ls-remote", "--symref", "origin", "HEAD")
+	cmd.Dir = repo
+	buf, err := cmd.Output()
+	if err != nil {
+		return "", mapExecError(err)
+	}
+
+	for _, l := range strings.Split(string(buf), "\n") {
+		fields := strings.Fields(l)
+		if len(fields) != 3 {
+			return "", errors.New("unexpected output from 'git ls-remote'")
+		}
+		if fields[0] == "ref:" {
+			return strings.TrimPrefix(fields[1], "refs/heads/"), nil
+		}
+	}
+
+	return "", errors.New("not found")
+}
+
+func MirrorGit(dst *url.URL, src *url.URL, description string) error {
+	if dst.Scheme != "" && dst.Scheme != "file://" {
+		return fmt.Errorf("'%s' scheme not supported", dst.Scheme)
+	}
+
+	if _, err := os.Stat(dst.Path); os.IsNotExist(err) {
+		err = os.MkdirAll(path.Join(dst.Path, ".."), 0750)
+		if err != nil {
+			return fmt.Errorf("creating new mirror: %w", err)
+		}
+
+		cmd := exec.Command("git", "clone", "--bare", "--single-branch", src.String(), dst.String())
+		cmd.Dir = path.Join(dst.Path, "..")
+		err := cmd.Run()
+		if err != nil {
+			err = mapExecError(err)
+			return fmt.Errorf("cloning: %s", err)
+		}
+	}
+
+	err := setDescription(dst.Path, description)
+	if err != nil {
+		return err
+	}
+
+	err = setRemoteOrigin(dst.Path, src)
+	if err != nil {
+		return err
+	}
+
+	branch, err := getRemoteHeadReference(dst.Path)
+	if err != nil {
+		return fmt.Errorf("guessing remote default branch: %w", err)
+	}
+
+	cmd := exec.Command("git", "fetch", "--tags", "origin", branch+":"+branch)
+	cmd.Dir = dst.Path
+	err = cmd.Run()
+	if err != nil {
+		err = mapExecError(err)
+		return fmt.Errorf("fetching project: %s", err)
+	}
+
+	return nil
+}
diff --git a/internal/service/git_test.go b/internal/service/git_test.go
new file mode 100644
index 0000000..6dc8f62
--- /dev/null
+++ b/internal/service/git_test.go
@@ -0,0 +1,34 @@
+package service
+
+import (
+	"os/exec"
+	"path"
+	"strings"
+	"testing"
+
+	"git.server.ky/slackcoder/mirror/internal"
+	"github.com/stretchr/testify/require"
+)
+
+func requireTagExist(t *testing.T, filePath string, tag string, msgAndArgs ...interface{}) {
+	cmd := exec.Command("git", "tag")
+	cmd.Dir = filePath
+
+	buf, err := cmd.Output()
+	require.NoError(t, err)
+
+	tags := strings.Fields(string(buf))
+	require.Contains(t, tags, tag, msgAndArgs...)
+}
+
+func TestMirrorGit(t *testing.T) {
+	d := t.TempDir()
+
+	dst := internal.MustURL(path.Join(d, "merchant"))
+	src := internal.MustURL("https://git.taler.net/merchant.git")
+
+	err := MirrorGit(dst, src, "GNU Taler Merchant")
+	require.NoError(t, err, "git mirror")
+
+	requireTagExist(t, dst.Path, "v0.11.3", "tag must exist")
+}
diff --git a/internal/service/rsync.go b/internal/service/rsync.go
new file mode 100644
index 0000000..8298589
--- /dev/null
+++ b/internal/service/rsync.go
@@ -0,0 +1,40 @@
+package service
+
+import (
+	"bytes"
+	"errors"
+	"net/url"
+	"os/exec"
+	"strings"
+)
+
+var rsyncOpts = []string{
+	"--delete-excluded",
+	"--hard-links",
+	"--links",
+	"--perms",
+	"--recursive",
+	"--safe-links",
+	"--sparse",
+	"--times",
+}
+
+func Rsync(dst *url.URL, src *url.URL) error {
+	src2 := *src
+	if !strings.HasSuffix(src2.Path, "/.") {
+		src2.Path = src2.Path + "/."
+	}
+
+	var stderr bytes.Buffer
+
+	args := append(rsyncOpts, src2.String(), dst.String())
+	cmd := exec.Command("rsync", args...)
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	if err != nil {
+		return errors.New(stderr.String())
+	}
+
+	return nil
+}
diff --git a/internal/service/rsync_test.go b/internal/service/rsync_test.go
new file mode 100644
index 0000000..c6c2341
--- /dev/null
+++ b/internal/service/rsync_test.go
@@ -0,0 +1,21 @@
+package service
+
+import (
+	"path"
+	"testing"
+
+	"git.server.ky/slackcoder/mirror/internal"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRsync(t *testing.T) {
+	d := t.TempDir()
+
+	dst := internal.MustURL(d)
+	src := internal.MustURL("rsync://mirror.cedia.org.ec/gnu/taler")
+
+	err := Rsync(dst, src)
+	require.NoError(t, err, "rsync")
+
+	require.FileExists(t, path.Join(dst.Path, "taler-merchant-0.11.3.tar.gz"))
+}
diff --git a/internal/service/service.go b/internal/service/service.go
new file mode 100644
index 0000000..c34bcd3
--- /dev/null
+++ b/internal/service/service.go
@@ -0,0 +1,195 @@
+package service
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math/rand"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"git.server.ky/slackcoder/mirror/internal/github"
+)
+
+type Mirror struct {
+	Method      string   `json:"method"`
+	From        *url.URL `json:"-"`
+	To          *url.URL `json:"-"`
+	Description string   `json:"description,omitempty"`
+}
+
+func (m *Mirror) Equal(arg *Mirror) bool {
+	return m.Method == arg.Method && m.From.String() == arg.From.String() && m.To.String() == arg.To.String()
+}
+
+func (m *Mirror) String() string {
+	buf, err := json.Marshal(m)
+	if err != nil {
+		panic(err)
+	}
+	return string(buf)
+}
+
+type Service struct {
+	cfg *Config
+
+	mirrorsLock sync.Mutex
+	mirrors     []*Mirror
+	schedule    []time.Time
+
+	stopCh chan struct{}
+}
+
+func NewService(cfg *Config) (*Service, error) {
+	cfg.Apply(DefaultConfig)
+
+	srv := &Service{
+		cfg:     cfg,
+		mirrors: nil,
+		stopCh:  make(chan struct{}),
+	}
+
+	for _, m := range cfg.Mirrors {
+		err := srv.AddMirror(m)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return srv, nil
+}
+
+func (s *Service) scheduleNextRun(arg *Mirror) {
+	// Be polite.
+	period := s.cfg.MaxInterval.Duration - s.cfg.MinInterval.Duration
+	at := time.Now().Add(s.cfg.MinInterval.Duration + time.Duration(rand.Intn(int(period))))
+
+	for i, m := range s.mirrors {
+		if m.Equal(arg) {
+			s.schedule[i] = at
+		}
+	}
+}
+
+func (s *Service) AddMirror(arg *Mirror) error {
+	defer s.mirrorsLock.Unlock()
+	s.mirrorsLock.Lock()
+
+	for _, m := range s.mirrors {
+		if m.Equal(arg) {
+			return errors.New("must be unique")
+		}
+	}
+
+	s.schedule = append(s.schedule, time.Time{})
+	s.mirrors = append(s.mirrors, arg)
+	s.scheduleNextRun(arg)
+
+	return nil
+}
+
+func (s *Service) scheduled(yield func(*Mirror) bool) {
+	defer s.mirrorsLock.Unlock()
+	s.mirrorsLock.Lock()
+
+	now := time.Now()
+	for i, m := range s.mirrors {
+		if s.schedule[i].After(now) {
+			continue
+		}
+
+		if !yield(m) {
+			break
+		}
+	}
+}
+
+func (s *Service) Mirror(arg *Mirror) error {
+	if arg.From.Path == "" || arg.To.Path == "" || arg.Method == "" {
+		return fmt.Errorf("badly formatted mirror '%s'", arg.String())
+	}
+
+	var err error
+
+	switch arg.Method {
+	case "git":
+		err = MirrorGit(arg.To, arg.From, arg.Description)
+	case "github-assets":
+		client := github.NewClient()
+		err = client.MirrorAssets(arg.To, arg.From)
+	case "rsync":
+		err = Rsync(arg.To, arg.From)
+	default:
+		err = fmt.Errorf("unknown method '%s'", arg.Method)
+	}
+
+	if err != nil {
+		return fmt.Errorf("could not clone from '%s': %w", arg.From, err)
+	}
+
+	return nil
+}
+
+func (s *Service) RemoveMirror(arg *Mirror) error {
+	defer s.mirrorsLock.Unlock()
+	s.mirrorsLock.Lock()
+
+	for i, m := range s.mirrors {
+		if m.Equal(arg) {
+			s.mirrors[i] = s.mirrors[len(s.mirrors)-1]
+			s.mirrors = s.mirrors[:len(s.mirrors)-1]
+
+			s.schedule[i] = s.schedule[len(s.schedule)-1]
+			s.schedule = s.schedule[:len(s.schedule)-1]
+
+			return nil
+		}
+	}
+
+	return errors.New("not found")
+}
+
+func (s *Service) Log(str string) {
+	escaped := []rune(strconv.Quote(strings.TrimSpace(str)))
+	escaped = escaped[1 : len(escaped)-1]
+
+	fmt.Fprintf(os.Stderr, "%s: %s\n", time.Now().Format(time.RFC822Z), string(escaped))
+}
+
+func (s *Service) Run() error {
+	wakeup := time.NewTicker(time.Second)
+
+mainLoop:
+	for {
+		select {
+		case <-wakeup.C:
+		case <-s.stopCh:
+			break mainLoop
+		}
+
+		s.scheduled(func(m *Mirror) bool {
+			err := s.Mirror(m)
+			if err != nil {
+				s.Log(err.Error())
+			}
+
+			s.scheduleNextRun(m)
+
+			return true
+		})
+	}
+
+	return nil
+}
+
+func (s *Service) Stop() {
+	select {
+	case <-s.stopCh:
+	default:
+		close(s.stopCh)
+	}
+}
diff --git a/internal/service/service_json.go b/internal/service/service_json.go
new file mode 100644
index 0000000..571eb30
--- /dev/null
+++ b/internal/service/service_json.go
@@ -0,0 +1,64 @@
+package service
+
+import (
+	"encoding/json"
+	"net/url"
+)
+
+type JsonURL struct {
+	*url.URL
+}
+
+func (m JsonURL) MarshalJSON() ([]byte, error) {
+	str := m.String()
+	return json.Marshal(str)
+}
+
+func (m *JsonURL) UnmarshalJSON(buf []byte) error {
+	var str string
+
+	err := json.Unmarshal(buf, &str)
+	if err != nil {
+		return err
+	}
+
+	m.URL, err = url.Parse(str)
+	return err
+}
+
+func (m Mirror) MarshalJSON() ([]byte, error) {
+	type Alias Mirror
+
+	m2 := struct {
+		*Alias
+		From JsonURL `json:"from"`
+		To   JsonURL `json:"to"`
+	}{
+		Alias: (*Alias)(&m),
+		From:  JsonURL{m.From},
+		To:    JsonURL{m.To},
+	}
+
+	return json.Marshal(m2)
+}
+
+func (m *Mirror) UnmarshalJSON(buf []byte) error {
+	type Alias Mirror
+
+	var m2 struct {
+		*Alias
+		From JsonURL `json:"from"`
+		To   JsonURL `json:"to"`
+	}
+
+	m2.Alias = (*Alias)(m)
+	err := json.Unmarshal(buf, &m2)
+	if err != nil {
+		return err
+	}
+
+	m.From = m2.From.URL
+	m.To = m2.To.URL
+
+	return nil
+}
diff --git a/internal/service/service_json_test.go b/internal/service/service_json_test.go
new file mode 100644
index 0000000..c8e073a
--- /dev/null
+++ b/internal/service/service_json_test.go
@@ -0,0 +1,37 @@
+package service
+
+import (
+	"encoding/json"
+	"testing"
+
+	"git.server.ky/slackcoder/mirror/internal"
+	"github.com/stretchr/testify/require"
+)
+
+func mustJSON(arg interface{}) string {
+	buf, err := json.Marshal(arg)
+	if err != nil {
+		panic(err)
+	}
+
+	return string(buf)
+}
+
+func TestMirrorUnmarshalJSON(t *testing.T) {
+	str := mustJSON(map[string]interface{}{
+		"method": "git",
+		"from":   "https://git.taler.net/merchant.git",
+		"to":     "/mirror/merchant",
+	})
+
+	exp := Mirror{
+		Method: "git",
+		From:   internal.MustURL("https://git.taler.net/merchant.git"),
+		To:     internal.MustURL("/mirror/merchant"),
+	}
+
+	var s Mirror
+	err := json.Unmarshal([]byte(str), &s)
+	require.NoError(t, err)
+	require.Equal(t, exp.String(), s.String())
+}