diff options
Diffstat (limited to 'internal')
| -rw-r--r-- | internal/github/filesystem.go | 63 | ||||
| -rw-r--r-- | internal/github/github.go | 262 | ||||
| -rw-r--r-- | internal/github/github_test.go | 34 | ||||
| -rw-r--r-- | internal/github/rest_client.go | 115 | ||||
| -rw-r--r-- | internal/service/config.go | 73 | ||||
| -rw-r--r-- | internal/service/config_test.go | 54 | ||||
| -rw-r--r-- | internal/service/git.go | 134 | ||||
| -rw-r--r-- | internal/service/git_test.go | 34 | ||||
| -rw-r--r-- | internal/service/rsync.go | 40 | ||||
| -rw-r--r-- | internal/service/rsync_test.go | 21 | ||||
| -rw-r--r-- | internal/service/service.go | 195 | ||||
| -rw-r--r-- | internal/service/service_json.go | 64 | ||||
| -rw-r--r-- | internal/service/service_json_test.go | 37 | ||||
| -rw-r--r-- | internal/url.go | 11 | 
14 files changed, 1137 insertions, 0 deletions
| diff --git a/internal/github/filesystem.go b/internal/github/filesystem.go new file mode 100644 index 0000000..3f069b4 --- /dev/null +++ b/internal/github/filesystem.go @@ -0,0 +1,63 @@ +package github + +import ( +	"fmt" +	"net/url" +	"os" +	"path" +	"strings" +) + +func listReleasesByTagName(dst *url.URL) ([]string, error) { +	entries, err := os.ReadDir(dst.Path) +	if err != nil { +		return nil, err +	} + +	var tagNames []string +	for _, entry := range entries { +		tagNames = append(tagNames, entry.Name()) +	} + +	return tagNames, nil +} + +// The path which project release assets are saved. +func localReleaseFilePath(dst *url.URL, tagName string) string { +	return path.Join(dst.Path, tagName) +} + +func releaseName(tagName string) string { +	version := tagName +	if strings.HasPrefix(version, "v") { +		version = strings.TrimLeft(version, "v") +	} + +	return version +} + +// The source filename for a Github release. +// +// # The source code URL provided by Github's API only references the tag name +// +// for the release.  To make it useful for users, we rename to file to include +// the project name as their website does. +func releaseSourceFileName(project string, tagName string, ext string) string { +	return fmt.Sprintf("%s-%s.%s", project, releaseName(tagName), ext) +} + +func removeRelease(dst *url.URL, tagName string) error { +	fp := localReleaseFilePath(dst, tagName) +	return os.RemoveAll(fp) +} + +func isFileExist(fp string) (bool, error) { +	_, err := os.Stat(fp) +	if os.IsNotExist(err) { +		return false, nil +	} else if err != nil { +		return false, err +	} + +	return true, nil +} diff --git a/internal/github/github.go b/internal/github/github.go new file mode 100644 index 0000000..bf35a6a --- /dev/null +++ b/internal/github/github.go @@ -0,0 +1,262 @@ +package github + +import ( +	"fmt" +	"io" +	"net/http" +	"net/url" +	"os" +	"path" +	"path/filepath" +	"regexp" +	"time" +) + +type Client struct { +	*jsonClient +} + +func NewClient() *Client { +	jsonClient := newJSONClient(http.DefaultClient, "https://api.github.com") +	return &Client{ +		jsonClient: jsonClient, +	} +} + +type Asset struct { +	Name               string `json:"name"` +	BrowserDownloadURL string `json:"browser_download_url"` +} + +type Release struct { +	ID          int       `json:"id"` +	TagName     string    `json:"tag_name"` +	PublishedAt time.Time `json:"published_at"` +	TarballURL  string    `json:"tarball_url"` +	ZipballURL  string    `json:"zipball_url"` +	Assets      []Asset   `json:"assets"` +} + +func intRef(v int) *int { +	return &v +} + +func (c *Client) ListReleases(owner string, project string) ([]Release, error) { +	var resp []Release + +	_, err := c.jsonClient.Request(http.MethodGet, intRef(http.StatusOK), path.Join("repos", owner, project, "releases"), nil, &resp) +	if err != nil { +		return nil, err +	} + +	return resp, nil +} + +func (c *Client) DownloadAsset(w io.Writer, owner string, project string, asset *Asset) error { +	resp, err := http.Get(asset.BrowserDownloadURL) +	if err != nil { +		return err +	} + +	_, err = io.Copy(w, resp.Body) +	if err != nil { +		return err +	} +	return nil + +} + +func (c *Client) DownloadRelease( +	dirPath string, +	org string, +	project string, +	release *Release, +) error { +	for _, asset := range release.Assets { +		fp := path.Join(dirPath, path.Base(asset.Name)) +		f, err := os.Create(fp) +		if err != nil { +			return err +		} +		defer f.Close() + +		resp, err := http.Get(asset.BrowserDownloadURL) +		if err != nil { +			return err +		} +		_, err = io.Copy(f, resp.Body) +		if err != nil { +			return err +		} +	} + +	fp := path.Join(dirPath, releaseSourceFileName(project, release.TagName, "tar.gz")) +	if exist, err := isFileExist(fp); err != nil { +		return fmt.Errorf("downloading tarball: %w", err) +	} else if !exist { +		f, err := os.Create(fp) +		if err != nil { +			return err +		} +		defer f.Close() + +		resp, err := http.Get(release.TarballURL) +		if err != nil { +			return err +		} +		_, err = io.Copy(f, resp.Body) +		if err != nil { +			return err +		} +	} + +	fp = path.Join(dirPath, releaseSourceFileName(project, release.TagName, "zip")) +	if exist, err := isFileExist(fp); err != nil { +		return err +	} else if !exist { +		f, err := os.Create(fp) +		if err != nil { +			return fmt.Errorf("downloading zipball: %w", err) +		} +		defer f.Close() + +		resp, err := http.Get(release.ZipballURL) +		if err != nil { +			return err +		} +		_, err = io.Copy(f, resp.Body) +		if err != nil { +			return err +		} +	} + +	return nil +} + +func releaseDownloads( +	project string, +	release *Release, +) map[string]string { +	files := make(map[string]string) + +	for _, asset := range release.Assets { +		files[path.Base(asset.Name)] = asset.BrowserDownloadURL +	} + +	fileName := releaseSourceFileName(project, release.TagName, "tar.gz") +	files[fileName] = release.TarballURL + +	fileName = releaseSourceFileName(project, release.TagName, "zip") +	files[fileName] = release.ZipballURL + +	return files +} + +func (c *Client) download(dst string, src string) error { +	resp, err := http.Head(src) +	if err != nil { +		return err +	} + +	info, err := os.Stat(dst) +	if !os.IsNotExist(err) && err != nil { +		return err +	} +	if info != nil { +		if info.Size() == resp.ContentLength { +			return nil +		} + +		err := os.Remove(dst) +		if !os.IsNotExist(err) && err != nil { +			return fmt.Errorf("could not remove '%s': %w", dst, err) +		} +	} + +	resp, err = http.Get(src) +	if err != nil { +		return err +	} + +	f, err := os.Create(dst) +	if err != nil { +		return fmt.Errorf("creating '%s': %w", dst, err) +	} +	defer f.Close() + +	_, err = io.Copy(f, resp.Body) +	if err != nil { +		return err +	} + +	return nil +} + +func (c *Client) MirrorAssets(dst *url.URL, src *url.URL) error { +	if src.Hostname() != "github.com" { +		return fmt.Errorf("host must be github.com") +	} +	if dst.Scheme != "file:///" && dst.Scheme != "" { +		return fmt.Errorf("unsupported destination scheme '%s'", dst.Scheme) +	} + +	matches := regexp.MustCompilePOSIX("/(.*?)/(.*?)").FindAllStringSubmatch(src.Path, 1) +	if len(matches) != 1 && len(matches[0]) != 2 { +		return fmt.Errorf("must be a full path to the project") +	} + +	owner := matches[0][1] +	project := matches[0][2] + +	releases, err := c.ListReleases(owner, project) +	if err != nil { +		return fmt.Errorf("fetching list of releases: %w", err) +	} + +	existingFiles := make(map[string]bool) + +	filepath.WalkDir(dst.Path, func(path string, _ os.DirEntry, err error) error { +		if err != nil { +			return err +		} + +		path, err = filepath.Abs(path) +		if err != nil { +			return err +		} +		existingFiles[path] = true + +		return nil +	}) +	delete(existingFiles, dst.Path) + +	for _, release := range releases { +		localDir := localReleaseFilePath(dst, release.TagName) +		localDir, err = filepath.Abs(localDir) +		if err != nil { +			return err +		} + +		err := os.MkdirAll(localDir, 0777) +		if err != nil { +			return fmt.Errorf("creating '%s': %w", localDir, err) +		} +		delete(existingFiles, localDir) + +		for fileName, srcURL := range releaseDownloads(project, &release) { +			localFile := path.Join(localDir, fileName) +			delete(existingFiles, localFile) + +			err := c.download(localFile, srcURL) +			if err != nil { +				return fmt.Errorf("downloading '%s': %w", srcURL, err) +			} +		} +	} + +	for fp := range existingFiles { +		_ = os.RemoveAll(fp) +	} + +	return nil +} diff --git a/internal/github/github_test.go b/internal/github/github_test.go new file mode 100644 index 0000000..5f3a270 --- /dev/null +++ b/internal/github/github_test.go @@ -0,0 +1,34 @@ +package github + +import ( +	"os" +	"path" +	"testing" + +	"git.server.ky/slackcoder/mirror/internal" +	"github.com/stretchr/testify/require" +) + +func TestMirrorDendrite(t *testing.T) { +	d := t.TempDir() + +	dst := internal.MustURL(d) +	src := internal.MustURL("https://github.com/matrix-org/dendrite") +	oldFile := "random_file.txt" + +	f, cErr := os.Create(path.Join(dst.Path, oldFile)) +	require.NoError(t, cErr) +	f.Close() + +	c := NewClient() +	err := c.MirrorAssets(dst, src) +	require.NoError(t, err, "dendrite assets") + +	require.FileExists(t, path.Join(dst.Path, "v0.13.7", "dendrite-0.13.7.tar.gz")) +	require.FileExists(t, path.Join(dst.Path, "v0.13.7", "dendrite-0.13.7.zip")) + +	err = c.MirrorAssets(dst, src) +	require.NoError(t, err, "dendrite assets") + +	require.NoFileExists(t, path.Join(dst.Path, oldFile), "only files from mirror should exist") +} diff --git a/internal/github/rest_client.go b/internal/github/rest_client.go new file mode 100644 index 0000000..6fdd31c --- /dev/null +++ b/internal/github/rest_client.go @@ -0,0 +1,115 @@ +package github + +import ( +	"bytes" +	"encoding/json" +	"fmt" +	"net/http" +) + +type HTTPRequester interface { +	Do(req *http.Request) (*http.Response, error) +} + +type BearerAuthClient struct { +	HTTPRequester +	Username string +	Password string +} + +func WithBearerAuth( +	cli HTTPRequester, +	username, password string, +) *BearerAuthClient { +	return &BearerAuthClient{cli, username, password} +} + +func (s *BearerAuthClient) Do(req *http.Request) (*http.Response, error) { +	if s.Username != "" && s.Password != "" { +		value := fmt.Sprintf("Bearer %s:%s", s.Username, s.Password) +		req.Header.Set("Authorization", value) +	} +	return s.HTTPRequester.Do(req) +} + +type jsonClient struct { +	Client   HTTPRequester +	basePath string +} + +func newJSONClient( +	cli HTTPRequester, +	basePath string, +) *jsonClient { +	return &jsonClient{ +		Client:   cli, +		basePath: basePath, +	} +} + +func newHTTPJSONReq( +	method string, +	url string, +	req interface{}, +) (*http.Request, error) { +	body := &bytes.Buffer{} +	if req != nil { +		buf, err := json.Marshal(req) +		if err != nil { +			return nil, err +		} +		body = bytes.NewBuffer(buf) +	} +	httpReq, err := http.NewRequest(method, url, body) +	if err != nil { +		return nil, err +	} +	httpReq.Header.Set("Content-Type", "application/json") +	httpReq.Header.Set("Accept", "application/json") +	return httpReq, nil +} + +func decodeJSONResponse(httpResp *http.Response, resp interface{}) error { +	if httpResp.StatusCode/100 != 2 { +		return fmt.Errorf( +			"received HTTP status code %d (%s)", +			httpResp.StatusCode, +			httpResp.Status, +		) +	} +	if resp == nil { +		return nil +	} +	err := json.NewDecoder(httpResp.Body).Decode(resp) +	if err != nil { +		return err +	} +	return err +} + +func (s *jsonClient) Request( +	method string, +	statusCode *int, +	path string, +	req, resp interface{}, +) (int, error) { +	url := fmt.Sprintf("%s/%s", s.basePath, path) +	httpReq, err := newHTTPJSONReq(method, url, req) +	if err != nil { +		return 0, err +	} +	httpResp, err := s.Client.Do(httpReq) +	if err != nil { +		return 0, err +	} +	defer httpResp.Body.Close() + +	err = decodeJSONResponse(httpResp, resp) +	if err != nil { +		return httpResp.StatusCode, err +	} +	if statusCode != nil && httpResp.StatusCode != *statusCode { +		return httpResp.StatusCode, fmt.Errorf("expected status code %d but got %d", *statusCode, httpResp.StatusCode) +	} +	return httpResp.StatusCode, nil +} diff --git a/internal/service/config.go b/internal/service/config.go new file mode 100644 index 0000000..bcb9efc --- /dev/null +++ b/internal/service/config.go @@ -0,0 +1,73 @@ +package service + +import ( +	"encoding/json" +	"fmt" +	"os" +	"time" + +	"dario.cat/mergo" +) + +type Duration struct { +	time.Duration +} + +func (s Duration) MarshalJSON() ([]byte, error) { +	return json.Marshal(s.Duration.String()) +} + +func (s *Duration) UnmarshalJSON(data []byte) error { +	var str string +	err := json.Unmarshal(data, &str) +	if err != nil { +		return err +	} + +	v, err := time.ParseDuration(str) +	if err != nil { +		return err +	} +	*s = Duration{v} + +	return nil +} + +type Config struct { +	MaxInterval Duration  `json:"max-interval,omitempty"` +	MinInterval Duration  `json:"min-interval,omitempty"` +	Mirrors     []*Mirror `json:"mirrors,omitempty"` +} + +var DefaultConfig = Config{ +	MaxInterval: Duration{24 * time.Hour}, +	MinInterval: Duration{time.Hour}, +} + +func (c *Config) Apply(arg Config) { +	err := mergo.Merge(c, &arg) +	if err != nil { +		panic(err) +	} +} + +// ApplyFileConfig loads the configuration described by the given yaml file. +func ApplyFileConfig(cfg *Config, filePath string) error { +	var ret Config + +	f, err := os.Open(filePath) +	if os.IsNotExist(err) { +		return nil +	} else if err != nil { +		return err +	} +	defer f.Close() + +	err = json.NewDecoder(f).Decode(&ret) +	if err != nil { +		return fmt.Errorf("loading configuration file: %w", err) +	} + +	cfg.Apply(ret) +	return nil +} diff --git a/internal/service/config_test.go b/internal/service/config_test.go new file mode 100644 index 0000000..c9af1e7 --- /dev/null +++ b/internal/service/config_test.go @@ -0,0 +1,54 @@ +package service + +import ( +	"encoding/json" +	"testing" +	"time" + +	"github.com/stretchr/testify/require" +) + +func TestDurationMarshal(t *testing.T) { +	tests := []struct { +		arg Duration +		exp string +	}{ +		{ +			Duration{time.Second}, +			"\"1s\"", +		}, +		{ +			Duration{time.Minute}, +			"\"1m0s\"", +		}, +	} + +	for _, test := range tests { +		v, err := json.Marshal(test.arg) +		require.NoError(t, err) +		require.Equal(t, test.exp, string(v)) +	} +} + +func TestDurationUnmarshal(t *testing.T) { +	tests := []struct { +		arg string +		exp Duration +	}{ +		{ +			"\"1s\"", +			Duration{time.Second}, +		}, +		{ +			"\"1m0s\"", +			Duration{time.Minute}, +		}, +	} + +	for _, test := range tests { +		var v Duration +		err := json.Unmarshal([]byte(test.arg), &v) +		require.NoError(t, err) +		require.Equal(t, test.exp.Duration, v.Duration) +	} +} diff --git a/internal/service/git.go b/internal/service/git.go new file mode 100644 index 0000000..ad3a653 --- /dev/null +++ b/internal/service/git.go @@ -0,0 +1,134 @@ +package service + +import ( +	"errors" +	"fmt" +	"net/url" +	"os" +	"os/exec" +	"path" +	"strings" +) + +const gitDescriptionFile = "description" + +func mapExecError(err error) error { +	if ee, ok := err.(*exec.ExitError); ok { +		return errors.New(string(ee.Stderr)) +	} +	return err +} + +// Set the description  for the projects repository. +func setDescription(repo string, desc string) error { +	descPath := path.Join(repo, gitDescriptionFile) + +	var curDesc string +	buf, err := os.ReadFile(descPath) +	if os.IsNotExist(err) { +		// empty +	} else if err != nil { +		return err +	} else { +		curDesc = string(buf) +	} + +	if curDesc != desc { +		err = os.WriteFile(descPath, []byte(desc), 0750) +		if err != nil { +			return err +		} +	} + +	return nil +} + +// Set the remote origin's URL  for the projects repository. +func setRemoteOrigin(repo string, origin *url.URL) error { +	cmd := exec.Command("git", "remote", "get-url", "origin") +	cmd.Dir = repo +	buf, err := cmd.Output() +	if err != nil { +		return fmt.Errorf("getting current project remote origin: %w", err) +	} +	currentOrigin := strings.TrimSpace(string(buf)) + +	if currentOrigin != origin.String() { +		cmd = exec.Command("git", "remote", "set-url", "origin", origin.String()) +		cmd.Dir = repo +		err = cmd.Run() +		if err != nil { +			err = mapExecError(err) +			return fmt.Errorf("setting project remote origin: %w", err) +		} +	} + +	return nil +} + +func getRemoteHeadReference(repo string) (string, error) { +	cmd := exec.Command("git", "ls-remote", "--symref", "origin", "HEAD") +	cmd.Dir = repo +	buf, err := cmd.Output() +	if err != nil { +		return "", mapExecError(err) +	} + +	for _, l := range strings.Split(string(buf), "\n") { +		fields := strings.Fields(l) +		if len(fields) != 3 { +			return "", errors.New("unexpected output from 'git ls-remote'") +		} +		if fields[0] == "ref:" { +			return strings.TrimPrefix(fields[1], "refs/heads/"), nil +		} +	} + +	return "", errors.New("not found") +} + +func MirrorGit(dst *url.URL, src *url.URL, description string) error { +	if dst.Scheme != "" && dst.Scheme != "file://" { +		return fmt.Errorf("'%s' scheme not supported", dst.Scheme) +	} + +	if _, err := os.Stat(dst.Path); os.IsNotExist(err) { +		err = os.MkdirAll(path.Join(dst.Path, ".."), 0750) +		if err != nil { +			return fmt.Errorf("creating new mirror: %w", err) +		} + +		cmd := exec.Command("git", "clone", "--bare", "--single-branch", src.String(), dst.String()) +		cmd.Dir = path.Join(dst.Path, "..") +		err := cmd.Run() +		if err != nil { +			err = mapExecError(err) +			return fmt.Errorf("cloning: %s", err) +		} +	} + +	err := setDescription(dst.Path, description) +	if err != nil { +		return err +	} + +	err = setRemoteOrigin(dst.Path, src) +	if err != nil { +		return err +	} + +	branch, err := getRemoteHeadReference(dst.Path) +	if err != nil { +		return fmt.Errorf("guessing remote default branch: %w", err) +	} + +	cmd := exec.Command("git", "fetch", "--tags", "origin", branch+":"+branch) +	cmd.Dir = dst.Path +	err = cmd.Run() +	if err != nil { +		err = mapExecError(err) +		return fmt.Errorf("fetching project: %s", err) +	} + +	return nil +} diff --git a/internal/service/git_test.go b/internal/service/git_test.go new file mode 100644 index 0000000..6dc8f62 --- /dev/null +++ b/internal/service/git_test.go @@ -0,0 +1,34 @@ +package service + +import ( +	"os/exec" +	"path" +	"strings" +	"testing" + +	"git.server.ky/slackcoder/mirror/internal" +	"github.com/stretchr/testify/require" +) + +func requireTagExist(t *testing.T, filePath string, tag string, msgAndArgs ...interface{}) { +	cmd := exec.Command("git", "tag") +	cmd.Dir = filePath + +	buf, err := cmd.Output() +	require.NoError(t, err) + +	tags := strings.Fields(string(buf)) +	require.Contains(t, tags, tag, msgAndArgs...) +} + +func TestMirrorGit(t *testing.T) { +	d := t.TempDir() + +	dst := internal.MustURL(path.Join(d, "merchant")) +	src := internal.MustURL("https://git.taler.net/merchant.git") + +	err := MirrorGit(dst, src, "GNU Taler Merchant") +	require.NoError(t, err, "git mirror") + +	requireTagExist(t, dst.Path, "v0.11.3", "tag must exist") +} diff --git a/internal/service/rsync.go b/internal/service/rsync.go new file mode 100644 index 0000000..8298589 --- /dev/null +++ b/internal/service/rsync.go @@ -0,0 +1,40 @@ +package service + +import ( +	"bytes" +	"errors" +	"net/url" +	"os/exec" +	"strings" +) + +var rsyncOpts = []string{ +	"--delete-excluded", +	"--hard-links", +	"--links", +	"--perms", +	"--recursive", +	"--safe-links", +	"--sparse", +	"--times", +} + +func Rsync(dst *url.URL, src *url.URL) error { +	src2 := *src +	if !strings.HasSuffix(src2.Path, "/.") { +		src2.Path = src2.Path + "/." +	} + +	var stderr bytes.Buffer + +	args := append(rsyncOpts, src2.String(), dst.String()) +	cmd := exec.Command("rsync", args...) +	cmd.Stderr = &stderr + +	err := cmd.Run() +	if err != nil { +		return errors.New(stderr.String()) +	} + +	return nil +} diff --git a/internal/service/rsync_test.go b/internal/service/rsync_test.go new file mode 100644 index 0000000..c6c2341 --- /dev/null +++ b/internal/service/rsync_test.go @@ -0,0 +1,21 @@ +package service + +import ( +	"path" +	"testing" + +	"git.server.ky/slackcoder/mirror/internal" +	"github.com/stretchr/testify/require" +) + +func TestRsync(t *testing.T) { +	d := t.TempDir() + +	dst := internal.MustURL(d) +	src := internal.MustURL("rsync://mirror.cedia.org.ec/gnu/taler") + +	err := Rsync(dst, src) +	require.NoError(t, err, "rsync") + +	require.FileExists(t, path.Join(dst.Path, "taler-merchant-0.11.3.tar.gz")) +} diff --git a/internal/service/service.go b/internal/service/service.go new file mode 100644 index 0000000..c34bcd3 --- /dev/null +++ b/internal/service/service.go @@ -0,0 +1,195 @@ +package service + +import ( +	"encoding/json" +	"errors" +	"fmt" +	"math/rand" +	"net/url" +	"os" +	"strconv" +	"strings" +	"sync" +	"time" + +	"git.server.ky/slackcoder/mirror/internal/github" +) + +type Mirror struct { +	Method      string   `json:"method"` +	From        *url.URL `json:"-"` +	To          *url.URL `json:"-"` +	Description string   `json:"description,omitempty"` +} + +func (m *Mirror) Equal(arg *Mirror) bool { +	return m.Method == arg.Method && m.From.String() == arg.From.String() && m.To.String() == arg.To.String() +} + +func (m *Mirror) String() string { +	buf, err := json.Marshal(m) +	if err != nil { +		panic(err) +	} +	return string(buf) +} + +type Service struct { +	cfg *Config + +	mirrorsLock sync.Mutex +	mirrors     []*Mirror +	schedule    []time.Time + +	stopCh chan struct{} +} + +func NewService(cfg *Config) (*Service, error) { +	cfg.Apply(DefaultConfig) + +	srv := &Service{ +		cfg:     cfg, +		mirrors: nil, +		stopCh:  make(chan struct{}), +	} + +	for _, m := range cfg.Mirrors { +		err := srv.AddMirror(m) +		if err != nil { +			return nil, err +		} +	} + +	return srv, nil +} + +func (s *Service) scheduleNextRun(arg *Mirror) { +	// Be polite. +	period := s.cfg.MaxInterval.Duration - s.cfg.MinInterval.Duration +	at := time.Now().Add(s.cfg.MinInterval.Duration + time.Duration(rand.Intn(int(period)))) + +	for i, m := range s.mirrors { +		if m.Equal(arg) { +			s.schedule[i] = at +		} +	} +} + +func (s *Service) AddMirror(arg *Mirror) error { +	defer s.mirrorsLock.Unlock() +	s.mirrorsLock.Lock() + +	for _, m := range s.mirrors { +		if m.Equal(arg) { +			return errors.New("must be unique") +		} +	} + +	s.schedule = append(s.schedule, time.Time{}) +	s.mirrors = append(s.mirrors, arg) +	s.scheduleNextRun(arg) + +	return nil +} + +func (s *Service) scheduled(yield func(*Mirror) bool) { +	defer s.mirrorsLock.Unlock() +	s.mirrorsLock.Lock() + +	now := time.Now() +	for i, m := range s.mirrors { +		if s.schedule[i].After(now) { +			continue +		} + +		if !yield(m) { +			break +		} +	} +} + +func (s *Service) Mirror(arg *Mirror) error { +	if arg.From.Path == "" || arg.To.Path == "" || arg.Method == "" { +		return fmt.Errorf("badly formatted mirror '%s'", arg.String()) +	} + +	var err error + +	switch arg.Method { +	case "git": +		err = MirrorGit(arg.To, arg.From, arg.Description) +	case "github-assets": +		client := github.NewClient() +		err = client.MirrorAssets(arg.To, arg.From) +	case "rsync": +		err = Rsync(arg.To, arg.From) +	default: +		err = fmt.Errorf("unknown method '%s'", arg.Method) +	} + +	if err != nil { +		return fmt.Errorf("could not clone from '%s': %w", arg.From, err) +	} + +	return nil +} + +func (s *Service) RemoveMirror(arg *Mirror) error { +	defer s.mirrorsLock.Unlock() +	s.mirrorsLock.Lock() + +	for i, m := range s.mirrors { +		if m.Equal(arg) { +			s.mirrors[i] = s.mirrors[len(s.mirrors)-1] +			s.mirrors = s.mirrors[:len(s.mirrors)-1] + +			s.schedule[i] = s.schedule[len(s.schedule)-1] +			s.schedule = s.schedule[:len(s.schedule)-1] + +			return nil +		} +	} + +	return errors.New("not found") +} + +func (s *Service) Log(str string) { +	escaped := []rune(strconv.Quote(strings.TrimSpace(str))) +	escaped = escaped[1 : len(escaped)-1] + +	fmt.Fprintf(os.Stderr, "%s: %s\n", time.Now().Format(time.RFC822Z), string(escaped)) +} + +func (s *Service) Run() error { +	wakeup := time.NewTicker(time.Second) + +mainLoop: +	for { +		select { +		case <-wakeup.C: +		case <-s.stopCh: +			break mainLoop +		} + +		s.scheduled(func(m *Mirror) bool { +			err := s.Mirror(m) +			if err != nil { +				s.Log(err.Error()) +			} + +			s.scheduleNextRun(m) + +			return true +		}) +	} + +	return nil +} + +func (s *Service) Stop() { +	select { +	case <-s.stopCh: +	default: +		close(s.stopCh) +	} +} diff --git a/internal/service/service_json.go b/internal/service/service_json.go new file mode 100644 index 0000000..571eb30 --- /dev/null +++ b/internal/service/service_json.go @@ -0,0 +1,64 @@ +package service + +import ( +	"encoding/json" +	"net/url" +) + +type JsonURL struct { +	*url.URL +} + +func (m JsonURL) MarshalJSON() ([]byte, error) { +	str := m.String() +	return json.Marshal(str) +} + +func (m *JsonURL) UnmarshalJSON(buf []byte) error { +	var str string + +	err := json.Unmarshal(buf, &str) +	if err != nil { +		return err +	} + +	m.URL, err = url.Parse(str) +	return err +} + +func (m Mirror) MarshalJSON() ([]byte, error) { +	type Alias Mirror + +	m2 := struct { +		*Alias +		From JsonURL `json:"from"` +		To   JsonURL `json:"to"` +	}{ +		Alias: (*Alias)(&m), +		From:  JsonURL{m.From}, +		To:    JsonURL{m.To}, +	} + +	return json.Marshal(m2) +} + +func (m *Mirror) UnmarshalJSON(buf []byte) error { +	type Alias Mirror + +	var m2 struct { +		*Alias +		From JsonURL `json:"from"` +		To   JsonURL `json:"to"` +	} + +	m2.Alias = (*Alias)(m) +	err := json.Unmarshal(buf, &m2) +	if err != nil { +		return err +	} + +	m.From = m2.From.URL +	m.To = m2.To.URL + +	return nil +} diff --git a/internal/service/service_json_test.go b/internal/service/service_json_test.go new file mode 100644 index 0000000..c8e073a --- /dev/null +++ b/internal/service/service_json_test.go @@ -0,0 +1,37 @@ +package service + +import ( +	"encoding/json" +	"testing" + +	"git.server.ky/slackcoder/mirror/internal" +	"github.com/stretchr/testify/require" +) + +func mustJSON(arg interface{}) string { +	buf, err := json.Marshal(arg) +	if err != nil { +		panic(err) +	} + +	return string(buf) +} + +func TestMirrorUnmarshalJSON(t *testing.T) { +	str := mustJSON(map[string]interface{}{ +		"method": "git", +		"from":   "https://git.taler.net/merchant.git", +		"to":     "/mirror/merchant", +	}) + +	exp := Mirror{ +		Method: "git", +		From:   internal.MustURL("https://git.taler.net/merchant.git"), +		To:     internal.MustURL("/mirror/merchant"), +	} + +	var s Mirror +	err := json.Unmarshal([]byte(str), &s) +	require.NoError(t, err) +	require.Equal(t, exp.String(), s.String()) +} diff --git a/internal/url.go b/internal/url.go new file mode 100644 index 0000000..3a8d20a --- /dev/null +++ b/internal/url.go @@ -0,0 +1,11 @@ +package internal + +import "net/url" + +func MustURL(arg string) *url.URL { +	u, err := url.Parse(arg) +	if err != nil { +		panic(err) +	} +	return u +} | 
