Port to Golangv0.0.1-dev

4 files changed, 474 insertions, 0 deletions

diff --git a/internal/github/filesystem.go b/internal/github/filesystem.go
new file mode 100644
index 0000000..3f069b4
--- /dev/null
+++ b/internal/github/filesystem.go
@@ -0,0 +1,63 @@
+package github
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+)
+
+func listReleasesByTagName(dst *url.URL) ([]string, error) {
+	entries, err := os.ReadDir(dst.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	var tagNames []string
+	for _, entry := range entries {
+		tagNames = append(tagNames, entry.Name())
+	}
+
+	return tagNames, nil
+}
+
+// The path which project release assets are saved.
+func localReleaseFilePath(dst *url.URL, tagName string) string {
+	return path.Join(dst.Path, tagName)
+}
+
+func releaseName(tagName string) string {
+	version := tagName
+	if strings.HasPrefix(version, "v") {
+		version = strings.TrimLeft(version, "v")
+	}
+
+	return version
+}
+
+// The source filename for a Github release.
+//
+// # The source code URL provided by Github's API only references the tag name
+//
+// for the release.  To make it useful for users, we rename to file to include
+// the project name as their website does.
+func releaseSourceFileName(project string, tagName string, ext string) string {
+	return fmt.Sprintf("%s-%s.%s", project, releaseName(tagName), ext)
+}
+
+func removeRelease(dst *url.URL, tagName string) error {
+	fp := localReleaseFilePath(dst, tagName)
+	return os.RemoveAll(fp)
+}
+
+func isFileExist(fp string) (bool, error) {
+	_, err := os.Stat(fp)
+	if os.IsNotExist(err) {
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
diff --git a/internal/github/github.go b/internal/github/github.go
new file mode 100644
index 0000000..bf35a6a
--- /dev/null
+++ b/internal/github/github.go
@@ -0,0 +1,262 @@
+package github
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"time"
+)
+
+type Client struct {
+	*jsonClient
+}
+
+func NewClient() *Client {
+	jsonClient := newJSONClient(http.DefaultClient, "https://api.github.com")
+	return &Client{
+		jsonClient: jsonClient,
+	}
+}
+
+type Asset struct {
+	Name               string `json:"name"`
+	BrowserDownloadURL string `json:"browser_download_url"`
+}
+
+type Release struct {
+	ID          int       `json:"id"`
+	TagName     string    `json:"tag_name"`
+	PublishedAt time.Time `json:"published_at"`
+	TarballURL  string    `json:"tarball_url"`
+	ZipballURL  string    `json:"zipball_url"`
+	Assets      []Asset   `json:"assets"`
+}
+
+func intRef(v int) *int {
+	return &v
+}
+
+func (c *Client) ListReleases(owner string, project string) ([]Release, error) {
+	var resp []Release
+
+	_, err := c.jsonClient.Request(http.MethodGet, intRef(http.StatusOK), path.Join("repos", owner, project, "releases"), nil, &resp)
+	if err != nil {
+		return nil, err
+	}
+
+	return resp, nil
+}
+
+func (c *Client) DownloadAsset(w io.Writer, owner string, project string, asset *Asset) error {
+	resp, err := http.Get(asset.BrowserDownloadURL)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(w, resp.Body)
+	if err != nil {
+		return err
+	}
+	return nil
+
+}
+
+func (c *Client) DownloadRelease(
+	dirPath string,
+	org string,
+	project string,
+	release *Release,
+) error {
+	for _, asset := range release.Assets {
+		fp := path.Join(dirPath, path.Base(asset.Name))
+		f, err := os.Create(fp)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
+		resp, err := http.Get(asset.BrowserDownloadURL)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(f, resp.Body)
+		if err != nil {
+			return err
+		}
+	}
+
+	fp := path.Join(dirPath, releaseSourceFileName(project, release.TagName, "tar.gz"))
+	if exist, err := isFileExist(fp); err != nil {
+		return fmt.Errorf("downloading tarball: %w", err)
+	} else if !exist {
+		f, err := os.Create(fp)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
+		resp, err := http.Get(release.TarballURL)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(f, resp.Body)
+		if err != nil {
+			return err
+		}
+	}
+
+	fp = path.Join(dirPath, releaseSourceFileName(project, release.TagName, "zip"))
+	if exist, err := isFileExist(fp); err != nil {
+		return err
+	} else if !exist {
+		f, err := os.Create(fp)
+		if err != nil {
+			return fmt.Errorf("downloading zipball: %w", err)
+		}
+		defer f.Close()
+
+		resp, err := http.Get(release.ZipballURL)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(f, resp.Body)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func releaseDownloads(
+	project string,
+	release *Release,
+) map[string]string {
+	files := make(map[string]string)
+
+	for _, asset := range release.Assets {
+		files[path.Base(asset.Name)] = asset.BrowserDownloadURL
+	}
+
+	fileName := releaseSourceFileName(project, release.TagName, "tar.gz")
+	files[fileName] = release.TarballURL
+
+	fileName = releaseSourceFileName(project, release.TagName, "zip")
+	files[fileName] = release.ZipballURL
+
+	return files
+}
+
+func (c *Client) download(dst string, src string) error {
+	resp, err := http.Head(src)
+	if err != nil {
+		return err
+	}
+
+	info, err := os.Stat(dst)
+	if !os.IsNotExist(err) && err != nil {
+		return err
+	}
+	if info != nil {
+		if info.Size() == resp.ContentLength {
+			return nil
+		}
+
+		err := os.Remove(dst)
+		if !os.IsNotExist(err) && err != nil {
+			return fmt.Errorf("could not remove '%s': %w", dst, err)
+		}
+	}
+
+	resp, err = http.Get(src)
+	if err != nil {
+		return err
+	}
+
+	f, err := os.Create(dst)
+	if err != nil {
+		return fmt.Errorf("creating '%s': %w", dst, err)
+	}
+	defer f.Close()
+
+	_, err = io.Copy(f, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Client) MirrorAssets(dst *url.URL, src *url.URL) error {
+	if src.Hostname() != "github.com" {
+		return fmt.Errorf("host must be github.com")
+	}
+	if dst.Scheme != "file:///" && dst.Scheme != "" {
+		return fmt.Errorf("unsupported destination scheme '%s'", dst.Scheme)
+	}
+
+	matches := regexp.MustCompilePOSIX("/(.*?)/(.*?)").FindAllStringSubmatch(src.Path, 1)
+	if len(matches) != 1 && len(matches[0]) != 2 {
+		return fmt.Errorf("must be a full path to the project")
+	}
+
+	owner := matches[0][1]
+	project := matches[0][2]
+
+	releases, err := c.ListReleases(owner, project)
+	if err != nil {
+		return fmt.Errorf("fetching list of releases: %w", err)
+	}
+
+	existingFiles := make(map[string]bool)
+
+	filepath.WalkDir(dst.Path, func(path string, _ os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		path, err = filepath.Abs(path)
+		if err != nil {
+			return err
+		}
+		existingFiles[path] = true
+
+		return nil
+	})
+	delete(existingFiles, dst.Path)
+
+	for _, release := range releases {
+		localDir := localReleaseFilePath(dst, release.TagName)
+		localDir, err = filepath.Abs(localDir)
+		if err != nil {
+			return err
+		}
+
+		err := os.MkdirAll(localDir, 0777)
+		if err != nil {
+			return fmt.Errorf("creating '%s': %w", localDir, err)
+		}
+		delete(existingFiles, localDir)
+
+		for fileName, srcURL := range releaseDownloads(project, &release) {
+			localFile := path.Join(localDir, fileName)
+			delete(existingFiles, localFile)
+
+			err := c.download(localFile, srcURL)
+			if err != nil {
+				return fmt.Errorf("downloading '%s': %w", srcURL, err)
+			}
+		}
+	}
+
+	for fp := range existingFiles {
+		_ = os.RemoveAll(fp)
+	}
+
+	return nil
+}
diff --git a/internal/github/github_test.go b/internal/github/github_test.go
new file mode 100644
index 0000000..5f3a270
--- /dev/null
+++ b/internal/github/github_test.go
@@ -0,0 +1,34 @@
+package github
+
+import (
+	"os"
+	"path"
+	"testing"
+
+	"git.server.ky/slackcoder/mirror/internal"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMirrorDendrite(t *testing.T) {
+	d := t.TempDir()
+
+	dst := internal.MustURL(d)
+	src := internal.MustURL("https://github.com/matrix-org/dendrite")
+	oldFile := "random_file.txt"
+
+	f, cErr := os.Create(path.Join(dst.Path, oldFile))
+	require.NoError(t, cErr)
+	f.Close()
+
+	c := NewClient()
+	err := c.MirrorAssets(dst, src)
+	require.NoError(t, err, "dendrite assets")
+
+	require.FileExists(t, path.Join(dst.Path, "v0.13.7", "dendrite-0.13.7.tar.gz"))
+	require.FileExists(t, path.Join(dst.Path, "v0.13.7", "dendrite-0.13.7.zip"))
+
+	err = c.MirrorAssets(dst, src)
+	require.NoError(t, err, "dendrite assets")
+
+	require.NoFileExists(t, path.Join(dst.Path, oldFile), "only files from mirror should exist")
+}
diff --git a/internal/github/rest_client.go b/internal/github/rest_client.go
new file mode 100644
index 0000000..6fdd31c
--- /dev/null
+++ b/internal/github/rest_client.go
@@ -0,0 +1,115 @@
+package github
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+type HTTPRequester interface {
+	Do(req *http.Request) (*http.Response, error)
+}
+
+type BearerAuthClient struct {
+	HTTPRequester
+	Username string
+	Password string
+}
+
+func WithBearerAuth(
+	cli HTTPRequester,
+	username, password string,
+) *BearerAuthClient {
+	return &BearerAuthClient{cli, username, password}
+}
+
+func (s *BearerAuthClient) Do(req *http.Request) (*http.Response, error) {
+	if s.Username != "" && s.Password != "" {
+		value := fmt.Sprintf("Bearer %s:%s", s.Username, s.Password)
+		req.Header.Set("Authorization", value)
+	}
+	return s.HTTPRequester.Do(req)
+}
+
+type jsonClient struct {
+	Client   HTTPRequester
+	basePath string
+}
+
+func newJSONClient(
+	cli HTTPRequester,
+	basePath string,
+) *jsonClient {
+	return &jsonClient{
+		Client:   cli,
+		basePath: basePath,
+	}
+}
+
+func newHTTPJSONReq(
+	method string,
+	url string,
+	req interface{},
+) (*http.Request, error) {
+	body := &bytes.Buffer{}
+	if req != nil {
+		buf, err := json.Marshal(req)
+		if err != nil {
+			return nil, err
+		}
+		body = bytes.NewBuffer(buf)
+	}
+	httpReq, err := http.NewRequest(method, url, body)
+	if err != nil {
+		return nil, err
+	}
+	httpReq.Header.Set("Content-Type", "application/json")
+	httpReq.Header.Set("Accept", "application/json")
+	return httpReq, nil
+}
+
+func decodeJSONResponse(httpResp *http.Response, resp interface{}) error {
+	if httpResp.StatusCode/100 != 2 {
+		return fmt.Errorf(
+			"received HTTP status code %d (%s)",
+			httpResp.StatusCode,
+			httpResp.Status,
+		)
+	}
+	if resp == nil {
+		return nil
+	}
+	err := json.NewDecoder(httpResp.Body).Decode(resp)
+	if err != nil {
+		return err
+	}
+	return err
+}
+
+func (s *jsonClient) Request(
+	method string,
+	statusCode *int,
+	path string,
+	req, resp interface{},
+) (int, error) {
+	url := fmt.Sprintf("%s/%s", s.basePath, path)
+	httpReq, err := newHTTPJSONReq(method, url, req)
+	if err != nil {
+		return 0, err
+	}
+	httpResp, err := s.Client.Do(httpReq)
+	if err != nil {
+		return 0, err
+	}
+	defer httpResp.Body.Close()
+
+	err = decodeJSONResponse(httpResp, resp)
+	if err != nil {
+		return httpResp.StatusCode, err
+	}
+	if statusCode != nil && httpResp.StatusCode != *statusCode {
+		return httpResp.StatusCode, fmt.Errorf("expected status code %d but got %d", *statusCode, httpResp.StatusCode)
+	}
+	return httpResp.StatusCode, nil
+}