From ae748859be8d6d3ed3c0929770f0c287ab6d6460 Mon Sep 17 00:00:00 2001 From: Slack Coder Date: Mon, 8 Apr 2024 15:29:11 -0500 Subject: Port to Golang --- internal/github/filesystem.go | 63 ++++++++++ internal/github/github.go | 262 +++++++++++++++++++++++++++++++++++++++++ internal/github/github_test.go | 34 ++++++ internal/github/rest_client.go | 115 ++++++++++++++++++ 4 files changed, 474 insertions(+) create mode 100644 internal/github/filesystem.go create mode 100644 internal/github/github.go create mode 100644 internal/github/github_test.go create mode 100644 internal/github/rest_client.go (limited to 'internal/github') diff --git a/internal/github/filesystem.go b/internal/github/filesystem.go new file mode 100644 index 0000000..3f069b4 --- /dev/null +++ b/internal/github/filesystem.go @@ -0,0 +1,63 @@ +package github + +import ( + "fmt" + "net/url" + "os" + "path" + "strings" +) + +func listReleasesByTagName(dst *url.URL) ([]string, error) { + entries, err := os.ReadDir(dst.Path) + if err != nil { + return nil, err + } + + var tagNames []string + for _, entry := range entries { + tagNames = append(tagNames, entry.Name()) + } + + return tagNames, nil +} + +// The path which project release assets are saved. +func localReleaseFilePath(dst *url.URL, tagName string) string { + return path.Join(dst.Path, tagName) +} + +func releaseName(tagName string) string { + version := tagName + if strings.HasPrefix(version, "v") { + version = strings.TrimLeft(version, "v") + } + + return version +} + +// The source filename for a Github release. +// +// # The source code URL provided by Github's API only references the tag name +// +// for the release. To make it useful for users, we rename to file to include +// the project name as their website does. +func releaseSourceFileName(project string, tagName string, ext string) string { + return fmt.Sprintf("%s-%s.%s", project, releaseName(tagName), ext) +} + +func removeRelease(dst *url.URL, tagName string) error { + fp := localReleaseFilePath(dst, tagName) + return os.RemoveAll(fp) +} + +func isFileExist(fp string) (bool, error) { + _, err := os.Stat(fp) + if os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, err + } + + return true, nil +} diff --git a/internal/github/github.go b/internal/github/github.go new file mode 100644 index 0000000..bf35a6a --- /dev/null +++ b/internal/github/github.go @@ -0,0 +1,262 @@ +package github + +import ( + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "regexp" + "time" +) + +type Client struct { + *jsonClient +} + +func NewClient() *Client { + jsonClient := newJSONClient(http.DefaultClient, "https://api.github.com") + return &Client{ + jsonClient: jsonClient, + } +} + +type Asset struct { + Name string `json:"name"` + BrowserDownloadURL string `json:"browser_download_url"` +} + +type Release struct { + ID int `json:"id"` + TagName string `json:"tag_name"` + PublishedAt time.Time `json:"published_at"` + TarballURL string `json:"tarball_url"` + ZipballURL string `json:"zipball_url"` + Assets []Asset `json:"assets"` +} + +func intRef(v int) *int { + return &v +} + +func (c *Client) ListReleases(owner string, project string) ([]Release, error) { + var resp []Release + + _, err := c.jsonClient.Request(http.MethodGet, intRef(http.StatusOK), path.Join("repos", owner, project, "releases"), nil, &resp) + if err != nil { + return nil, err + } + + return resp, nil +} + +func (c *Client) DownloadAsset(w io.Writer, owner string, project string, asset *Asset) error { + resp, err := http.Get(asset.BrowserDownloadURL) + if err != nil { + return err + } + + _, err = io.Copy(w, resp.Body) + if err != nil { + return err + } + return nil + +} + +func (c *Client) DownloadRelease( + dirPath string, + org string, + project string, + release *Release, +) error { + for _, asset := range release.Assets { + fp := path.Join(dirPath, path.Base(asset.Name)) + f, err := os.Create(fp) + if err != nil { + return err + } + defer f.Close() + + resp, err := http.Get(asset.BrowserDownloadURL) + if err != nil { + return err + } + _, err = io.Copy(f, resp.Body) + if err != nil { + return err + } + } + + fp := path.Join(dirPath, releaseSourceFileName(project, release.TagName, "tar.gz")) + if exist, err := isFileExist(fp); err != nil { + return fmt.Errorf("downloading tarball: %w", err) + } else if !exist { + f, err := os.Create(fp) + if err != nil { + return err + } + defer f.Close() + + resp, err := http.Get(release.TarballURL) + if err != nil { + return err + } + _, err = io.Copy(f, resp.Body) + if err != nil { + return err + } + } + + fp = path.Join(dirPath, releaseSourceFileName(project, release.TagName, "zip")) + if exist, err := isFileExist(fp); err != nil { + return err + } else if !exist { + f, err := os.Create(fp) + if err != nil { + return fmt.Errorf("downloading zipball: %w", err) + } + defer f.Close() + + resp, err := http.Get(release.ZipballURL) + if err != nil { + return err + } + _, err = io.Copy(f, resp.Body) + if err != nil { + return err + } + } + + return nil +} + +func releaseDownloads( + project string, + release *Release, +) map[string]string { + files := make(map[string]string) + + for _, asset := range release.Assets { + files[path.Base(asset.Name)] = asset.BrowserDownloadURL + } + + fileName := releaseSourceFileName(project, release.TagName, "tar.gz") + files[fileName] = release.TarballURL + + fileName = releaseSourceFileName(project, release.TagName, "zip") + files[fileName] = release.ZipballURL + + return files +} + +func (c *Client) download(dst string, src string) error { + resp, err := http.Head(src) + if err != nil { + return err + } + + info, err := os.Stat(dst) + if !os.IsNotExist(err) && err != nil { + return err + } + if info != nil { + if info.Size() == resp.ContentLength { + return nil + } + + err := os.Remove(dst) + if !os.IsNotExist(err) && err != nil { + return fmt.Errorf("could not remove '%s': %w", dst, err) + } + } + + resp, err = http.Get(src) + if err != nil { + return err + } + + f, err := os.Create(dst) + if err != nil { + return fmt.Errorf("creating '%s': %w", dst, err) + } + defer f.Close() + + _, err = io.Copy(f, resp.Body) + if err != nil { + return err + } + + return nil +} + +func (c *Client) MirrorAssets(dst *url.URL, src *url.URL) error { + if src.Hostname() != "github.com" { + return fmt.Errorf("host must be github.com") + } + if dst.Scheme != "file:///" && dst.Scheme != "" { + return fmt.Errorf("unsupported destination scheme '%s'", dst.Scheme) + } + + matches := regexp.MustCompilePOSIX("/(.*?)/(.*?)").FindAllStringSubmatch(src.Path, 1) + if len(matches) != 1 && len(matches[0]) != 2 { + return fmt.Errorf("must be a full path to the project") + } + + owner := matches[0][1] + project := matches[0][2] + + releases, err := c.ListReleases(owner, project) + if err != nil { + return fmt.Errorf("fetching list of releases: %w", err) + } + + existingFiles := make(map[string]bool) + + filepath.WalkDir(dst.Path, func(path string, _ os.DirEntry, err error) error { + if err != nil { + return err + } + + path, err = filepath.Abs(path) + if err != nil { + return err + } + existingFiles[path] = true + + return nil + }) + delete(existingFiles, dst.Path) + + for _, release := range releases { + localDir := localReleaseFilePath(dst, release.TagName) + localDir, err = filepath.Abs(localDir) + if err != nil { + return err + } + + err := os.MkdirAll(localDir, 0777) + if err != nil { + return fmt.Errorf("creating '%s': %w", localDir, err) + } + delete(existingFiles, localDir) + + for fileName, srcURL := range releaseDownloads(project, &release) { + localFile := path.Join(localDir, fileName) + delete(existingFiles, localFile) + + err := c.download(localFile, srcURL) + if err != nil { + return fmt.Errorf("downloading '%s': %w", srcURL, err) + } + } + } + + for fp := range existingFiles { + _ = os.RemoveAll(fp) + } + + return nil +} diff --git a/internal/github/github_test.go b/internal/github/github_test.go new file mode 100644 index 0000000..5f3a270 --- /dev/null +++ b/internal/github/github_test.go @@ -0,0 +1,34 @@ +package github + +import ( + "os" + "path" + "testing" + + "git.server.ky/slackcoder/mirror/internal" + "github.com/stretchr/testify/require" +) + +func TestMirrorDendrite(t *testing.T) { + d := t.TempDir() + + dst := internal.MustURL(d) + src := internal.MustURL("https://github.com/matrix-org/dendrite") + oldFile := "random_file.txt" + + f, cErr := os.Create(path.Join(dst.Path, oldFile)) + require.NoError(t, cErr) + f.Close() + + c := NewClient() + err := c.MirrorAssets(dst, src) + require.NoError(t, err, "dendrite assets") + + require.FileExists(t, path.Join(dst.Path, "v0.13.7", "dendrite-0.13.7.tar.gz")) + require.FileExists(t, path.Join(dst.Path, "v0.13.7", "dendrite-0.13.7.zip")) + + err = c.MirrorAssets(dst, src) + require.NoError(t, err, "dendrite assets") + + require.NoFileExists(t, path.Join(dst.Path, oldFile), "only files from mirror should exist") +} diff --git a/internal/github/rest_client.go b/internal/github/rest_client.go new file mode 100644 index 0000000..6fdd31c --- /dev/null +++ b/internal/github/rest_client.go @@ -0,0 +1,115 @@ +package github + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" +) + +type HTTPRequester interface { + Do(req *http.Request) (*http.Response, error) +} + +type BearerAuthClient struct { + HTTPRequester + Username string + Password string +} + +func WithBearerAuth( + cli HTTPRequester, + username, password string, +) *BearerAuthClient { + return &BearerAuthClient{cli, username, password} +} + +func (s *BearerAuthClient) Do(req *http.Request) (*http.Response, error) { + if s.Username != "" && s.Password != "" { + value := fmt.Sprintf("Bearer %s:%s", s.Username, s.Password) + req.Header.Set("Authorization", value) + } + return s.HTTPRequester.Do(req) +} + +type jsonClient struct { + Client HTTPRequester + basePath string +} + +func newJSONClient( + cli HTTPRequester, + basePath string, +) *jsonClient { + return &jsonClient{ + Client: cli, + basePath: basePath, + } +} + +func newHTTPJSONReq( + method string, + url string, + req interface{}, +) (*http.Request, error) { + body := &bytes.Buffer{} + if req != nil { + buf, err := json.Marshal(req) + if err != nil { + return nil, err + } + body = bytes.NewBuffer(buf) + } + httpReq, err := http.NewRequest(method, url, body) + if err != nil { + return nil, err + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Accept", "application/json") + return httpReq, nil +} + +func decodeJSONResponse(httpResp *http.Response, resp interface{}) error { + if httpResp.StatusCode/100 != 2 { + return fmt.Errorf( + "received HTTP status code %d (%s)", + httpResp.StatusCode, + httpResp.Status, + ) + } + if resp == nil { + return nil + } + err := json.NewDecoder(httpResp.Body).Decode(resp) + if err != nil { + return err + } + return err +} + +func (s *jsonClient) Request( + method string, + statusCode *int, + path string, + req, resp interface{}, +) (int, error) { + url := fmt.Sprintf("%s/%s", s.basePath, path) + httpReq, err := newHTTPJSONReq(method, url, req) + if err != nil { + return 0, err + } + httpResp, err := s.Client.Do(httpReq) + if err != nil { + return 0, err + } + defer httpResp.Body.Close() + + err = decodeJSONResponse(httpResp, resp) + if err != nil { + return httpResp.StatusCode, err + } + if statusCode != nil && httpResp.StatusCode != *statusCode { + return httpResp.StatusCode, fmt.Errorf("expected status code %d but got %d", *statusCode, httpResp.StatusCode) + } + return httpResp.StatusCode, nil +} -- cgit v1.2.3