diff options
Diffstat (limited to 'internal/github')
-rw-r--r-- | internal/github/filesystem.go | 63 | ||||
-rw-r--r-- | internal/github/github.go | 262 | ||||
-rw-r--r-- | internal/github/github_test.go | 34 | ||||
-rw-r--r-- | internal/github/rest_client.go | 115 |
4 files changed, 0 insertions, 474 deletions
diff --git a/internal/github/filesystem.go b/internal/github/filesystem.go deleted file mode 100644 index 3f069b4..0000000 --- a/internal/github/filesystem.go +++ /dev/null @@ -1,63 +0,0 @@ -package github - -import ( - "fmt" - "net/url" - "os" - "path" - "strings" -) - -func listReleasesByTagName(dst *url.URL) ([]string, error) { - entries, err := os.ReadDir(dst.Path) - if err != nil { - return nil, err - } - - var tagNames []string - for _, entry := range entries { - tagNames = append(tagNames, entry.Name()) - } - - return tagNames, nil -} - -// The path which project release assets are saved. -func localReleaseFilePath(dst *url.URL, tagName string) string { - return path.Join(dst.Path, tagName) -} - -func releaseName(tagName string) string { - version := tagName - if strings.HasPrefix(version, "v") { - version = strings.TrimLeft(version, "v") - } - - return version -} - -// The source filename for a Github release. -// -// # The source code URL provided by Github's API only references the tag name -// -// for the release. To make it useful for users, we rename to file to include -// the project name as their website does. -func releaseSourceFileName(project string, tagName string, ext string) string { - return fmt.Sprintf("%s-%s.%s", project, releaseName(tagName), ext) -} - -func removeRelease(dst *url.URL, tagName string) error { - fp := localReleaseFilePath(dst, tagName) - return os.RemoveAll(fp) -} - -func isFileExist(fp string) (bool, error) { - _, err := os.Stat(fp) - if os.IsNotExist(err) { - return false, nil - } else if err != nil { - return false, err - } - - return true, nil -} diff --git a/internal/github/github.go b/internal/github/github.go deleted file mode 100644 index bf35a6a..0000000 --- a/internal/github/github.go +++ /dev/null @@ -1,262 +0,0 @@ -package github - -import ( - "fmt" - "io" - "net/http" - "net/url" - "os" - "path" - "path/filepath" - "regexp" - "time" -) - -type Client struct { - *jsonClient -} - -func NewClient() *Client { - jsonClient := newJSONClient(http.DefaultClient, "https://api.github.com") - return &Client{ - jsonClient: jsonClient, - } -} - -type Asset struct { - Name string `json:"name"` - BrowserDownloadURL string `json:"browser_download_url"` -} - -type Release struct { - ID int `json:"id"` - TagName string `json:"tag_name"` - PublishedAt time.Time `json:"published_at"` - TarballURL string `json:"tarball_url"` - ZipballURL string `json:"zipball_url"` - Assets []Asset `json:"assets"` -} - -func intRef(v int) *int { - return &v -} - -func (c *Client) ListReleases(owner string, project string) ([]Release, error) { - var resp []Release - - _, err := c.jsonClient.Request(http.MethodGet, intRef(http.StatusOK), path.Join("repos", owner, project, "releases"), nil, &resp) - if err != nil { - return nil, err - } - - return resp, nil -} - -func (c *Client) DownloadAsset(w io.Writer, owner string, project string, asset *Asset) error { - resp, err := http.Get(asset.BrowserDownloadURL) - if err != nil { - return err - } - - _, err = io.Copy(w, resp.Body) - if err != nil { - return err - } - return nil - -} - -func (c *Client) DownloadRelease( - dirPath string, - org string, - project string, - release *Release, -) error { - for _, asset := range release.Assets { - fp := path.Join(dirPath, path.Base(asset.Name)) - f, err := os.Create(fp) - if err != nil { - return err - } - defer f.Close() - - resp, err := http.Get(asset.BrowserDownloadURL) - if err != nil { - return err - } - _, err = io.Copy(f, resp.Body) - if err != nil { - return err - } - } - - fp := path.Join(dirPath, releaseSourceFileName(project, release.TagName, "tar.gz")) - if exist, err := isFileExist(fp); err != nil { - return fmt.Errorf("downloading tarball: %w", err) - } else if !exist { - f, err := os.Create(fp) - if err != nil { - return err - } - defer f.Close() - - resp, err := http.Get(release.TarballURL) - if err != nil { - return err - } - _, err = io.Copy(f, resp.Body) - if err != nil { - return err - } - } - - fp = path.Join(dirPath, releaseSourceFileName(project, release.TagName, "zip")) - if exist, err := isFileExist(fp); err != nil { - return err - } else if !exist { - f, err := os.Create(fp) - if err != nil { - return fmt.Errorf("downloading zipball: %w", err) - } - defer f.Close() - - resp, err := http.Get(release.ZipballURL) - if err != nil { - return err - } - _, err = io.Copy(f, resp.Body) - if err != nil { - return err - } - } - - return nil -} - -func releaseDownloads( - project string, - release *Release, -) map[string]string { - files := make(map[string]string) - - for _, asset := range release.Assets { - files[path.Base(asset.Name)] = asset.BrowserDownloadURL - } - - fileName := releaseSourceFileName(project, release.TagName, "tar.gz") - files[fileName] = release.TarballURL - - fileName = releaseSourceFileName(project, release.TagName, "zip") - files[fileName] = release.ZipballURL - - return files -} - -func (c *Client) download(dst string, src string) error { - resp, err := http.Head(src) - if err != nil { - return err - } - - info, err := os.Stat(dst) - if !os.IsNotExist(err) && err != nil { - return err - } - if info != nil { - if info.Size() == resp.ContentLength { - return nil - } - - err := os.Remove(dst) - if !os.IsNotExist(err) && err != nil { - return fmt.Errorf("could not remove '%s': %w", dst, err) - } - } - - resp, err = http.Get(src) - if err != nil { - return err - } - - f, err := os.Create(dst) - if err != nil { - return fmt.Errorf("creating '%s': %w", dst, err) - } - defer f.Close() - - _, err = io.Copy(f, resp.Body) - if err != nil { - return err - } - - return nil -} - -func (c *Client) MirrorAssets(dst *url.URL, src *url.URL) error { - if src.Hostname() != "github.com" { - return fmt.Errorf("host must be github.com") - } - if dst.Scheme != "file:///" && dst.Scheme != "" { - return fmt.Errorf("unsupported destination scheme '%s'", dst.Scheme) - } - - matches := regexp.MustCompilePOSIX("/(.*?)/(.*?)").FindAllStringSubmatch(src.Path, 1) - if len(matches) != 1 && len(matches[0]) != 2 { - return fmt.Errorf("must be a full path to the project") - } - - owner := matches[0][1] - project := matches[0][2] - - releases, err := c.ListReleases(owner, project) - if err != nil { - return fmt.Errorf("fetching list of releases: %w", err) - } - - existingFiles := make(map[string]bool) - - filepath.WalkDir(dst.Path, func(path string, _ os.DirEntry, err error) error { - if err != nil { - return err - } - - path, err = filepath.Abs(path) - if err != nil { - return err - } - existingFiles[path] = true - - return nil - }) - delete(existingFiles, dst.Path) - - for _, release := range releases { - localDir := localReleaseFilePath(dst, release.TagName) - localDir, err = filepath.Abs(localDir) - if err != nil { - return err - } - - err := os.MkdirAll(localDir, 0777) - if err != nil { - return fmt.Errorf("creating '%s': %w", localDir, err) - } - delete(existingFiles, localDir) - - for fileName, srcURL := range releaseDownloads(project, &release) { - localFile := path.Join(localDir, fileName) - delete(existingFiles, localFile) - - err := c.download(localFile, srcURL) - if err != nil { - return fmt.Errorf("downloading '%s': %w", srcURL, err) - } - } - } - - for fp := range existingFiles { - _ = os.RemoveAll(fp) - } - - return nil -} diff --git a/internal/github/github_test.go b/internal/github/github_test.go deleted file mode 100644 index 5f3a270..0000000 --- a/internal/github/github_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package github - -import ( - "os" - "path" - "testing" - - "git.server.ky/slackcoder/mirror/internal" - "github.com/stretchr/testify/require" -) - -func TestMirrorDendrite(t *testing.T) { - d := t.TempDir() - - dst := internal.MustURL(d) - src := internal.MustURL("https://github.com/matrix-org/dendrite") - oldFile := "random_file.txt" - - f, cErr := os.Create(path.Join(dst.Path, oldFile)) - require.NoError(t, cErr) - f.Close() - - c := NewClient() - err := c.MirrorAssets(dst, src) - require.NoError(t, err, "dendrite assets") - - require.FileExists(t, path.Join(dst.Path, "v0.13.7", "dendrite-0.13.7.tar.gz")) - require.FileExists(t, path.Join(dst.Path, "v0.13.7", "dendrite-0.13.7.zip")) - - err = c.MirrorAssets(dst, src) - require.NoError(t, err, "dendrite assets") - - require.NoFileExists(t, path.Join(dst.Path, oldFile), "only files from mirror should exist") -} diff --git a/internal/github/rest_client.go b/internal/github/rest_client.go deleted file mode 100644 index 6fdd31c..0000000 --- a/internal/github/rest_client.go +++ /dev/null @@ -1,115 +0,0 @@ -package github - -import ( - "bytes" - "encoding/json" - "fmt" - "net/http" -) - -type HTTPRequester interface { - Do(req *http.Request) (*http.Response, error) -} - -type BearerAuthClient struct { - HTTPRequester - Username string - Password string -} - -func WithBearerAuth( - cli HTTPRequester, - username, password string, -) *BearerAuthClient { - return &BearerAuthClient{cli, username, password} -} - -func (s *BearerAuthClient) Do(req *http.Request) (*http.Response, error) { - if s.Username != "" && s.Password != "" { - value := fmt.Sprintf("Bearer %s:%s", s.Username, s.Password) - req.Header.Set("Authorization", value) - } - return s.HTTPRequester.Do(req) -} - -type jsonClient struct { - Client HTTPRequester - basePath string -} - -func newJSONClient( - cli HTTPRequester, - basePath string, -) *jsonClient { - return &jsonClient{ - Client: cli, - basePath: basePath, - } -} - -func newHTTPJSONReq( - method string, - url string, - req interface{}, -) (*http.Request, error) { - body := &bytes.Buffer{} - if req != nil { - buf, err := json.Marshal(req) - if err != nil { - return nil, err - } - body = bytes.NewBuffer(buf) - } - httpReq, err := http.NewRequest(method, url, body) - if err != nil { - return nil, err - } - httpReq.Header.Set("Content-Type", "application/json") - httpReq.Header.Set("Accept", "application/json") - return httpReq, nil -} - -func decodeJSONResponse(httpResp *http.Response, resp interface{}) error { - if httpResp.StatusCode/100 != 2 { - return fmt.Errorf( - "received HTTP status code %d (%s)", - httpResp.StatusCode, - httpResp.Status, - ) - } - if resp == nil { - return nil - } - err := json.NewDecoder(httpResp.Body).Decode(resp) - if err != nil { - return err - } - return err -} - -func (s *jsonClient) Request( - method string, - statusCode *int, - path string, - req, resp interface{}, -) (int, error) { - url := fmt.Sprintf("%s/%s", s.basePath, path) - httpReq, err := newHTTPJSONReq(method, url, req) - if err != nil { - return 0, err - } - httpResp, err := s.Client.Do(httpReq) - if err != nil { - return 0, err - } - defer httpResp.Body.Close() - - err = decodeJSONResponse(httpResp, resp) - if err != nil { - return httpResp.StatusCode, err - } - if statusCode != nil && httpResp.StatusCode != *statusCode { - return httpResp.StatusCode, fmt.Errorf("expected status code %d but got %d", *statusCode, httpResp.StatusCode) - } - return httpResp.StatusCode, nil -} |