feat: improve gitlab/github readme url (#456)

* Use GitHub API to find readme filename

* Fix lint errors and typos

* Bring back "tries to find" instead of "finds"

* Rename `readmeURL` to `apiURL`

* Don't close body

* Use GitLab API to find readme filename

* feat: improve gitlab/github readme url

Signed-off-by: Carlos A Becker <caarlos0@users.noreply.github.com>

---------

Signed-off-by: Carlos A Becker <caarlos0@users.noreply.github.com>
Signed-off-by: Carlos Alexandro Becker <caarlos0@users.noreply.github.com>
Co-authored-by: danielwerg <35052399+danielwerg@users.noreply.github.com>

github.go

@@ -1,50 +1,55 @@
 package main
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
 )
 
-// isGitHubURL tests a string to determine if it is a well-structured GitHub URL.
+// findGitHubREADME tries to find the correct README filename in a repository using GitHub API.
-func isGitHubURL(s string) (string, bool) {
+func findGitHubREADME(u *url.URL) (*source, error) {
-	if strings.HasPrefix(s, "github.com/") {
+	owner, repo, ok := strings.Cut(strings.TrimPrefix(u.Path, "/"), "/")
-		s = "https://" + s
+	if !ok {
+		return nil, fmt.Errorf("invalid url: %s", u.String())
 	}
 
-	u, err := url.ParseRequestURI(s)
+	type readme struct {
-	if err != nil {
+		DownloadURL string `json:"download_url"`
-		return "", false
 	}
 
-	return u.String(), strings.ToLower(u.Host) == "github.com"
+	apiURL := fmt.Sprintf("https://api.%s/repos/%s/%s/readme", u.Hostname(), owner, repo)
-}
 
-// findGitHubREADME tries to find the correct README filename in a repository.
+	// nolint:bodyclose
-func findGitHubREADME(s string) (*source, error) {
+	// it is closed on the caller
-	u, err := url.ParseRequestURI(s)
+	res, err := http.Get(apiURL) // nolint: gosec
 	if err != nil {
 		return nil, err
 	}
-	u.Host = "raw.githubusercontent.com"
 
-	for _, b := range readmeBranches {
+	body, err := io.ReadAll(res.Body)
-		for _, r := range readmeNames {
+	if err != nil {
-			v := *u
+		return nil, err
-			v.Path += fmt.Sprintf("/%s/%s", b, r)
+	}
 
-			// nolint:bodyclose
+	var result readme
-			// it is closed on the caller
+	if err := json.Unmarshal(body, &result); err != nil {
-			resp, err := http.Get(v.String())
+		return nil, err
-			if err != nil {
+	}
-				return nil, err
-			}
 
-			if resp.StatusCode == http.StatusOK {
+	if res.StatusCode == http.StatusOK {
-				return &source{resp.Body, v.String()}, nil
+		// nolint:bodyclose
-			}
+		// it is closed on the caller
+		resp, err := http.Get(result.DownloadURL)
+		if err != nil {
+			return nil, err
+		}
+
+		if resp.StatusCode == http.StatusOK {
+			return &source{resp.Body, result.DownloadURL}, nil
 		}
 	}
 

gitlab.go

@@ -1,49 +1,59 @@
 package main
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
 )
 
-// isGitLabURL tests a string to determine if it is a well-structured GitLab URL.
+// findGitLabREADME tries to find the correct README filename in a repository using GitLab API.
-func isGitLabURL(s string) (string, bool) {
+func findGitLabREADME(u *url.URL) (*source, error) {
-	if strings.HasPrefix(s, "gitlab.com/") {
+	owner, repo, ok := strings.Cut(strings.TrimPrefix(u.Path, "/"), "/")
-		s = "https://" + s
+	if !ok {
+		return nil, fmt.Errorf("invalid url: %s", u.String())
 	}
 
-	u, err := url.ParseRequestURI(s)
+	projectPath := url.QueryEscape(owner + "/" + repo)
-	if err != nil {
+
-		return "", false
+	type readme struct {
+		ReadmeURL string `json:"readme_url"`
 	}
 
-	return u.String(), strings.ToLower(u.Host) == "gitlab.com"
+	apiURL := fmt.Sprintf("https://%s/api/v4/projects/%s", u.Hostname(), projectPath)
-}
 
-// findGitLabREADME tries to find the correct README filename in a repository.
+	// nolint:bodyclose
-func findGitLabREADME(s string) (*source, error) {
+	// it is closed on the caller
-	u, err := url.ParseRequestURI(s)
+	res, err := http.Get(apiURL) // nolint: gosec
 	if err != nil {
 		return nil, err
 	}
 
-	for _, b := range readmeBranches {
+	body, err := io.ReadAll(res.Body)
-		for _, r := range readmeNames {
+	if err != nil {
-			v := *u
+		return nil, err
-			v.Path += fmt.Sprintf("/raw/%s/%s", b, r)
+	}
 
-			// nolint:bodyclose
+	var result readme
-			// it is closed on the caller
+	if err := json.Unmarshal(body, &result); err != nil {
-			resp, err := http.Get(v.String())
+		return nil, err
-			if err != nil {
+	}
-				return nil, err
-			}
 
-			if resp.StatusCode == http.StatusOK {
+	readmeRawURL := strings.Replace(result.ReadmeURL, "blob", "raw", -1)
-				return &source{resp.Body, v.String()}, nil
+
-			}
+	if res.StatusCode == http.StatusOK {
+		// nolint:bodyclose
+		// it is closed on the caller
+		resp, err := http.Get(readmeRawURL) // nolint: gosec
+		if err != nil {
+			return nil, err
+		}
+
+		if resp.StatusCode == http.StatusOK {
+			return &source{resp.Body, readmeRawURL}, nil
 		}
 	}
 

main.go

@@ -72,19 +72,9 @@ func sourceFromArg(arg string) (*source, error) {
 	}
 
 	// a GitHub or GitLab URL (even without the protocol):
-	if u, ok := isGitHubURL(arg); ok {
+	src, err := readmeURL(arg)
-		src, err := findGitHubREADME(u)
+	if src != nil || err != nil {
-		if err != nil {
+		return src, err
-			return nil, err
-		}
-		return src, nil
-	}
-	if u, ok := isGitLabURL(arg); ok {
-		src, err := findGitLabREADME(u)
-		if err != nil {
-			return nil, err
-		}
-		return src, nil
 	}
 
 	// HTTP(S) URLs:

url.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"net/url"
+	"strings"
+	"sync"
+)
+
+const (
+	protoGithub = "github://"
+	protoGitlab = "gitlab://"
+	protoHTTPS  = "https://"
+)
+
+var (
+	githubURL *url.URL
+	gitlabURL *url.URL
+	urlsOnce  sync.Once
+)
+
+func init() {
+	urlsOnce.Do(func() {
+		githubURL, _ = url.Parse("https://github.com")
+		gitlabURL, _ = url.Parse("https://gitlab.com")
+	})
+}
+
+func readmeURL(path string) (*source, error) {
+	switch {
+	case strings.HasPrefix(path, protoGithub):
+		if u := githubReadmeURL(path); u != nil {
+			return readmeURL(u.String())
+		}
+		return nil, nil
+	case strings.HasPrefix(path, protoGitlab):
+		if u := gitlabReadmeURL(path); u != nil {
+			return readmeURL(u.String())
+		}
+		return nil, nil
+	}
+
+	if !strings.HasPrefix(path, protoHTTPS) {
+		path = protoHTTPS + path
+	}
+	u, err := url.Parse(path)
+	if err != nil {
+		return nil, err
+	}
+
+	switch {
+	case u.Hostname() == githubURL.Hostname():
+		return findGitHubREADME(u)
+	case u.Hostname() == gitlabURL.Hostname():
+		return findGitLabREADME(u)
+	}
+
+	return nil, nil
+}
+
+func githubReadmeURL(path string) *url.URL {
+	path = strings.TrimPrefix(path, protoGithub)
+	parts := strings.Split(path, "/")
+	if len(parts) != 2 {
+		// custom hostnames are not supported yet
+		return nil
+	}
+	u, _ := url.Parse(githubURL.String())
+	return u.JoinPath(path)
+}
+
+func gitlabReadmeURL(path string) *url.URL {
+	path = strings.TrimPrefix(path, protoGitlab)
+	parts := strings.Split(path, "/")
+	if len(parts) != 2 {
+		// custom hostnames are not supported yet
+		return nil
+	}
+	u, _ := url.Parse(gitlabURL.String())
+	return u.JoinPath(path)
+}

url_test.go

@@ -0,0 +1,29 @@
+package main
+
+import "testing"
+
+func TestURLParser(t *testing.T) {
+	for path, url := range map[string]string{
+		"github.com/charmbracelet/glow":             "https://raw.githubusercontent.com/charmbracelet/glow/master/README.md",
+		"github://charmbracelet/glow":               "https://raw.githubusercontent.com/charmbracelet/glow/master/README.md",
+		"github://caarlos0/dotfiles.fish":           "https://raw.githubusercontent.com/caarlos0/dotfiles.fish/main/README.md",
+		"github://tj/git-extras":                    "https://raw.githubusercontent.com/tj/git-extras/main/Readme.md",
+		"https://github.com/goreleaser/nfpm":        "https://raw.githubusercontent.com/goreleaser/nfpm/main/README.md",
+		"gitlab.com/caarlos0/test":                  "https://gitlab.com/caarlos0/test/-/raw/master/README.md",
+		"gitlab://caarlos0/test":                    "https://gitlab.com/caarlos0/test/-/raw/master/README.md",
+		"https://gitlab.com/terrakok/gitlab-client": "https://gitlab.com/terrakok/gitlab-client/-/raw/develop/Readme.md",
+	} {
+		t.Run(path, func(t *testing.T) {
+			got, err := readmeURL(path)
+			if err != nil {
+				t.Fatalf("expected no error, got %v", err)
+			}
+			if got == nil {
+				t.Fatalf("should not be nil")
+			}
+			if url != got.URL {
+				t.Errorf("expected url for %s to be %s, was %s", path, url, got.URL)
+			}
+		})
+	}
+}