Get Netrc machine from clone url (#800)

We previously got the machine hostname for Netrc from the url of the remote, but in cases where the clone-url does not match the api url this can lead to errors.

cmd/server/setup.go

@@ -285,7 +285,6 @@ func setupCoding(c *cli.Context) (remote.Remote, error) {
 		Client:     c.String("coding-client"),
 		Secret:     c.String("coding-secret"),
 		Scopes:     c.StringSlice("coding-scope"),
-		Machine:    c.String("coding-git-machine"),
 		Username:   c.String("coding-git-username"),
 		Password:   c.String("coding-git-password"),
 		SkipVerify: c.Bool("coding-skip-verify"),

server/remote/bitbucket/bitbucket_test.go

@@ -52,7 +52,7 @@ func Test_bitbucket(t *testing.T) {
 
 		g.It("Should return the netrc file", func() {
 			remote, _ := New(&Opts{})
-			netrc, _ := remote.Netrc(fakeUser, nil)
+			netrc, _ := remote.Netrc(fakeUser, fakeRepo)
 			g.Assert(netrc.Machine).Equal("bitbucket.org")
 			g.Assert(netrc.Login).Equal("x-token-auth")
 			g.Assert(netrc.Password).Equal(fakeUser.Token)

server/remote/bitbucketserver/bitbucketserver.go

@@ -26,8 +26,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"net/url"
-	"strings"
 
 	"github.com/mrjones/oauth"
 
@@ -201,21 +199,15 @@ func (c *Config) Status(ctx context.Context, user *model.User, repo *model.Repo,
 }
 
 func (c *Config) Netrc(user *model.User, r *model.Repo) (*model.Netrc, error) {
-	u, err := url.Parse(c.URL)
+	host, err := common.ExtractHostFromCloneURL(r.Clone)
 	if err != nil {
 		return nil, err
 	}
-	// remove the port
-	tmp := strings.Split(u.Host, ":")
-	host := tmp[0]
 
-	if err != nil {
-		return nil, err
-	}
 	return &model.Netrc{
-		Machine:  host,
 		Login:    c.Username,
 		Password: c.Password,
+		Machine:  host,
 	}, nil
 }
 

server/remote/coding/coding.go

@@ -27,6 +27,7 @@ import (
 	"github.com/woodpecker-ci/woodpecker/server/model"
 	"github.com/woodpecker-ci/woodpecker/server/remote"
 	"github.com/woodpecker-ci/woodpecker/server/remote/coding/internal"
+	"github.com/woodpecker-ci/woodpecker/server/remote/common"
 )
 
 const (
@@ -39,7 +40,6 @@ type Opts struct {
 	Client     string   // Coding oauth client id.
 	Secret     string   // Coding oauth client secret.
 	Scopes     []string // Coding oauth scopes.
-	Machine    string   // Optional machine name.
 	Username   string   // Optional machine account username.
 	Password   string   // Optional machine account password.
 	SkipVerify bool     // Skip ssl verification.
@@ -53,7 +53,6 @@ func New(opts Opts) (remote.Remote, error) {
 		Client:     opts.Client,
 		Secret:     opts.Secret,
 		Scopes:     opts.Scopes,
-		Machine:    opts.Machine,
 		Username:   opts.Username,
 		Password:   opts.Password,
 		SkipVerify: opts.SkipVerify,
@@ -70,7 +69,6 @@ type Coding struct {
 	Client     string
 	Secret     string
 	Scopes     []string
-	Machine    string
 	Username   string
 	Password   string
 	SkipVerify bool
@@ -251,17 +249,23 @@ func (c *Coding) Status(ctx context.Context, u *model.User, r *model.Repo, b *mo
 // Netrc returns a .netrc file that can be used to clone
 // private repositories from a remote system.
 func (c *Coding) Netrc(u *model.User, r *model.Repo) (*model.Netrc, error) {
+	host, err := common.ExtractHostFromCloneURL(r.Clone)
+	if err != nil {
+		return nil, err
+	}
+
 	if c.Password != "" {
 		return &model.Netrc{
 			Login:    c.Username,
 			Password: c.Password,
-			Machine:  c.Machine,
+			Machine:  host,
 		}, nil
 	}
+
 	return &model.Netrc{
 		Login:    u.Token,
 		Password: "x-oauth-basic",
-		Machine:  c.Machine,
+		Machine:  host,
 	}, nil
 }
 

server/remote/coding/coding_test.go

@@ -48,7 +48,6 @@ func Test_coding(t *testing.T) {
 					Client:     "KTNF2ALdm3ofbtxLh6IbV95Ro5AKWJUP",
 					Secret:     "zVtxJrKhNhBcNyqCz1NggNAAmehAxnRO3Z0fXmCp",
 					Scopes:     []string{"user", "project", "project:depot"},
-					Machine:    "git.coding.net",
 					Username:   "someuser",
 					Password:   "password",
 					SkipVerify: true,
@@ -57,7 +56,6 @@ func Test_coding(t *testing.T) {
 				g.Assert(remote.(*Coding).Client).Equal("KTNF2ALdm3ofbtxLh6IbV95Ro5AKWJUP")
 				g.Assert(remote.(*Coding).Secret).Equal("zVtxJrKhNhBcNyqCz1NggNAAmehAxnRO3Z0fXmCp")
 				g.Assert(remote.(*Coding).Scopes).Equal([]string{"user", "project", "project:depot"})
-				g.Assert(remote.(*Coding).Machine).Equal("git.coding.net")
 				g.Assert(remote.(*Coding).Username).Equal("someuser")
 				g.Assert(remote.(*Coding).Password).Equal("password")
 				g.Assert(remote.(*Coding).SkipVerify).Equal(true)
@@ -172,21 +170,18 @@ func Test_coding(t *testing.T) {
 		g.Describe("When requesting a netrc config", func() {
 			g.It("Should return the netrc file for global credential", func() {
 				remote, _ := New(Opts{
-					Machine:  "git.coding.net",
 					Username: "someuser",
 					Password: "password",
 				})
-				netrc, err := remote.Netrc(fakeUser, nil)
+				netrc, err := remote.Netrc(fakeUser, fakeRepo)
 				g.Assert(err).IsNil()
 				g.Assert(netrc.Login).Equal("someuser")
 				g.Assert(netrc.Password).Equal("password")
 				g.Assert(netrc.Machine).Equal("git.coding.net")
 			})
 			g.It("Should return the netrc file for specified user", func() {
-				remote, _ := New(Opts{
+				remote, _ := New(Opts{})
-					Machine: "git.coding.net",
+				netrc, err := remote.Netrc(fakeUser, fakeRepo)
-				})
-				netrc, err := remote.Netrc(fakeUser, nil)
 				g.Assert(err).IsNil()
 				g.Assert(netrc.Login).Equal(fakeUser.Token)
 				g.Assert(netrc.Password).Equal("x-oauth-basic")

server/remote/common/utils.go

@@ -0,0 +1,25 @@
+package common
+
+import (
+	"net"
+	"net/url"
+	"strings"
+)
+
+func ExtractHostFromCloneURL(cloneURL string) (string, error) {
+	u, err := url.Parse(cloneURL)
+	if err != nil {
+		return "", err
+	}
+
+	if !strings.Contains(u.Host, ":") {
+		return u.Host, nil
+	}
+
+	host, _, err := net.SplitHostPort(u.Host)
+	if err != nil {
+		return "", err
+	}
+
+	return host, nil
+}

server/remote/common/utils_test.go

@@ -0,0 +1,18 @@
+package common_test
+
+import (
+	"testing"
+
+	"github.com/woodpecker-ci/woodpecker/server/remote/common"
+)
+
+func Test_Netrc(t *testing.T) {
+	host, err := common.ExtractHostFromCloneURL("https://git.example.com/foo/bar.git")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if host != "git.example.com" {
+		t.Errorf("Expected host to be git.example.com, got %s", host)
+	}
+}

server/remote/gitea/gitea.go

@@ -44,7 +44,6 @@ const (
 
 type Gitea struct {
 	URL          string
-	Machine      string
 	ClientID     string
 	ClientSecret string
 	SkipVerify   bool
@@ -71,7 +70,6 @@ func New(opts Opts) (remote.Remote, error) {
 	}
 	return &Gitea{
 		URL:          opts.URL,
-		Machine:      u.Host,
 		ClientID:     opts.Client,
 		ClientSecret: opts.Secret,
 		SkipVerify:   opts.SkipVerify,
@@ -355,10 +353,15 @@ func (c *Gitea) Netrc(u *model.User, r *model.Repo) (*model.Netrc, error) {
 		token = u.Token
 	}
 
+	host, err := common.ExtractHostFromCloneURL(r.Clone)
+	if err != nil {
+		return nil, err
+	}
+
 	return &model.Netrc{
 		Login:    login,
 		Password: token,
-		Machine:  c.Machine,
+		Machine:  host,
 	}, nil
 }
 

server/remote/gitea/gitea_test.go

@@ -49,7 +49,6 @@ func Test_gitea(t *testing.T) {
 					SkipVerify: true,
 				})
 				g.Assert(remote.(*Gitea).URL).Equal("http://localhost:8080")
-				g.Assert(remote.(*Gitea).Machine).Equal("localhost")
 				g.Assert(remote.(*Gitea).SkipVerify).Equal(true)
 			})
 			g.It("Should handle malformed url", func() {
@@ -60,19 +59,15 @@ func Test_gitea(t *testing.T) {
 
 		g.Describe("Generating a netrc file", func() {
 			g.It("Should return a netrc with the user token", func() {
-				remote, _ := New(Opts{
+				remote, _ := New(Opts{})
-					URL: "http://gitea.com",
+				netrc, _ := remote.Netrc(fakeUser, fakeRepo)
-				})
-				netrc, _ := remote.Netrc(fakeUser, nil)
 				g.Assert(netrc.Machine).Equal("gitea.com")
 				g.Assert(netrc.Login).Equal(fakeUser.Login)
 				g.Assert(netrc.Password).Equal(fakeUser.Token)
 			})
 			g.It("Should return a netrc with the machine account", func() {
-				remote, _ := New(Opts{
+				remote, _ := New(Opts{})
-					URL: "http://gitea.com",
+				netrc, _ := remote.Netrc(nil, fakeRepo)
-				})
-				netrc, _ := remote.Netrc(nil, nil)
 				g.Assert(netrc.Machine).Equal("gitea.com")
 				g.Assert(netrc.Login).Equal("")
 				g.Assert(netrc.Password).Equal("")
@@ -172,6 +167,7 @@ var (
 	}
 
 	fakeRepo = &model.Repo{
+		Clone:    "http://gitea.com/test_name/repo_name.git",
 		Owner:    "test_name",
 		Name:     "repo_name",
 		FullName: "test_name/repo_name",

server/remote/github/github.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"net"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -54,14 +53,6 @@ type Opts struct {
 // New returns a Remote implementation that integrates with a GitHub Cloud or
 // GitHub Enterprise version control hosting provider.
 func New(opts Opts) (remote.Remote, error) {
-	u, err := url.Parse(opts.URL)
-	if err != nil {
-		return nil, err
-	}
-	host, _, err := net.SplitHostPort(u.Host)
-	if err == nil {
-		u.Host = host
-	}
 	r := &client{
 		API:        defaultAPI,
 		URL:        defaultURL,
@@ -69,7 +60,6 @@ func New(opts Opts) (remote.Remote, error) {
 		Secret:     opts.Secret,
 		SkipVerify: opts.SkipVerify,
 		MergeRef:   opts.MergeRef,
-		Machine:    u.Host,
 	}
 	if opts.URL != defaultURL {
 		r.URL = strings.TrimSuffix(opts.URL, "/")
@@ -84,7 +74,6 @@ type client struct {
 	API        string
 	Client     string
 	Secret     string
-	Machine    string
 	SkipVerify bool
 	MergeRef   bool
 }
@@ -283,10 +272,15 @@ func (c *client) Netrc(u *model.User, r *model.Repo) (*model.Netrc, error) {
 		token = "x-oauth-basic"
 	}
 
+	host, err := common.ExtractHostFromCloneURL(r.Clone)
+	if err != nil {
+		return nil, err
+	}
+
 	return &model.Netrc{
 		Login:    login,
 		Password: token,
-		Machine:  c.Machine,
+		Machine:  host,
 	}, nil
 }
 

server/remote/github/github_test.go

@@ -52,32 +52,23 @@ func Test_github(t *testing.T) {
 				})
 				g.Assert(remote.(*client).URL).Equal("http://localhost:8080")
 				g.Assert(remote.(*client).API).Equal("http://localhost:8080/api/v3/")
-				g.Assert(remote.(*client).Machine).Equal("localhost")
 				g.Assert(remote.(*client).Client).Equal("0ZXh0IjoiI")
 				g.Assert(remote.(*client).Secret).Equal("I1NiIsInR5")
 				g.Assert(remote.(*client).SkipVerify).Equal(true)
 			})
-			g.It("Should handle malformed url", func() {
-				_, err := New(Opts{URL: "%gh&%ij"})
-				g.Assert(err).IsNotNil()
-			})
 		})
 
 		g.Describe("Generating a netrc file", func() {
 			g.It("Should return a netrc with the user token", func() {
-				remote, _ := New(Opts{
+				remote, _ := New(Opts{})
-					URL: "http://github.com:443",
+				netrc, _ := remote.Netrc(fakeUser, fakeRepo)
-				})
-				netrc, _ := remote.Netrc(fakeUser, nil)
 				g.Assert(netrc.Machine).Equal("github.com")
 				g.Assert(netrc.Login).Equal(fakeUser.Token)
 				g.Assert(netrc.Password).Equal("x-oauth-basic")
 			})
 			g.It("Should return a netrc with the machine account", func() {
-				remote, _ := New(Opts{
+				remote, _ := New(Opts{})
-					URL: "http://github.com:443",
+				netrc, _ := remote.Netrc(nil, fakeRepo)
-				})
-				netrc, _ := remote.Netrc(nil, nil)
 				g.Assert(netrc.Machine).Equal("github.com")
 				g.Assert(netrc.Login).Equal("")
 				g.Assert(netrc.Password).Equal("")

server/remote/gitlab/gitlab.go

@@ -19,7 +19,6 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
-	"net"
 	"net/http"
 	"net/url"
 	"strings"
@@ -54,7 +53,6 @@ type Gitlab struct {
 	URL          string
 	ClientID     string
 	ClientSecret string
-	Machine      string
 	SkipVerify   bool
 	HideArchives bool
 	Search       bool
@@ -63,19 +61,10 @@ type Gitlab struct {
 // New returns a Remote implementation that integrates with Gitlab, an open
 // source Git service. See https://gitlab.com
 func New(opts Opts) (remote.Remote, error) {
-	u, err := url.Parse(opts.URL)
-	if err != nil {
-		return nil, err
-	}
-	host, _, err := net.SplitHostPort(u.Host)
-	if err == nil {
-		u.Host = host
-	}
 	return &Gitlab{
 		URL:          opts.URL,
 		ClientID:     opts.ClientID,
 		ClientSecret: opts.ClientSecret,
-		Machine:      u.Host,
 		SkipVerify:   opts.SkipVerify,
 	}, nil
 }
@@ -386,10 +375,15 @@ func (g *Gitlab) Netrc(u *model.User, r *model.Repo) (*model.Netrc, error) {
 		token = u.Token
 	}
 
+	host, err := common.ExtractHostFromCloneURL(r.Clone)
+	if err != nil {
+		return nil, err
+	}
+
 	return &model.Netrc{
 		Login:    login,
 		Password: token,
-		Machine:  g.Machine,
+		Machine:  host,
 	}, nil
 }
 

server/remote/gogs/gogs.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/woodpecker-ci/woodpecker/server/model"
 	"github.com/woodpecker-ci/woodpecker/server/remote"
+	"github.com/woodpecker-ci/woodpecker/server/remote/common"
 )
 
 // Opts defines configuration options.
@@ -40,7 +41,6 @@ type Opts struct {
 
 type client struct {
 	URL         string
-	Machine     string
 	Username    string
 	Password    string
 	PrivateMode bool
@@ -60,7 +60,6 @@ func New(opts Opts) (remote.Remote, error) {
 	}
 	return &client{
 		URL:         opts.URL,
-		Machine:     u.Host,
 		Username:    opts.Username,
 		Password:    opts.Password,
 		PrivateMode: opts.PrivateMode,
@@ -217,17 +216,22 @@ func (c *client) Status(ctx context.Context, u *model.User, r *model.Repo, b *mo
 // cloning Gogs repositories. The netrc will use the global machine account
 // when configured.
 func (c *client) Netrc(u *model.User, r *model.Repo) (*model.Netrc, error) {
+	host, err := common.ExtractHostFromCloneURL(r.Clone)
+	if err != nil {
+		return nil, err
+	}
+
 	if c.Password != "" {
 		return &model.Netrc{
 			Login:    c.Username,
 			Password: c.Password,
-			Machine:  c.Machine,
+			Machine:  host,
 		}, nil
 	}
 	return &model.Netrc{
 		Login:    u.Token,
 		Password: "x-oauth-basic",
-		Machine:  c.Machine,
+		Machine:  host,
 	}, nil
 }
 

server/remote/gogs/gogs_test.go

@@ -52,7 +52,6 @@ func Test_gogs(t *testing.T) {
 					PrivateMode: true,
 				})
 				g.Assert(remote.(*client).URL).Equal("http://localhost:8080")
-				g.Assert(remote.(*client).Machine).Equal("localhost")
 				g.Assert(remote.(*client).Username).Equal("someuser")
 				g.Assert(remote.(*client).Password).Equal("password")
 				g.Assert(remote.(*client).SkipVerify).Equal(true)
@@ -66,21 +65,18 @@ func Test_gogs(t *testing.T) {
 
 		g.Describe("Generating a netrc file", func() {
 			g.It("Should return a netrc with the user token", func() {
-				remote, _ := New(Opts{
+				remote, _ := New(Opts{})
-					URL: "http://gogs.com",
+				netrc, _ := remote.Netrc(fakeUser, fakeRepo)
-				})
-				netrc, _ := remote.Netrc(fakeUser, nil)
 				g.Assert(netrc.Machine).Equal("gogs.com")
 				g.Assert(netrc.Login).Equal(fakeUser.Token)
 				g.Assert(netrc.Password).Equal("x-oauth-basic")
 			})
 			g.It("Should return a netrc with the machine account", func() {
 				remote, _ := New(Opts{
-					URL:      "http://gogs.com",
 					Username: "someuser",
 					Password: "password",
 				})
-				netrc, _ := remote.Netrc(nil, nil)
+				netrc, _ := remote.Netrc(nil, fakeRepo)
 				g.Assert(netrc.Machine).Equal("gogs.com")
 				g.Assert(netrc.Login).Equal("someuser")
 				g.Assert(netrc.Password).Equal("password")
@@ -185,6 +181,7 @@ var (
 	}
 
 	fakeRepo = &model.Repo{
+		Clone:    "http://gogs.com/test_name/repo_name.git",
 		Owner:    "test_name",
 		Name:     "repo_name",
 		FullName: "test_name/repo_name",