From 52d80972903f07f2be350fadeda6aae3d5a5159c Mon Sep 17 00:00:00 2001 From: Anbraten Date: Sat, 26 Feb 2022 02:54:15 +0100 Subject: [PATCH] Get Netrc machine from clone url (#800) We previously got the machine hostname for Netrc from the url of the remote, but in cases where the clone-url does not match the api url this can lead to errors. --- cmd/server/setup.go | 1 - server/remote/bitbucket/bitbucket_test.go | 2 +- .../remote/bitbucketserver/bitbucketserver.go | 12 ++------- server/remote/coding/coding.go | 14 +++++++---- server/remote/coding/coding_test.go | 11 +++----- server/remote/common/utils.go | 25 +++++++++++++++++++ server/remote/common/utils_test.go | 18 +++++++++++++ server/remote/gitea/gitea.go | 9 ++++--- server/remote/gitea/gitea_test.go | 14 ++++------- server/remote/github/github.go | 18 +++++-------- server/remote/github/github_test.go | 17 +++---------- server/remote/gitlab/gitlab.go | 18 +++++-------- server/remote/gogs/gogs.go | 12 ++++++--- server/remote/gogs/gogs_test.go | 11 +++----- 14 files changed, 97 insertions(+), 85 deletions(-) create mode 100644 server/remote/common/utils.go create mode 100644 server/remote/common/utils_test.go diff --git a/cmd/server/setup.go b/cmd/server/setup.go index e113ffe1c..e021501a4 100644 --- a/cmd/server/setup.go +++ b/cmd/server/setup.go @@ -285,7 +285,6 @@ func setupCoding(c *cli.Context) (remote.Remote, error) { Client: c.String("coding-client"), Secret: c.String("coding-secret"), Scopes: c.StringSlice("coding-scope"), - Machine: c.String("coding-git-machine"), Username: c.String("coding-git-username"), Password: c.String("coding-git-password"), SkipVerify: c.Bool("coding-skip-verify"), diff --git a/server/remote/bitbucket/bitbucket_test.go b/server/remote/bitbucket/bitbucket_test.go index 216109a47..412b93e01 100644 --- a/server/remote/bitbucket/bitbucket_test.go +++ b/server/remote/bitbucket/bitbucket_test.go @@ -52,7 +52,7 @@ func Test_bitbucket(t *testing.T) { g.It("Should return the netrc file", func() { remote, _ := New(&Opts{}) - netrc, _ := remote.Netrc(fakeUser, nil) + netrc, _ := remote.Netrc(fakeUser, fakeRepo) g.Assert(netrc.Machine).Equal("bitbucket.org") g.Assert(netrc.Login).Equal("x-token-auth") g.Assert(netrc.Password).Equal(fakeUser.Token) diff --git a/server/remote/bitbucketserver/bitbucketserver.go b/server/remote/bitbucketserver/bitbucketserver.go index 2dd75e68c..60ae2c2ac 100644 --- a/server/remote/bitbucketserver/bitbucketserver.go +++ b/server/remote/bitbucketserver/bitbucketserver.go @@ -26,8 +26,6 @@ import ( "fmt" "io/ioutil" "net/http" - "net/url" - "strings" "github.com/mrjones/oauth" @@ -201,21 +199,15 @@ func (c *Config) Status(ctx context.Context, user *model.User, repo *model.Repo, } func (c *Config) Netrc(user *model.User, r *model.Repo) (*model.Netrc, error) { - u, err := url.Parse(c.URL) + host, err := common.ExtractHostFromCloneURL(r.Clone) if err != nil { return nil, err } - // remove the port - tmp := strings.Split(u.Host, ":") - host := tmp[0] - if err != nil { - return nil, err - } return &model.Netrc{ - Machine: host, Login: c.Username, Password: c.Password, + Machine: host, }, nil } diff --git a/server/remote/coding/coding.go b/server/remote/coding/coding.go index d7a2a1376..7cf2c620d 100644 --- a/server/remote/coding/coding.go +++ b/server/remote/coding/coding.go @@ -27,6 +27,7 @@ import ( "github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/remote" "github.com/woodpecker-ci/woodpecker/server/remote/coding/internal" + "github.com/woodpecker-ci/woodpecker/server/remote/common" ) const ( @@ -39,7 +40,6 @@ type Opts struct { Client string // Coding oauth client id. Secret string // Coding oauth client secret. Scopes []string // Coding oauth scopes. - Machine string // Optional machine name. Username string // Optional machine account username. Password string // Optional machine account password. SkipVerify bool // Skip ssl verification. @@ -53,7 +53,6 @@ func New(opts Opts) (remote.Remote, error) { Client: opts.Client, Secret: opts.Secret, Scopes: opts.Scopes, - Machine: opts.Machine, Username: opts.Username, Password: opts.Password, SkipVerify: opts.SkipVerify, @@ -70,7 +69,6 @@ type Coding struct { Client string Secret string Scopes []string - Machine string Username string Password string SkipVerify bool @@ -251,17 +249,23 @@ func (c *Coding) Status(ctx context.Context, u *model.User, r *model.Repo, b *mo // Netrc returns a .netrc file that can be used to clone // private repositories from a remote system. func (c *Coding) Netrc(u *model.User, r *model.Repo) (*model.Netrc, error) { + host, err := common.ExtractHostFromCloneURL(r.Clone) + if err != nil { + return nil, err + } + if c.Password != "" { return &model.Netrc{ Login: c.Username, Password: c.Password, - Machine: c.Machine, + Machine: host, }, nil } + return &model.Netrc{ Login: u.Token, Password: "x-oauth-basic", - Machine: c.Machine, + Machine: host, }, nil } diff --git a/server/remote/coding/coding_test.go b/server/remote/coding/coding_test.go index 9cd1b2170..3296fe788 100644 --- a/server/remote/coding/coding_test.go +++ b/server/remote/coding/coding_test.go @@ -48,7 +48,6 @@ func Test_coding(t *testing.T) { Client: "KTNF2ALdm3ofbtxLh6IbV95Ro5AKWJUP", Secret: "zVtxJrKhNhBcNyqCz1NggNAAmehAxnRO3Z0fXmCp", Scopes: []string{"user", "project", "project:depot"}, - Machine: "git.coding.net", Username: "someuser", Password: "password", SkipVerify: true, @@ -57,7 +56,6 @@ func Test_coding(t *testing.T) { g.Assert(remote.(*Coding).Client).Equal("KTNF2ALdm3ofbtxLh6IbV95Ro5AKWJUP") g.Assert(remote.(*Coding).Secret).Equal("zVtxJrKhNhBcNyqCz1NggNAAmehAxnRO3Z0fXmCp") g.Assert(remote.(*Coding).Scopes).Equal([]string{"user", "project", "project:depot"}) - g.Assert(remote.(*Coding).Machine).Equal("git.coding.net") g.Assert(remote.(*Coding).Username).Equal("someuser") g.Assert(remote.(*Coding).Password).Equal("password") g.Assert(remote.(*Coding).SkipVerify).Equal(true) @@ -172,21 +170,18 @@ func Test_coding(t *testing.T) { g.Describe("When requesting a netrc config", func() { g.It("Should return the netrc file for global credential", func() { remote, _ := New(Opts{ - Machine: "git.coding.net", Username: "someuser", Password: "password", }) - netrc, err := remote.Netrc(fakeUser, nil) + netrc, err := remote.Netrc(fakeUser, fakeRepo) g.Assert(err).IsNil() g.Assert(netrc.Login).Equal("someuser") g.Assert(netrc.Password).Equal("password") g.Assert(netrc.Machine).Equal("git.coding.net") }) g.It("Should return the netrc file for specified user", func() { - remote, _ := New(Opts{ - Machine: "git.coding.net", - }) - netrc, err := remote.Netrc(fakeUser, nil) + remote, _ := New(Opts{}) + netrc, err := remote.Netrc(fakeUser, fakeRepo) g.Assert(err).IsNil() g.Assert(netrc.Login).Equal(fakeUser.Token) g.Assert(netrc.Password).Equal("x-oauth-basic") diff --git a/server/remote/common/utils.go b/server/remote/common/utils.go new file mode 100644 index 000000000..224f03b7b --- /dev/null +++ b/server/remote/common/utils.go @@ -0,0 +1,25 @@ +package common + +import ( + "net" + "net/url" + "strings" +) + +func ExtractHostFromCloneURL(cloneURL string) (string, error) { + u, err := url.Parse(cloneURL) + if err != nil { + return "", err + } + + if !strings.Contains(u.Host, ":") { + return u.Host, nil + } + + host, _, err := net.SplitHostPort(u.Host) + if err != nil { + return "", err + } + + return host, nil +} diff --git a/server/remote/common/utils_test.go b/server/remote/common/utils_test.go new file mode 100644 index 000000000..35a400060 --- /dev/null +++ b/server/remote/common/utils_test.go @@ -0,0 +1,18 @@ +package common_test + +import ( + "testing" + + "github.com/woodpecker-ci/woodpecker/server/remote/common" +) + +func Test_Netrc(t *testing.T) { + host, err := common.ExtractHostFromCloneURL("https://git.example.com/foo/bar.git") + if err != nil { + t.Fatal(err) + } + + if host != "git.example.com" { + t.Errorf("Expected host to be git.example.com, got %s", host) + } +} diff --git a/server/remote/gitea/gitea.go b/server/remote/gitea/gitea.go index fb29823b2..fea560dc3 100644 --- a/server/remote/gitea/gitea.go +++ b/server/remote/gitea/gitea.go @@ -44,7 +44,6 @@ const ( type Gitea struct { URL string - Machine string ClientID string ClientSecret string SkipVerify bool @@ -71,7 +70,6 @@ func New(opts Opts) (remote.Remote, error) { } return &Gitea{ URL: opts.URL, - Machine: u.Host, ClientID: opts.Client, ClientSecret: opts.Secret, SkipVerify: opts.SkipVerify, @@ -355,10 +353,15 @@ func (c *Gitea) Netrc(u *model.User, r *model.Repo) (*model.Netrc, error) { token = u.Token } + host, err := common.ExtractHostFromCloneURL(r.Clone) + if err != nil { + return nil, err + } + return &model.Netrc{ Login: login, Password: token, - Machine: c.Machine, + Machine: host, }, nil } diff --git a/server/remote/gitea/gitea_test.go b/server/remote/gitea/gitea_test.go index 6cd2bdf4d..c2c3d7ae5 100644 --- a/server/remote/gitea/gitea_test.go +++ b/server/remote/gitea/gitea_test.go @@ -49,7 +49,6 @@ func Test_gitea(t *testing.T) { SkipVerify: true, }) g.Assert(remote.(*Gitea).URL).Equal("http://localhost:8080") - g.Assert(remote.(*Gitea).Machine).Equal("localhost") g.Assert(remote.(*Gitea).SkipVerify).Equal(true) }) g.It("Should handle malformed url", func() { @@ -60,19 +59,15 @@ func Test_gitea(t *testing.T) { g.Describe("Generating a netrc file", func() { g.It("Should return a netrc with the user token", func() { - remote, _ := New(Opts{ - URL: "http://gitea.com", - }) - netrc, _ := remote.Netrc(fakeUser, nil) + remote, _ := New(Opts{}) + netrc, _ := remote.Netrc(fakeUser, fakeRepo) g.Assert(netrc.Machine).Equal("gitea.com") g.Assert(netrc.Login).Equal(fakeUser.Login) g.Assert(netrc.Password).Equal(fakeUser.Token) }) g.It("Should return a netrc with the machine account", func() { - remote, _ := New(Opts{ - URL: "http://gitea.com", - }) - netrc, _ := remote.Netrc(nil, nil) + remote, _ := New(Opts{}) + netrc, _ := remote.Netrc(nil, fakeRepo) g.Assert(netrc.Machine).Equal("gitea.com") g.Assert(netrc.Login).Equal("") g.Assert(netrc.Password).Equal("") @@ -172,6 +167,7 @@ var ( } fakeRepo = &model.Repo{ + Clone: "http://gitea.com/test_name/repo_name.git", Owner: "test_name", Name: "repo_name", FullName: "test_name/repo_name", diff --git a/server/remote/github/github.go b/server/remote/github/github.go index f4633b50f..90f50b5a0 100644 --- a/server/remote/github/github.go +++ b/server/remote/github/github.go @@ -18,7 +18,6 @@ import ( "context" "crypto/tls" "fmt" - "net" "net/http" "net/url" "regexp" @@ -54,14 +53,6 @@ type Opts struct { // New returns a Remote implementation that integrates with a GitHub Cloud or // GitHub Enterprise version control hosting provider. func New(opts Opts) (remote.Remote, error) { - u, err := url.Parse(opts.URL) - if err != nil { - return nil, err - } - host, _, err := net.SplitHostPort(u.Host) - if err == nil { - u.Host = host - } r := &client{ API: defaultAPI, URL: defaultURL, @@ -69,7 +60,6 @@ func New(opts Opts) (remote.Remote, error) { Secret: opts.Secret, SkipVerify: opts.SkipVerify, MergeRef: opts.MergeRef, - Machine: u.Host, } if opts.URL != defaultURL { r.URL = strings.TrimSuffix(opts.URL, "/") @@ -84,7 +74,6 @@ type client struct { API string Client string Secret string - Machine string SkipVerify bool MergeRef bool } @@ -283,10 +272,15 @@ func (c *client) Netrc(u *model.User, r *model.Repo) (*model.Netrc, error) { token = "x-oauth-basic" } + host, err := common.ExtractHostFromCloneURL(r.Clone) + if err != nil { + return nil, err + } + return &model.Netrc{ Login: login, Password: token, - Machine: c.Machine, + Machine: host, }, nil } diff --git a/server/remote/github/github_test.go b/server/remote/github/github_test.go index 6e0ff430f..4f8bf23c5 100644 --- a/server/remote/github/github_test.go +++ b/server/remote/github/github_test.go @@ -52,32 +52,23 @@ func Test_github(t *testing.T) { }) g.Assert(remote.(*client).URL).Equal("http://localhost:8080") g.Assert(remote.(*client).API).Equal("http://localhost:8080/api/v3/") - g.Assert(remote.(*client).Machine).Equal("localhost") g.Assert(remote.(*client).Client).Equal("0ZXh0IjoiI") g.Assert(remote.(*client).Secret).Equal("I1NiIsInR5") g.Assert(remote.(*client).SkipVerify).Equal(true) }) - g.It("Should handle malformed url", func() { - _, err := New(Opts{URL: "%gh&%ij"}) - g.Assert(err).IsNotNil() - }) }) g.Describe("Generating a netrc file", func() { g.It("Should return a netrc with the user token", func() { - remote, _ := New(Opts{ - URL: "http://github.com:443", - }) - netrc, _ := remote.Netrc(fakeUser, nil) + remote, _ := New(Opts{}) + netrc, _ := remote.Netrc(fakeUser, fakeRepo) g.Assert(netrc.Machine).Equal("github.com") g.Assert(netrc.Login).Equal(fakeUser.Token) g.Assert(netrc.Password).Equal("x-oauth-basic") }) g.It("Should return a netrc with the machine account", func() { - remote, _ := New(Opts{ - URL: "http://github.com:443", - }) - netrc, _ := remote.Netrc(nil, nil) + remote, _ := New(Opts{}) + netrc, _ := remote.Netrc(nil, fakeRepo) g.Assert(netrc.Machine).Equal("github.com") g.Assert(netrc.Login).Equal("") g.Assert(netrc.Password).Equal("") diff --git a/server/remote/gitlab/gitlab.go b/server/remote/gitlab/gitlab.go index 5e54c9887..26f2877a1 100644 --- a/server/remote/gitlab/gitlab.go +++ b/server/remote/gitlab/gitlab.go @@ -19,7 +19,6 @@ import ( "crypto/tls" "fmt" "io/ioutil" - "net" "net/http" "net/url" "strings" @@ -54,7 +53,6 @@ type Gitlab struct { URL string ClientID string ClientSecret string - Machine string SkipVerify bool HideArchives bool Search bool @@ -63,19 +61,10 @@ type Gitlab struct { // New returns a Remote implementation that integrates with Gitlab, an open // source Git service. See https://gitlab.com func New(opts Opts) (remote.Remote, error) { - u, err := url.Parse(opts.URL) - if err != nil { - return nil, err - } - host, _, err := net.SplitHostPort(u.Host) - if err == nil { - u.Host = host - } return &Gitlab{ URL: opts.URL, ClientID: opts.ClientID, ClientSecret: opts.ClientSecret, - Machine: u.Host, SkipVerify: opts.SkipVerify, }, nil } @@ -386,10 +375,15 @@ func (g *Gitlab) Netrc(u *model.User, r *model.Repo) (*model.Netrc, error) { token = u.Token } + host, err := common.ExtractHostFromCloneURL(r.Clone) + if err != nil { + return nil, err + } + return &model.Netrc{ Login: login, Password: token, - Machine: g.Machine, + Machine: host, }, nil } diff --git a/server/remote/gogs/gogs.go b/server/remote/gogs/gogs.go index 553c40d83..b458314dd 100644 --- a/server/remote/gogs/gogs.go +++ b/server/remote/gogs/gogs.go @@ -27,6 +27,7 @@ import ( "github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/remote" + "github.com/woodpecker-ci/woodpecker/server/remote/common" ) // Opts defines configuration options. @@ -40,7 +41,6 @@ type Opts struct { type client struct { URL string - Machine string Username string Password string PrivateMode bool @@ -60,7 +60,6 @@ func New(opts Opts) (remote.Remote, error) { } return &client{ URL: opts.URL, - Machine: u.Host, Username: opts.Username, Password: opts.Password, PrivateMode: opts.PrivateMode, @@ -217,17 +216,22 @@ func (c *client) Status(ctx context.Context, u *model.User, r *model.Repo, b *mo // cloning Gogs repositories. The netrc will use the global machine account // when configured. func (c *client) Netrc(u *model.User, r *model.Repo) (*model.Netrc, error) { + host, err := common.ExtractHostFromCloneURL(r.Clone) + if err != nil { + return nil, err + } + if c.Password != "" { return &model.Netrc{ Login: c.Username, Password: c.Password, - Machine: c.Machine, + Machine: host, }, nil } return &model.Netrc{ Login: u.Token, Password: "x-oauth-basic", - Machine: c.Machine, + Machine: host, }, nil } diff --git a/server/remote/gogs/gogs_test.go b/server/remote/gogs/gogs_test.go index 2b9d5b015..52874ee50 100644 --- a/server/remote/gogs/gogs_test.go +++ b/server/remote/gogs/gogs_test.go @@ -52,7 +52,6 @@ func Test_gogs(t *testing.T) { PrivateMode: true, }) g.Assert(remote.(*client).URL).Equal("http://localhost:8080") - g.Assert(remote.(*client).Machine).Equal("localhost") g.Assert(remote.(*client).Username).Equal("someuser") g.Assert(remote.(*client).Password).Equal("password") g.Assert(remote.(*client).SkipVerify).Equal(true) @@ -66,21 +65,18 @@ func Test_gogs(t *testing.T) { g.Describe("Generating a netrc file", func() { g.It("Should return a netrc with the user token", func() { - remote, _ := New(Opts{ - URL: "http://gogs.com", - }) - netrc, _ := remote.Netrc(fakeUser, nil) + remote, _ := New(Opts{}) + netrc, _ := remote.Netrc(fakeUser, fakeRepo) g.Assert(netrc.Machine).Equal("gogs.com") g.Assert(netrc.Login).Equal(fakeUser.Token) g.Assert(netrc.Password).Equal("x-oauth-basic") }) g.It("Should return a netrc with the machine account", func() { remote, _ := New(Opts{ - URL: "http://gogs.com", Username: "someuser", Password: "password", }) - netrc, _ := remote.Netrc(nil, nil) + netrc, _ := remote.Netrc(nil, fakeRepo) g.Assert(netrc.Machine).Equal("gogs.com") g.Assert(netrc.Login).Equal("someuser") g.Assert(netrc.Password).Equal("password") @@ -185,6 +181,7 @@ var ( } fakeRepo = &model.Repo{ + Clone: "http://gogs.com/test_name/repo_name.git", Owner: "test_name", Name: "repo_name", FullName: "test_name/repo_name",