From 2be83fdca5c440d45b8cd92bda9315757463d6c7 Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Tue, 1 Aug 2023 19:50:17 +0200 Subject: [PATCH] [feature] Allow users to skip http client tls verification for testing purposes (with appropriately loud warnings) (#2052) --- CONTRIBUTING.md | 16 ++++++++++------ cmd/gotosocial/action/server/server.go | 7 ++++--- docs/configuration/httpclient.md | 12 ++++++++++++ example/config.yaml | 12 ++++++++++++ internal/config/config.go | 7 ++++--- internal/config/defaults.go | 7 ++++--- internal/config/flags.go | 1 + internal/config/helpers.gen.go | 25 +++++++++++++++++++++++++ internal/httpclient/client.go | 24 ++++++++++++++++++++++++ test/envparsing.sh | 3 ++- 10 files changed, 98 insertions(+), 16 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 853bf614..6738a99d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -437,17 +437,21 @@ Although this test *is* part of the CI/CD testing process, you probably won't ne #### Federation -By using the support for loading TLS files from disk it is possible to have two local instances with TLS to allow for (manually) testing federation. +By using the support for loading TLS files from disk it is possible to have two or more local instances with TLS to allow for (manually) testing federation. You'll need to set the following configuration options: -* `GTS_TLS_CERTIFICATE_CHAIN`: poiting to a PEM-encoded certificate chain including the public certificate -* `GTS_TLS_CERTIFICATE_KEY`: pointing to a PEM-encoded private key + +- `GTS_TLS_CERTIFICATE_CHAIN`: poiting to a PEM-encoded certificate chain including the public certificate. +- `GTS_TLS_CERTIFICATE_KEY`: pointing to a PEM-encoded private key. Additionally, for the Go HTTP client to recognise certificates issued by a custom CA as valid, you'll need to set one of: -* `SSL_CERT_FILE`: pointing to the public key of your custom CA -* `SSL_CERT_DIR`: a `:`-separated list of directories to load CA certificates from -You'll additionally need functioning DNS for your two instance names which you can achieve through entries in `/etc/hosts` or by running a local DNS server like [dnsmasq](https://thekelleys.org.uk/dnsmasq/doc.html). +- `SSL_CERT_FILE`: pointing to the public key of your custom CA. +- `SSL_CERT_DIR`: a `:`-separated list of directories to load CA certificates from. + +The above `SSL_CERT` variables work on Unix-like systems only, excluding Mac. See https://pkg.go.dev/crypto/x509#SystemCertPool. If you are running your tests on an architecture that doesn't support setting the above variables, you can instead disable TLS certificate verification for the HTTP client entirely by setting `http-client.tls-insecure-skip-verify` to `true` in the config.yaml file. + +You'll additionally need functioning DNS for your two instance names, which you can achieve through entries in `/etc/hosts` or by running a local DNS server like [dnsmasq](https://thekelleys.org.uk/dnsmasq/doc.html). ### Updating Swagger docs diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index cdcedd47..ad1fe576 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -107,9 +107,10 @@ var Start action.GTSAction = func(ctx context.Context) error { // Build HTTP client client := httpclient.New(httpclient.Config{ - AllowRanges: config.MustParseIPPrefixes(config.GetHTTPClientAllowIPs()), - BlockRanges: config.MustParseIPPrefixes(config.GetHTTPClientBlockIPs()), - Timeout: config.GetHTTPClientTimeout(), + AllowRanges: config.MustParseIPPrefixes(config.GetHTTPClientAllowIPs()), + BlockRanges: config.MustParseIPPrefixes(config.GetHTTPClientBlockIPs()), + Timeout: config.GetHTTPClientTimeout(), + TLSInsecureSkipVerify: config.GetHTTPClientTLSInsecureSkipVerify(), }) // Initialize workers. diff --git a/docs/configuration/httpclient.md b/docs/configuration/httpclient.md index 1fcf2d06..b76f434d 100644 --- a/docs/configuration/httpclient.md +++ b/docs/configuration/httpclient.md @@ -53,4 +53,16 @@ http-client: # Both allow-ips and block-ips default to an empty array. allow-ips: [] block-ips: [] + + # Bool. Disable verification of TLS certificates of remote servers. + # With this set to 'true', GoToSocial will not error when a remote + # server presents an invalid or self-signed certificate. + # + # THIS SETTING SHOULD BE USED FOR TESTING ONLY! IF YOU TURN THIS + # ON WHILE RUNNING IN PRODUCTION YOU ARE LEAVING YOUR SERVER WIDE + # OPEN TO MAN IN THE MIDDLE ATTACKS! DO NOT CHANGE THIS SETTING + # UNLESS YOU KNOW EXACTLY WHAT YOU'RE DOING AND WHY YOU'RE DOING IT. + # + # Default: false + tls-insecure-skip-verify: false ``` diff --git a/example/config.yaml b/example/config.yaml index ceef6dc9..b0956147 100644 --- a/example/config.yaml +++ b/example/config.yaml @@ -880,6 +880,18 @@ http-client: allow-ips: [] block-ips: [] + # Bool. Disable verification of TLS certificates of remote servers. + # With this set to 'true', GoToSocial will not error when a remote + # server presents an invalid or self-signed certificate. + # + # THIS SETTING SHOULD BE USED FOR TESTING ONLY! IF YOU TURN THIS + # ON WHILE RUNNING IN PRODUCTION YOU ARE LEAVING YOUR SERVER WIDE + # OPEN TO MAN IN THE MIDDLE ATTACKS! DO NOT CHANGE THIS SETTING + # UNLESS YOU KNOW EXACTLY WHAT YOU'RE DOING AND WHY YOU'RE DOING IT. + # + # Default: false + tls-insecure-skip-verify: false + ############################# ##### ADVANCED SETTINGS ##### ############################# diff --git a/internal/config/config.go b/internal/config/config.go index 9397379b..7f09b5fc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -168,9 +168,10 @@ type Configuration struct { } type HTTPClientConfiguration struct { - AllowIPs []string `name:"allow-ips"` - BlockIPs []string `name:"block-ips"` - Timeout time.Duration `name:"timeout"` + AllowIPs []string `name:"allow-ips"` + BlockIPs []string `name:"block-ips"` + Timeout time.Duration `name:"timeout"` + TLSInsecureSkipVerify bool `name:"tls-insecure-skip-verify"` } type CacheConfiguration struct { diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 7729840f..e8cb3932 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -234,9 +234,10 @@ var Defaults = Configuration{ }, HTTPClient: HTTPClientConfiguration{ - AllowIPs: make([]string, 0), - BlockIPs: make([]string, 0), - Timeout: 10 * time.Second, + AllowIPs: make([]string, 0), + BlockIPs: make([]string, 0), + Timeout: 10 * time.Second, + TLSInsecureSkipVerify: false, }, AdminMediaPruneDryRun: true, diff --git a/internal/config/flags.go b/internal/config/flags.go index c42b5c7b..32140025 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -60,6 +60,7 @@ func (s *ConfigState) AddGlobalFlags(cmd *cobra.Command) { cmd.PersistentFlags().StringSlice(HTTPClientAllowIPsFlag(), cfg.HTTPClient.AllowIPs, "no usage string") cmd.PersistentFlags().StringSlice(HTTPClientBlockIPsFlag(), cfg.HTTPClient.BlockIPs, "no usage string") cmd.PersistentFlags().Duration(HTTPClientTimeoutFlag(), cfg.HTTPClient.Timeout, "no usage string") + cmd.PersistentFlags().Bool(HTTPClientTLSInsecureSkipVerifyFlag(), cfg.HTTPClient.TLSInsecureSkipVerify, "no usage string") }) } diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index e4b82edd..4c2f1d05 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -2399,6 +2399,31 @@ func GetHTTPClientTimeout() time.Duration { return global.GetHTTPClientTimeout() // SetHTTPClientTimeout safely sets the value for global configuration 'HTTPClient.Timeout' field func SetHTTPClientTimeout(v time.Duration) { global.SetHTTPClientTimeout(v) } +// GetHTTPClientTLSInsecureSkipVerify safely fetches the Configuration value for state's 'HTTPClient.TLSInsecureSkipVerify' field +func (st *ConfigState) GetHTTPClientTLSInsecureSkipVerify() (v bool) { + st.mutex.RLock() + v = st.config.HTTPClient.TLSInsecureSkipVerify + st.mutex.RUnlock() + return +} + +// SetHTTPClientTLSInsecureSkipVerify safely sets the Configuration value for state's 'HTTPClient.TLSInsecureSkipVerify' field +func (st *ConfigState) SetHTTPClientTLSInsecureSkipVerify(v bool) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.HTTPClient.TLSInsecureSkipVerify = v + st.reloadToViper() +} + +// HTTPClientTLSInsecureSkipVerifyFlag returns the flag name for the 'HTTPClient.TLSInsecureSkipVerify' field +func HTTPClientTLSInsecureSkipVerifyFlag() string { return "httpclient-tls-insecure-skip-verify" } + +// GetHTTPClientTLSInsecureSkipVerify safely fetches the value for global configuration 'HTTPClient.TLSInsecureSkipVerify' field +func GetHTTPClientTLSInsecureSkipVerify() bool { return global.GetHTTPClientTLSInsecureSkipVerify() } + +// SetHTTPClientTLSInsecureSkipVerify safely sets the value for global configuration 'HTTPClient.TLSInsecureSkipVerify' field +func SetHTTPClientTLSInsecureSkipVerify(v bool) { global.SetHTTPClientTLSInsecureSkipVerify(v) } + // GetCacheGTSAccountMaxSize safely fetches the Configuration value for state's 'Cache.GTS.AccountMaxSize' field func (st *ConfigState) GetCacheGTSAccountMaxSize() (v int) { st.mutex.RLock() diff --git a/internal/httpclient/client.go b/internal/httpclient/client.go index 18bbe1ee..f5701d6f 100644 --- a/internal/httpclient/client.go +++ b/internal/httpclient/client.go @@ -19,6 +19,7 @@ package httpclient import ( "context" + "crypto/tls" "errors" "fmt" "io" @@ -86,6 +87,14 @@ type Config struct { // BlockRanges blocks outgoing communiciations to given IP nets. BlockRanges []netip.Prefix + + // TLSInsecureSkipVerify can be set to true to + // skip validation of remote TLS certificates. + // + // THIS SHOULD BE USED FOR TESTING ONLY, IF YOU + // TURN THIS ON WHILE RUNNING IN PRODUCTION YOU + // ARE LEAVING YOUR SERVER WIDE OPEN TO ATTACKS! + TLSInsecureSkipVerify bool } // Client wraps an underlying http.Client{} to provide the following: @@ -139,11 +148,26 @@ func New(cfg Config) *Client { c.client.Timeout = cfg.Timeout c.bodyMax = cfg.MaxBodySize + // Prepare TLS config for transport. + tlsClientConfig := &tls.Config{ + InsecureSkipVerify: cfg.TLSInsecureSkipVerify, //nolint:gosec + } + + if tlsClientConfig.InsecureSkipVerify { + // Warn against playing silly buggers. + log.Warn(nil, "http-client.tls-insecure-skip-verify was set to TRUE. "+ + "*****THIS SHOULD BE USED FOR TESTING ONLY, IF YOU TURN THIS ON WHILE "+ + "RUNNING IN PRODUCTION YOU ARE LEAVING YOUR SERVER WIDE OPEN TO ATTACKS! "+ + "IF IN DOUBT, STOP YOUR SERVER *NOW* AND ADJUST YOUR CONFIGURATION!*****", + ) + } + // Set underlying HTTP client roundtripper. c.client.Transport = &http.Transport{ Proxy: http.ProxyFromEnvironment, ForceAttemptHTTP2: true, DialContext: d.DialContext, + TLSClientConfig: tlsClientConfig, MaxIdleConns: cfg.MaxIdleConns, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, diff --git a/test/envparsing.sh b/test/envparsing.sh index 6506d0f2..14b265b3 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -119,7 +119,8 @@ EXPECT=$(cat <<"EOF" "http-client": { "allow-ips": [], "block-ips": [], - "timeout": 10000000000 + "timeout": 10000000000, + "tls-insecure-skip-verify": false }, "instance-deliver-to-shared-inboxes": false, "instance-expose-peers": true,