[feature] Allow users to skip http client tls verification for testing purposes (with appropriately loud warnings) (#2052)

CONTRIBUTING.md

@@ -437,17 +437,21 @@ Although this test *is* part of the CI/CD testing process, you probably won't ne
 
 #### Federation
 
-By using the support for loading TLS files from disk it is possible to have two local instances with TLS to allow for (manually) testing federation.
+By using the support for loading TLS files from disk it is possible to have two or more local instances with TLS to allow for (manually) testing federation.
 
 You'll need to set the following configuration options:
-* `GTS_TLS_CERTIFICATE_CHAIN`: poiting to a PEM-encoded certificate chain including the public certificate
+
-* `GTS_TLS_CERTIFICATE_KEY`: pointing to a PEM-encoded private key
+- `GTS_TLS_CERTIFICATE_CHAIN`: poiting to a PEM-encoded certificate chain including the public certificate.
+- `GTS_TLS_CERTIFICATE_KEY`: pointing to a PEM-encoded private key.
 
 Additionally, for the Go HTTP client to recognise certificates issued by a custom CA as valid, you'll need to set one of:
-* `SSL_CERT_FILE`: pointing to the public key of your custom CA
-* `SSL_CERT_DIR`: a `:`-separated list of directories to load CA certificates from
 
-You'll additionally need functioning DNS for your two instance names which you can achieve through entries in `/etc/hosts` or by running a local DNS server like [dnsmasq](https://thekelleys.org.uk/dnsmasq/doc.html).
+- `SSL_CERT_FILE`: pointing to the public key of your custom CA.
+- `SSL_CERT_DIR`: a `:`-separated list of directories to load CA certificates from.
+
+The above `SSL_CERT` variables work on Unix-like systems only, excluding Mac. See https://pkg.go.dev/crypto/x509#SystemCertPool. If you are running your tests on an architecture that doesn't support setting the above variables, you can instead disable TLS certificate verification for the HTTP client entirely by setting `http-client.tls-insecure-skip-verify` to `true` in the config.yaml file.
+
+You'll additionally need functioning DNS for your two instance names, which you can achieve through entries in `/etc/hosts` or by running a local DNS server like [dnsmasq](https://thekelleys.org.uk/dnsmasq/doc.html).
 
 ### Updating Swagger docs
 

cmd/gotosocial/action/server/server.go

@@ -107,9 +107,10 @@ var Start action.GTSAction = func(ctx context.Context) error {
 
 	// Build HTTP client
 	client := httpclient.New(httpclient.Config{
-		AllowRanges: config.MustParseIPPrefixes(config.GetHTTPClientAllowIPs()),
+		AllowRanges:           config.MustParseIPPrefixes(config.GetHTTPClientAllowIPs()),
-		BlockRanges: config.MustParseIPPrefixes(config.GetHTTPClientBlockIPs()),
+		BlockRanges:           config.MustParseIPPrefixes(config.GetHTTPClientBlockIPs()),
-		Timeout:     config.GetHTTPClientTimeout(),
+		Timeout:               config.GetHTTPClientTimeout(),
+		TLSInsecureSkipVerify: config.GetHTTPClientTLSInsecureSkipVerify(),
 	})
 
 	// Initialize workers.

docs/configuration/httpclient.md

@@ -53,4 +53,16 @@ http-client:
   # Both allow-ips and block-ips default to an empty array.
   allow-ips: []
   block-ips: []
+
+  # Bool. Disable verification of TLS certificates of remote servers.
+  # With this set to 'true', GoToSocial will not error when a remote
+  # server presents an invalid or self-signed certificate.
+  #
+  # THIS SETTING SHOULD BE USED FOR TESTING ONLY! IF YOU TURN THIS
+  # ON WHILE RUNNING IN PRODUCTION YOU ARE LEAVING YOUR SERVER WIDE
+  # OPEN TO MAN IN THE MIDDLE ATTACKS! DO NOT CHANGE THIS SETTING 
+  # UNLESS YOU KNOW EXACTLY WHAT YOU'RE DOING AND WHY YOU'RE DOING IT.
+  #
+  # Default: false
+  tls-insecure-skip-verify: false
 ```

example/config.yaml

@@ -880,6 +880,18 @@ http-client:
   allow-ips: []
   block-ips: []
 
+  # Bool. Disable verification of TLS certificates of remote servers.
+  # With this set to 'true', GoToSocial will not error when a remote
+  # server presents an invalid or self-signed certificate.
+  #
+  # THIS SETTING SHOULD BE USED FOR TESTING ONLY! IF YOU TURN THIS
+  # ON WHILE RUNNING IN PRODUCTION YOU ARE LEAVING YOUR SERVER WIDE
+  # OPEN TO MAN IN THE MIDDLE ATTACKS! DO NOT CHANGE THIS SETTING 
+  # UNLESS YOU KNOW EXACTLY WHAT YOU'RE DOING AND WHY YOU'RE DOING IT.
+  #
+  # Default: false
+  tls-insecure-skip-verify: false
+
 #############################
 ##### ADVANCED SETTINGS #####
 #############################

internal/config/config.go

@@ -168,9 +168,10 @@ type Configuration struct {
 }
 
 type HTTPClientConfiguration struct {
-	AllowIPs []string      `name:"allow-ips"`
+	AllowIPs              []string      `name:"allow-ips"`
-	BlockIPs []string      `name:"block-ips"`
+	BlockIPs              []string      `name:"block-ips"`
-	Timeout  time.Duration `name:"timeout"`
+	Timeout               time.Duration `name:"timeout"`
+	TLSInsecureSkipVerify bool          `name:"tls-insecure-skip-verify"`
 }
 
 type CacheConfiguration struct {

internal/config/defaults.go

@@ -234,9 +234,10 @@ var Defaults = Configuration{
 	},
 
 	HTTPClient: HTTPClientConfiguration{
-		AllowIPs: make([]string, 0),
+		AllowIPs:              make([]string, 0),
-		BlockIPs: make([]string, 0),
+		BlockIPs:              make([]string, 0),
-		Timeout:  10 * time.Second,
+		Timeout:               10 * time.Second,
+		TLSInsecureSkipVerify: false,
 	},
 
 	AdminMediaPruneDryRun: true,

internal/config/flags.go

@@ -60,6 +60,7 @@ func (s *ConfigState) AddGlobalFlags(cmd *cobra.Command) {
 		cmd.PersistentFlags().StringSlice(HTTPClientAllowIPsFlag(), cfg.HTTPClient.AllowIPs, "no usage string")
 		cmd.PersistentFlags().StringSlice(HTTPClientBlockIPsFlag(), cfg.HTTPClient.BlockIPs, "no usage string")
 		cmd.PersistentFlags().Duration(HTTPClientTimeoutFlag(), cfg.HTTPClient.Timeout, "no usage string")
+		cmd.PersistentFlags().Bool(HTTPClientTLSInsecureSkipVerifyFlag(), cfg.HTTPClient.TLSInsecureSkipVerify, "no usage string")
 	})
 }
 

internal/config/helpers.gen.go

@@ -2399,6 +2399,31 @@ func GetHTTPClientTimeout() time.Duration { return global.GetHTTPClientTimeout()
 // SetHTTPClientTimeout safely sets the value for global configuration 'HTTPClient.Timeout' field
 func SetHTTPClientTimeout(v time.Duration) { global.SetHTTPClientTimeout(v) }
 
+// GetHTTPClientTLSInsecureSkipVerify safely fetches the Configuration value for state's 'HTTPClient.TLSInsecureSkipVerify' field
+func (st *ConfigState) GetHTTPClientTLSInsecureSkipVerify() (v bool) {
+	st.mutex.RLock()
+	v = st.config.HTTPClient.TLSInsecureSkipVerify
+	st.mutex.RUnlock()
+	return
+}
+
+// SetHTTPClientTLSInsecureSkipVerify safely sets the Configuration value for state's 'HTTPClient.TLSInsecureSkipVerify' field
+func (st *ConfigState) SetHTTPClientTLSInsecureSkipVerify(v bool) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.HTTPClient.TLSInsecureSkipVerify = v
+	st.reloadToViper()
+}
+
+// HTTPClientTLSInsecureSkipVerifyFlag returns the flag name for the 'HTTPClient.TLSInsecureSkipVerify' field
+func HTTPClientTLSInsecureSkipVerifyFlag() string { return "httpclient-tls-insecure-skip-verify" }
+
+// GetHTTPClientTLSInsecureSkipVerify safely fetches the value for global configuration 'HTTPClient.TLSInsecureSkipVerify' field
+func GetHTTPClientTLSInsecureSkipVerify() bool { return global.GetHTTPClientTLSInsecureSkipVerify() }
+
+// SetHTTPClientTLSInsecureSkipVerify safely sets the value for global configuration 'HTTPClient.TLSInsecureSkipVerify' field
+func SetHTTPClientTLSInsecureSkipVerify(v bool) { global.SetHTTPClientTLSInsecureSkipVerify(v) }
+
 // GetCacheGTSAccountMaxSize safely fetches the Configuration value for state's 'Cache.GTS.AccountMaxSize' field
 func (st *ConfigState) GetCacheGTSAccountMaxSize() (v int) {
 	st.mutex.RLock()

internal/httpclient/client.go

@@ -19,6 +19,7 @@ package httpclient
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
@@ -86,6 +87,14 @@ type Config struct {
 
 	// BlockRanges blocks outgoing communiciations to given IP nets.
 	BlockRanges []netip.Prefix
+
+	// TLSInsecureSkipVerify can be set to true to
+	// skip validation of remote TLS certificates.
+	//
+	// THIS SHOULD BE USED FOR TESTING ONLY, IF YOU
+	// TURN THIS ON WHILE RUNNING IN PRODUCTION YOU
+	// ARE LEAVING YOUR SERVER WIDE OPEN TO ATTACKS!
+	TLSInsecureSkipVerify bool
 }
 
 // Client wraps an underlying http.Client{} to provide the following:
@@ -139,11 +148,26 @@ func New(cfg Config) *Client {
 	c.client.Timeout = cfg.Timeout
 	c.bodyMax = cfg.MaxBodySize
 
+	// Prepare TLS config for transport.
+	tlsClientConfig := &tls.Config{
+		InsecureSkipVerify: cfg.TLSInsecureSkipVerify, //nolint:gosec
+	}
+
+	if tlsClientConfig.InsecureSkipVerify {
+		// Warn against playing silly buggers.
+		log.Warn(nil, "http-client.tls-insecure-skip-verify was set to TRUE. "+
+			"*****THIS SHOULD BE USED FOR TESTING ONLY, IF YOU TURN THIS ON WHILE "+
+			"RUNNING IN PRODUCTION YOU ARE LEAVING YOUR SERVER WIDE OPEN TO ATTACKS! "+
+			"IF IN DOUBT, STOP YOUR SERVER *NOW* AND ADJUST YOUR CONFIGURATION!*****",
+		)
+	}
+
 	// Set underlying HTTP client roundtripper.
 	c.client.Transport = &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
 		ForceAttemptHTTP2:     true,
 		DialContext:           d.DialContext,
+		TLSClientConfig:       tlsClientConfig,
 		MaxIdleConns:          cfg.MaxIdleConns,
 		IdleConnTimeout:       90 * time.Second,
 		TLSHandshakeTimeout:   10 * time.Second,

test/envparsing.sh

@@ -119,7 +119,8 @@ EXPECT=$(cat <<"EOF"
     "http-client": {
         "allow-ips": [],
         "block-ips": [],
-        "timeout": 10000000000
+        "timeout": 10000000000,
+        "tls-insecure-skip-verify": false
     },
     "instance-deliver-to-shared-inboxes": false,
     "instance-expose-peers": true,