2023-03-12 15:00:57 +00:00
|
|
|
// GoToSocial
|
|
|
|
// Copyright (C) GoToSocial Authors admin@gotosocial.org
|
|
|
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
//
|
|
|
|
// This program is free software: you can redistribute it and/or modify
|
|
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
|
|
// (at your option) any later version.
|
|
|
|
//
|
|
|
|
// This program is distributed in the hope that it will be useful,
|
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
// GNU Affero General Public License for more details.
|
|
|
|
//
|
|
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
2022-05-15 09:16:43 +00:00
|
|
|
|
|
|
|
package httpclient
|
|
|
|
|
|
|
|
import (
|
2023-04-28 15:45:21 +00:00
|
|
|
"context"
|
|
|
|
"crypto/x509"
|
2022-05-15 09:16:43 +00:00
|
|
|
"errors"
|
2023-04-28 15:45:21 +00:00
|
|
|
"fmt"
|
2022-05-15 09:16:43 +00:00
|
|
|
"io"
|
|
|
|
"net"
|
|
|
|
"net/http"
|
|
|
|
"net/netip"
|
|
|
|
"runtime"
|
2023-04-28 15:45:21 +00:00
|
|
|
"strconv"
|
|
|
|
"strings"
|
2022-05-15 09:16:43 +00:00
|
|
|
"time"
|
2022-11-08 09:35:24 +00:00
|
|
|
|
|
|
|
"codeberg.org/gruf/go-bytesize"
|
2023-04-28 15:45:21 +00:00
|
|
|
"codeberg.org/gruf/go-byteutil"
|
|
|
|
"codeberg.org/gruf/go-cache/v3"
|
|
|
|
errorsv2 "codeberg.org/gruf/go-errors/v2"
|
2022-11-08 09:35:24 +00:00
|
|
|
"codeberg.org/gruf/go-kv"
|
2023-04-28 15:45:21 +00:00
|
|
|
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
|
|
|
|
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
|
2022-11-08 09:35:24 +00:00
|
|
|
"github.com/superseriousbusiness/gotosocial/internal/log"
|
2022-05-15 09:16:43 +00:00
|
|
|
)
|
|
|
|
|
2023-04-28 15:45:21 +00:00
|
|
|
var (
|
|
|
|
// ErrInvalidNetwork is returned if the request would not be performed over TCP
|
|
|
|
ErrInvalidNetwork = errors.New("invalid network type")
|
2022-05-26 11:38:41 +00:00
|
|
|
|
2023-04-28 15:45:21 +00:00
|
|
|
// ErrReservedAddr is returned if a dialed address resolves to an IP within a blocked or reserved net.
|
|
|
|
ErrReservedAddr = errors.New("dial within blocked / reserved IP range")
|
2022-11-26 11:09:55 +00:00
|
|
|
|
2023-04-28 15:45:21 +00:00
|
|
|
// ErrBodyTooLarge is returned when a received response body is above predefined limit (default 40MB).
|
|
|
|
ErrBodyTooLarge = errors.New("body size too large")
|
|
|
|
)
|
2022-05-15 09:16:43 +00:00
|
|
|
|
|
|
|
// Config provides configuration details for setting up a new
|
|
|
|
// instance of httpclient.Client{}. Within are a subset of the
|
|
|
|
// configuration values passed to initialized http.Transport{}
|
|
|
|
// and http.Client{}, along with httpclient.Client{} specific.
|
|
|
|
type Config struct {
|
2022-11-08 09:35:24 +00:00
|
|
|
// MaxOpenConnsPerHost limits the max number of open connections to a host.
|
|
|
|
MaxOpenConnsPerHost int
|
2022-05-15 09:16:43 +00:00
|
|
|
|
|
|
|
// MaxIdleConns: see http.Transport{}.MaxIdleConns.
|
|
|
|
MaxIdleConns int
|
|
|
|
|
|
|
|
// ReadBufferSize: see http.Transport{}.ReadBufferSize.
|
|
|
|
ReadBufferSize int
|
|
|
|
|
|
|
|
// WriteBufferSize: see http.Transport{}.WriteBufferSize.
|
|
|
|
WriteBufferSize int
|
|
|
|
|
|
|
|
// MaxBodySize determines the maximum fetchable body size.
|
|
|
|
MaxBodySize int64
|
|
|
|
|
|
|
|
// Timeout: see http.Client{}.Timeout.
|
|
|
|
Timeout time.Duration
|
|
|
|
|
|
|
|
// DisableCompression: see http.Transport{}.DisableCompression.
|
|
|
|
DisableCompression bool
|
|
|
|
|
|
|
|
// AllowRanges allows outgoing communications to given IP nets.
|
|
|
|
AllowRanges []netip.Prefix
|
|
|
|
|
|
|
|
// BlockRanges blocks outgoing communiciations to given IP nets.
|
|
|
|
BlockRanges []netip.Prefix
|
|
|
|
}
|
|
|
|
|
|
|
|
// Client wraps an underlying http.Client{} to provide the following:
|
2022-09-28 17:30:40 +00:00
|
|
|
// - setting a maximum received request body size, returning error on
|
|
|
|
// large content lengths, and using a limited reader in all other
|
|
|
|
// cases to protect against forged / unknown content-lengths
|
|
|
|
// - protection from server side request forgery (SSRF) by only dialing
|
|
|
|
// out to known public IP prefixes, configurable with allows/blocks
|
2022-05-15 09:16:43 +00:00
|
|
|
type Client struct {
|
2023-04-28 15:45:21 +00:00
|
|
|
client http.Client
|
|
|
|
badHosts cache.Cache[string, struct{}]
|
|
|
|
bodyMax int64
|
2022-05-15 09:16:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// New returns a new instance of Client initialized using configuration.
|
|
|
|
func New(cfg Config) *Client {
|
|
|
|
var c Client
|
|
|
|
|
2022-07-18 09:25:26 +00:00
|
|
|
d := &net.Dialer{
|
2022-11-23 21:40:07 +00:00
|
|
|
Timeout: 15 * time.Second,
|
2022-07-18 09:25:26 +00:00
|
|
|
KeepAlive: 30 * time.Second,
|
|
|
|
Resolver: &net.Resolver{},
|
|
|
|
}
|
2022-05-15 09:16:43 +00:00
|
|
|
|
2022-11-08 09:35:24 +00:00
|
|
|
if cfg.MaxOpenConnsPerHost <= 0 {
|
2022-05-15 09:16:43 +00:00
|
|
|
// By default base this value on GOMAXPROCS.
|
|
|
|
maxprocs := runtime.GOMAXPROCS(0)
|
2022-11-08 09:35:24 +00:00
|
|
|
cfg.MaxOpenConnsPerHost = maxprocs * 20
|
2022-05-15 09:16:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if cfg.MaxIdleConns <= 0 {
|
2023-04-28 15:45:21 +00:00
|
|
|
// By default base this value on MaxOpenConns.
|
2022-11-08 09:35:24 +00:00
|
|
|
cfg.MaxIdleConns = cfg.MaxOpenConnsPerHost * 10
|
2022-05-15 09:16:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if cfg.MaxBodySize <= 0 {
|
2023-04-28 15:45:21 +00:00
|
|
|
// By default set this to a reasonable 40MB.
|
2022-11-08 09:35:24 +00:00
|
|
|
cfg.MaxBodySize = int64(40 * bytesize.MiB)
|
2022-05-15 09:16:43 +00:00
|
|
|
}
|
|
|
|
|
2023-04-28 15:45:21 +00:00
|
|
|
// Protect dialer with IP range sanitizer.
|
2022-05-15 09:16:43 +00:00
|
|
|
d.Control = (&sanitizer{
|
|
|
|
allow: cfg.AllowRanges,
|
|
|
|
block: cfg.BlockRanges,
|
|
|
|
}).Sanitize
|
|
|
|
|
2023-04-28 15:45:21 +00:00
|
|
|
// Prepare client fields.
|
2022-05-15 09:16:43 +00:00
|
|
|
c.client.Timeout = cfg.Timeout
|
2023-04-28 15:45:21 +00:00
|
|
|
c.bodyMax = cfg.MaxBodySize
|
2022-05-15 09:16:43 +00:00
|
|
|
|
2023-04-28 15:45:21 +00:00
|
|
|
// Set underlying HTTP client roundtripper.
|
2022-05-15 09:16:43 +00:00
|
|
|
c.client.Transport = &http.Transport{
|
|
|
|
Proxy: http.ProxyFromEnvironment,
|
|
|
|
ForceAttemptHTTP2: true,
|
|
|
|
DialContext: d.DialContext,
|
|
|
|
MaxIdleConns: cfg.MaxIdleConns,
|
|
|
|
IdleConnTimeout: 90 * time.Second,
|
|
|
|
TLSHandshakeTimeout: 10 * time.Second,
|
|
|
|
ExpectContinueTimeout: 1 * time.Second,
|
|
|
|
ReadBufferSize: cfg.ReadBufferSize,
|
|
|
|
WriteBufferSize: cfg.WriteBufferSize,
|
|
|
|
DisableCompression: cfg.DisableCompression,
|
|
|
|
}
|
|
|
|
|
2023-04-28 15:45:21 +00:00
|
|
|
// Initiate outgoing bad hosts lookup cache.
|
|
|
|
c.badHosts = cache.New[string, struct{}](0, 1000, 0)
|
|
|
|
c.badHosts.SetTTL(15*time.Minute, false)
|
|
|
|
if !c.badHosts.Start(time.Minute) {
|
|
|
|
log.Panic(nil, "failed to start transport controller cache")
|
|
|
|
}
|
|
|
|
|
2022-05-15 09:16:43 +00:00
|
|
|
return &c
|
|
|
|
}
|
|
|
|
|
2023-04-28 15:45:21 +00:00
|
|
|
// Do ...
|
|
|
|
func (c *Client) Do(r *http.Request) (*http.Response, error) {
|
|
|
|
return c.DoSigned(r, func(r *http.Request) error {
|
|
|
|
return nil // no request signing
|
|
|
|
})
|
|
|
|
}
|
2023-02-19 11:01:15 +00:00
|
|
|
|
2023-04-28 15:45:21 +00:00
|
|
|
// DoSigned ...
|
|
|
|
func (c *Client) DoSigned(r *http.Request, sign SignFunc) (*http.Response, error) {
|
|
|
|
const (
|
|
|
|
// max no. attempts.
|
|
|
|
maxRetries = 5
|
|
|
|
|
|
|
|
// starting backoff duration.
|
|
|
|
baseBackoff = 2 * time.Second
|
|
|
|
)
|
|
|
|
|
|
|
|
// Get request hostname.
|
|
|
|
host := r.URL.Hostname()
|
|
|
|
|
|
|
|
// Check whether request should fast fail.
|
|
|
|
fastFail := gtscontext.IsFastfail(r.Context())
|
|
|
|
if !fastFail {
|
|
|
|
// Check if recently reached max retries for this host
|
|
|
|
// so we don't bother with a retry-backoff loop. The only
|
|
|
|
// errors that are retried upon are server failure and
|
|
|
|
// domain resolution type errors, so this cached result
|
|
|
|
// indicates this server is likely having issues.
|
|
|
|
fastFail = c.badHosts.Has(host)
|
2022-11-08 09:35:24 +00:00
|
|
|
}
|
|
|
|
|
2023-04-28 15:45:21 +00:00
|
|
|
// Start a log entry for this request
|
|
|
|
l := log.WithContext(r.Context()).
|
|
|
|
WithFields(kv.Fields{
|
|
|
|
{"method", r.Method},
|
|
|
|
{"url", r.URL.String()},
|
|
|
|
}...)
|
|
|
|
|
|
|
|
for i := 0; i < maxRetries; i++ {
|
|
|
|
var backoff time.Duration
|
|
|
|
|
|
|
|
// Reset signing header fields
|
|
|
|
now := time.Now().UTC()
|
|
|
|
r.Header.Set("Date", now.Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
|
|
|
|
r.Header.Del("Signature")
|
|
|
|
r.Header.Del("Digest")
|
|
|
|
|
|
|
|
// Rewind body reader and content-length if set.
|
|
|
|
if rc, ok := r.Body.(*byteutil.ReadNopCloser); ok {
|
|
|
|
r.ContentLength = int64(rc.Len())
|
|
|
|
rc.Rewind()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Sign the outgoing request.
|
|
|
|
if err := sign(r); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
l.Infof("performing request")
|
|
|
|
|
|
|
|
// Perform the request.
|
|
|
|
rsp, err := c.do(r)
|
|
|
|
if err == nil { //nolint:gocritic
|
|
|
|
|
|
|
|
// TooManyRequest means we need to slow
|
|
|
|
// down and retry our request. Codes over
|
|
|
|
// 500 generally indicate temp. outages.
|
|
|
|
if code := rsp.StatusCode; code < 500 &&
|
|
|
|
code != http.StatusTooManyRequests {
|
|
|
|
return rsp, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Generate error from status code for logging
|
|
|
|
err = errors.New(`http response "` + rsp.Status + `"`)
|
|
|
|
|
|
|
|
// Search for a provided "Retry-After" header value.
|
|
|
|
if after := rsp.Header.Get("Retry-After"); after != "" {
|
|
|
|
|
|
|
|
if u, _ := strconv.ParseUint(after, 10, 32); u != 0 {
|
|
|
|
// An integer number of backoff seconds was provided.
|
|
|
|
backoff = time.Duration(u) * time.Second
|
|
|
|
} else if at, _ := http.ParseTime(after); !at.Before(now) {
|
|
|
|
// An HTTP formatted future date-time was provided.
|
|
|
|
backoff = at.Sub(now)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Don't let their provided backoff exceed our max.
|
|
|
|
if max := baseBackoff * maxRetries; backoff > max {
|
|
|
|
backoff = max
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
} else if errorsv2.Is(err,
|
|
|
|
context.DeadlineExceeded,
|
|
|
|
context.Canceled,
|
|
|
|
ErrBodyTooLarge,
|
|
|
|
ErrReservedAddr,
|
|
|
|
) {
|
|
|
|
// Return on non-retryable errors
|
|
|
|
return nil, err
|
|
|
|
} else if strings.Contains(err.Error(), "stopped after 10 redirects") {
|
|
|
|
// Don't bother if net/http returned after too many redirects
|
|
|
|
return nil, err
|
|
|
|
} else if errors.As(err, &x509.UnknownAuthorityError{}) {
|
|
|
|
// Unknown authority errors we do NOT recover from
|
|
|
|
return nil, err
|
|
|
|
} else if dnserr := (*net.DNSError)(nil); // nocollapse
|
|
|
|
errors.As(err, &dnserr) && dnserr.IsNotFound {
|
|
|
|
// DNS lookup failure, this domain does not exist
|
|
|
|
return nil, gtserror.SetNotFound(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if fastFail {
|
|
|
|
// on fast-fail, don't bother backoff/retry
|
|
|
|
return nil, fmt.Errorf("%w (fast fail)", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if backoff == 0 {
|
|
|
|
// No retry-after found, set our predefined
|
|
|
|
// backoff according to a multiplier of 2^n.
|
|
|
|
backoff = baseBackoff * 1 << (i + 1)
|
|
|
|
}
|
|
|
|
|
|
|
|
l.Errorf("backing off for %s after http request error: %v", backoff, err)
|
2022-11-08 09:35:24 +00:00
|
|
|
|
|
|
|
select {
|
2023-04-28 15:45:21 +00:00
|
|
|
// Request ctx cancelled
|
|
|
|
case <-r.Context().Done():
|
|
|
|
return nil, r.Context().Err()
|
|
|
|
|
|
|
|
// Backoff for some time
|
|
|
|
case <-time.After(backoff):
|
2022-11-08 09:35:24 +00:00
|
|
|
}
|
2022-05-15 09:16:43 +00:00
|
|
|
}
|
|
|
|
|
2023-04-28 15:45:21 +00:00
|
|
|
// Add "bad" entry for this host.
|
|
|
|
c.badHosts.Set(host, struct{}{})
|
|
|
|
|
|
|
|
return nil, errors.New("transport reached max retries")
|
|
|
|
}
|
|
|
|
|
|
|
|
// do ...
|
|
|
|
func (c *Client) do(req *http.Request) (*http.Response, error) {
|
|
|
|
// Perform the HTTP request.
|
2022-05-15 09:16:43 +00:00
|
|
|
rsp, err := c.client.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2023-04-28 15:45:21 +00:00
|
|
|
// Check response body not too large.
|
|
|
|
if rsp.ContentLength > c.bodyMax {
|
2022-05-15 09:16:43 +00:00
|
|
|
return nil, ErrBodyTooLarge
|
|
|
|
}
|
|
|
|
|
2023-04-28 15:45:21 +00:00
|
|
|
// Seperate the body implementers.
|
2022-05-15 09:16:43 +00:00
|
|
|
rbody := (io.Reader)(rsp.Body)
|
|
|
|
cbody := (io.Closer)(rsp.Body)
|
|
|
|
|
|
|
|
var limit int64
|
|
|
|
|
|
|
|
if limit = rsp.ContentLength; limit < 0 {
|
2023-04-28 15:45:21 +00:00
|
|
|
// If unknown, use max as reader limit.
|
|
|
|
limit = c.bodyMax
|
2022-05-15 09:16:43 +00:00
|
|
|
}
|
|
|
|
|
2023-04-28 15:45:21 +00:00
|
|
|
// Don't trust them, limit body reads.
|
2022-05-15 09:16:43 +00:00
|
|
|
rbody = io.LimitReader(rbody, limit)
|
|
|
|
|
2023-04-28 15:45:21 +00:00
|
|
|
// Wrap body with limit.
|
2022-05-15 09:16:43 +00:00
|
|
|
rsp.Body = &struct {
|
|
|
|
io.Reader
|
|
|
|
io.Closer
|
|
|
|
}{rbody, cbody}
|
|
|
|
|
|
|
|
return rsp, nil
|
|
|
|
}
|