[feature] Add rate limit exceptions option, use ISO8601 for rate limit reset (#2151)

* start updating rate limiting, add exceptions

* tests, comments, tidying up

* add rate limiting exceptions to example config

* envparsing

* nolint

* apply kimbediff

* add examples

cmd/gotosocial/action/server/server.go

@@ -266,10 +266,11 @@ var Start action.GTSAction = func(ctx context.Context) error {
 
 	// create required middleware
 	// rate limiting
-	limit := config.GetAdvancedRateLimitRequests()
+	rlLimit := config.GetAdvancedRateLimitRequests()
-	clLimit := middleware.RateLimit(limit)  // client api
+	rlExceptions := config.GetAdvancedRateLimitExceptions()
-	s2sLimit := middleware.RateLimit(limit) // server-to-server (AP)
+	clLimit := middleware.RateLimit(rlLimit, rlExceptions)  // client api
-	fsLimit := middleware.RateLimit(limit)  // fileserver / web templates
+	s2sLimit := middleware.RateLimit(rlLimit, rlExceptions) // server-to-server (AP)
+	fsLimit := middleware.RateLimit(rlLimit, rlExceptions)  // fileserver / web templates
 
 	// throttling
 	cpuMultiplier := config.GetAdvancedThrottlingMultiplier()

docs/api/ratelimiting.md

@@ -16,7 +16,7 @@ Every response will include the current status of the rate limit with the follow
 
 - `X-Ratelimit-Limit`: maximum number of requests allowed per time period.
 - `X-Ratelimit-Remaining`: number of remaining requests that can still be performed within.
-- `X-Ratelimit-Reset`: unix timestamp indicating when the rate limit will reset.
+- `X-Ratelimit-Reset`: ISO8601 timestamp indicating when the rate limit will reset.
 
 In case the rate limit is exceeded, an [HTTP 429 Too Many Requests](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429) error is returned to the caller.
 
@@ -35,3 +35,7 @@ If you don't have an HTTP proxy, then it's likely caused by NAT. In this case yo
 ### Can I configure the rate limit? Can I just turn it off?
 
 Yes! Set `advanced-rate-limit-requests: 0` in the config.
+
+### Can I exclude one or more IP addresses from rate limiting, but leave the rest in place?
+
+Yes! Set `advanced-rate-limit-exceptions` in the config.

docs/configuration/advanced.md

@@ -52,6 +52,34 @@ advanced-cookies-samesite: "lax"
 # Default: 300
 advanced-rate-limit-requests: 300
 
+# Array of string. CIDRs to except from rate limit restrictions.
+# Any IPs inside the CIDR range(s) will not have rate limiting
+# applied on their requests, and rate limit headers will not be
+# set for those requests.
+#
+# This can be useful in the following example cases (and probably
+# a bunch of others as well):
+#
+# 1. You've set up an automated service that uses the API, and
+#    it keeps getting rate limited, even though you trust it's
+#    not abusing the instance.
+#
+# 2. You live with multiple people who use the same instance,
+#    and you're all using the same router/NAT, so you all have
+#    the same IP address, and you keep rate limiting each other.
+#
+# 3. You mostly use your own home internet to access your instance,
+#    and you want to exempt your home internet from rate limiting.
+#
+# You should be careful when adjusting this setting, since you
+# might inadvertently make rate limiting useless if you set too
+# wide a range. If in doubt, be too restrictive rather than too
+# lenient, and adjust as you go.
+#
+# Example: ["192.168.0.0/16"]
+# Default: []
+advanced-rate-limit-exceptions: []
+
 # Int. Amount of open requests to permit per CPU, per router grouping, before applying http
 # request throttling. Any requests beyond the calculated limit are held in a backlog queue for
 # up to 30 seconds before either being processed or timing out. Requests that don't fit in the backlog

example/config.yaml

@@ -848,6 +848,34 @@ advanced-cookies-samesite: "lax"
 # Default: 300
 advanced-rate-limit-requests: 300
 
+# Array of string. CIDRs to except from rate limit restrictions.
+# Any IPs inside the CIDR range(s) will not have rate limiting
+# applied on their requests, and rate limit headers will not be
+# set for those requests.
+#
+# This can be useful in the following example cases (and probably
+# a bunch of others as well):
+#
+# 1. You've set up an automated service that uses the API, and
+#    it keeps getting rate limited, even though you trust it's
+#    not abusing the instance.
+#
+# 2. You live with multiple people who use the same instance,
+#    and you're all using the same router/NAT, so you all have
+#    the same IP address, and you keep rate limiting each other.
+#
+# 3. You mostly use your own home internet to access your instance,
+#    and you want to exempt your home internet from rate limiting.
+#
+# You should be careful when adjusting this setting, since you
+# might inadvertently make rate limiting useless if you set too
+# wide a range. If in doubt, be too restrictive rather than too
+# lenient, and adjust as you go.
+#
+# Example: ["192.168.0.0/16"]
+# Default: []
+advanced-rate-limit-exceptions: []
+
 # Int. Amount of open requests to permit per CPU, per router grouping, before applying http
 # request throttling. Any requests beyond the calculated limit are held in a backlog queue for
 # up to 30 seconds before either being processed or timing out. Requests that don't fit in the backlog

internal/config/config.go

@@ -148,6 +148,7 @@ type Configuration struct {
 
 	AdvancedCookiesSamesite      string        `name:"advanced-cookies-samesite" usage:"'strict' or 'lax', see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite"`
 	AdvancedRateLimitRequests    int           `name:"advanced-rate-limit-requests" usage:"Amount of HTTP requests to permit within a 5 minute window. 0 or less turns rate limiting off."`
+	AdvancedRateLimitExceptions  []string      `name:"advanced-rate-limit-exceptions" usage:"Slice of CIDRs to exclude from rate limit restrictions."`
 	AdvancedThrottlingMultiplier int           `name:"advanced-throttling-multiplier" usage:"Multiplier to use per cpu for http request throttling. 0 or less turns throttling off."`
 	AdvancedThrottlingRetryAfter time.Duration `name:"advanced-throttling-retry-after" usage:"Retry-After duration response to send for throttled requests."`
 	AdvancedSenderMultiplier     int           `name:"advanced-sender-multiplier" usage:"Multiplier to use per cpu for batching outgoing fedi messages. 0 or less turns batching off (not recommended)."`

internal/config/defaults.go

@@ -122,6 +122,7 @@ var Defaults = Configuration{
 
 	AdvancedCookiesSamesite:      "lax",
 	AdvancedRateLimitRequests:    300, // 1 per second per 5 minutes
+	AdvancedRateLimitExceptions:  []string{},
 	AdvancedThrottlingMultiplier: 8, // 8 open requests per CPU
 	AdvancedThrottlingRetryAfter: time.Second * 30,
 	AdvancedSenderMultiplier:     2, // 2 senders per CPU

internal/config/flags.go

@@ -149,6 +149,7 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) {
 		// Advanced flags
 		cmd.Flags().String(AdvancedCookiesSamesiteFlag(), cfg.AdvancedCookiesSamesite, fieldtag("AdvancedCookiesSamesite", "usage"))
 		cmd.Flags().Int(AdvancedRateLimitRequestsFlag(), cfg.AdvancedRateLimitRequests, fieldtag("AdvancedRateLimitRequests", "usage"))
+		cmd.Flags().StringSlice(AdvancedRateLimitExceptionsFlag(), cfg.AdvancedRateLimitExceptions, fieldtag("AdvancedRateLimitExceptions", "usage"))
 		cmd.Flags().Int(AdvancedThrottlingMultiplierFlag(), cfg.AdvancedThrottlingMultiplier, fieldtag("AdvancedThrottlingMultiplier", "usage"))
 		cmd.Flags().Duration(AdvancedThrottlingRetryAfterFlag(), cfg.AdvancedThrottlingRetryAfter, fieldtag("AdvancedThrottlingRetryAfter", "usage"))
 		cmd.Flags().Int(AdvancedSenderMultiplierFlag(), cfg.AdvancedSenderMultiplier, fieldtag("AdvancedSenderMultiplier", "usage"))

internal/config/helpers.gen.go

@@ -2274,6 +2274,31 @@ func GetAdvancedRateLimitRequests() int { return global.GetAdvancedRateLimitRequ
 // SetAdvancedRateLimitRequests safely sets the value for global configuration 'AdvancedRateLimitRequests' field
 func SetAdvancedRateLimitRequests(v int) { global.SetAdvancedRateLimitRequests(v) }
 
+// GetAdvancedRateLimitExceptions safely fetches the Configuration value for state's 'AdvancedRateLimitExceptions' field
+func (st *ConfigState) GetAdvancedRateLimitExceptions() (v []string) {
+	st.mutex.RLock()
+	v = st.config.AdvancedRateLimitExceptions
+	st.mutex.RUnlock()
+	return
+}
+
+// SetAdvancedRateLimitExceptions safely sets the Configuration value for state's 'AdvancedRateLimitExceptions' field
+func (st *ConfigState) SetAdvancedRateLimitExceptions(v []string) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.AdvancedRateLimitExceptions = v
+	st.reloadToViper()
+}
+
+// AdvancedRateLimitExceptionsFlag returns the flag name for the 'AdvancedRateLimitExceptions' field
+func AdvancedRateLimitExceptionsFlag() string { return "advanced-rate-limit-exceptions" }
+
+// GetAdvancedRateLimitExceptions safely fetches the value for global configuration 'AdvancedRateLimitExceptions' field
+func GetAdvancedRateLimitExceptions() []string { return global.GetAdvancedRateLimitExceptions() }
+
+// SetAdvancedRateLimitExceptions safely sets the value for global configuration 'AdvancedRateLimitExceptions' field
+func SetAdvancedRateLimitExceptions(v []string) { global.SetAdvancedRateLimitExceptions(v) }
+
 // GetAdvancedThrottlingMultiplier safely fetches the Configuration value for state's 'AdvancedThrottlingMultiplier' field
 func (st *ConfigState) GetAdvancedThrottlingMultiplier() (v int) {
 	st.mutex.RLock()

internal/middleware/ratelimit.go

@@ -20,47 +20,136 @@ package middleware
 import (
 	"net"
 	"net/http"
+	"net/netip"
+	"strconv"
 	"time"
 
 	"github.com/gin-gonic/gin"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 	"github.com/ulule/limiter/v3"
-	limitergin "github.com/ulule/limiter/v3/drivers/middleware/gin"
 	"github.com/ulule/limiter/v3/drivers/store/memory"
 )
 
 const rateLimitPeriod = 5 * time.Minute
 
-// RateLimit returns a gin middleware that will automatically rate limit caller (by IP address),
+// RateLimit returns a gin middleware that will automatically rate
-// and enrich the response header with the following headers:
+// limit caller (by IP address), and enrich the response header with
+// the following headers:
 //
-//   - `x-ratelimit-limit`     - maximum number of requests allowed per time period (fixed).
+//   - `X-Ratelimit-Limit`     - max requests allowed per time period (fixed).
-//   - `x-ratelimit-remaining` - number of remaining requests that can still be performed.
+//   - `X-Ratelimit-Remaining` - requests remaining for this IP before reset.
-//   - `x-ratelimit-reset`     - unix timestamp when the rate limit will reset.
+//   - `X-Ratelimit-Reset`     - ISO8601 timestamp when the rate limit will reset.
 //
-// If `x-ratelimit-limit` is exceeded, the request is aborted and an HTTP 429 TooManyRequests
+// If `X-Ratelimit-Limit` is exceeded, the request is aborted and an
-// status is returned.
+// HTTP 429 TooManyRequests status is returned.
 //
-// If the config AdvancedRateLimitRequests value is <= 0, then a noop handler will be returned,
+// If the config AdvancedRateLimitRequests value is <= 0, then a noop
-// which performs no rate limiting.
+// handler will be returned, which performs no rate limiting.
-func RateLimit(limit int) gin.HandlerFunc {
+func RateLimit(limit int, exceptions []string) gin.HandlerFunc {
 	if limit <= 0 {
-		// use noop middleware if ratelimiting is disabled
+		// Rate limiting is disabled.
+		// Return noop middleware.
 		return func(ctx *gin.Context) {}
 	}
 
 	limiter := limiter.New(
 		memory.NewStore(),
-		limiter.Rate{Period: rateLimitPeriod, Limit: int64(limit)},
+		limiter.Rate{
-		limiter.WithIPv6Mask(net.CIDRMask(64, 128)), // apply /64 mask to IPv6 addresses
+			Period: rateLimitPeriod,
+			Limit:  int64(limit),
+		},
 	)
 
-	// use custom rate limit reached error
+	// Convert exceptions IP ranges into prefixes.
-	handler := func(c *gin.Context) {
+	exceptPrefs := make([]netip.Prefix, len(exceptions))
-		c.AbortWithStatusJSON(http.StatusTooManyRequests, gin.H{"error": "rate limit reached"})
+	for i, str := range exceptions {
+		exceptPrefs[i] = netip.MustParsePrefix(str)
 	}
 
-	return limitergin.NewMiddleware(
+	// It's prettymuch impossible to effectively
-		limiter,
+	// rate limit the immense IPv6 address space
-		limitergin.WithLimitReachedHandler(handler),
+	// unless we mask some of the bytes.
-	)
+	//
+	// This mask is pretty coarse, and puts IPv6
+	// blocking on more or less the same footing
+	// as IPv4 blocking in terms of how likely it
+	// is to prevent abuse while still allowing
+	// legit users access to the service.
+	ipv6Mask := net.CIDRMask(64, 128)
+
+	return func(c *gin.Context) {
+		// Use Gin's heuristic for determining
+		// clientIP, which accounts for reverse
+		// proxies and trusted proxies setting.
+		clientIP := netip.MustParseAddr(c.ClientIP())
+
+		// Check if this IP is exempt from rate
+		// limits and skip further checks if so.
+		for _, prefix := range exceptPrefs {
+			if prefix.Contains(clientIP) {
+				c.Next()
+				return
+			}
+		}
+
+		if clientIP.Is6() {
+			// Convert to "net" package IP for mask.
+			asIP := net.IP(clientIP.AsSlice())
+
+			// Apply coarse IPv6 mask.
+			asIP = asIP.Mask(ipv6Mask)
+
+			// Convert back to netip.Addr from net.IP.
+			clientIP, _ = netip.AddrFromSlice(asIP)
+		}
+
+		// Fetch rate limit info for this (masked) clientIP.
+		context, err := limiter.Get(c, clientIP.String())
+		if err != nil {
+			// Since we use an in-memory cache now,
+			// it's actually impossible for this to
+			// error, but handle it nicely anyway in
+			// case we switch implementation in future.
+			errWithCode := gtserror.NewErrorInternalError(err)
+
+			// Set error on gin context so it'll
+			// be picked up by logging middleware.
+			c.Error(errWithCode) //nolint:errcheck
+
+			// Bail with 500.
+			c.AbortWithStatusJSON(
+				errWithCode.Code(),
+				gin.H{"error": errWithCode.Safe()},
+			)
+			return
+		}
+
+		// Provide reset in same format used by
+		// Mastodon. There's no real standard as
+		// to what format X-RateLimit-Reset SHOULD
+		// use, but since most clients interacting
+		// with us will expect the Mastodon version,
+		// it makes sense to take this.
+		resetT := time.Unix(context.Reset, 0)
+		reset := util.FormatISO8601(resetT)
+
+		c.Header("X-RateLimit-Limit", strconv.FormatInt(context.Limit, 10))
+		c.Header("X-RateLimit-Remaining", strconv.FormatInt(context.Remaining, 10))
+		c.Header("X-RateLimit-Reset", reset)
+
+		if context.Reached {
+			// Return JSON error message for
+			// consistency with other endpoints.
+			c.AbortWithStatusJSON(
+				http.StatusTooManyRequests,
+				gin.H{"error": "rate limit reached"},
+			)
+			return
+		}
+
+		// Allow the request
+		// to continue.
+		c.Next()
+	}
 }

internal/middleware/ratelimit_test.go

@@ -0,0 +1,192 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package middleware_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/middleware"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+type RateLimitTestSuite struct {
+	suite.Suite
+}
+
+func (suite *RateLimitTestSuite) TestRateLimit() {
+	// Suppress warnings about debug mode.
+	gin.SetMode(gin.ReleaseMode)
+
+	const (
+		trustedPlatform = "X-Test-IP"
+		rlLimit         = "X-RateLimit-Limit"
+		rlRemaining     = "X-RateLimit-Remaining"
+		rlReset         = "X-RateLimit-Reset"
+	)
+
+	type rlTest struct {
+		limit        int
+		exceptions   []string
+		clientIP     string
+		shouldPanic  bool
+		shouldExcept bool
+	}
+
+	for _, test := range []rlTest{
+		{
+			limit:        10,
+			exceptions:   []string{},
+			clientIP:     "192.0.2.0",
+			shouldPanic:  false,
+			shouldExcept: false,
+		},
+		{
+			limit:        10,
+			exceptions:   []string{},
+			clientIP:     "192.0.2.0",
+			shouldPanic:  false,
+			shouldExcept: false,
+		},
+		{
+			limit:        10,
+			exceptions:   []string{"192.0.2.0/24"},
+			clientIP:     "192.0.2.0",
+			shouldPanic:  false,
+			shouldExcept: true,
+		},
+		{
+			limit:        10,
+			exceptions:   []string{"192.0.2.0/32"},
+			clientIP:     "192.0.2.1",
+			shouldPanic:  false,
+			shouldExcept: false,
+		},
+		{
+			limit:        10,
+			exceptions:   []string{"Ceci n'est pas une CIDR"},
+			clientIP:     "192.0.2.0",
+			shouldPanic:  true,
+			shouldExcept: false,
+		},
+	} {
+		if test.shouldPanic {
+			// Try to trigger panic.
+			suite.Panics(func() {
+				_ = middleware.RateLimit(
+					test.limit,
+					test.exceptions,
+				)
+			})
+			continue
+		}
+
+		rlMiddleware := middleware.RateLimit(
+			test.limit,
+			test.exceptions,
+		)
+
+		// Approximate time when this limiter will reset.
+		resetAt := time.Now().Add(5 * time.Minute)
+
+		// Make requests up to +
+		// just over the limit.
+		limitedAt := test.limit + 1
+		for requestsCount := 1; requestsCount < limitedAt; requestsCount++ {
+			var (
+				recorder = httptest.NewRecorder()
+				ctx, e   = gin.CreateTestContext(recorder)
+			)
+
+			// Instruct engine to derive
+			// clientIP from test header.
+			e.TrustedPlatform = trustedPlatform
+			ctx.Request = httptest.NewRequest(http.MethodGet, "/example", nil)
+			ctx.Request.Header.Add(trustedPlatform, test.clientIP)
+
+			// Call the rate limiter.
+			rlMiddleware(ctx)
+
+			// Fetch RL headers if present.
+			var (
+				limitStr     = recorder.Header().Get(rlLimit)
+				remainingStr = recorder.Header().Get(rlRemaining)
+				resetStr     = recorder.Header().Get(rlReset)
+			)
+
+			if test.shouldExcept {
+				// Request should be allowed through,
+				// no rate-limit headers should be written.
+				suite.Equal(http.StatusOK, recorder.Code)
+				suite.Empty(limitStr)
+				suite.Empty(remainingStr)
+				suite.Empty(resetStr)
+				continue
+			}
+
+			suite.Equal(strconv.Itoa(test.limit), limitStr)
+			suite.Equal(strconv.Itoa(test.limit-requestsCount), remainingStr)
+
+			// Ensure reset is ISO8601, and resets at
+			// approximate reset time (+/- 10 seconds).
+			reset, err := util.ParseISO8601(resetStr)
+			if err != nil {
+				suite.FailNow("", "couldn't parse %s as ISO8601: %q", resetStr, err.Error())
+			}
+			suite.WithinDuration(resetAt, reset, 10*time.Second)
+
+			if requestsCount < limitedAt {
+				// Request should be allowed through.
+				suite.Equal(http.StatusOK, recorder.Code)
+				continue
+			}
+
+			// Request should be denied.
+			suite.Equal(http.StatusTooManyRequests, recorder.Code)
+
+			// Make a final request with an unrelated IP to
+			// ensure it's only the one IP being limited.
+			var (
+				unrelatedRecorder        = httptest.NewRecorder()
+				unrelatedCtx, unrelatedE = gin.CreateTestContext(unrelatedRecorder)
+			)
+
+			// Instruct engine to derive
+			// clientIP from test header.
+			unrelatedE.TrustedPlatform = trustedPlatform
+			unrelatedCtx.Request = httptest.NewRequest(http.MethodGet, "/example", nil)
+			unrelatedCtx.Request.Header.Add(trustedPlatform, "192.0.2.255")
+
+			// Call the rate limiter.
+			rlMiddleware(unrelatedCtx)
+
+			// Request should be allowed through.
+			suite.Equal(http.StatusOK, unrelatedRecorder.Code)
+
+		}
+	}
+}
+
+func TestRateLimitTestSuite(t *testing.T) {
+	suite.Run(t, new(RateLimitTestSuite))
+}

test/envparsing.sh

@@ -12,6 +12,10 @@ EXPECT=$(cat << "EOF"
     "accounts-registration-open": true,
     "advanced-cookies-samesite": "strict",
     "advanced-csp-extra-uris": [],
+    "advanced-rate-limit-exceptions": [
+        "192.0.2.0/24",
+        "127.0.0.1/32"
+    ],
     "advanced-rate-limit-requests": 6969,
     "advanced-sender-multiplier": -1,
     "advanced-throttling-multiplier": -1,
@@ -237,6 +241,7 @@ GTS_SYSLOG_PROTOCOL='udp' \
 GTS_SYSLOG_ADDRESS='127.0.0.1:6969' \
 GTS_TRACING_ENDPOINT='localhost:4317' \
 GTS_ADVANCED_COOKIES_SAMESITE='strict' \
+GTS_ADVANCED_RATE_LIMIT_EXCEPTIONS="192.0.2.0/24,127.0.0.1/32" \
 GTS_ADVANCED_RATE_LIMIT_REQUESTS=6969 \
 GTS_ADVANCED_SENDER_MULTIPLIER=-1 \
 GTS_ADVANCED_THROTTLING_MULTIPLIER=-1 \