From a089a98ea9874f339e635700dfdb00c91f10b4d5 Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Sat, 19 Feb 2022 12:12:41 +0100 Subject: [PATCH] [feature] Gin enable gzip encoding (#405) * add gin gzip dependency * add gzip middleware to router * go mod tidy --- README.md | 1 + go.mod | 1 + go.sum | 2 + internal/router/gzip.go | 30 ++++ internal/router/router.go | 15 +- vendor/github.com/gin-contrib/gzip/LICENSE | 21 +++ vendor/github.com/gin-contrib/gzip/README.md | 135 ++++++++++++++++++ vendor/github.com/gin-contrib/gzip/gzip.go | 39 +++++ vendor/github.com/gin-contrib/gzip/handler.go | 84 +++++++++++ vendor/github.com/gin-contrib/gzip/options.go | 116 +++++++++++++++ vendor/modules.txt | 3 + 11 files changed, 442 insertions(+), 5 deletions(-) create mode 100644 internal/router/gzip.go create mode 100644 vendor/github.com/gin-contrib/gzip/LICENSE create mode 100644 vendor/github.com/gin-contrib/gzip/README.md create mode 100644 vendor/github.com/gin-contrib/gzip/gzip.go create mode 100644 vendor/github.com/gin-contrib/gzip/handler.go create mode 100644 vendor/github.com/gin-contrib/gzip/options.go diff --git a/README.md b/README.md index eace37a1..0d0709c5 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,7 @@ The following libraries and frameworks are used by GoToSocial, with gratitude - [coreos/go-oidc](https://github.com/coreos/go-oidc); OIDC client library. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html). - [gin-gonic/gin](https://github.com/gin-gonic/gin); speedy router engine. [MIT License](https://spdx.org/licenses/MIT.html). - [gin-contrib/cors](https://github.com/gin-contrib/cors); Gin CORS middleware. [MIT License](https://spdx.org/licenses/MIT.html). + - [gin-contrib/gzip](https://github.com/gin-contrib/gzip); Gin gzip middleware. [MIT License](https://spdx.org/licenses/MIT.html). - [gin-contrib/sessions](https://github.com/gin-contrib/sessions); Gin sessions middleware. [MIT License](https://spdx.org/licenses/MIT.html) - [gin-contrib/static](https://github.com/gin-contrib/static); Gin static page middleware. [MIT License](https://spdx.org/licenses/MIT.html) - [go-fed/httpsig](https://github.com/go-fed/httpsig); secure HTTP signature library. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html). diff --git a/go.mod b/go.mod index e04f3088..3c234240 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/buckket/go-blurhash v1.1.0 github.com/coreos/go-oidc/v3 v3.1.0 github.com/gin-contrib/cors v1.3.1 + github.com/gin-contrib/gzip v0.0.5 github.com/gin-contrib/sessions v0.0.4 github.com/gin-gonic/gin v1.7.7 github.com/go-fed/httpsig v1.1.0 diff --git a/go.sum b/go.sum index a878363c..6395601d 100644 --- a/go.sum +++ b/go.sum @@ -196,6 +196,8 @@ github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/cors v1.3.1 h1:doAsuITavI4IOcd0Y19U4B+O0dNWihRyX//nn4sEmgA= github.com/gin-contrib/cors v1.3.1/go.mod h1:jjEJ4268OPZUcU7k9Pm653S7lXUGcqMADzFA61xsmDk= +github.com/gin-contrib/gzip v0.0.5 h1:mhnVU32YnnBh2LPH2iqRqsA/eR7SAqRaD388jL2s/j0= +github.com/gin-contrib/gzip v0.0.5/go.mod h1:OPIK6HR0Um2vNmBUTlayD7qle4yVVRZT0PyhdUigrKk= github.com/gin-contrib/sessions v0.0.4 h1:gq4fNa1Zmp564iHP5G6EBuktilEos8VKhe2sza1KMgo= github.com/gin-contrib/sessions v0.0.4/go.mod h1:pQ3sIyviBBGcxgyR8mkeJuXbeV3h3NYmhJADQTq5+Vo= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= diff --git a/internal/router/gzip.go b/internal/router/gzip.go new file mode 100644 index 00000000..4c5c99ee --- /dev/null +++ b/internal/router/gzip.go @@ -0,0 +1,30 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package router + +import ( + ginGzip "github.com/gin-contrib/gzip" + "github.com/gin-gonic/gin" +) + +func useGzip(engine *gin.Engine) error { + gzipMiddleware := ginGzip.Gzip(ginGzip.DefaultCompression) + engine.Use(gzipMiddleware) + return nil +} diff --git a/internal/router/router.go b/internal/router/router.go index f1247d27..4b00f0eb 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -137,11 +137,8 @@ func New(ctx context.Context, db db.DB) (Router, error) { return nil, err } - // set template functions - LoadTemplateFunctions(engine) - - // load templates onto the engine - if err := loadTemplates(engine); err != nil { + // enable gzip compression on the engine + if err := useGzip(engine); err != nil { return nil, err } @@ -150,6 +147,14 @@ func New(ctx context.Context, db db.DB) (Router, error) { return nil, err } + // set template functions + LoadTemplateFunctions(engine) + + // load templates onto the engine + if err := loadTemplates(engine); err != nil { + return nil, err + } + // create the http server here, passing the gin engine as handler bindAddress := viper.GetString(keys.BindAddress) port := viper.GetInt(keys.Port) diff --git a/vendor/github.com/gin-contrib/gzip/LICENSE b/vendor/github.com/gin-contrib/gzip/LICENSE new file mode 100644 index 00000000..a863f57c --- /dev/null +++ b/vendor/github.com/gin-contrib/gzip/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Gin-Gonic + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/gin-contrib/gzip/README.md b/vendor/github.com/gin-contrib/gzip/README.md new file mode 100644 index 00000000..2f94d053 --- /dev/null +++ b/vendor/github.com/gin-contrib/gzip/README.md @@ -0,0 +1,135 @@ +# GZIP gin's middleware + +[![Run Tests](https://github.com/gin-contrib/gzip/actions/workflows/go.yml/badge.svg)](https://github.com/gin-contrib/gzip/actions/workflows/go.yml) +[![codecov](https://codecov.io/gh/gin-contrib/gzip/branch/master/graph/badge.svg)](https://codecov.io/gh/gin-contrib/gzip) +[![Go Report Card](https://goreportcard.com/badge/github.com/gin-contrib/gzip)](https://goreportcard.com/report/github.com/gin-contrib/gzip) +[![GoDoc](https://godoc.org/github.com/gin-contrib/gzip?status.svg)](https://godoc.org/github.com/gin-contrib/gzip) +[![Join the chat at https://gitter.im/gin-gonic/gin](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/gin-gonic/gin) + +Gin middleware to enable `GZIP` support. + +## Usage + +Download and install it: + +```sh +go get github.com/gin-contrib/gzip +``` + +Import it in your code: + +```go +import "github.com/gin-contrib/gzip" +``` + +Canonical example: + +```go +package main + +import ( + "fmt" + "net/http" + "time" + + "github.com/gin-contrib/gzip" + "github.com/gin-gonic/gin" +) + +func main() { + r := gin.Default() + r.Use(gzip.Gzip(gzip.DefaultCompression)) + r.GET("/ping", func(c *gin.Context) { + c.String(http.StatusOK, "pong "+fmt.Sprint(time.Now().Unix())) + }) + + // Listen and Server in 0.0.0.0:8080 + if err := r.Run(":8080"); err != nil { + log.Fatal(err) + } +} +``` + +Customized Excluded Extensions + +```go +package main + +import ( + "fmt" + "net/http" + "time" + + "github.com/gin-contrib/gzip" + "github.com/gin-gonic/gin" +) + +func main() { + r := gin.Default() + r.Use(gzip.Gzip(gzip.DefaultCompression, gzip.WithExcludedExtensions([]string{".pdf", ".mp4"}))) + r.GET("/ping", func(c *gin.Context) { + c.String(http.StatusOK, "pong "+fmt.Sprint(time.Now().Unix())) + }) + + // Listen and Server in 0.0.0.0:8080 + if err := r.Run(":8080"); err != nil { + log.Fatal(err) + } +} +``` + +Customized Excluded Paths + +```go +package main + +import ( + "fmt" + "net/http" + "time" + + "github.com/gin-contrib/gzip" + "github.com/gin-gonic/gin" +) + +func main() { + r := gin.Default() + r.Use(gzip.Gzip(gzip.DefaultCompression, gzip.WithExcludedPaths([]string{"/api/"}))) + r.GET("/ping", func(c *gin.Context) { + c.String(http.StatusOK, "pong "+fmt.Sprint(time.Now().Unix())) + }) + + // Listen and Server in 0.0.0.0:8080 + if err := r.Run(":8080"); err != nil { + log.Fatal(err) + } +} +``` + +Customized Excluded Paths + +```go +package main + +import ( + "fmt" + "net/http" + "time" + + "github.com/gin-contrib/gzip" + "github.com/gin-gonic/gin" +) + +func main() { + r := gin.Default() + r.Use(gzip.Gzip(gzip.DefaultCompression, gzip.WithExcludedPathsRegexs([]string{".*"}))) + r.GET("/ping", func(c *gin.Context) { + c.String(http.StatusOK, "pong "+fmt.Sprint(time.Now().Unix())) + }) + + // Listen and Server in 0.0.0.0:8080 + if err := r.Run(":8080"); err != nil { + log.Fatal(err) + } +} +``` diff --git a/vendor/github.com/gin-contrib/gzip/gzip.go b/vendor/github.com/gin-contrib/gzip/gzip.go new file mode 100644 index 00000000..529c62df --- /dev/null +++ b/vendor/github.com/gin-contrib/gzip/gzip.go @@ -0,0 +1,39 @@ +package gzip + +import ( + "compress/gzip" + + "github.com/gin-gonic/gin" +) + +const ( + BestCompression = gzip.BestCompression + BestSpeed = gzip.BestSpeed + DefaultCompression = gzip.DefaultCompression + NoCompression = gzip.NoCompression +) + +func Gzip(level int, options ...Option) gin.HandlerFunc { + return newGzipHandler(level, options...).Handle +} + +type gzipWriter struct { + gin.ResponseWriter + writer *gzip.Writer +} + +func (g *gzipWriter) WriteString(s string) (int, error) { + g.Header().Del("Content-Length") + return g.writer.Write([]byte(s)) +} + +func (g *gzipWriter) Write(data []byte) (int, error) { + g.Header().Del("Content-Length") + return g.writer.Write(data) +} + +// Fix: https://github.com/mholt/caddy/issues/38 +func (g *gzipWriter) WriteHeader(code int) { + g.Header().Del("Content-Length") + g.ResponseWriter.WriteHeader(code) +} diff --git a/vendor/github.com/gin-contrib/gzip/handler.go b/vendor/github.com/gin-contrib/gzip/handler.go new file mode 100644 index 00000000..58f87db9 --- /dev/null +++ b/vendor/github.com/gin-contrib/gzip/handler.go @@ -0,0 +1,84 @@ +package gzip + +import ( + "compress/gzip" + "fmt" + "io/ioutil" + "net/http" + "path/filepath" + "strings" + "sync" + + "github.com/gin-gonic/gin" +) + +type gzipHandler struct { + *Options + gzPool sync.Pool +} + +func newGzipHandler(level int, options ...Option) *gzipHandler { + handler := &gzipHandler{ + Options: DefaultOptions, + gzPool: sync.Pool{ + New: func() interface{} { + gz, err := gzip.NewWriterLevel(ioutil.Discard, level) + if err != nil { + panic(err) + } + return gz + }, + }, + } + for _, setter := range options { + setter(handler.Options) + } + return handler +} + +func (g *gzipHandler) Handle(c *gin.Context) { + if fn := g.DecompressFn; fn != nil && c.Request.Header.Get("Content-Encoding") == "gzip" { + fn(c) + } + + if !g.shouldCompress(c.Request) { + return + } + + gz := g.gzPool.Get().(*gzip.Writer) + defer g.gzPool.Put(gz) + defer gz.Reset(ioutil.Discard) + gz.Reset(c.Writer) + + c.Header("Content-Encoding", "gzip") + c.Header("Vary", "Accept-Encoding") + c.Writer = &gzipWriter{c.Writer, gz} + defer func() { + gz.Close() + c.Header("Content-Length", fmt.Sprint(c.Writer.Size())) + }() + c.Next() +} + +func (g *gzipHandler) shouldCompress(req *http.Request) bool { + if !strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") || + strings.Contains(req.Header.Get("Connection"), "Upgrade") || + strings.Contains(req.Header.Get("Accept"), "text/event-stream") { + + return false + } + + extension := filepath.Ext(req.URL.Path) + if g.ExcludedExtensions.Contains(extension) { + return false + } + + if g.ExcludedPaths.Contains(req.URL.Path) { + return false + } + if g.ExcludedPathesRegexs.Contains(req.URL.Path) { + return false + } + + return true +} diff --git a/vendor/github.com/gin-contrib/gzip/options.go b/vendor/github.com/gin-contrib/gzip/options.go new file mode 100644 index 00000000..6b3bc3f4 --- /dev/null +++ b/vendor/github.com/gin-contrib/gzip/options.go @@ -0,0 +1,116 @@ +package gzip + +import ( + "compress/gzip" + "net/http" + "regexp" + "strings" + + "github.com/gin-gonic/gin" +) + +var ( + DefaultExcludedExtentions = NewExcludedExtensions([]string{ + ".png", ".gif", ".jpeg", ".jpg", + }) + DefaultOptions = &Options{ + ExcludedExtensions: DefaultExcludedExtentions, + } +) + +type Options struct { + ExcludedExtensions ExcludedExtensions + ExcludedPaths ExcludedPaths + ExcludedPathesRegexs ExcludedPathesRegexs + DecompressFn func(c *gin.Context) +} + +type Option func(*Options) + +func WithExcludedExtensions(args []string) Option { + return func(o *Options) { + o.ExcludedExtensions = NewExcludedExtensions(args) + } +} + +func WithExcludedPaths(args []string) Option { + return func(o *Options) { + o.ExcludedPaths = NewExcludedPaths(args) + } +} + +func WithExcludedPathsRegexs(args []string) Option { + return func(o *Options) { + o.ExcludedPathesRegexs = NewExcludedPathesRegexs(args) + } +} + +func WithDecompressFn(decompressFn func(c *gin.Context)) Option { + return func(o *Options) { + o.DecompressFn = decompressFn + } +} + +// Using map for better lookup performance +type ExcludedExtensions map[string]bool + +func NewExcludedExtensions(extensions []string) ExcludedExtensions { + res := make(ExcludedExtensions) + for _, e := range extensions { + res[e] = true + } + return res +} + +func (e ExcludedExtensions) Contains(target string) bool { + _, ok := e[target] + return ok +} + +type ExcludedPaths []string + +func NewExcludedPaths(paths []string) ExcludedPaths { + return ExcludedPaths(paths) +} + +func (e ExcludedPaths) Contains(requestURI string) bool { + for _, path := range e { + if strings.HasPrefix(requestURI, path) { + return true + } + } + return false +} + +type ExcludedPathesRegexs []*regexp.Regexp + +func NewExcludedPathesRegexs(regexs []string) ExcludedPathesRegexs { + result := make([]*regexp.Regexp, len(regexs)) + for i, reg := range regexs { + result[i] = regexp.MustCompile(reg) + } + return result +} + +func (e ExcludedPathesRegexs) Contains(requestURI string) bool { + for _, reg := range e { + if reg.MatchString(requestURI) { + return true + } + } + return false +} + +func DefaultDecompressHandle(c *gin.Context) { + if c.Request.Body == nil { + return + } + r, err := gzip.NewReader(c.Request.Body) + if err != nil { + _ = c.AbortWithError(http.StatusBadRequest, err) + return + } + c.Request.Header.Del("Content-Encoding") + c.Request.Header.Del("Content-Length") + c.Request.Body = r +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 2ef54775..981ad15a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -74,6 +74,9 @@ github.com/fsnotify/fsnotify # github.com/gin-contrib/cors v1.3.1 ## explicit; go 1.13 github.com/gin-contrib/cors +# github.com/gin-contrib/gzip v0.0.5 +## explicit; go 1.13 +github.com/gin-contrib/gzip # github.com/gin-contrib/sessions v0.0.4 ## explicit; go 1.13 github.com/gin-contrib/sessions