[feature] Make log format configurable (#2130)

* [feature] Don't emit timestamp in log lines

When running gotosocial with a service manager like systemd, or a
container runtime, the associated log driver usually emits timestamps
itself. In those cases, having the extra timestamp from our own log
lines ends up being a bit noisy and when centrally ingesting logs is
duplicate information.

This introduces a configuration flag that allows disabling emitting the
timestamp. It's only wired up for "daemonised" processes, meaning server
and testrig.

* [chore] Add docs for log-timestamp

* [feature] Simplify timestamp handling

Co-Authored-By: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com>

* [chore] Less escaped double-quotes

* [chore] Fix help string

---------

Co-authored-by: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com>
This commit is contained in:
Daenney 2023-08-21 20:07:55 +02:00 committed by GitHub
parent 638f023a1c
commit 4ae16bce8c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 128 additions and 47 deletions

View file

@ -63,6 +63,7 @@ func preRun(a preRunArgs) error {
// The idea here is to take a GTSAction and run it with the given // The idea here is to take a GTSAction and run it with the given
// context, after initializing any last-minute things like loggers etc. // context, after initializing any last-minute things like loggers etc.
func run(ctx context.Context, action action.GTSAction) error { func run(ctx context.Context, action action.GTSAction) error {
log.SetTimeFormat(config.GetLogTimestampFormat())
// Set the global log level from configuration // Set the global log level from configuration
if err := log.ParseLevel(config.GetLogLevel()); err != nil { if err := log.ParseLevel(config.GetLogLevel()); err != nil {
return fmt.Errorf("error parsing log level: %w", err) return fmt.Errorf("error parsing log level: %w", err)

View file

@ -28,6 +28,17 @@ log-db-queries: false
# Default: true # Default: true
log-client-ip: true log-client-ip: true
# String. Format to use for the timestamp in log lines.
# If set to the empty string, the timestamp will be
# ommitted from the logs entirely.
#
# The format must be compatible with Go's time.Layout, as
# documented on https://pkg.go.dev/time#pkg-constants.
#
# Examples: [true, false]
# Default: "02/01/2006 15:04:05.000"
log-timestamp-format: "02/01/2006 15:04:05.000"
# String. Application name to use internally. # String. Application name to use internally.
# Examples: ["My Application","gotosocial"] # Examples: ["My Application","gotosocial"]
# Default: "gotosocial" # Default: "gotosocial"

View file

@ -35,6 +35,17 @@ log-db-queries: false
# Default: true # Default: true
log-client-ip: true log-client-ip: true
# String. Format to use for the timestamp in log lines.
# If set to the empty string, the timestamp will be
# ommitted from the logs entirely.
#
# The format must be compatible with Go's time.Layout, as
# documented on https://pkg.go.dev/time#pkg-constants.
#
# Examples: [true, false]
# Default: "02/01/2006 15:04:05.000"
log-timestamp-format: "02/01/2006 15:04:05.000"
# String. Application name to use internally. # String. Application name to use internally.
# Examples: ["My Application","gotosocial"] # Examples: ["My Application","gotosocial"]
# Default: "gotosocial" # Default: "gotosocial"

View file

@ -44,19 +44,20 @@ func fieldtag(field, tag string) string {
// will need to regenerate the global Getter/Setter helpers by running: // will need to regenerate the global Getter/Setter helpers by running:
// `go run ./internal/config/gen/ -out ./internal/config/helpers.gen.go` // `go run ./internal/config/gen/ -out ./internal/config/helpers.gen.go`
type Configuration struct { type Configuration struct {
LogLevel string `name:"log-level" usage:"Log level to run at: [trace, debug, info, warn, fatal]"` LogLevel string `name:"log-level" usage:"Log level to run at: [trace, debug, info, warn, fatal]"`
LogDbQueries bool `name:"log-db-queries" usage:"Log database queries verbosely when log-level is trace or debug"` LogTimestampFormat string `name:"log-timestamp-format" usage:"Format to use for the log timestamp, as supported by Go's time.Layout"`
LogClientIP bool `name:"log-client-ip" usage:"Include the client IP in logs"` LogDbQueries bool `name:"log-db-queries" usage:"Log database queries verbosely when log-level is trace or debug"`
ApplicationName string `name:"application-name" usage:"Name of the application, used in various places internally"` LogClientIP bool `name:"log-client-ip" usage:"Include the client IP in logs"`
LandingPageUser string `name:"landing-page-user" usage:"the user that should be shown on the instance's landing page"` ApplicationName string `name:"application-name" usage:"Name of the application, used in various places internally"`
ConfigPath string `name:"config-path" usage:"Path to a file containing gotosocial configuration. Values set in this file will be overwritten by values set as env vars or arguments"` LandingPageUser string `name:"landing-page-user" usage:"the user that should be shown on the instance's landing page"`
Host string `name:"host" usage:"Hostname to use for the server (eg., example.org, gotosocial.whatever.com). DO NOT change this on a server that's already run!"` ConfigPath string `name:"config-path" usage:"Path to a file containing gotosocial configuration. Values set in this file will be overwritten by values set as env vars or arguments"`
AccountDomain string `name:"account-domain" usage:"Domain to use in account names (eg., example.org, whatever.com). If not set, will default to the setting for host. DO NOT change this on a server that's already run!"` Host string `name:"host" usage:"Hostname to use for the server (eg., example.org, gotosocial.whatever.com). DO NOT change this on a server that's already run!"`
Protocol string `name:"protocol" usage:"Protocol to use for the REST api of the server (only use http if you are debugging or behind a reverse proxy!)"` AccountDomain string `name:"account-domain" usage:"Domain to use in account names (eg., example.org, whatever.com). If not set, will default to the setting for host. DO NOT change this on a server that's already run!"`
BindAddress string `name:"bind-address" usage:"Bind address to use for the GoToSocial server (eg., 0.0.0.0, 172.138.0.9, [::], localhost). For ipv6, enclose the address in square brackets, eg [2001:db8::fed1]. Default binds to all interfaces."` Protocol string `name:"protocol" usage:"Protocol to use for the REST api of the server (only use http if you are debugging or behind a reverse proxy!)"`
Port int `name:"port" usage:"Port to use for GoToSocial. Change this to 443 if you're running the binary directly on the host machine."` BindAddress string `name:"bind-address" usage:"Bind address to use for the GoToSocial server (eg., 0.0.0.0, 172.138.0.9, [::], localhost). For ipv6, enclose the address in square brackets, eg [2001:db8::fed1]. Default binds to all interfaces."`
TrustedProxies []string `name:"trusted-proxies" usage:"Proxies to trust when parsing x-forwarded headers into real IPs."` Port int `name:"port" usage:"Port to use for GoToSocial. Change this to 443 if you're running the binary directly on the host machine."`
SoftwareVersion string `name:"software-version" usage:""` TrustedProxies []string `name:"trusted-proxies" usage:"Proxies to trust when parsing x-forwarded headers into real IPs."`
SoftwareVersion string `name:"software-version" usage:""`
DbType string `name:"db-type" usage:"Database type: eg., postgres"` DbType string `name:"db-type" usage:"Database type: eg., postgres"`
DbAddress string `name:"db-address" usage:"Database ipv4 address, hostname, or filename"` DbAddress string `name:"db-address" usage:"Database ipv4 address, hostname, or filename"`

View file

@ -27,17 +27,18 @@ import (
// Defaults contains a populated Configuration with reasonable defaults. Note that // Defaults contains a populated Configuration with reasonable defaults. Note that
// if you use this, you will still need to set Host, and, if desired, ConfigPath. // if you use this, you will still need to set Host, and, if desired, ConfigPath.
var Defaults = Configuration{ var Defaults = Configuration{
LogLevel: "info", LogLevel: "info",
LogDbQueries: false, LogTimestampFormat: "02/01/2006 15:04:05.000",
ApplicationName: "gotosocial", LogDbQueries: false,
LandingPageUser: "", ApplicationName: "gotosocial",
ConfigPath: "", LandingPageUser: "",
Host: "", ConfigPath: "",
AccountDomain: "", Host: "",
Protocol: "https", AccountDomain: "",
BindAddress: "0.0.0.0", Protocol: "https",
Port: 8080, BindAddress: "0.0.0.0",
TrustedProxies: []string{"127.0.0.1/32", "::1"}, // localhost Port: 8080,
TrustedProxies: []string{"127.0.0.1/32", "::1"}, // localhost
DbType: "postgres", DbType: "postgres",
DbAddress: "", DbAddress: "",

View file

@ -38,6 +38,7 @@ func (s *ConfigState) AddGlobalFlags(cmd *cobra.Command) {
cmd.PersistentFlags().String(AccountDomainFlag(), cfg.AccountDomain, fieldtag("AccountDomain", "usage")) cmd.PersistentFlags().String(AccountDomainFlag(), cfg.AccountDomain, fieldtag("AccountDomain", "usage"))
cmd.PersistentFlags().String(ProtocolFlag(), cfg.Protocol, fieldtag("Protocol", "usage")) cmd.PersistentFlags().String(ProtocolFlag(), cfg.Protocol, fieldtag("Protocol", "usage"))
cmd.PersistentFlags().String(LogLevelFlag(), cfg.LogLevel, fieldtag("LogLevel", "usage")) cmd.PersistentFlags().String(LogLevelFlag(), cfg.LogLevel, fieldtag("LogLevel", "usage"))
cmd.PersistentFlags().String(LogTimestampFormatFlag(), cfg.LogTimestampFormat, fieldtag("LogTimestampFormat", "usage"))
cmd.PersistentFlags().Bool(LogDbQueriesFlag(), cfg.LogDbQueries, fieldtag("LogDbQueries", "usage")) cmd.PersistentFlags().Bool(LogDbQueriesFlag(), cfg.LogDbQueries, fieldtag("LogDbQueries", "usage"))
cmd.PersistentFlags().String(ConfigPathFlag(), cfg.ConfigPath, fieldtag("ConfigPath", "usage")) cmd.PersistentFlags().String(ConfigPathFlag(), cfg.ConfigPath, fieldtag("ConfigPath", "usage"))

View file

@ -49,6 +49,31 @@ func GetLogLevel() string { return global.GetLogLevel() }
// SetLogLevel safely sets the value for global configuration 'LogLevel' field // SetLogLevel safely sets the value for global configuration 'LogLevel' field
func SetLogLevel(v string) { global.SetLogLevel(v) } func SetLogLevel(v string) { global.SetLogLevel(v) }
// GetLogTimestampFormat safely fetches the Configuration value for state's 'LogTimestampFormat' field
func (st *ConfigState) GetLogTimestampFormat() (v string) {
st.mutex.RLock()
v = st.config.LogTimestampFormat
st.mutex.RUnlock()
return
}
// SetLogTimestampFormat safely sets the Configuration value for state's 'LogTimestampFormat' field
func (st *ConfigState) SetLogTimestampFormat(v string) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.LogTimestampFormat = v
st.reloadToViper()
}
// LogTimestampFormatFlag returns the flag name for the 'LogTimestampFormat' field
func LogTimestampFormatFlag() string { return "log-timestamp-format" }
// GetLogTimestampFormat safely fetches the value for global configuration 'LogTimestampFormat' field
func GetLogTimestampFormat() string { return global.GetLogTimestampFormat() }
// SetLogTimestampFormat safely sets the value for global configuration 'LogTimestampFormat' field
func SetLogTimestampFormat(v string) { global.SetLogTimestampFormat(v) }
// GetLogDbQueries safely fetches the Configuration value for state's 'LogDbQueries' field // GetLogDbQueries safely fetches the Configuration value for state's 'LogDbQueries' field
func (st *ConfigState) GetLogDbQueries() (v bool) { func (st *ConfigState) GetLogDbQueries() (v bool) {
st.mutex.RLock() st.mutex.RLock()

View file

@ -23,7 +23,6 @@ import (
"log/syslog" "log/syslog"
"os" "os"
"strings" "strings"
"sync/atomic"
"syscall" "syscall"
"time" "time"
@ -33,7 +32,7 @@ import (
var ( var (
// loglvl is the currently set logging level. // loglvl is the currently set logging level.
loglvl atomic.Uint32 loglvl level.LEVEL
// lvlstrs is the lookup table of log levels to strings. // lvlstrs is the lookup table of log levels to strings.
lvlstrs = level.Default() lvlstrs = level.Default()
@ -41,8 +40,9 @@ var (
// syslog output, only set if enabled. // syslog output, only set if enabled.
sysout *syslog.Writer sysout *syslog.Writer
// timefmt is the logging time format used. // timefmt is the logging time format used, which includes
timefmt = "02/01/2006 15:04:05.000" // the full field and required quoting
timefmt = `timestamp="02/01/2006 15:04:05.000" `
// ctxhooks allows modifying log content based on context. // ctxhooks allows modifying log content based on context.
ctxhooks []func(context.Context, []kv.Field) []kv.Field ctxhooks []func(context.Context, []kv.Field) []kv.Field
@ -55,12 +55,26 @@ func Hook(hook func(ctx context.Context, kvs []kv.Field) []kv.Field) {
// Level returns the currently set log level. // Level returns the currently set log level.
func Level() level.LEVEL { func Level() level.LEVEL {
return level.LEVEL(loglvl.Load()) return loglvl
} }
// SetLevel sets the max logging level. // SetLevel sets the max logging level.
func SetLevel(lvl level.LEVEL) { func SetLevel(lvl level.LEVEL) {
loglvl.Store(uint32(lvl)) loglvl = lvl
}
// TimeFormat returns the currently-set timestamp format.
func TimeFormat() string {
return timefmt
}
// SetTimeFormat sets the timestamp format to the given string.
func SetTimeFormat(format string) {
if format == "" {
timefmt = format
return
}
timefmt = `timestamp="` + format + `" `
} }
// New starts a new log entry. // New starts a new log entry.
@ -164,10 +178,8 @@ func printf(depth int, fields []kv.Field, s string, a ...interface{}) {
// Acquire buffer // Acquire buffer
buf := getBuf() buf := getBuf()
// Append formatted timestamp // Append formatted timestamp according to `timefmt`
buf.B = append(buf.B, `timestamp="`...)
buf.B = time.Now().AppendFormat(buf.B, timefmt) buf.B = time.Now().AppendFormat(buf.B, timefmt)
buf.B = append(buf.B, `" `...)
// Append formatted caller func // Append formatted caller func
buf.B = append(buf.B, `func=`...) buf.B = append(buf.B, `func=`...)
@ -217,10 +229,8 @@ func logf(ctx context.Context, depth int, lvl level.LEVEL, fields []kv.Field, s
// Acquire buffer // Acquire buffer
buf := getBuf() buf := getBuf()
// Append formatted timestamp // Append formatted timestamp according to `timefmt`
buf.B = append(buf.B, `timestamp="`...)
buf.B = time.Now().AppendFormat(buf.B, timefmt) buf.B = time.Now().AppendFormat(buf.B, timefmt)
buf.B = append(buf.B, `" `...)
// Append formatted caller func // Append formatted caller func
buf.B = append(buf.B, `func=`...) buf.B = append(buf.B, `func=`...)

View file

@ -71,6 +71,22 @@ func (suite *SyslogTestSuite) TestSyslog() {
suite.Regexp(regexp.MustCompile(`timestamp=.* func=.* level=INFO msg="this is a test of the emergency broadcast system!"`), entry["content"]) suite.Regexp(regexp.MustCompile(`timestamp=.* func=.* level=INFO msg="this is a test of the emergency broadcast system!"`), entry["content"])
} }
func (suite *SyslogTestSuite) TestSyslogDisableTimestamp() {
// Get the current format.
timefmt := log.TimeFormat()
// Set an empty timestamp.
log.SetTimeFormat("")
// Set old timestamp on return.
defer log.SetTimeFormat(timefmt)
log.Info(nil, "this is a test of the emergency broadcast system!")
entry := <-suite.syslogChannel
suite.Regexp(regexp.MustCompile(`func=.* level=INFO msg="this is a test of the emergency broadcast system!"`), entry["content"])
}
func (suite *SyslogTestSuite) TestSyslogLongMessage() { func (suite *SyslogTestSuite) TestSyslogLongMessage() {
log.Warn(nil, longMessage) log.Warn(nil, longMessage)

View file

@ -86,6 +86,7 @@ EXPECT=$(cat << "EOF"
"log-client-ip": false, "log-client-ip": false,
"log-db-queries": true, "log-db-queries": true,
"log-level": "info", "log-level": "info",
"log-timestamp-format": "banana",
"media-description-max-chars": 5000, "media-description-max-chars": 5000,
"media-description-min-chars": 69, "media-description-min-chars": 69,
"media-emoji-local-max-size": 420, "media-emoji-local-max-size": 420,
@ -155,6 +156,7 @@ EOF
# Set all the environment variables to # Set all the environment variables to
# ensure that these are parsed without panic # ensure that these are parsed without panic
OUTPUT=$(GTS_LOG_LEVEL='info' \ OUTPUT=$(GTS_LOG_LEVEL='info' \
GTS_LOG_TIMESTAMP_FORMAT="banana" \
GTS_LOG_DB_QUERIES=true \ GTS_LOG_DB_QUERIES=true \
GTS_LOG_CLIENT_IP=false \ GTS_LOG_CLIENT_IP=false \
GTS_APPLICATION_NAME=gts \ GTS_APPLICATION_NAME=gts \

View file

@ -33,17 +33,18 @@ func InitTestConfig() {
} }
var testDefaults = config.Configuration{ var testDefaults = config.Configuration{
LogLevel: "info", LogLevel: "info",
LogDbQueries: true, LogTimestampFormat: "02/01/2006 15:04:05.000",
ApplicationName: "gotosocial", LogDbQueries: true,
LandingPageUser: "", ApplicationName: "gotosocial",
ConfigPath: "", LandingPageUser: "",
Host: "localhost:8080", ConfigPath: "",
AccountDomain: "localhost:8080", Host: "localhost:8080",
Protocol: "http", AccountDomain: "localhost:8080",
BindAddress: "127.0.0.1", Protocol: "http",
Port: 8080, BindAddress: "127.0.0.1",
TrustedProxies: []string{"127.0.0.1/32", "::1"}, Port: 8080,
TrustedProxies: []string{"127.0.0.1/32", "::1"},
DbType: "sqlite", DbType: "sqlite",
DbAddress: ":memory:", DbAddress: ":memory:",

View file

@ -26,6 +26,7 @@ import (
// InitTestLog sets the global logger to trace level for logging // InitTestLog sets the global logger to trace level for logging
func InitTestLog() { func InitTestLog() {
log.SetTimeFormat(config.GetLogTimestampFormat())
// Set the global log level from configuration // Set the global log level from configuration
if err := log.ParseLevel(config.GetLogLevel()); err != nil { if err := log.ParseLevel(config.GetLogLevel()); err != nil {
log.Panicf(nil, "error parsing log level: %v", err) log.Panicf(nil, "error parsing log level: %v", err)