feat(build): add linter for missing msgid definitions

Signed-off-by: Ellen Emilia Anna Zscheile <fogti+devel@ytrizja.de>

Makefile

@@ -413,7 +413,7 @@ lint-frontend: lint-js lint-css
 lint-frontend-fix: lint-js-fix lint-css-fix
 
 .PHONY: lint-backend
-lint-backend: lint-go lint-go-vet lint-editorconfig lint-renovate lint-locale lint-disposable-emails
+lint-backend: lint-go lint-go-vet lint-editorconfig lint-renovate lint-locale lint-locale-usage lint-disposable-emails
 
 .PHONY: lint-backend-fix
 lint-backend-fix: lint-go-fix lint-go-vet lint-editorconfig lint-disposable-emails-fix
@@ -460,6 +460,10 @@ lint-renovate: node_modules
 lint-locale:
 	$(GO) run build/lint-locale.go
 
+.PHONY: lint-locale-usage
+lint-locale-usage:
+	$(GO) run build/lint-locale-usage.go
+
 .PHONY: lint-md
 lint-md: node_modules
 	npx markdownlint docs *.md

build/lint-locale-usage.go

@@ -0,0 +1,305 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// Copyright 2025 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build ignore
+
+package main
+
+import (
+	"fmt"
+	"go/ast"
+	goParser "go/parser"
+	"go/token"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"text/template"
+	tmplParser "text/template/parse"
+
+	"code.gitea.io/gitea/modules/container"
+	"code.gitea.io/gitea/modules/locale"
+	fjTemplates "code.gitea.io/gitea/modules/templates"
+	"code.gitea.io/gitea/modules/util"
+)
+
+// this works by first gathering all valid source string IDs from `en-US` reference files
+// and then checking if all used source strings are actually defined
+
+type OnMsgidHandler func(fset *token.FileSet, pos token.Pos, msgid string)
+
+type LocatedError struct {
+	Location string
+	Kind     string
+	Err      error
+}
+
+func (e LocatedError) Error() string {
+	var sb strings.Builder
+
+	sb.WriteString(e.Location)
+	sb.WriteString(":\t")
+	if e.Kind != "" {
+		sb.WriteString(e.Kind)
+		sb.WriteString(": ")
+	}
+	sb.WriteString("ERROR: ")
+	sb.WriteString(e.Err.Error())
+
+	return sb.String()
+}
+
+func isLocaleTrFunction(funcname string) bool {
+	return funcname == "Tr" || funcname == "TrN"
+}
+
+// the `Handle*File` functions follow the following calling convention:
+// * `fname` is the name of the input file
+// * `src` is either `nil` (then the function invokes `ReadFile` to read the file)
+//   or the contents of the file as {`[]byte`, or a `string`}
+
+func (omh OnMsgidHandler) HandleGoFile(fname string, src any) error {
+	fset := token.NewFileSet()
+	node, err := goParser.ParseFile(fset, fname, src, goParser.SkipObjectResolution)
+	if err != nil {
+		return LocatedError{
+			Location: fname,
+			Kind:     "Go parser",
+			Err:      err,
+		}
+	}
+
+	ast.Inspect(node, func(n ast.Node) bool {
+		// search for function calls of the form `anything.Tr(any-string-lit)`
+
+		call, ok := n.(*ast.CallExpr)
+		if !ok || len(call.Args) != 1 {
+			return true
+		}
+
+		funSel, ok := call.Fun.(*ast.SelectorExpr)
+		if (!ok) || !isLocaleTrFunction(funSel.Sel.Name) {
+			return true
+		}
+
+		argLit, ok := call.Args[0].(*ast.BasicLit)
+		if (!ok) || argLit.Kind != token.STRING {
+			return true
+		}
+
+		// extract string content
+		arg, err := strconv.Unquote(argLit.Value)
+		if err != nil {
+			return true
+		}
+
+		// found interesting string
+		omh(fset, argLit.ValuePos, arg)
+
+		return true
+	})
+
+	return nil
+}
+
+// derived from source: modules/templates/scopedtmpl/scopedtmpl.go, L169-L213
+func (omh OnMsgidHandler) handleTemplateNode(fset *token.FileSet, node tmplParser.Node) {
+	switch node.Type() {
+	case tmplParser.NodeAction:
+		omh.handleTemplatePipeNode(fset, node.(*tmplParser.ActionNode).Pipe)
+	case tmplParser.NodeList:
+		nodeList := node.(*tmplParser.ListNode)
+		omh.handleTemplateFileNodes(fset, nodeList.Nodes)
+	case tmplParser.NodePipe:
+		omh.handleTemplatePipeNode(fset, node.(*tmplParser.PipeNode))
+	case tmplParser.NodeTemplate:
+		omh.handleTemplatePipeNode(fset, node.(*tmplParser.TemplateNode).Pipe)
+	case tmplParser.NodeIf:
+		nodeIf := node.(*tmplParser.IfNode)
+		omh.handleTemplateBranchNode(fset, nodeIf.BranchNode)
+	case tmplParser.NodeRange:
+		nodeRange := node.(*tmplParser.RangeNode)
+		omh.handleTemplateBranchNode(fset, nodeRange.BranchNode)
+	case tmplParser.NodeWith:
+		nodeWith := node.(*tmplParser.WithNode)
+		omh.handleTemplateBranchNode(fset, nodeWith.BranchNode)
+
+	case tmplParser.NodeCommand:
+		nodeCommand := node.(*tmplParser.CommandNode)
+
+		omh.handleTemplateFileNodes(fset, nodeCommand.Args)
+
+		if len(nodeCommand.Args) != 2 {
+			return
+		}
+
+		nodeChain, ok := nodeCommand.Args[0].(*tmplParser.ChainNode)
+		if !ok {
+			return
+		}
+
+		nodeString, ok := nodeCommand.Args[1].(*tmplParser.StringNode)
+		if !ok {
+			return
+		}
+
+		nodeIdent, ok := nodeChain.Node.(*tmplParser.IdentifierNode)
+		if !ok || nodeIdent.Ident != "ctx" {
+			return
+		}
+
+		if len(nodeChain.Field) != 2 || nodeChain.Field[0] != "Locale" || !isLocaleTrFunction(nodeChain.Field[1]) {
+			return
+		}
+
+		// found interesting string
+		// the column numbers are a bit "off", but much better than nothing
+		omh(fset, token.Pos(nodeString.Pos), nodeString.Text)
+
+	default:
+	}
+}
+
+func (omh OnMsgidHandler) handleTemplatePipeNode(fset *token.FileSet, pipeNode *tmplParser.PipeNode) {
+	if pipeNode == nil {
+		return
+	}
+
+	// NOTE: we can't pass `pipeNode.Cmds` to handleTemplateFileNodes due to incompatible argument types
+	for _, node := range pipeNode.Cmds {
+		omh.handleTemplateNode(fset, node)
+	}
+}
+
+func (omh OnMsgidHandler) handleTemplateBranchNode(fset *token.FileSet, branchNode tmplParser.BranchNode) {
+	omh.handleTemplatePipeNode(fset, branchNode.Pipe)
+	omh.handleTemplateFileNodes(fset, branchNode.List.Nodes)
+	if branchNode.ElseList != nil {
+		omh.handleTemplateFileNodes(fset, branchNode.ElseList.Nodes)
+	}
+}
+
+func (omh OnMsgidHandler) handleTemplateFileNodes(fset *token.FileSet, nodes []tmplParser.Node) {
+	for _, node := range nodes {
+		omh.handleTemplateNode(fset, node)
+	}
+}
+
+func (omh OnMsgidHandler) HandleTemplateFile(fname string, src any) error {
+	var tmplContent []byte
+	switch src2 := src.(type) {
+	case nil:
+		var err error
+		tmplContent, err = os.ReadFile(fname)
+		if err != nil {
+			return LocatedError{
+				Location: fname,
+				Kind:     "ReadFile",
+				Err:      err,
+			}
+		}
+	case []byte:
+		tmplContent = src2
+	case string:
+		// SAFETY: we do not modify tmplContent below
+		tmplContent = util.UnsafeStringToBytes(src2)
+	default:
+		panic("invalid type for 'src'")
+	}
+
+	fset := token.NewFileSet()
+	fset.AddFile(fname, 1, len(tmplContent)).SetLinesForContent(tmplContent)
+	// SAFETY: we do not modify tmplContent2 below
+	tmplContent2 := util.UnsafeBytesToString(tmplContent)
+
+	tmpl := template.New(fname)
+	tmpl.Funcs(fjTemplates.NewFuncMap())
+	tmplParsed, err := tmpl.Parse(tmplContent2)
+	if err != nil {
+		return LocatedError{
+			Location: fname,
+			Kind:     "Template parser",
+			Err:      err,
+		}
+	} else {
+		omh.handleTemplateFileNodes(fset, tmplParsed.Tree.Root.Nodes)
+		return nil
+	}
+}
+
+// This command assumes that we get started from the project root directory
+func main() {
+	onError := func(err error) {
+		if err == nil {
+			return
+		}
+		fmt.Println(err.Error())
+	}
+
+	msgids := make(container.Set[string])
+	onMsgid := func(trKey, trValue string) error {
+		msgids[trKey] = struct{}{}
+		return nil
+	}
+
+	localeFile := filepath.Join(filepath.Join("options", "locale"), "locale_en-US.ini")
+	localeContent, err := os.ReadFile(localeFile)
+	if err != nil {
+		fmt.Printf("%s:\tERROR: %s\n", localeFile, err.Error())
+		os.Exit(2)
+	}
+
+	if err = locale.IterateMessagesContent(localeContent, onMsgid); err != nil {
+		fmt.Printf("%s:\tERROR: %s\n", localeFile, err.Error())
+		os.Exit(2)
+	}
+
+	localeFile = filepath.Join(filepath.Join("options", "locale_next"), "locale_en-US.json")
+	localeContent, err = os.ReadFile(localeFile)
+	if err != nil {
+		fmt.Printf("%s:\tERROR: %s\n", localeFile, err.Error())
+		os.Exit(2)
+	}
+
+	if err := locale.IterateMessagesNextContent(localeContent, onMsgid); err != nil {
+		fmt.Printf("%s:\tERROR: %s\n", localeFile, err.Error())
+		os.Exit(2)
+	}
+
+	omh := OnMsgidHandler(func(fset *token.FileSet, pos token.Pos, msgid string) {
+		if !msgids.Contains(msgid) {
+			fmt.Printf("%s:\tmissing msgid: %s\n", fset.Position(pos).String(), msgid)
+		}
+	})
+
+	if err := filepath.WalkDir(".", func(fpath string, d fs.DirEntry, err error) error {
+		if err != nil {
+			if os.IsNotExist(err) {
+				return nil
+			}
+			return err
+		}
+		name := d.Name()
+		if d.IsDir() {
+			if name == "docker" || name == ".git" || name == "node_modules" {
+				return fs.SkipDir
+			}
+		} else if name == "bindata.go" {
+			// skip false positives
+		} else if strings.HasSuffix(name, ".go") {
+			onError(omh.HandleGoFile(fpath, nil))
+		} else if strings.HasSuffix(name, ".tmpl") {
+			if strings.HasPrefix(fpath, "tests") && strings.HasSuffix(name, ".ini.tmpl") {
+				// skip false positives
+			} else {
+				onError(omh.HandleTemplateFile(fpath, nil))
+			}
+		}
+		return nil
+	}); err != nil {
+		fmt.Printf("walkdir ERROR: %s\n", err.Error())
+		os.Exit(1)
+	}
+}

build/lint-locale-usage_test.go

@@ -0,0 +1,46 @@
+// Copyright 2025 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build ignore
+
+package main
+
+import (
+	"go/token"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func HandleGoFileWrapped(t *testing.T, fname, src string) []string {
+	var ret []string
+	omh := OnMsgidHandler(func(fset *token.FileSet, pos token.Pos, msgid string) {
+		ret = append(ret, msgid)
+	})
+	require.NoError(t, omh.HandleGoFile(fname, src))
+	return ret
+}
+
+func HandleTemplateFileWrapped(t *testing.T, fname, src string) []string {
+	var ret []string
+	omh := OnMsgidHandler(func(fset *token.FileSet, pos token.Pos, msgid string) {
+		ret = append(ret, msgid)
+	})
+	require.NoError(t, omh.HandleTemplateFile(fname, src))
+	return ret
+}
+
+func TestUsagesParser(t *testing.T) {
+	t.Run("go, simple", func(t *testing.T) {
+		assert.EqualValues(t,
+			HandleGoFileWrapped(t, "<g1>", "package main\nfunc Render(ctx *context.Context) string { return ctx.Tr(\"what.an.example\"); }\n"),
+			[]string{"what.an.example"})
+	})
+
+	t.Run("template, simple", func(t *testing.T) {
+		assert.EqualValues(t,
+			HandleTemplateFileWrapped(t, "<t1>", "{{ ctx.Locale.Tr \"what.an.example\" }}\n"),
+			[]string{"what.an.example"})
+	})
+}

build/lint-locale.go

@@ -5,7 +5,6 @@
 package main
 
 import (
-	"encoding/json" //nolint:depguard
 	"fmt"
 	"html"
 	"io/fs"
@@ -15,9 +14,10 @@ import (
 	"slices"
 	"strings"
 
+	"code.gitea.io/gitea/modules/locale"
+
 	"github.com/microcosm-cc/bluemonday"
 	"github.com/sergi/go-diff/diffmatchpatch"
-	"gopkg.in/ini.v1" //nolint:depguard
 )
 
 var (
@@ -98,49 +98,28 @@ func checkValue(trKey, value string) []string {
 }
 
 func checkLocaleContent(localeContent []byte) []string {
-	// Same configuration as Forgejo uses.
-	cfg := ini.Empty(ini.LoadOptions{
-		IgnoreContinuation: true,
-	})
-	cfg.NameMapper = ini.SnackCase
+	errors := []string{}
 
-	if err := cfg.Append(localeContent); err != nil {
+	if err := locale.IterateMessagesContent(localeContent, func(trKey, trValue string) error {
+		errors = append(errors, checkValue(trKey, trValue)...)
+		return nil
+	}); err != nil {
 		panic(err)
 	}
 
-	errors := []string{}
-	for _, section := range cfg.Sections() {
-		for _, key := range section.Keys() {
-			var trKey string
-			if section.Name() == "" || section.Name() == "DEFAULT" || section.Name() == "common" {
-				trKey = key.Name()
-			} else {
-				trKey = section.Name() + "." + key.Name()
-			}
-
-			errors = append(errors, checkValue(trKey, key.Value())...)
-		}
-	}
 	return errors
 }
 
-func checkLocaleNextContent(data map[string]any, trKey ...string) []string {
+func checkLocaleNextContent(localeContent []byte) []string {
 	errors := []string{}
-	for key, value := range data {
-		currentKey := key
-		if len(trKey) == 1 {
-			currentKey = trKey[0] + "." + key
-		}
 
-		switch value := value.(type) {
-		case string:
-			errors = append(errors, checkValue(currentKey, value)...)
-		case map[string]any:
-			errors = append(errors, checkLocaleNextContent(value, currentKey)...)
-		default:
-			panic(fmt.Sprintf("Unexpected type during linting locale next: %s - %T", currentKey, value))
-		}
+	if err := locale.IterateMessagesNextContent(localeContent, func(trKey, trValue string) error {
+		errors = append(errors, checkValue(trKey, trValue)...)
+		return nil
+	}); err != nil {
+		panic(err)
 	}
+
 	return errors
 }
 
@@ -168,6 +147,7 @@ func main() {
 
 		localeContent, err := os.ReadFile(filepath.Join(localeDir, localeFile.Name()))
 		if err != nil {
+			fmt.Println(localeFile.Name())
 			panic(err)
 		}
 
@@ -195,15 +175,11 @@ func main() {
 	for _, localeFile := range localeFiles {
 		localeContent, err := os.ReadFile(filepath.Join(localeDir, localeFile.Name()))
 		if err != nil {
+			fmt.Println(localeFile.Name())
 			panic(err)
 		}
 
-		var localeData map[string]any
-		if err := json.Unmarshal(localeContent, &localeData); err != nil {
-			panic(err)
-		}
-
-		if err := checkLocaleNextContent(localeData); len(err) > 0 {
+		if err := checkLocaleNextContent(localeContent); len(err) > 0 {
 			fmt.Println(localeFile.Name())
 			fmt.Println(strings.Join(err, "\n"))
 			fmt.Println()

build/lint-locale_test.go

@@ -69,26 +69,26 @@ func TestNextLocalizationPolicy(t *testing.T) {
 	initRemoveTags()
 
 	t.Run("Nested locales", func(t *testing.T) {
-		assert.Empty(t, checkLocaleNextContent(map[string]any{
-			"settings": map[string]any{
-				"hidden_comment_types_description": `Comment types checked here will not be shown inside issue pages. Checking "Label" for example removes all "<user> added/removed <label>" comments.`,
-			},
-		}))
+		assert.Empty(t, checkLocaleNextContent([]byte(`{
+			"settings": {
+				"hidden_comment_types_description": "Comment types checked here will not be shown inside issue pages. Checking \"Label\" for example removes all \"<user> added/removed <label>\" comments."
+			}
+		}`)))
 
-		assert.EqualValues(t, []string{"settings.hidden_comment_types_description: \"\x1b[31m<not-an-allowed-key>\x1b[0m REPLACED-TAG\""}, checkLocaleNextContent(map[string]any{
-			"settings": map[string]any{
-				"hidden_comment_types_description": `"<not-an-allowed-key> <label>"`,
-			},
-		}))
+		assert.EqualValues(t, []string{"settings.hidden_comment_types_description: \"\x1b[31m<not-an-allowed-key>\x1b[0m REPLACED-TAG\""}, checkLocaleNextContent([]byte(`{
+			"settings": {
+				"hidden_comment_types_description": "\"<not-an-allowed-key> <label>\""
+			}
+		}`)))
 	})
 
 	t.Run("Flat locales", func(t *testing.T) {
-		assert.Empty(t, checkLocaleNextContent(map[string]any{
-			"settings.hidden_comment_types_description": `Comment types checked here will not be shown inside issue pages. Checking "Label" for example removes all "<user> added/removed <label>" comments.`,
-		}))
+		assert.Empty(t, checkLocaleNextContent([]byte(`{
+			"settings.hidden_comment_types_description": "Comment types checked here will not be shown inside issue pages. Checking \"Label\" for example removes all \"<user> added/removed <label>\" comments."
+		}`)))
 
-		assert.EqualValues(t, []string{"settings.hidden_comment_types_description: \"\x1b[31m<not-an-allowed-key>\x1b[0m REPLACED-TAG\""}, checkLocaleNextContent(map[string]any{
-			"settings.hidden_comment_types_description": `"<not-an-allowed-key> <label>"`,
-		}))
+		assert.EqualValues(t, []string{"settings.hidden_comment_types_description: \"\x1b[31m<not-an-allowed-key>\x1b[0m REPLACED-TAG\""}, checkLocaleNextContent([]byte(`{
+			"settings.hidden_comment_types_description": "\"<not-an-allowed-key> <label>\""
+		}`)))
 	})
 }

modules/locale/utils.go

@@ -0,0 +1,74 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// Copyright 2025 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+// extracted from `/build/lint-locale.go`, `/build/lint-locale-usage.go`
+
+package locale
+
+import (
+	"encoding/json" //nolint:depguard
+	"fmt"
+
+	"gopkg.in/ini.v1" //nolint:depguard
+)
+
+func IterateMessagesContent(localeContent []byte, onMsgid func(string, string) error) error {
+	// Same configuration as Forgejo uses.
+	cfg := ini.Empty(ini.LoadOptions{
+		IgnoreContinuation: true,
+	})
+	cfg.NameMapper = ini.SnackCase
+
+	if err := cfg.Append(localeContent); err != nil {
+		return err
+	}
+
+	for _, section := range cfg.Sections() {
+		for _, key := range section.Keys() {
+			var trKey string
+			if section.Name() == "" || section.Name() == "DEFAULT" || section.Name() == "common" {
+				trKey = key.Name()
+			} else {
+				trKey = section.Name() + "." + key.Name()
+			}
+			if err := onMsgid(trKey, key.Value()); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func iterateMessagesNextInner(onMsgid func(string, string) error, data map[string]any, trKey ...string) error {
+	for key, value := range data {
+		currentKey := key
+		if len(trKey) == 1 {
+			currentKey = trKey[0] + "." + key
+		}
+
+		switch value := value.(type) {
+		case string:
+			if err := onMsgid(currentKey, value); err != nil {
+				return err
+			}
+		case map[string]any:
+			if err := iterateMessagesNextInner(onMsgid, value, currentKey); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("unexpected type: %s - %T", currentKey, value)
+		}
+	}
+
+	return nil
+}
+
+func IterateMessagesNextContent(localeContent []byte, onMsgid func(string, string) error) error {
+	var localeData map[string]any
+	if err := json.Unmarshal(localeContent, &localeData); err != nil {
+		return err
+	}
+	return iterateMessagesNextInner(onMsgid, localeData)
+}