5 files changed, 474 insertions, 0 deletions
diff --git a/modules/translation/i18n/errors.go b/modules/translation/i18n/errors.go
new file mode 100644
index 00000000..7f64ccf9
--- /dev/null
+++ b/modules/translation/i18n/errors.go
@@ -0,0 +1,13 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package i18n
+
+import (
+	"code.gitea.io/gitea/modules/util"
+)
+
+var (
+	ErrLocaleAlreadyExist = util.SilentWrap{Message: "lang already exists", Err: util.ErrAlreadyExist}
+	ErrUncertainArguments = util.SilentWrap{Message: "arguments to i18n should not contain uncertain slices", Err: util.ErrInvalidArgument}
+)
diff --git a/modules/translation/i18n/format.go b/modules/translation/i18n/format.go
new file mode 100644
index 00000000..e5e22183
--- /dev/null
+++ b/modules/translation/i18n/format.go
@@ -0,0 +1,41 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package i18n
+
+import (
+	"fmt"
+	"reflect"
+)
+
+// Format formats provided arguments for a given translated message
+func Format(format string, args ...any) (msg string, err error) {
+	if len(args) == 0 {
+		return format, nil
+	}
+
+	fmtArgs := make([]any, 0, len(args))
+	for _, arg := range args {
+		val := reflect.ValueOf(arg)
+		if val.Kind() == reflect.Slice {
+			// Previously, we would accept Tr(lang, key, a, [b, c], d, [e, f]) as Sprintf(msg, a, b, c, d, e, f)
+			// but this is an unstable behavior.
+			//
+			// So we restrict the accepted arguments to either:
+			//
+			// 1. Tr(lang, key, [slice-items]) as Sprintf(msg, items...)
+			// 2. Tr(lang, key, args...) as Sprintf(msg, args...)
+			if len(args) == 1 {
+				for i := 0; i < val.Len(); i++ {
+					fmtArgs = append(fmtArgs, val.Index(i).Interface())
+				}
+			} else {
+				err = ErrUncertainArguments
+				break
+			}
+		} else {
+			fmtArgs = append(fmtArgs, arg)
+		}
+	}
+	return fmt.Sprintf(format, fmtArgs...), err
+}
diff --git a/modules/translation/i18n/i18n.go b/modules/translation/i18n/i18n.go
new file mode 100644
index 00000000..1555cd96
--- /dev/null
+++ b/modules/translation/i18n/i18n.go
@@ -0,0 +1,50 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package i18n
+
+import (
+	"html/template"
+	"io"
+)
+
+var DefaultLocales = NewLocaleStore()
+
+type Locale interface {
+	// TrString translates a given key and arguments for a language
+	TrString(trKey string, trArgs ...any) string
+	// TrHTML translates a given key and arguments for a language, string arguments are escaped to HTML
+	TrHTML(trKey string, trArgs ...any) template.HTML
+	// HasKey reports if a locale has a translation for a given key
+	HasKey(trKey string) bool
+}
+
+// LocaleStore provides the functions common to all locale stores
+type LocaleStore interface {
+	io.Closer
+
+	// SetDefaultLang sets the default language to fall back to
+	SetDefaultLang(lang string)
+	// ListLangNameDesc provides paired slices of language names to descriptors
+	ListLangNameDesc() (names, desc []string)
+	// Locale return the locale for the provided language or the default language if not found
+	Locale(langName string) (Locale, bool)
+	// HasLang returns whether a given language is present in the store
+	HasLang(langName string) bool
+	// AddLocaleByIni adds a new language to the store
+	AddLocaleByIni(langName, langDesc string, source, moreSource []byte) error
+}
+
+// ResetDefaultLocales resets the current default locales
+// NOTE: this is not synchronized
+func ResetDefaultLocales() {
+	if DefaultLocales != nil {
+		_ = DefaultLocales.Close()
+	}
+	DefaultLocales = NewLocaleStore()
+}
+
+// GetLocale returns the locale from the default locales
+func GetLocale(lang string) (Locale, bool) {
+	return DefaultLocales.Locale(lang)
+}
diff --git a/modules/translation/i18n/i18n_test.go b/modules/translation/i18n/i18n_test.go
new file mode 100644
index 00000000..244f6ffb
--- /dev/null
+++ b/modules/translation/i18n/i18n_test.go
@@ -0,0 +1,204 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package i18n
+
+import (
+	"html/template"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLocaleStore(t *testing.T) {
+	testData1 := []byte(`
+.dot.name = Dot Name
+fmt = %[1]s %[2]s
+
+[section]
+sub = Sub String
+mixed = test value; <span style="color: red\; background: none;">%s</span>
+`)
+
+	testData2 := []byte(`
+fmt = %[2]s %[1]s
+
+[section]
+sub = Changed Sub String
+`)
+
+	ls := NewLocaleStore()
+	require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1, nil))
+	require.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2, nil))
+	ls.SetDefaultLang("lang1")
+
+	lang1, _ := ls.Locale("lang1")
+	lang2, _ := ls.Locale("lang2")
+
+	result := lang1.TrString("fmt", "a", "b")
+	assert.Equal(t, "a b", result)
+
+	result = lang2.TrString("fmt", "a", "b")
+	assert.Equal(t, "b a", result)
+
+	result = lang1.TrString("section.sub")
+	assert.Equal(t, "Sub String", result)
+
+	result = lang2.TrString("section.sub")
+	assert.Equal(t, "Changed Sub String", result)
+
+	langNone, _ := ls.Locale("none")
+	result = langNone.TrString(".dot.name")
+	assert.Equal(t, "Dot Name", result)
+
+	result2 := lang2.TrHTML("section.mixed", "a&b")
+	assert.EqualValues(t, `test value; <span style="color: red; background: none;">a&amp;b</span>`, result2)
+
+	langs, descs := ls.ListLangNameDesc()
+	assert.ElementsMatch(t, []string{"lang1", "lang2"}, langs)
+	assert.ElementsMatch(t, []string{"Lang1", "Lang2"}, descs)
+
+	found := lang1.HasKey("no-such")
+	assert.False(t, found)
+	require.NoError(t, ls.Close())
+}
+
+func TestLocaleStoreMoreSource(t *testing.T) {
+	testData1 := []byte(`
+a=11
+b=12
+`)
+
+	testData2 := []byte(`
+b=21
+c=22
+`)
+
+	ls := NewLocaleStore()
+	require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1, testData2))
+	lang1, _ := ls.Locale("lang1")
+	assert.Equal(t, "11", lang1.TrString("a"))
+	assert.Equal(t, "21", lang1.TrString("b"))
+	assert.Equal(t, "22", lang1.TrString("c"))
+}
+
+type stringerPointerReceiver struct {
+	s string
+}
+
+func (s *stringerPointerReceiver) String() string {
+	return s.s
+}
+
+type stringerStructReceiver struct {
+	s string
+}
+
+func (s stringerStructReceiver) String() string {
+	return s.s
+}
+
+type errorStructReceiver struct {
+	s string
+}
+
+func (e errorStructReceiver) Error() string {
+	return e.s
+}
+
+type errorPointerReceiver struct {
+	s string
+}
+
+func (e *errorPointerReceiver) Error() string {
+	return e.s
+}
+
+func TestLocaleWithTemplate(t *testing.T) {
+	ls := NewLocaleStore()
+	require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", []byte(`key=<a>%s</a>`), nil))
+	lang1, _ := ls.Locale("lang1")
+
+	tmpl := template.New("test").Funcs(template.FuncMap{"tr": lang1.TrHTML})
+	tmpl = template.Must(tmpl.Parse(`{{tr "key" .var}}`))
+
+	cases := []struct {
+		in   any
+		want string
+	}{
+		{"<str>", "<a>&lt;str&gt;</a>"},
+		{[]byte("<bytes>"), "<a>[60 98 121 116 101 115 62]</a>"},
+		{template.HTML("<html>"), "<a><html></a>"},
+		{stringerPointerReceiver{"<stringerPointerReceiver>"}, "<a>{&lt;stringerPointerReceiver&gt;}</a>"},
+		{&stringerPointerReceiver{"<stringerPointerReceiver ptr>"}, "<a>&lt;stringerPointerReceiver ptr&gt;</a>"},
+		{stringerStructReceiver{"<stringerStructReceiver>"}, "<a>&lt;stringerStructReceiver&gt;</a>"},
+		{&stringerStructReceiver{"<stringerStructReceiver ptr>"}, "<a>&lt;stringerStructReceiver ptr&gt;</a>"},
+		{errorStructReceiver{"<errorStructReceiver>"}, "<a>&lt;errorStructReceiver&gt;</a>"},
+		{&errorStructReceiver{"<errorStructReceiver ptr>"}, "<a>&lt;errorStructReceiver ptr&gt;</a>"},
+		{errorPointerReceiver{"<errorPointerReceiver>"}, "<a>{&lt;errorPointerReceiver&gt;}</a>"},
+		{&errorPointerReceiver{"<errorPointerReceiver ptr>"}, "<a>&lt;errorPointerReceiver ptr&gt;</a>"},
+	}
+
+	buf := &strings.Builder{}
+	for _, c := range cases {
+		buf.Reset()
+		require.NoError(t, tmpl.Execute(buf, map[string]any{"var": c.in}))
+		assert.Equal(t, c.want, buf.String())
+	}
+}
+
+func TestLocaleStoreQuirks(t *testing.T) {
+	const nl = "\n"
+	q := func(q1, s string, q2 ...string) string {
+		return q1 + s + strings.Join(q2, "")
+	}
+	testDataList := []struct {
+		in   string
+		out  string
+		hint string
+	}{
+		{` xx`, `xx`, "simple, no quote"},
+		{`" xx"`, ` xx`, "simple, double-quote"},
+		{`' xx'`, ` xx`, "simple, single-quote"},
+		{"` xx`", ` xx`, "simple, back-quote"},
+
+		{`x\"y`, `x\"y`, "no unescape, simple"},
+		{q(`"`, `x\"y`, `"`), `"x\"y"`, "unescape, double-quote"},
+		{q(`'`, `x\"y`, `'`), `x\"y`, "no unescape, single-quote"},
+		{q("`", `x\"y`, "`"), `x\"y`, "no unescape, back-quote"},
+
+		{q(`"`, `x\"y`) + nl + "b=", `"x\"y`, "half open, double-quote"},
+		{q(`'`, `x\"y`) + nl + "b=", `'x\"y`, "half open, single-quote"},
+		{q("`", `x\"y`) + nl + "b=`", `x\"y` + nl + "b=", "half open, back-quote, multi-line"},
+
+		{`x ; y`, `x ; y`, "inline comment (;)"},
+		{`x # y`, `x # y`, "inline comment (#)"},
+		{`x \; y`, `x ; y`, `inline comment (\;)`},
+		{`x \# y`, `x # y`, `inline comment (\#)`},
+	}
+
+	for _, testData := range testDataList {
+		ls := NewLocaleStore()
+		err := ls.AddLocaleByIni("lang1", "Lang1", []byte("a="+testData.in), nil)
+		lang1, _ := ls.Locale("lang1")
+		require.NoError(t, err, testData.hint)
+		assert.Equal(t, testData.out, lang1.TrString("a"), testData.hint)
+		require.NoError(t, ls.Close())
+	}
+
+	// TODO: Crowdin needs the strings to be quoted correctly and doesn't like incomplete quotes
+	//       and Crowdin always outputs quoted strings if there are quotes in the strings.
+	//       So, Gitea's `key="quoted" unquoted` content shouldn't be used on Crowdin directly,
+	//       it should be converted to `key="\"quoted\" unquoted"` first.
+	// TODO: We can not use UnescapeValueDoubleQuotes=true, because there are a lot of back-quotes in en-US.ini,
+	//       then Crowdin will output:
+	//       > key = "`x \" y`"
+	//       Then Gitea will read a string with back-quotes, which is incorrect.
+	// TODO: Crowdin might generate multi-line strings, quoted by double-quote, it's not supported by LocaleStore
+	//       LocaleStore uses back-quote for multi-line strings, it's not supported by Crowdin.
+	// TODO: Crowdin doesn't support back-quote as string quoter, it mainly uses double-quote
+	//       so, the following line will be parsed as: value="`first", comment="second`" on Crowdin
+	//       > a = `first; second`
+}
diff --git a/modules/translation/i18n/localestore.go b/modules/translation/i18n/localestore.go
new file mode 100644
index 00000000..0e6ddab4
--- /dev/null
+++ b/modules/translation/i18n/localestore.go
@@ -0,0 +1,166 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package i18n
+
+import (
+	"fmt"
+	"html/template"
+	"slices"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// This file implements the static LocaleStore that will not watch for changes
+
+type locale struct {
+	store       *localeStore
+	langName    string
+	idxToMsgMap map[int]string // the map idx is generated by store's trKeyToIdxMap
+}
+
+var _ Locale = (*locale)(nil)
+
+type localeStore struct {
+	// After initializing has finished, these fields are read-only.
+	langNames []string
+	langDescs []string
+
+	localeMap     map[string]*locale
+	trKeyToIdxMap map[string]int
+
+	defaultLang string
+}
+
+// NewLocaleStore creates a static locale store
+func NewLocaleStore() LocaleStore {
+	return &localeStore{localeMap: make(map[string]*locale), trKeyToIdxMap: make(map[string]int)}
+}
+
+// AddLocaleByIni adds locale by ini into the store
+func (store *localeStore) AddLocaleByIni(langName, langDesc string, source, moreSource []byte) error {
+	if _, ok := store.localeMap[langName]; ok {
+		return ErrLocaleAlreadyExist
+	}
+
+	store.langNames = append(store.langNames, langName)
+	store.langDescs = append(store.langDescs, langDesc)
+
+	l := &locale{store: store, langName: langName, idxToMsgMap: make(map[int]string)}
+	store.localeMap[l.langName] = l
+
+	iniFile, err := setting.NewConfigProviderForLocale(source, moreSource)
+	if err != nil {
+		return fmt.Errorf("unable to load ini: %w", err)
+	}
+
+	for _, section := range iniFile.Sections() {
+		for _, key := range section.Keys() {
+			var trKey string
+			// see https://codeberg.org/forgejo/discussions/issues/104
+			//     https://github.com/WeblateOrg/weblate/issues/10831
+			// for an explanation of why "common" is an alternative
+			if section.Name() == "" || section.Name() == "DEFAULT" || section.Name() == "common" {
+				trKey = key.Name()
+			} else {
+				trKey = section.Name() + "." + key.Name()
+			}
+			idx, ok := store.trKeyToIdxMap[trKey]
+			if !ok {
+				idx = len(store.trKeyToIdxMap)
+				store.trKeyToIdxMap[trKey] = idx
+			}
+			l.idxToMsgMap[idx] = key.Value()
+		}
+	}
+
+	return nil
+}
+
+func (store *localeStore) HasLang(langName string) bool {
+	_, ok := store.localeMap[langName]
+	return ok
+}
+
+func (store *localeStore) ListLangNameDesc() (names, desc []string) {
+	return store.langNames, store.langDescs
+}
+
+// SetDefaultLang sets default language as a fallback
+func (store *localeStore) SetDefaultLang(lang string) {
+	store.defaultLang = lang
+}
+
+// Locale returns the locale for the lang or the default language
+func (store *localeStore) Locale(lang string) (Locale, bool) {
+	l, found := store.localeMap[lang]
+	if !found {
+		var ok bool
+		l, ok = store.localeMap[store.defaultLang]
+		if !ok {
+			// no default - return an empty locale
+			l = &locale{store: store, idxToMsgMap: make(map[int]string)}
+		}
+	}
+	return l, found
+}
+
+func (store *localeStore) Close() error {
+	return nil
+}
+
+func (l *locale) TrString(trKey string, trArgs ...any) string {
+	format := trKey
+
+	idx, ok := l.store.trKeyToIdxMap[trKey]
+	found := false
+	if ok {
+		if msg, ok := l.idxToMsgMap[idx]; ok {
+			format = msg // use the found translation
+			found = true
+		} else if def, ok := l.store.localeMap[l.store.defaultLang]; ok {
+			// try to use default locale's translation
+			if msg, ok := def.idxToMsgMap[idx]; ok {
+				format = msg
+				found = true
+			}
+		}
+	}
+	if !found {
+		log.Error("Missing translation %q", trKey)
+	}
+
+	msg, err := Format(format, trArgs...)
+	if err != nil {
+		log.Error("Error whilst formatting %q in %s: %v", trKey, l.langName, err)
+	}
+	return msg
+}
+
+func (l *locale) TrHTML(trKey string, trArgs ...any) template.HTML {
+	args := slices.Clone(trArgs)
+	for i, v := range args {
+		switch v := v.(type) {
+		case nil, bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, template.HTML:
+			// for most basic types (including template.HTML which is safe), just do nothing and use it
+		case string:
+			args[i] = template.HTMLEscapeString(v)
+		case fmt.Stringer:
+			args[i] = template.HTMLEscapeString(v.String())
+		default:
+			args[i] = template.HTMLEscapeString(fmt.Sprint(v))
+		}
+	}
+	return template.HTML(l.TrString(trKey, args...))
+}
+
+// HasKey returns whether a key is present in this locale or not
+func (l *locale) HasKey(trKey string) bool {
+	idx, ok := l.store.trKeyToIdxMap[trKey]
+	if !ok {
+		return false
+	}
+	_, ok = l.idxToMsgMap[idx]
+	return ok
+}