diff options
Diffstat (limited to '')
63 files changed, 9634 insertions, 0 deletions
diff --git a/modules/markup/asciicast/asciicast.go b/modules/markup/asciicast/asciicast.go new file mode 100644 index 00000000..06780623 --- /dev/null +++ b/modules/markup/asciicast/asciicast.go @@ -0,0 +1,64 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package asciicast + +import ( + "fmt" + "io" + "net/url" + "regexp" + + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/setting" +) + +func init() { + markup.RegisterRenderer(Renderer{}) +} + +// Renderer implements markup.Renderer for asciicast files. +// See https://github.com/asciinema/asciinema/blob/develop/doc/asciicast-v2.md +type Renderer struct{} + +// Name implements markup.Renderer +func (Renderer) Name() string { + return "asciicast" +} + +// Extensions implements markup.Renderer +func (Renderer) Extensions() []string { + return []string{".cast"} +} + +const ( + playerClassName = "asciinema-player-container" + playerSrcAttr = "data-asciinema-player-src" +) + +// SanitizerRules implements markup.Renderer +func (Renderer) SanitizerRules() []setting.MarkupSanitizerRule { + return []setting.MarkupSanitizerRule{ + {Element: "div", AllowAttr: "class", Regexp: regexp.MustCompile(playerClassName)}, + {Element: "div", AllowAttr: playerSrcAttr}, + } +} + +// Render implements markup.Renderer +func (Renderer) Render(ctx *markup.RenderContext, _ io.Reader, output io.Writer) error { + rawURL := fmt.Sprintf("%s/%s/%s/raw/%s/%s", + setting.AppSubURL, + url.PathEscape(ctx.Metas["user"]), + url.PathEscape(ctx.Metas["repo"]), + ctx.Metas["BranchNameSubURL"], + url.PathEscape(ctx.RelativePath), + ) + + _, err := io.WriteString(output, fmt.Sprintf( + `<div class="%s" %s="%s"></div>`, + playerClassName, + playerSrcAttr, + rawURL, + )) + return err +} diff --git a/modules/markup/camo.go b/modules/markup/camo.go new file mode 100644 index 00000000..e93797de --- /dev/null +++ b/modules/markup/camo.go @@ -0,0 +1,46 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markup + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "net/url" + "strings" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +// CamoEncode encodes a lnk to fit with the go-camo and camo proxy links. The purposes of camo-proxy are: +// 1. Allow accessing "http://" images on a HTTPS site by using the "https://" URLs provided by camo-proxy. +// 2. Hide the visitor's real IP (protect privacy) when accessing external images. +func CamoEncode(link string) string { + if strings.HasPrefix(link, setting.Camo.ServerURL) { + return link + } + + mac := hmac.New(sha1.New, []byte(setting.Camo.HMACKey)) + _, _ = mac.Write([]byte(link)) // hmac does not return errors + macSum := b64encode(mac.Sum(nil)) + encodedURL := b64encode([]byte(link)) + + return util.URLJoin(setting.Camo.ServerURL, macSum, encodedURL) +} + +func b64encode(data []byte) string { + return strings.TrimRight(base64.URLEncoding.EncodeToString(data), "=") +} + +func camoHandleLink(link string) string { + if setting.Camo.Enabled { + lnkURL, err := url.Parse(link) + if err == nil && lnkURL.IsAbs() && !strings.HasPrefix(link, setting.AppURL) && + (setting.Camo.Allways || lnkURL.Scheme != "https") { + return CamoEncode(link) + } + } + return link +} diff --git a/modules/markup/camo_test.go b/modules/markup/camo_test.go new file mode 100644 index 00000000..ba588352 --- /dev/null +++ b/modules/markup/camo_test.go @@ -0,0 +1,44 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markup + +import ( + "testing" + + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" +) + +func TestCamoHandleLink(t *testing.T) { + setting.AppURL = "https://gitea.com" + // Test media proxy + setting.Camo.Enabled = true + setting.Camo.ServerURL = "https://image.proxy" + setting.Camo.HMACKey = "geheim" + + assert.Equal(t, + "https://gitea.com/img.jpg", + camoHandleLink("https://gitea.com/img.jpg")) + assert.Equal(t, + "https://testimages.org/img.jpg", + camoHandleLink("https://testimages.org/img.jpg")) + assert.Equal(t, + "https://image.proxy/eivin43gJwGVIjR9MiYYtFIk0mw/aHR0cDovL3Rlc3RpbWFnZXMub3JnL2ltZy5qcGc", + camoHandleLink("http://testimages.org/img.jpg")) + + setting.Camo.Allways = true + assert.Equal(t, + "https://gitea.com/img.jpg", + camoHandleLink("https://gitea.com/img.jpg")) + assert.Equal(t, + "https://image.proxy/tkdlvmqpbIr7SjONfHNgEU622y0/aHR0cHM6Ly90ZXN0aW1hZ2VzLm9yZy9pbWcuanBn", + camoHandleLink("https://testimages.org/img.jpg")) + assert.Equal(t, + "https://image.proxy/eivin43gJwGVIjR9MiYYtFIk0mw/aHR0cDovL3Rlc3RpbWFnZXMub3JnL2ltZy5qcGc", + camoHandleLink("http://testimages.org/img.jpg")) + + // Restore previous settings + setting.Camo.Enabled = false +} diff --git a/modules/markup/common/footnote.go b/modules/markup/common/footnote.go new file mode 100644 index 00000000..0e75e2ad --- /dev/null +++ b/modules/markup/common/footnote.go @@ -0,0 +1,498 @@ +// Copyright 2019 Yusuke Inuzuka +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +// Most of what follows is a subtly changed version of github.com/yuin/goldmark/extension/footnote.go + +package common + +import ( + "bytes" + "fmt" + "strconv" + "unicode" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +// CleanValue will clean a value to make it safe to be an id +// This function is quite different from the original goldmark function +// and more closely matches the output from the shurcooL sanitizer +// In particular Unicode letters and numbers are a lot more than a-zA-Z0-9... +func CleanValue(value []byte) []byte { + value = bytes.TrimSpace(value) + rs := bytes.Runes(value) + result := make([]rune, 0, len(rs)) + needsDash := false + for _, r := range rs { + switch { + case unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_': + if needsDash && len(result) > 0 { + result = append(result, '-') + } + needsDash = false + result = append(result, unicode.ToLower(r)) + default: + needsDash = true + } + } + return []byte(string(result)) +} + +// Most of what follows is a subtly changed version of github.com/yuin/goldmark/extension/footnote.go + +// A FootnoteLink struct represents a link to a footnote of Markdown +// (PHP Markdown Extra) text. +type FootnoteLink struct { + ast.BaseInline + Index int + Name []byte +} + +// Dump implements Node.Dump. +func (n *FootnoteLink) Dump(source []byte, level int) { + m := map[string]string{} + m["Index"] = fmt.Sprintf("%v", n.Index) + m["Name"] = fmt.Sprintf("%v", n.Name) + ast.DumpHelper(n, source, level, m, nil) +} + +// KindFootnoteLink is a NodeKind of the FootnoteLink node. +var KindFootnoteLink = ast.NewNodeKind("GiteaFootnoteLink") + +// Kind implements Node.Kind. +func (n *FootnoteLink) Kind() ast.NodeKind { + return KindFootnoteLink +} + +// NewFootnoteLink returns a new FootnoteLink node. +func NewFootnoteLink(index int, name []byte) *FootnoteLink { + return &FootnoteLink{ + Index: index, + Name: name, + } +} + +// A FootnoteBackLink struct represents a link to a footnote of Markdown +// (PHP Markdown Extra) text. +type FootnoteBackLink struct { + ast.BaseInline + Index int + Name []byte +} + +// Dump implements Node.Dump. +func (n *FootnoteBackLink) Dump(source []byte, level int) { + m := map[string]string{} + m["Index"] = fmt.Sprintf("%v", n.Index) + m["Name"] = fmt.Sprintf("%v", n.Name) + ast.DumpHelper(n, source, level, m, nil) +} + +// KindFootnoteBackLink is a NodeKind of the FootnoteBackLink node. +var KindFootnoteBackLink = ast.NewNodeKind("GiteaFootnoteBackLink") + +// Kind implements Node.Kind. +func (n *FootnoteBackLink) Kind() ast.NodeKind { + return KindFootnoteBackLink +} + +// NewFootnoteBackLink returns a new FootnoteBackLink node. +func NewFootnoteBackLink(index int, name []byte) *FootnoteBackLink { + return &FootnoteBackLink{ + Index: index, + Name: name, + } +} + +// A Footnote struct represents a footnote of Markdown +// (PHP Markdown Extra) text. +type Footnote struct { + ast.BaseBlock + Ref []byte + Index int + Name []byte +} + +// Dump implements Node.Dump. +func (n *Footnote) Dump(source []byte, level int) { + m := map[string]string{} + m["Index"] = strconv.Itoa(n.Index) + m["Ref"] = string(n.Ref) + m["Name"] = string(n.Name) + ast.DumpHelper(n, source, level, m, nil) +} + +// KindFootnote is a NodeKind of the Footnote node. +var KindFootnote = ast.NewNodeKind("GiteaFootnote") + +// Kind implements Node.Kind. +func (n *Footnote) Kind() ast.NodeKind { + return KindFootnote +} + +// NewFootnote returns a new Footnote node. +func NewFootnote(ref []byte) *Footnote { + return &Footnote{ + Ref: ref, + Index: -1, + Name: ref, + } +} + +// A FootnoteList struct represents footnotes of Markdown +// (PHP Markdown Extra) text. +type FootnoteList struct { + ast.BaseBlock + Count int +} + +// Dump implements Node.Dump. +func (n *FootnoteList) Dump(source []byte, level int) { + m := map[string]string{} + m["Count"] = fmt.Sprintf("%v", n.Count) + ast.DumpHelper(n, source, level, m, nil) +} + +// KindFootnoteList is a NodeKind of the FootnoteList node. +var KindFootnoteList = ast.NewNodeKind("GiteaFootnoteList") + +// Kind implements Node.Kind. +func (n *FootnoteList) Kind() ast.NodeKind { + return KindFootnoteList +} + +// NewFootnoteList returns a new FootnoteList node. +func NewFootnoteList() *FootnoteList { + return &FootnoteList{ + Count: 0, + } +} + +var footnoteListKey = parser.NewContextKey() + +type footnoteBlockParser struct{} + +var defaultFootnoteBlockParser = &footnoteBlockParser{} + +// NewFootnoteBlockParser returns a new parser.BlockParser that can parse +// footnotes of the Markdown(PHP Markdown Extra) text. +func NewFootnoteBlockParser() parser.BlockParser { + return defaultFootnoteBlockParser +} + +func (b *footnoteBlockParser) Trigger() []byte { + return []byte{'['} +} + +func (b *footnoteBlockParser) Open(parent ast.Node, reader text.Reader, pc parser.Context) (ast.Node, parser.State) { + line, segment := reader.PeekLine() + pos := pc.BlockOffset() + if pos < 0 || line[pos] != '[' { + return nil, parser.NoChildren + } + pos++ + if pos > len(line)-1 || line[pos] != '^' { + return nil, parser.NoChildren + } + open := pos + 1 + closure := util.FindClosure(line[pos+1:], '[', ']', false, false) //nolint + closes := pos + 1 + closure + next := closes + 1 + if closure > -1 { + if next >= len(line) || line[next] != ':' { + return nil, parser.NoChildren + } + } else { + return nil, parser.NoChildren + } + padding := segment.Padding + label := reader.Value(text.NewSegment(segment.Start+open-padding, segment.Start+closes-padding)) + if util.IsBlank(label) { + return nil, parser.NoChildren + } + item := NewFootnote(label) + + pos = next + 1 - padding + if pos >= len(line) { + reader.Advance(pos) + return item, parser.NoChildren + } + reader.AdvanceAndSetPadding(pos, padding) + return item, parser.HasChildren +} + +func (b *footnoteBlockParser) Continue(node ast.Node, reader text.Reader, pc parser.Context) parser.State { + line, _ := reader.PeekLine() + if util.IsBlank(line) { + return parser.Continue | parser.HasChildren + } + childpos, padding := util.IndentPosition(line, reader.LineOffset(), 4) + if childpos < 0 { + return parser.Close + } + reader.AdvanceAndSetPadding(childpos, padding) + return parser.Continue | parser.HasChildren +} + +func (b *footnoteBlockParser) Close(node ast.Node, reader text.Reader, pc parser.Context) { + var list *FootnoteList + if tlist := pc.Get(footnoteListKey); tlist != nil { + list = tlist.(*FootnoteList) + } else { + list = NewFootnoteList() + pc.Set(footnoteListKey, list) + node.Parent().InsertBefore(node.Parent(), node, list) + } + node.Parent().RemoveChild(node.Parent(), node) + list.AppendChild(list, node) +} + +func (b *footnoteBlockParser) CanInterruptParagraph() bool { + return true +} + +func (b *footnoteBlockParser) CanAcceptIndentedLine() bool { + return false +} + +type footnoteParser struct{} + +var defaultFootnoteParser = &footnoteParser{} + +// NewFootnoteParser returns a new parser.InlineParser that can parse +// footnote links of the Markdown(PHP Markdown Extra) text. +func NewFootnoteParser() parser.InlineParser { + return defaultFootnoteParser +} + +func (s *footnoteParser) Trigger() []byte { + // footnote syntax probably conflict with the image syntax. + // So we need trigger this parser with '!'. + return []byte{'!', '['} +} + +func (s *footnoteParser) Parse(parent ast.Node, block text.Reader, pc parser.Context) ast.Node { + line, segment := block.PeekLine() + pos := 1 + if len(line) > 0 && line[0] == '!' { + pos++ + } + if pos >= len(line) || line[pos] != '^' { + return nil + } + pos++ + if pos >= len(line) { + return nil + } + open := pos + closure := util.FindClosure(line[pos:], '[', ']', false, false) //nolint + if closure < 0 { + return nil + } + closes := pos + closure + value := block.Value(text.NewSegment(segment.Start+open, segment.Start+closes)) + block.Advance(closes + 1) + + var list *FootnoteList + if tlist := pc.Get(footnoteListKey); tlist != nil { + list = tlist.(*FootnoteList) + } + if list == nil { + return nil + } + index := 0 + name := []byte{} + for def := list.FirstChild(); def != nil; def = def.NextSibling() { + d := def.(*Footnote) + if bytes.Equal(d.Ref, value) { + if d.Index < 0 { + list.Count++ + d.Index = list.Count + val := CleanValue(d.Name) + if len(val) == 0 { + val = []byte(strconv.Itoa(d.Index)) + } + d.Name = pc.IDs().Generate(val, KindFootnote) + } + index = d.Index + name = d.Name + break + } + } + if index == 0 { + return nil + } + + return NewFootnoteLink(index, name) +} + +type footnoteASTTransformer struct{} + +var defaultFootnoteASTTransformer = &footnoteASTTransformer{} + +// NewFootnoteASTTransformer returns a new parser.ASTTransformer that +// insert a footnote list to the last of the document. +func NewFootnoteASTTransformer() parser.ASTTransformer { + return defaultFootnoteASTTransformer +} + +func (a *footnoteASTTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) { + var list *FootnoteList + if tlist := pc.Get(footnoteListKey); tlist != nil { + list = tlist.(*FootnoteList) + } else { + return + } + pc.Set(footnoteListKey, nil) + for footnote := list.FirstChild(); footnote != nil; { + container := footnote + next := footnote.NextSibling() + if fc := container.LastChild(); fc != nil && ast.IsParagraph(fc) { + container = fc + } + footnoteNode := footnote.(*Footnote) + index := footnoteNode.Index + name := footnoteNode.Name + if index < 0 { + list.RemoveChild(list, footnote) + } else { + container.AppendChild(container, NewFootnoteBackLink(index, name)) + } + footnote = next + } + list.SortChildren(func(n1, n2 ast.Node) int { + if n1.(*Footnote).Index < n2.(*Footnote).Index { + return -1 + } + return 1 + }) + if list.Count <= 0 { + list.Parent().RemoveChild(list.Parent(), list) + return + } + + node.AppendChild(node, list) +} + +// FootnoteHTMLRenderer is a renderer.NodeRenderer implementation that +// renders FootnoteLink nodes. +type FootnoteHTMLRenderer struct { + html.Config +} + +// NewFootnoteHTMLRenderer returns a new FootnoteHTMLRenderer. +func NewFootnoteHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { + r := &FootnoteHTMLRenderer{ + Config: html.NewConfig(), + } + for _, opt := range opts { + opt.SetHTMLOption(&r.Config) + } + return r +} + +// RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs. +func (r *FootnoteHTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(KindFootnoteLink, r.renderFootnoteLink) + reg.Register(KindFootnoteBackLink, r.renderFootnoteBackLink) + reg.Register(KindFootnote, r.renderFootnote) + reg.Register(KindFootnoteList, r.renderFootnoteList) +} + +func (r *FootnoteHTMLRenderer) renderFootnoteLink(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + n := node.(*FootnoteLink) + is := strconv.Itoa(n.Index) + _, _ = w.WriteString(`<sup id="fnref:`) + _, _ = w.Write(n.Name) + _, _ = w.WriteString(`"><a href="#fn:`) + _, _ = w.Write(n.Name) + _, _ = w.WriteString(`" class="footnote-ref" role="doc-noteref">`) + _, _ = w.WriteString(is) + _, _ = w.WriteString(`</a></sup>`) + } + return ast.WalkContinue, nil +} + +func (r *FootnoteHTMLRenderer) renderFootnoteBackLink(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + n := node.(*FootnoteBackLink) + _, _ = w.WriteString(` <a href="#fnref:`) + _, _ = w.Write(n.Name) + _, _ = w.WriteString(`" class="footnote-backref" role="doc-backlink">`) + _, _ = w.WriteString("↩︎") + _, _ = w.WriteString(`</a>`) + } + return ast.WalkContinue, nil +} + +func (r *FootnoteHTMLRenderer) renderFootnote(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*Footnote) + if entering { + _, _ = w.WriteString(`<li id="fn:`) + _, _ = w.Write(n.Name) + _, _ = w.WriteString(`" role="doc-endnote"`) + if node.Attributes() != nil { + html.RenderAttributes(w, node, html.ListItemAttributeFilter) + } + _, _ = w.WriteString(">\n") + } else { + _, _ = w.WriteString("</li>\n") + } + return ast.WalkContinue, nil +} + +func (r *FootnoteHTMLRenderer) renderFootnoteList(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + tag := "div" + if entering { + _, _ = w.WriteString("<") + _, _ = w.WriteString(tag) + _, _ = w.WriteString(` class="footnotes" role="doc-endnotes"`) + if node.Attributes() != nil { + html.RenderAttributes(w, node, html.GlobalAttributeFilter) + } + _ = w.WriteByte('>') + if r.Config.XHTML { + _, _ = w.WriteString("\n<hr />\n") + } else { + _, _ = w.WriteString("\n<hr>\n") + } + _, _ = w.WriteString("<ol>\n") + } else { + _, _ = w.WriteString("</ol>\n") + _, _ = w.WriteString("</") + _, _ = w.WriteString(tag) + _, _ = w.WriteString(">\n") + } + return ast.WalkContinue, nil +} + +type footnoteExtension struct{} + +// FootnoteExtension represents the Gitea Footnote +var FootnoteExtension = &footnoteExtension{} + +// Extend extends the markdown converter with the Gitea Footnote parser +func (e *footnoteExtension) Extend(m goldmark.Markdown) { + m.Parser().AddOptions( + parser.WithBlockParsers( + util.Prioritized(NewFootnoteBlockParser(), 999), + ), + parser.WithInlineParsers( + util.Prioritized(NewFootnoteParser(), 101), + ), + parser.WithASTTransformers( + util.Prioritized(NewFootnoteASTTransformer(), 999), + ), + ) + m.Renderer().AddOptions(renderer.WithNodeRenderers( + util.Prioritized(NewFootnoteHTMLRenderer(), 500), + )) +} diff --git a/modules/markup/common/footnote_test.go b/modules/markup/common/footnote_test.go new file mode 100644 index 00000000..62763c56 --- /dev/null +++ b/modules/markup/common/footnote_test.go @@ -0,0 +1,62 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// Copyright 2023 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: MIT +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCleanValue(t *testing.T) { + tests := []struct { + param string + expect string + }{ + // Github behavior test cases + {"", ""}, + {"test.0.1", "test-0-1"}, + {"test(0)", "test-0"}, + {"test!1", "test-1"}, + {"test:2", "test-2"}, + {"test*3", "test-3"}, + {"test！4", "test-4"}, + {"test：5", "test-5"}, + {"test*6", "test-6"}, + {"test：6 a", "test-6-a"}, + {"test：6 !b", "test-6-b"}, + {"test：ad # df", "test-ad-df"}, + {"test：ad #23 df 2*/*", "test-ad-23-df-2"}, + {"test：ad 23 df 2*/*", "test-ad-23-df-2"}, + {"test：ad # 23 df 2*/*", "test-ad-23-df-2"}, + {"Anchors in Markdown", "anchors-in-markdown"}, + {"a_b_c", "a_b_c"}, + {"a-b-c", "a-b-c"}, + {"a-b-c----", "a-b-c"}, + {"test：6a", "test-6a"}, + {"test：a6", "test-a6"}, + {"tes a a a a", "tes-a-a-a-a"}, + {" tes a a a a ", "tes-a-a-a-a"}, + {"Header with \"double quotes\"", "header-with-double-quotes"}, + {"Placeholder to force scrolling on link's click", "placeholder-to-force-scrolling-on-link-s-click"}, + {"tes（）", "tes"}, + {"tes（0）", "tes-0"}, + {"tes{0}", "tes-0"}, + {"tes[0]", "tes-0"}, + {"test【0】", "test-0"}, + {"tes…@a", "tes-a"}, + {"tes￥& a", "tes-a"}, + {"tes= a", "tes-a"}, + {"tes|a", "tes-a"}, + {"tes\\a", "tes-a"}, + {"tes/a", "tes-a"}, + {"a啊啊b", "a啊啊b"}, + {"c🤔️🤔️d", "c-d"}, + {"a⚡a", "a-a"}, + {"e.~f", "e-f"}, + } + for _, test := range tests { + assert.Equal(t, []byte(test.expect), CleanValue([]byte(test.param)), test.param) + } +} diff --git a/modules/markup/common/html.go b/modules/markup/common/html.go new file mode 100644 index 00000000..5658839c --- /dev/null +++ b/modules/markup/common/html.go @@ -0,0 +1,16 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package common + +import ( + "mvdan.cc/xurls/v2" +) + +// NOTE: All below regex matching do not perform any extra validation. +// Thus a link is produced even if the linked entity does not exist. +// While fast, this is also incorrect and lead to false positives. +// TODO: fix invalid linking issue + +// LinkRegex is a regexp matching a valid link +var LinkRegex, _ = xurls.StrictMatchingScheme("https?://") diff --git a/modules/markup/common/linkify.go b/modules/markup/common/linkify.go new file mode 100644 index 00000000..f8468020 --- /dev/null +++ b/modules/markup/common/linkify.go @@ -0,0 +1,153 @@ +// Copyright 2019 Yusuke Inuzuka +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +// Most of this file is a subtly changed version of github.com/yuin/goldmark/extension/linkify.go + +package common + +import ( + "bytes" + "regexp" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +var wwwURLRegxp = regexp.MustCompile(`^www\.[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}((?:/|[#?])[-a-zA-Z0-9@:%_\+.~#!?&//=\(\);,'">\^{}\[\]` + "`" + `]*)?`) + +type linkifyParser struct{} + +var defaultLinkifyParser = &linkifyParser{} + +// NewLinkifyParser return a new InlineParser can parse +// text that seems like a URL. +func NewLinkifyParser() parser.InlineParser { + return defaultLinkifyParser +} + +func (s *linkifyParser) Trigger() []byte { + // ' ' indicates any white spaces and a line head + return []byte{' ', '*', '_', '~', '('} +} + +var ( + protoHTTP = []byte("http:") + protoHTTPS = []byte("https:") + protoFTP = []byte("ftp:") + domainWWW = []byte("www.") +) + +func (s *linkifyParser) Parse(parent ast.Node, block text.Reader, pc parser.Context) ast.Node { + if pc.IsInLinkLabel() { + return nil + } + line, segment := block.PeekLine() + consumes := 0 + start := segment.Start + c := line[0] + // advance if current position is not a line head. + if c == ' ' || c == '*' || c == '_' || c == '~' || c == '(' { + consumes++ + start++ + line = line[1:] + } + + var m []int + var protocol []byte + typ := ast.AutoLinkURL + if bytes.HasPrefix(line, protoHTTP) || bytes.HasPrefix(line, protoHTTPS) || bytes.HasPrefix(line, protoFTP) { + m = LinkRegex.FindSubmatchIndex(line) + } + if m == nil && bytes.HasPrefix(line, domainWWW) { + m = wwwURLRegxp.FindSubmatchIndex(line) + protocol = []byte("http") + } + if m != nil { + lastChar := line[m[1]-1] + if lastChar == '.' { + m[1]-- + } else if lastChar == ')' { + closing := 0 + for i := m[1] - 1; i >= m[0]; i-- { + if line[i] == ')' { + closing++ + } else if line[i] == '(' { + closing-- + } + } + if closing > 0 { + m[1] -= closing + } + } else if lastChar == ';' { + i := m[1] - 2 + for ; i >= m[0]; i-- { + if util.IsAlphaNumeric(line[i]) { + continue + } + break + } + if i != m[1]-2 { + if line[i] == '&' { + m[1] -= m[1] - i + } + } + } + } + if m == nil { + if len(line) > 0 && util.IsPunct(line[0]) { + return nil + } + typ = ast.AutoLinkEmail + stop := util.FindEmailIndex(line) + if stop < 0 { + return nil + } + at := bytes.IndexByte(line, '@') + m = []int{0, stop, at, stop - 1} + if bytes.IndexByte(line[m[2]:m[3]], '.') < 0 { + return nil + } + lastChar := line[m[1]-1] + if lastChar == '.' { + m[1]-- + } + if m[1] < len(line) { + nextChar := line[m[1]] + if nextChar == '-' || nextChar == '_' { + return nil + } + } + } + + if consumes != 0 { + s := segment.WithStop(segment.Start + 1) + ast.MergeOrAppendTextSegment(parent, s) + } + consumes += m[1] + block.Advance(consumes) + n := ast.NewTextSegment(text.NewSegment(start, start+m[1])) + link := ast.NewAutoLink(typ, n) + link.Protocol = protocol + return link +} + +func (s *linkifyParser) CloseBlock(parent ast.Node, pc parser.Context) { + // nothing to do +} + +type linkify struct{} + +// Linkify is an extension that allow you to parse text that seems like a URL. +var Linkify = &linkify{} + +func (e *linkify) Extend(m goldmark.Markdown) { + m.Parser().AddOptions( + parser.WithInlineParsers( + util.Prioritized(NewLinkifyParser(), 999), + ), + ) +} diff --git a/modules/markup/console/console.go b/modules/markup/console/console.go new file mode 100644 index 00000000..cf42c9cc --- /dev/null +++ b/modules/markup/console/console.go @@ -0,0 +1,86 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package console + +import ( + "bytes" + "io" + "path/filepath" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/setting" + + trend "github.com/buildkite/terminal-to-html/v3" + "github.com/go-enry/go-enry/v2" +) + +// MarkupName describes markup's name +var MarkupName = "console" + +func init() { + markup.RegisterRenderer(Renderer{}) +} + +// Renderer implements markup.Renderer +type Renderer struct{} + +// Name implements markup.Renderer +func (Renderer) Name() string { + return MarkupName +} + +// Extensions implements markup.Renderer +func (Renderer) Extensions() []string { + return []string{".sh-session"} +} + +// SanitizerRules implements markup.Renderer +func (Renderer) SanitizerRules() []setting.MarkupSanitizerRule { + return []setting.MarkupSanitizerRule{ + {Element: "span", AllowAttr: "class", Regexp: regexp.MustCompile(`^term-((fg[ix]?|bg)\d+|container)$`)}, + } +} + +// CanRender implements markup.RendererContentDetector +func (Renderer) CanRender(filename string, input io.Reader) bool { + buf, err := io.ReadAll(input) + if err != nil { + return false + } + if enry.GetLanguage(filepath.Base(filename), buf) != enry.OtherLanguage { + return false + } + return bytes.ContainsRune(buf, '\x1b') +} + +// Render renders terminal colors to HTML with all specific handling stuff. +func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + buf, err := io.ReadAll(input) + if err != nil { + return err + } + buf = trend.Render(buf) + buf = bytes.ReplaceAll(buf, []byte("\n"), []byte(`<br>`)) + _, err = output.Write(buf) + return err +} + +// Render renders terminal colors to HTML with all specific handling stuff. +func Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + if ctx.Type == "" { + ctx.Type = MarkupName + } + return markup.Render(ctx, input, output) +} + +// RenderString renders terminal colors in string to HTML with all specific handling stuff and return string +func RenderString(ctx *markup.RenderContext, content string) (string, error) { + var buf strings.Builder + if err := Render(ctx, strings.NewReader(content), &buf); err != nil { + return "", err + } + return buf.String(), nil +} diff --git a/modules/markup/console/console_test.go b/modules/markup/console/console_test.go new file mode 100644 index 00000000..0d4a2bbe --- /dev/null +++ b/modules/markup/console/console_test.go @@ -0,0 +1,33 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package console + +import ( + "strings" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/markup" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRenderConsole(t *testing.T) { + var render Renderer + kases := map[string]string{ + "\x1b[37m\x1b[40mnpm\x1b[0m \x1b[0m\x1b[32minfo\x1b[0m \x1b[0m\x1b[35mit worked if it ends with\x1b[0m ok": "<span class=\"term-fg37 term-bg40\">npm</span> <span class=\"term-fg32\">info</span> <span class=\"term-fg35\">it worked if it ends with</span> ok", + } + + for k, v := range kases { + var buf strings.Builder + canRender := render.CanRender("test", strings.NewReader(k)) + assert.True(t, canRender) + + err := render.Render(&markup.RenderContext{Ctx: git.DefaultContext}, + strings.NewReader(k), &buf) + require.NoError(t, err) + assert.EqualValues(t, v, buf.String()) + } +} diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go new file mode 100644 index 00000000..3d952b0d --- /dev/null +++ b/modules/markup/csv/csv.go @@ -0,0 +1,157 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markup + +import ( + "bufio" + "html" + "io" + "regexp" + "strconv" + + "code.gitea.io/gitea/modules/csv" + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/translation" + "code.gitea.io/gitea/modules/util" +) + +func init() { + markup.RegisterRenderer(Renderer{}) +} + +// Renderer implements markup.Renderer for csv files +type Renderer struct{} + +// Name implements markup.Renderer +func (Renderer) Name() string { + return "csv" +} + +// Extensions implements markup.Renderer +func (Renderer) Extensions() []string { + return []string{".csv", ".tsv"} +} + +// SanitizerRules implements markup.Renderer +func (Renderer) SanitizerRules() []setting.MarkupSanitizerRule { + return []setting.MarkupSanitizerRule{ + {Element: "table", AllowAttr: "class", Regexp: regexp.MustCompile(`data-table`)}, + {Element: "th", AllowAttr: "class", Regexp: regexp.MustCompile(`line-num`)}, + {Element: "td", AllowAttr: "class", Regexp: regexp.MustCompile(`line-num`)}, + } +} + +func writeField(w io.Writer, element, class, field string) error { + if _, err := io.WriteString(w, "<"); err != nil { + return err + } + if _, err := io.WriteString(w, element); err != nil { + return err + } + if len(class) > 0 { + if _, err := io.WriteString(w, " class=\""); err != nil { + return err + } + if _, err := io.WriteString(w, class); err != nil { + return err + } + if _, err := io.WriteString(w, "\""); err != nil { + return err + } + } + if _, err := io.WriteString(w, ">"); err != nil { + return err + } + if _, err := io.WriteString(w, html.EscapeString(field)); err != nil { + return err + } + if _, err := io.WriteString(w, "</"); err != nil { + return err + } + if _, err := io.WriteString(w, element); err != nil { + return err + } + _, err := io.WriteString(w, ">") + return err +} + +// Render implements markup.Renderer +func (r Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + tmpBlock := bufio.NewWriter(output) + maxSize := setting.UI.CSV.MaxFileSize + maxRows := setting.UI.CSV.MaxRows + + if maxSize != 0 { + input = io.LimitReader(input, maxSize+1) + } + + rd, err := csv.CreateReaderAndDetermineDelimiter(ctx, input) + if err != nil { + return err + } + if _, err := tmpBlock.WriteString(`<table class="data-table">`); err != nil { + return err + } + + row := 0 + for { + fields, err := rd.Read() + if err == io.EOF || (row >= maxRows && maxRows != 0) { + break + } + if err != nil { + continue + } + + if _, err := tmpBlock.WriteString("<tr>"); err != nil { + return err + } + element := "td" + if row == 0 { + element = "th" + } + if err := writeField(tmpBlock, element, "line-num", strconv.Itoa(row+1)); err != nil { + return err + } + for _, field := range fields { + if err := writeField(tmpBlock, element, "", field); err != nil { + return err + } + } + if _, err := tmpBlock.WriteString("</tr>"); err != nil { + return err + } + + row++ + } + + if _, err = tmpBlock.WriteString("</table>"); err != nil { + return err + } + + // Check if maxRows or maxSize is reached, and if true, warn. + if (row >= maxRows && maxRows != 0) || (rd.InputOffset() >= maxSize && maxSize != 0) { + warn := `<table class="data-table"><tr><td>` + rawLink := ` <a href="` + ctx.Links.RawLink() + `/` + util.PathEscapeSegments(ctx.RelativePath) + `">` + + // Try to get the user translation + if locale, ok := ctx.Ctx.Value(translation.ContextKey).(translation.Locale); ok { + warn += locale.TrString("repo.file_too_large") + rawLink += locale.TrString("repo.file_view_raw") + } else { + warn += "The file is too large to be shown." + rawLink += "View Raw" + } + + warn += rawLink + `</a></td></tr></table>` + + // Write the HTML string to the output + if _, err := tmpBlock.WriteString(warn); err != nil { + return err + } + } + + return tmpBlock.Flush() +} diff --git a/modules/markup/csv/csv_test.go b/modules/markup/csv/csv_test.go new file mode 100644 index 00000000..383f1341 --- /dev/null +++ b/modules/markup/csv/csv_test.go @@ -0,0 +1,33 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markup + +import ( + "strings" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/markup" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRenderCSV(t *testing.T) { + var render Renderer + kases := map[string]string{ + "a": "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>a</th></tr></table>", + "1,2": "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>1</th><th>2</th></tr></table>", + "1;2\n3;4": "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>1</th><th>2</th></tr><tr><td class=\"line-num\">2</td><td>3</td><td>4</td></tr></table>", + "<br/>": "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th><br/></th></tr></table>", + } + + for k, v := range kases { + var buf strings.Builder + err := render.Render(&markup.RenderContext{Ctx: git.DefaultContext}, + strings.NewReader(k), &buf) + require.NoError(t, err) + assert.EqualValues(t, v, buf.String()) + } +} diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go new file mode 100644 index 00000000..122517ed --- /dev/null +++ b/modules/markup/external/external.go @@ -0,0 +1,146 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package external + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "runtime" + "strings" + + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +// RegisterRenderers registers all supported third part renderers according settings +func RegisterRenderers() { + for _, renderer := range setting.ExternalMarkupRenderers { + if renderer.Enabled && renderer.Command != "" && len(renderer.FileExtensions) > 0 { + markup.RegisterRenderer(&Renderer{renderer}) + } + } +} + +// Renderer implements markup.Renderer for external tools +type Renderer struct { + *setting.MarkupRenderer +} + +var ( + _ markup.PostProcessRenderer = (*Renderer)(nil) + _ markup.ExternalRenderer = (*Renderer)(nil) +) + +// Name returns the external tool name +func (p *Renderer) Name() string { + return p.MarkupName +} + +// NeedPostProcess implements markup.Renderer +func (p *Renderer) NeedPostProcess() bool { + return p.MarkupRenderer.NeedPostProcess +} + +// Extensions returns the supported extensions of the tool +func (p *Renderer) Extensions() []string { + return p.FileExtensions +} + +// SanitizerRules implements markup.Renderer +func (p *Renderer) SanitizerRules() []setting.MarkupSanitizerRule { + return p.MarkupSanitizerRules +} + +// SanitizerDisabled disabled sanitize if return true +func (p *Renderer) SanitizerDisabled() bool { + return p.RenderContentMode == setting.RenderContentModeNoSanitizer || p.RenderContentMode == setting.RenderContentModeIframe +} + +// DisplayInIFrame represents whether render the content with an iframe +func (p *Renderer) DisplayInIFrame() bool { + return p.RenderContentMode == setting.RenderContentModeIframe +} + +func envMark(envName string) string { + if runtime.GOOS == "windows" { + return "%" + envName + "%" + } + return "$" + envName +} + +// Render renders the data of the document to HTML via the external tool. +func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + var ( + command = strings.NewReplacer( + envMark("GITEA_PREFIX_SRC"), ctx.Links.SrcLink(), + envMark("GITEA_PREFIX_RAW"), ctx.Links.RawLink(), + ).Replace(p.Command) + commands = strings.Fields(command) + args = commands[1:] + ) + + if p.IsInputFile { + // write to temp file + f, err := os.CreateTemp("", "gitea_input") + if err != nil { + return fmt.Errorf("%s create temp file when rendering %s failed: %w", p.Name(), p.Command, err) + } + tmpPath := f.Name() + defer func() { + if err := util.Remove(tmpPath); err != nil { + log.Warn("Unable to remove temporary file: %s: Error: %v", tmpPath, err) + } + }() + + _, err = io.Copy(f, input) + if err != nil { + f.Close() + return fmt.Errorf("%s write data to temp file when rendering %s failed: %w", p.Name(), p.Command, err) + } + + err = f.Close() + if err != nil { + return fmt.Errorf("%s close temp file when rendering %s failed: %w", p.Name(), p.Command, err) + } + args = append(args, f.Name()) + } + + if ctx == nil || ctx.Ctx == nil { + if ctx == nil { + log.Warn("RenderContext not provided defaulting to empty ctx") + ctx = &markup.RenderContext{} + } + log.Warn("RenderContext did not provide context, defaulting to Shutdown context") + ctx.Ctx = graceful.GetManager().ShutdownContext() + } + + processCtx, _, finished := process.GetManager().AddContext(ctx.Ctx, fmt.Sprintf("Render [%s] for %s", commands[0], ctx.Links.SrcLink())) + defer finished() + + cmd := exec.CommandContext(processCtx, commands[0], args...) + cmd.Env = append( + os.Environ(), + "GITEA_PREFIX_SRC="+ctx.Links.SrcLink(), + "GITEA_PREFIX_RAW="+ctx.Links.RawLink(), + ) + if !p.IsInputFile { + cmd.Stdin = input + } + var stderr bytes.Buffer + cmd.Stdout = output + cmd.Stderr = &stderr + process.SetSysProcAttribute(cmd) + + if err := cmd.Run(); err != nil { + return fmt.Errorf("%s render run command %s %v failed: %w\nStderr: %s", p.Name(), commands[0], args, err, stderr.String()) + } + return nil +} diff --git a/modules/markup/file_preview.go b/modules/markup/file_preview.go new file mode 100644 index 00000000..993df717 --- /dev/null +++ b/modules/markup/file_preview.go @@ -0,0 +1,361 @@ +// Copyright The Forgejo Authors. +// SPDX-License-Identifier: MIT + +package markup + +import ( + "bufio" + "bytes" + "html/template" + "regexp" + "slices" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/charset" + "code.gitea.io/gitea/modules/highlight" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/translation" + + "golang.org/x/net/html" + "golang.org/x/net/html/atom" +) + +// filePreviewPattern matches "http://domain/org/repo/src/commit/COMMIT/filepath#L1-L2" +var filePreviewPattern = regexp.MustCompile(`https?://((?:\S+/){3})src/commit/([0-9a-f]{4,64})/(\S+)#(L\d+(?:-L\d+)?)`) + +type FilePreview struct { + fileContent []template.HTML + title template.HTML + subTitle template.HTML + lineOffset int + start int + end int + isTruncated bool +} + +func NewFilePreviews(ctx *RenderContext, node *html.Node, locale translation.Locale) []*FilePreview { + if setting.FilePreviewMaxLines == 0 { + // Feature is disabled + return nil + } + + mAll := filePreviewPattern.FindAllStringSubmatchIndex(node.Data, -1) + if mAll == nil { + return nil + } + + result := make([]*FilePreview, 0) + + for _, m := range mAll { + if slices.Contains(m, -1) { + continue + } + + preview := newFilePreview(ctx, node, locale, m) + if preview != nil { + result = append(result, preview) + } + } + + return result +} + +func newFilePreview(ctx *RenderContext, node *html.Node, locale translation.Locale, m []int) *FilePreview { + preview := &FilePreview{} + + urlFull := node.Data[m[0]:m[1]] + + // Ensure that we only use links to local repositories + if !strings.HasPrefix(urlFull, setting.AppURL) { + return nil + } + + projPath := strings.TrimPrefix(strings.TrimSuffix(node.Data[m[0]:m[3]], "/"), setting.AppURL) + + commitSha := node.Data[m[4]:m[5]] + filePath := node.Data[m[6]:m[7]] + hash := node.Data[m[8]:m[9]] + + preview.start = m[0] + preview.end = m[1] + + projPathSegments := strings.Split(projPath, "/") + if len(projPathSegments) != 2 { + return nil + } + + ownerName := projPathSegments[len(projPathSegments)-2] + repoName := projPathSegments[len(projPathSegments)-1] + + var language string + fileBlob, err := DefaultProcessorHelper.GetRepoFileBlob( + ctx.Ctx, + ownerName, + repoName, + commitSha, filePath, + &language, + ) + if err != nil { + return nil + } + + titleBuffer := new(bytes.Buffer) + + isExternRef := ownerName != ctx.Metas["user"] || repoName != ctx.Metas["repo"] + if isExternRef { + err = html.Render(titleBuffer, createLink(node.Data[m[0]:m[3]], ownerName+"/"+repoName, "")) + if err != nil { + log.Error("failed to render repoLink: %v", err) + } + titleBuffer.WriteString(" – ") + } + + err = html.Render(titleBuffer, createLink(urlFull, filePath, "muted")) + if err != nil { + log.Error("failed to render filepathLink: %v", err) + } + + preview.title = template.HTML(titleBuffer.String()) + + lineSpecs := strings.Split(hash, "-") + + commitLinkBuffer := new(bytes.Buffer) + commitLinkText := commitSha[0:7] + if isExternRef { + commitLinkText = ownerName + "/" + repoName + "@" + commitLinkText + } + + err = html.Render(commitLinkBuffer, createLink(node.Data[m[0]:m[5]], commitLinkText, "text black")) + if err != nil { + log.Error("failed to render commitLink: %v", err) + } + + var startLine, endLine int + + if len(lineSpecs) == 1 { + startLine, _ = strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) + endLine = startLine + preview.subTitle = locale.Tr( + "markup.filepreview.line", startLine, + template.HTML(commitLinkBuffer.String()), + ) + + preview.lineOffset = startLine - 1 + } else { + startLine, _ = strconv.Atoi(strings.TrimPrefix(lineSpecs[0], "L")) + endLine, _ = strconv.Atoi(strings.TrimPrefix(lineSpecs[1], "L")) + preview.subTitle = locale.Tr( + "markup.filepreview.lines", startLine, endLine, + template.HTML(commitLinkBuffer.String()), + ) + + preview.lineOffset = startLine - 1 + } + + lineCount := endLine - (startLine - 1) + if startLine < 1 || endLine < 1 || lineCount < 1 { + return nil + } + + if setting.FilePreviewMaxLines > 0 && lineCount > setting.FilePreviewMaxLines { + preview.isTruncated = true + lineCount = setting.FilePreviewMaxLines + } + + dataRc, err := fileBlob.DataAsync() + if err != nil { + return nil + } + defer dataRc.Close() + + reader := bufio.NewReader(dataRc) + + // skip all lines until we find our startLine + for i := 1; i < startLine; i++ { + _, err := reader.ReadBytes('\n') + if err != nil { + return nil + } + } + + // capture the lines we're interested in + lineBuffer := new(bytes.Buffer) + for i := 0; i < lineCount; i++ { + buf, err := reader.ReadBytes('\n') + if err != nil { + break + } + lineBuffer.Write(buf) + } + + // highlight the file... + fileContent, _, err := highlight.File(fileBlob.Name(), language, lineBuffer.Bytes()) + if err != nil { + log.Error("highlight.File failed, fallback to plain text: %v", err) + fileContent = highlight.PlainText(lineBuffer.Bytes()) + } + preview.fileContent = fileContent + + return preview +} + +func (p *FilePreview) CreateHTML(locale translation.Locale) *html.Node { + table := &html.Node{ + Type: html.ElementNode, + Data: atom.Table.String(), + Attr: []html.Attribute{{Key: "class", Val: "file-preview"}}, + } + tbody := &html.Node{ + Type: html.ElementNode, + Data: atom.Tbody.String(), + } + + status := &charset.EscapeStatus{} + statuses := make([]*charset.EscapeStatus, len(p.fileContent)) + for i, line := range p.fileContent { + statuses[i], p.fileContent[i] = charset.EscapeControlHTML(line, locale, charset.FileviewContext) + status = status.Or(statuses[i]) + } + + for idx, code := range p.fileContent { + tr := &html.Node{ + Type: html.ElementNode, + Data: atom.Tr.String(), + } + + lineNum := strconv.Itoa(p.lineOffset + idx + 1) + + tdLinesnum := &html.Node{ + Type: html.ElementNode, + Data: atom.Td.String(), + Attr: []html.Attribute{ + {Key: "class", Val: "lines-num"}, + }, + } + spanLinesNum := &html.Node{ + Type: html.ElementNode, + Data: atom.Span.String(), + Attr: []html.Attribute{ + {Key: "data-line-number", Val: lineNum}, + }, + } + tdLinesnum.AppendChild(spanLinesNum) + tr.AppendChild(tdLinesnum) + + if status.Escaped { + tdLinesEscape := &html.Node{ + Type: html.ElementNode, + Data: atom.Td.String(), + Attr: []html.Attribute{ + {Key: "class", Val: "lines-escape"}, + }, + } + + if statuses[idx].Escaped { + btnTitle := "" + if statuses[idx].HasInvisible { + btnTitle += locale.TrString("repo.invisible_runes_line") + " " + } + if statuses[idx].HasAmbiguous { + btnTitle += locale.TrString("repo.ambiguous_runes_line") + } + + escapeBtn := &html.Node{ + Type: html.ElementNode, + Data: atom.Button.String(), + Attr: []html.Attribute{ + {Key: "class", Val: "toggle-escape-button btn interact-bg"}, + {Key: "title", Val: btnTitle}, + }, + } + tdLinesEscape.AppendChild(escapeBtn) + } + + tr.AppendChild(tdLinesEscape) + } + + tdCode := &html.Node{ + Type: html.ElementNode, + Data: atom.Td.String(), + Attr: []html.Attribute{ + {Key: "class", Val: "lines-code chroma"}, + }, + } + codeInner := &html.Node{ + Type: html.ElementNode, + Data: atom.Code.String(), + Attr: []html.Attribute{{Key: "class", Val: "code-inner"}}, + } + codeText := &html.Node{ + Type: html.RawNode, + Data: string(code), + } + codeInner.AppendChild(codeText) + tdCode.AppendChild(codeInner) + tr.AppendChild(tdCode) + + tbody.AppendChild(tr) + } + + table.AppendChild(tbody) + + twrapper := &html.Node{ + Type: html.ElementNode, + Data: atom.Div.String(), + Attr: []html.Attribute{{Key: "class", Val: "ui table"}}, + } + twrapper.AppendChild(table) + + header := &html.Node{ + Type: html.ElementNode, + Data: atom.Div.String(), + Attr: []html.Attribute{{Key: "class", Val: "header"}}, + } + + ptitle := &html.Node{ + Type: html.ElementNode, + Data: atom.Div.String(), + } + ptitle.AppendChild(&html.Node{ + Type: html.RawNode, + Data: string(p.title), + }) + header.AppendChild(ptitle) + + psubtitle := &html.Node{ + Type: html.ElementNode, + Data: atom.Span.String(), + Attr: []html.Attribute{{Key: "class", Val: "text small grey"}}, + } + psubtitle.AppendChild(&html.Node{ + Type: html.RawNode, + Data: string(p.subTitle), + }) + header.AppendChild(psubtitle) + + node := &html.Node{ + Type: html.ElementNode, + Data: atom.Div.String(), + Attr: []html.Attribute{{Key: "class", Val: "file-preview-box"}}, + } + node.AppendChild(header) + + if p.isTruncated { + warning := &html.Node{ + Type: html.ElementNode, + Data: atom.Div.String(), + Attr: []html.Attribute{{Key: "class", Val: "ui warning message tw-text-left"}}, + } + warning.AppendChild(&html.Node{ + Type: html.TextNode, + Data: locale.TrString("markup.filepreview.truncated"), + }) + node.AppendChild(warning) + } + + node.AppendChild(twrapper) + + return node +} diff --git a/modules/markup/html.go b/modules/markup/html.go new file mode 100644 index 00000000..b5aadb2a --- /dev/null +++ b/modules/markup/html.go @@ -0,0 +1,1300 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markup + +import ( + "bytes" + "io" + "net/url" + "path" + "path/filepath" + "regexp" + "strings" + "sync" + + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/emoji" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup/common" + "code.gitea.io/gitea/modules/references" + "code.gitea.io/gitea/modules/regexplru" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates/vars" + "code.gitea.io/gitea/modules/translation" + "code.gitea.io/gitea/modules/util" + + "golang.org/x/net/html" + "golang.org/x/net/html/atom" + "mvdan.cc/xurls/v2" +) + +// Issue name styles +const ( + IssueNameStyleNumeric = "numeric" + IssueNameStyleAlphanumeric = "alphanumeric" + IssueNameStyleRegexp = "regexp" +) + +var ( + // NOTE: All below regex matching do not perform any extra validation. + // Thus a link is produced even if the linked entity does not exist. + // While fast, this is also incorrect and lead to false positives. + // TODO: fix invalid linking issue + + // valid chars in encoded path and parameter: [-+~_%.a-zA-Z0-9/] + + // hashCurrentPattern matches string that represents a commit SHA, e.g. d8a994ef243349f321568f9e36d5c3f444b99cae + // Although SHA1 hashes are 40 chars long, SHA256 are 64, the regex matches the hash from 7 to 64 chars in length + // so that abbreviated hash links can be used as well. This matches git and GitHub usability. + hashCurrentPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-f]{7,64})(?:\s|$|\)|\]|[.,:](\s|$))`) + + // shortLinkPattern matches short but difficult to parse [[name|link|arg=test]] syntax + shortLinkPattern = regexp.MustCompile(`\[\[(.*?)\]\](\w*)`) + + // anySHA1Pattern splits url containing SHA into parts + anyHashPattern = regexp.MustCompile(`https?://(?:\S+/){4,5}([0-9a-f]{40,64})(/[-+~_%.a-zA-Z0-9/]+)?(\?[-+~_%\.a-zA-Z0-9=&]+)?(#[-+~_%.a-zA-Z0-9]+)?`) + + // comparePattern matches "http://domain/org/repo/compare/COMMIT1...COMMIT2#hash" + comparePattern = regexp.MustCompile(`https?://(?:\S+/){4,5}([0-9a-f]{7,64})(\.\.\.?)([0-9a-f]{7,64})?(#[-+~_%.a-zA-Z0-9]+)?`) + + validLinksPattern = regexp.MustCompile(`^[a-z][\w-]+://`) + + // While this email regex is definitely not perfect and I'm sure you can come up + // with edge cases, it is still accepted by the CommonMark specification, as + // well as the HTML5 spec: + // http://spec.commonmark.org/0.28/#email-address + // https://html.spec.whatwg.org/multipage/input.html#e-mail-state-(type%3Demail) + emailRegex = regexp.MustCompile("(?:\\s|^|\\(|\\[)([a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9]{2,}(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)(?:\\s|$|\\)|\\]|;|,|\\?|!|\\.(\\s|$))") + + // blackfriday extensions create IDs like fn:user-content-footnote + blackfridayExtRegex = regexp.MustCompile(`[^:]*:user-content-`) + + // EmojiShortCodeRegex find emoji by alias like :smile: + EmojiShortCodeRegex = regexp.MustCompile(`:[-+\w]+:`) +) + +// CSS class for action keywords (e.g. "closes: #1") +const keywordClass = "issue-keyword" + +// IsLink reports whether link fits valid format. +func IsLink(link []byte) bool { + return validLinksPattern.Match(link) +} + +func IsLinkStr(link string) bool { + return validLinksPattern.MatchString(link) +} + +// regexp for full links to issues/pulls +var issueFullPattern *regexp.Regexp + +// Once for to prevent races +var issueFullPatternOnce sync.Once + +// regexp for full links to hash comment in pull request files changed tab +var filesChangedFullPattern *regexp.Regexp + +// Once for to prevent races +var filesChangedFullPatternOnce sync.Once + +func getIssueFullPattern() *regexp.Regexp { + issueFullPatternOnce.Do(func() { + // example: https://domain/org/repo/pulls/27#hash + issueFullPattern = regexp.MustCompile(regexp.QuoteMeta(setting.AppURL) + + `[\w_.-]+/[\w_.-]+/(?:issues|pulls)/((?:\w{1,10}-)?[1-9][0-9]*)([\?|#](\S+)?)?\b`) + }) + return issueFullPattern +} + +func getFilesChangedFullPattern() *regexp.Regexp { + filesChangedFullPatternOnce.Do(func() { + // example: https://domain/org/repo/pulls/27/files#hash + filesChangedFullPattern = regexp.MustCompile(regexp.QuoteMeta(setting.AppURL) + + `[\w_.-]+/[\w_.-]+/pulls/((?:\w{1,10}-)?[1-9][0-9]*)/files([\?|#](\S+)?)?\b`) + }) + return filesChangedFullPattern +} + +// CustomLinkURLSchemes allows for additional schemes to be detected when parsing links within text +func CustomLinkURLSchemes(schemes []string) { + schemes = append(schemes, "http", "https") + withAuth := make([]string, 0, len(schemes)) + validScheme := regexp.MustCompile(`^[a-z]+$`) + for _, s := range schemes { + if !validScheme.MatchString(s) { + continue + } + without := false + for _, sna := range xurls.SchemesNoAuthority { + if s == sna { + without = true + break + } + } + if without { + s += ":" + } else { + s += "://" + } + withAuth = append(withAuth, s) + } + common.LinkRegex, _ = xurls.StrictMatchingScheme(strings.Join(withAuth, "|")) +} + +type postProcessError struct { + context string + err error +} + +func (p *postProcessError) Error() string { + return "PostProcess: " + p.context + ", " + p.err.Error() +} + +type processor func(ctx *RenderContext, node *html.Node) + +var defaultProcessors = []processor{ + fullIssuePatternProcessor, + comparePatternProcessor, + filePreviewPatternProcessor, + fullHashPatternProcessor, + shortLinkProcessor, + linkProcessor, + mentionProcessor, + issueIndexPatternProcessor, + commitCrossReferencePatternProcessor, + hashCurrentPatternProcessor, + emailAddressProcessor, + emojiProcessor, + emojiShortCodeProcessor, +} + +// PostProcess does the final required transformations to the passed raw HTML +// data, and ensures its validity. Transformations include: replacing links and +// emails with HTML links, parsing shortlinks in the format of [[Link]], like +// MediaWiki, linking issues in the format #ID, and mentions in the format +// @user, and others. +func PostProcess( + ctx *RenderContext, + input io.Reader, + output io.Writer, +) error { + return postProcess(ctx, defaultProcessors, input, output) +} + +var commitMessageProcessors = []processor{ + fullIssuePatternProcessor, + comparePatternProcessor, + fullHashPatternProcessor, + linkProcessor, + mentionProcessor, + issueIndexPatternProcessor, + commitCrossReferencePatternProcessor, + hashCurrentPatternProcessor, + emailAddressProcessor, + emojiProcessor, + emojiShortCodeProcessor, +} + +// RenderCommitMessage will use the same logic as PostProcess, but will disable +// the shortLinkProcessor and will add a defaultLinkProcessor if defaultLink is +// set, which changes every text node into a link to the passed default link. +func RenderCommitMessage( + ctx *RenderContext, + content string, +) (string, error) { + procs := commitMessageProcessors + if ctx.DefaultLink != "" { + // we don't have to fear data races, because being + // commitMessageProcessors of fixed len and cap, every time we append + // something to it the slice is realloc+copied, so append always + // generates the slice ex-novo. + procs = append(procs, genDefaultLinkProcessor(ctx.DefaultLink)) + } + return renderProcessString(ctx, procs, content) +} + +var commitMessageSubjectProcessors = []processor{ + fullIssuePatternProcessor, + comparePatternProcessor, + fullHashPatternProcessor, + linkProcessor, + mentionProcessor, + issueIndexPatternProcessor, + commitCrossReferencePatternProcessor, + hashCurrentPatternProcessor, + emojiShortCodeProcessor, + emojiProcessor, +} + +var emojiProcessors = []processor{ + emojiShortCodeProcessor, + emojiProcessor, +} + +// RenderCommitMessageSubject will use the same logic as PostProcess and +// RenderCommitMessage, but will disable the shortLinkProcessor and +// emailAddressProcessor, will add a defaultLinkProcessor if defaultLink is set, +// which changes every text node into a link to the passed default link. +func RenderCommitMessageSubject( + ctx *RenderContext, + content string, +) (string, error) { + procs := commitMessageSubjectProcessors + if ctx.DefaultLink != "" { + // we don't have to fear data races, because being + // commitMessageSubjectProcessors of fixed len and cap, every time we + // append something to it the slice is realloc+copied, so append always + // generates the slice ex-novo. + procs = append(procs, genDefaultLinkProcessor(ctx.DefaultLink)) + } + return renderProcessString(ctx, procs, content) +} + +// RenderIssueTitle to process title on individual issue/pull page +func RenderIssueTitle( + ctx *RenderContext, + title string, +) (string, error) { + return renderProcessString(ctx, []processor{ + issueIndexPatternProcessor, + commitCrossReferencePatternProcessor, + hashCurrentPatternProcessor, + emojiShortCodeProcessor, + emojiProcessor, + }, title) +} + +func renderProcessString(ctx *RenderContext, procs []processor, content string) (string, error) { + var buf strings.Builder + if err := postProcess(ctx, procs, strings.NewReader(content), &buf); err != nil { + return "", err + } + return buf.String(), nil +} + +// RenderDescriptionHTML will use similar logic as PostProcess, but will +// use a single special linkProcessor. +func RenderDescriptionHTML( + ctx *RenderContext, + content string, +) (string, error) { + return renderProcessString(ctx, []processor{ + descriptionLinkProcessor, + emojiShortCodeProcessor, + emojiProcessor, + }, content) +} + +// RenderEmoji for when we want to just process emoji and shortcodes +// in various places it isn't already run through the normal markdown processor +func RenderEmoji( + ctx *RenderContext, + content string, +) (string, error) { + return renderProcessString(ctx, emojiProcessors, content) +} + +var ( + tagCleaner = regexp.MustCompile(`<((?:/?\w+/\w+)|(?:/[\w ]+/)|(/?[hH][tT][mM][lL]\b)|(/?[hH][eE][aA][dD]\b))`) + nulCleaner = strings.NewReplacer("\000", "") +) + +func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output io.Writer) error { + defer ctx.Cancel() + // FIXME: don't read all content to memory + rawHTML, err := io.ReadAll(input) + if err != nil { + return err + } + + // parse the HTML + node, err := html.Parse(io.MultiReader( + // prepend "<html><body>" + strings.NewReader("<html><body>"), + // Strip out nuls - they're always invalid + bytes.NewReader(tagCleaner.ReplaceAll([]byte(nulCleaner.Replace(string(rawHTML))), []byte("<$1"))), + // close the tags + strings.NewReader("</body></html>"), + )) + if err != nil { + return &postProcessError{"invalid HTML", err} + } + + if node.Type == html.DocumentNode { + node = node.FirstChild + } + + visitNode(ctx, procs, node) + + newNodes := make([]*html.Node, 0, 5) + + if node.Data == "html" { + node = node.FirstChild + for node != nil && node.Data != "body" { + node = node.NextSibling + } + } + if node != nil { + if node.Data == "body" { + child := node.FirstChild + for child != nil { + newNodes = append(newNodes, child) + child = child.NextSibling + } + } else { + newNodes = append(newNodes, node) + } + } + + // Render everything to buf. + for _, node := range newNodes { + if err := html.Render(output, node); err != nil { + return &postProcessError{"error rendering processed HTML", err} + } + } + return nil +} + +func visitNode(ctx *RenderContext, procs []processor, node *html.Node) { + // Add user-content- to IDs and "#" links if they don't already have them + for idx, attr := range node.Attr { + val := strings.TrimPrefix(attr.Val, "#") + notHasPrefix := !(strings.HasPrefix(val, "user-content-") || blackfridayExtRegex.MatchString(val)) + + if attr.Key == "id" && notHasPrefix { + node.Attr[idx].Val = "user-content-" + attr.Val + } + + if attr.Key == "href" && strings.HasPrefix(attr.Val, "#") && notHasPrefix { + node.Attr[idx].Val = "#user-content-" + val + } + + if attr.Key == "class" && attr.Val == "emoji" { + procs = nil + } + } + + // We ignore code and pre. + switch node.Type { + case html.TextNode: + processTextNodes(ctx, procs, node) + case html.ElementNode: + if node.Data == "img" { + for i, attr := range node.Attr { + if attr.Key != "src" { + continue + } + if len(attr.Val) > 0 && !IsLinkStr(attr.Val) && !strings.HasPrefix(attr.Val, "data:image/") { + attr.Val = util.URLJoin(ctx.Links.ResolveMediaLink(ctx.IsWiki), attr.Val) + } + attr.Val = camoHandleLink(attr.Val) + node.Attr[i] = attr + } + } else if node.Data == "a" { + // Restrict text in links to emojis + procs = emojiProcessors + } else if node.Data == "code" || node.Data == "pre" { + return + } else if node.Data == "i" { + for _, attr := range node.Attr { + if attr.Key != "class" { + continue + } + classes := strings.Split(attr.Val, " ") + for i, class := range classes { + if class == "icon" { + classes[0], classes[i] = classes[i], classes[0] + attr.Val = strings.Join(classes, " ") + + // Remove all children of icons + child := node.FirstChild + for child != nil { + node.RemoveChild(child) + child = node.FirstChild + } + break + } + } + } + } + for n := node.FirstChild; n != nil; n = n.NextSibling { + visitNode(ctx, procs, n) + } + default: + } + // ignore everything else +} + +// processTextNodes runs the passed node through various processors, in order to handle +// all kinds of special links handled by the post-processing. +func processTextNodes(ctx *RenderContext, procs []processor, node *html.Node) { + for _, p := range procs { + p(ctx, node) + } +} + +// createKeyword() renders a highlighted version of an action keyword +func createKeyword(content string) *html.Node { + span := &html.Node{ + Type: html.ElementNode, + Data: atom.Span.String(), + Attr: []html.Attribute{}, + } + span.Attr = append(span.Attr, html.Attribute{Key: "class", Val: keywordClass}) + + text := &html.Node{ + Type: html.TextNode, + Data: content, + } + span.AppendChild(text) + + return span +} + +func createEmoji(content, class, name string) *html.Node { + span := &html.Node{ + Type: html.ElementNode, + Data: atom.Span.String(), + Attr: []html.Attribute{}, + } + if class != "" { + span.Attr = append(span.Attr, html.Attribute{Key: "class", Val: class}) + } + if name != "" { + span.Attr = append(span.Attr, html.Attribute{Key: "aria-label", Val: name}) + } + + text := &html.Node{ + Type: html.TextNode, + Data: content, + } + + span.AppendChild(text) + return span +} + +func createCustomEmoji(alias string) *html.Node { + span := &html.Node{ + Type: html.ElementNode, + Data: atom.Span.String(), + Attr: []html.Attribute{}, + } + span.Attr = append(span.Attr, html.Attribute{Key: "class", Val: "emoji"}) + span.Attr = append(span.Attr, html.Attribute{Key: "aria-label", Val: alias}) + + img := &html.Node{ + Type: html.ElementNode, + DataAtom: atom.Img, + Data: "img", + Attr: []html.Attribute{}, + } + img.Attr = append(img.Attr, html.Attribute{Key: "alt", Val: ":" + alias + ":"}) + img.Attr = append(img.Attr, html.Attribute{Key: "src", Val: setting.StaticURLPrefix + "/assets/img/emoji/" + alias + ".png"}) + + span.AppendChild(img) + return span +} + +func createLink(href, content, class string) *html.Node { + a := &html.Node{ + Type: html.ElementNode, + Data: atom.A.String(), + Attr: []html.Attribute{{Key: "href", Val: href}}, + } + + if class != "" { + a.Attr = append(a.Attr, html.Attribute{Key: "class", Val: class}) + } + + text := &html.Node{ + Type: html.TextNode, + Data: content, + } + + a.AppendChild(text) + return a +} + +func createCodeLink(href, content, class string) *html.Node { + a := &html.Node{ + Type: html.ElementNode, + Data: atom.A.String(), + Attr: []html.Attribute{{Key: "href", Val: href}}, + } + + if class != "" { + a.Attr = append(a.Attr, html.Attribute{Key: "class", Val: class}) + } + + text := &html.Node{ + Type: html.TextNode, + Data: content, + } + + code := &html.Node{ + Type: html.ElementNode, + Data: atom.Code.String(), + Attr: []html.Attribute{{Key: "class", Val: "nohighlight"}}, + } + + code.AppendChild(text) + a.AppendChild(code) + return a +} + +// replaceContent takes text node, and in its content it replaces a section of +// it with the specified newNode. +func replaceContent(node *html.Node, i, j int, newNode *html.Node) { + replaceContentList(node, i, j, []*html.Node{newNode}) +} + +// replaceContentList takes text node, and in its content it replaces a section of +// it with the specified newNodes. An example to visualize how this can work can +// be found here: https://play.golang.org/p/5zP8NnHZ03s +func replaceContentList(node *html.Node, i, j int, newNodes []*html.Node) { + // get the data before and after the match + before := node.Data[:i] + after := node.Data[j:] + + // Replace in the current node the text, so that it is only what it is + // supposed to have. + node.Data = before + + // Get the current next sibling, before which we place the replaced data, + // and after that we place the new text node. + nextSibling := node.NextSibling + for _, n := range newNodes { + node.Parent.InsertBefore(n, nextSibling) + } + if after != "" { + node.Parent.InsertBefore(&html.Node{ + Type: html.TextNode, + Data: after, + }, nextSibling) + } +} + +func mentionProcessor(ctx *RenderContext, node *html.Node) { + start := 0 + next := node.NextSibling + for node != nil && node != next && start < len(node.Data) { + // We replace only the first mention; other mentions will be addressed later + found, loc := references.FindFirstMentionBytes([]byte(node.Data[start:])) + if !found { + return + } + loc.Start += start + loc.End += start + mention := node.Data[loc.Start:loc.End] + var teams string + teams, ok := ctx.Metas["teams"] + // FIXME: util.URLJoin may not be necessary here: + // - setting.AppURL is defined to have a terminal '/' so unless mention[1:] + // is an AppSubURL link we can probably fallback to concatenation. + // team mention should follow @orgName/teamName style + if ok && strings.Contains(mention, "/") { + mentionOrgAndTeam := strings.Split(mention, "/") + if mentionOrgAndTeam[0][1:] == ctx.Metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") { + replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(ctx.Links.Prefix(), "org", ctx.Metas["org"], "teams", mentionOrgAndTeam[1]), mention, "mention")) + node = node.NextSibling.NextSibling + start = 0 + continue + } + start = loc.End + continue + } + mentionedUsername := mention[1:] + + if DefaultProcessorHelper.IsUsernameMentionable != nil && DefaultProcessorHelper.IsUsernameMentionable(ctx.Ctx, mentionedUsername) { + replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(ctx.Links.Prefix(), mentionedUsername), mention, "mention")) + node = node.NextSibling.NextSibling + start = 0 + } else { + start = loc.End + } + } +} + +func shortLinkProcessor(ctx *RenderContext, node *html.Node) { + next := node.NextSibling + for node != nil && node != next { + m := shortLinkPattern.FindStringSubmatchIndex(node.Data) + if m == nil { + return + } + + content := node.Data[m[2]:m[3]] + tail := node.Data[m[4]:m[5]] + props := make(map[string]string) + + // MediaWiki uses [[link|text]], while GitHub uses [[text|link]] + // It makes page handling terrible, but we prefer GitHub syntax + // And fall back to MediaWiki only when it is obvious from the look + // Of text and link contents + sl := strings.Split(content, "|") + for _, v := range sl { + if equalPos := strings.IndexByte(v, '='); equalPos == -1 { + // There is no equal in this argument; this is a mandatory arg + if props["name"] == "" { + if IsLinkStr(v) { + // If we clearly see it is a link, we save it so + + // But first we need to ensure, that if both mandatory args provided + // look like links, we stick to GitHub syntax + if props["link"] != "" { + props["name"] = props["link"] + } + + props["link"] = strings.TrimSpace(v) + } else { + props["name"] = v + } + } else { + props["link"] = strings.TrimSpace(v) + } + } else { + // There is an equal; optional argument. + + sep := strings.IndexByte(v, '=') + key, val := v[:sep], html.UnescapeString(v[sep+1:]) + + // When parsing HTML, x/net/html will change all quotes which are + // not used for syntax into UTF-8 quotes. So checking val[0] won't + // be enough, since that only checks a single byte. + if len(val) > 1 { + if (strings.HasPrefix(val, "“") && strings.HasSuffix(val, "”")) || + (strings.HasPrefix(val, "‘") && strings.HasSuffix(val, "’")) { + const lenQuote = len("‘") + val = val[lenQuote : len(val)-lenQuote] + } else if (strings.HasPrefix(val, "\"") && strings.HasSuffix(val, "\"")) || + (strings.HasPrefix(val, "'") && strings.HasSuffix(val, "'")) { + val = val[1 : len(val)-1] + } else if strings.HasPrefix(val, "'") && strings.HasSuffix(val, "’") { + const lenQuote = len("‘") + val = val[1 : len(val)-lenQuote] + } + } + props[key] = val + } + } + + var name, link string + if props["link"] != "" { + link = props["link"] + } else if props["name"] != "" { + link = props["name"] + } + if props["title"] != "" { + name = props["title"] + } else if props["name"] != "" { + name = props["name"] + } else { + name = link + } + + name += tail + image := false + switch ext := filepath.Ext(link); ext { + // fast path: empty string, ignore + case "": + // leave image as false + case ".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp", ".gif", ".bmp", ".ico", ".svg": + image = true + } + + childNode := &html.Node{} + linkNode := &html.Node{ + FirstChild: childNode, + LastChild: childNode, + Type: html.ElementNode, + Data: "a", + DataAtom: atom.A, + } + childNode.Parent = linkNode + absoluteLink := IsLinkStr(link) + if !absoluteLink { + if image { + link = strings.ReplaceAll(link, " ", "+") + } else { + link = strings.ReplaceAll(link, " ", "-") + } + if !strings.Contains(link, "/") { + link = url.PathEscape(link) + } + } + if image { + if !absoluteLink { + link = util.URLJoin(ctx.Links.ResolveMediaLink(ctx.IsWiki), link) + } + title := props["title"] + if title == "" { + title = props["alt"] + } + if title == "" { + title = path.Base(name) + } + alt := props["alt"] + if alt == "" { + alt = name + } + + // make the childNode an image - if we can, we also place the alt + childNode.Type = html.ElementNode + childNode.Data = "img" + childNode.DataAtom = atom.Img + childNode.Attr = []html.Attribute{ + {Key: "src", Val: link}, + {Key: "title", Val: title}, + {Key: "alt", Val: alt}, + } + if alt == "" { + childNode.Attr = childNode.Attr[:2] + } + } else { + if !absoluteLink { + if ctx.IsWiki { + link = util.URLJoin(ctx.Links.WikiLink(), link) + } else { + link = util.URLJoin(ctx.Links.SrcLink(), link) + } + } + childNode.Type = html.TextNode + childNode.Data = name + } + linkNode.Attr = []html.Attribute{{Key: "href", Val: link}} + replaceContent(node, m[0], m[1], linkNode) + node = node.NextSibling.NextSibling + } +} + +func fullIssuePatternProcessor(ctx *RenderContext, node *html.Node) { + if ctx.Metas == nil { + return + } + next := node.NextSibling + for node != nil && node != next { + m := getIssueFullPattern().FindStringSubmatchIndex(node.Data) + if m == nil { + return + } + + mDiffView := getFilesChangedFullPattern().FindStringSubmatchIndex(node.Data) + // leave it as it is if the link is from "Files Changed" tab in PR Diff View https://domain/org/repo/pulls/27/files + if mDiffView != nil { + return + } + + link := node.Data[m[0]:m[1]] + text := "#" + node.Data[m[2]:m[3]] + // if m[4] and m[5] is not -1, then link is to a comment + // indicate that in the text by appending (comment) + if m[4] != -1 && m[5] != -1 { + if locale, ok := ctx.Ctx.Value(translation.ContextKey).(translation.Locale); ok { + text += " " + locale.TrString("repo.from_comment") + } else { + text += " (comment)" + } + } + + // extract repo and org name from matched link like + // http://localhost:3000/gituser/myrepo/issues/1 + linkParts := strings.Split(link, "/") + matchOrg := linkParts[len(linkParts)-4] + matchRepo := linkParts[len(linkParts)-3] + + if matchOrg == ctx.Metas["user"] && matchRepo == ctx.Metas["repo"] { + replaceContent(node, m[0], m[1], createLink(link, text, "ref-issue")) + } else { + text = matchOrg + "/" + matchRepo + text + replaceContent(node, m[0], m[1], createLink(link, text, "ref-issue")) + } + node = node.NextSibling.NextSibling + } +} + +func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { + if ctx.Metas == nil { + return + } + + // FIXME: the use of "mode" is quite dirty and hacky, for example: what is a "document"? how should it be rendered? + // The "mode" approach should be refactored to some other more clear&reliable way. + crossLinkOnly := (ctx.Metas["mode"] == "document" && !ctx.IsWiki) + + var ( + found bool + ref *references.RenderizableReference + ) + + next := node.NextSibling + + for node != nil && node != next { + _, hasExtTrackFormat := ctx.Metas["format"] + + // Repos with external issue trackers might still need to reference local PRs + // We need to concern with the first one that shows up in the text, whichever it is + isNumericStyle := ctx.Metas["style"] == "" || ctx.Metas["style"] == IssueNameStyleNumeric + foundNumeric, refNumeric := references.FindRenderizableReferenceNumeric(node.Data, hasExtTrackFormat && !isNumericStyle, crossLinkOnly) + + switch ctx.Metas["style"] { + case "", IssueNameStyleNumeric: + found, ref = foundNumeric, refNumeric + case IssueNameStyleAlphanumeric: + found, ref = references.FindRenderizableReferenceAlphanumeric(node.Data) + case IssueNameStyleRegexp: + pattern, err := regexplru.GetCompiled(ctx.Metas["regexp"]) + if err != nil { + return + } + found, ref = references.FindRenderizableReferenceRegexp(node.Data, pattern) + } + + // Repos with external issue trackers might still need to reference local PRs + // We need to concern with the first one that shows up in the text, whichever it is + if hasExtTrackFormat && !isNumericStyle && refNumeric != nil { + // If numeric (PR) was found, and it was BEFORE the non-numeric pattern, use that + // Allow a free-pass when non-numeric pattern wasn't found. + if found && (ref == nil || refNumeric.RefLocation.Start < ref.RefLocation.Start) { + found = foundNumeric + ref = refNumeric + } + } + if !found { + return + } + + var link *html.Node + reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End] + if hasExtTrackFormat && !ref.IsPull && ref.Owner == "" { + ctx.Metas["index"] = ref.Issue + + res, err := vars.Expand(ctx.Metas["format"], ctx.Metas) + if err != nil { + // here we could just log the error and continue the rendering + log.Error("unable to expand template vars for ref %s, err: %v", ref.Issue, err) + } + + link = createLink(res, reftext, "ref-issue ref-external-issue") + } else { + // Path determines the type of link that will be rendered. It's unknown at this point whether + // the linked item is actually a PR or an issue. Luckily it's of no real consequence because + // Forgejo will redirect on click as appropriate. + path := "issues" + if ref.IsPull { + path = "pulls" + } + if ref.Owner == "" { + link = createLink(util.URLJoin(ctx.Links.Prefix(), ctx.Metas["user"], ctx.Metas["repo"], path, ref.Issue), reftext, "ref-issue") + } else { + link = createLink(util.URLJoin(ctx.Links.Prefix(), ref.Owner, ref.Name, path, ref.Issue), reftext, "ref-issue") + } + } + + if ref.Action == references.XRefActionNone { + replaceContent(node, ref.RefLocation.Start, ref.RefLocation.End, link) + node = node.NextSibling.NextSibling + continue + } + + // Decorate action keywords if actionable + var keyword *html.Node + if references.IsXrefActionable(ref, hasExtTrackFormat) { + keyword = createKeyword(node.Data[ref.ActionLocation.Start:ref.ActionLocation.End]) + } else { + keyword = &html.Node{ + Type: html.TextNode, + Data: node.Data[ref.ActionLocation.Start:ref.ActionLocation.End], + } + } + spaces := &html.Node{ + Type: html.TextNode, + Data: node.Data[ref.ActionLocation.End:ref.RefLocation.Start], + } + replaceContentList(node, ref.ActionLocation.Start, ref.RefLocation.End, []*html.Node{keyword, spaces, link}) + node = node.NextSibling.NextSibling.NextSibling.NextSibling + } +} + +func commitCrossReferencePatternProcessor(ctx *RenderContext, node *html.Node) { + next := node.NextSibling + + for node != nil && node != next { + found, ref := references.FindRenderizableCommitCrossReference(node.Data) + if !found { + return + } + + reftext := ref.Owner + "/" + ref.Name + "@" + base.ShortSha(ref.CommitSha) + link := createLink(util.URLJoin(ctx.Links.Prefix(), ref.Owner, ref.Name, "commit", ref.CommitSha), reftext, "commit") + + replaceContent(node, ref.RefLocation.Start, ref.RefLocation.End, link) + node = node.NextSibling.NextSibling + } +} + +// fullHashPatternProcessor renders SHA containing URLs +func fullHashPatternProcessor(ctx *RenderContext, node *html.Node) { + if ctx.Metas == nil { + return + } + + next := node.NextSibling + for node != nil && node != next { + m := anyHashPattern.FindStringSubmatchIndex(node.Data) + if m == nil { + return + } + + urlFull := node.Data[m[0]:m[1]] + text := base.ShortSha(node.Data[m[2]:m[3]]) + + // 3rd capture group matches a optional path + subpath := "" + if m[5] > 0 { + subpath = node.Data[m[4]:m[5]] + } + + // 5th capture group matches a optional url hash + hash := "" + if m[9] > 0 { + hash = node.Data[m[8]:m[9]][1:] + } + + start := m[0] + end := m[1] + + // If url ends in '.', it's very likely that it is not part of the + // actual url but used to finish a sentence. + if strings.HasSuffix(urlFull, ".") { + end-- + urlFull = urlFull[:len(urlFull)-1] + if hash != "" { + hash = hash[:len(hash)-1] + } else if subpath != "" { + subpath = subpath[:len(subpath)-1] + } + } + + if subpath != "" { + text += subpath + } + + if hash != "" { + text += " (" + hash + ")" + } + replaceContent(node, start, end, createCodeLink(urlFull, text, "commit")) + node = node.NextSibling.NextSibling + } +} + +func comparePatternProcessor(ctx *RenderContext, node *html.Node) { + if ctx.Metas == nil { + return + } + + next := node.NextSibling + for node != nil && node != next { + m := comparePattern.FindStringSubmatchIndex(node.Data) + if m == nil { + return + } + + // Ensure that every group (m[0]...m[7]) has a match + for i := 0; i < 8; i++ { + if m[i] == -1 { + return + } + } + + urlFull := node.Data[m[0]:m[1]] + text1 := base.ShortSha(node.Data[m[2]:m[3]]) + textDots := base.ShortSha(node.Data[m[4]:m[5]]) + text2 := base.ShortSha(node.Data[m[6]:m[7]]) + + hash := "" + if m[9] > 0 { + hash = node.Data[m[8]:m[9]][1:] + } + + start := m[0] + end := m[1] + + // If url ends in '.', it's very likely that it is not part of the + // actual url but used to finish a sentence. + if strings.HasSuffix(urlFull, ".") { + end-- + urlFull = urlFull[:len(urlFull)-1] + if hash != "" { + hash = hash[:len(hash)-1] + } else if text2 != "" { + text2 = text2[:len(text2)-1] + } + } + + text := text1 + textDots + text2 + if hash != "" { + text += " (" + hash + ")" + } + replaceContent(node, start, end, createCodeLink(urlFull, text, "compare")) + node = node.NextSibling.NextSibling + } +} + +func filePreviewPatternProcessor(ctx *RenderContext, node *html.Node) { + if ctx.Metas == nil || ctx.Metas["user"] == "" || ctx.Metas["repo"] == "" { + return + } + if DefaultProcessorHelper.GetRepoFileBlob == nil { + return + } + + locale := translation.NewLocale("en-US") + if ctx.Ctx != nil { + ctxLocale, ok := ctx.Ctx.Value(translation.ContextKey).(translation.Locale) + if ok { + locale = ctxLocale + } + } + + next := node.NextSibling + for node != nil && node != next { + previews := NewFilePreviews(ctx, node, locale) + if previews == nil { + node = node.NextSibling + continue + } + + offset := 0 + for _, preview := range previews { + previewNode := preview.CreateHTML(locale) + + // Specialized version of replaceContent, so the parent paragraph element is not destroyed from our div + before := node.Data[:(preview.start - offset)] + after := node.Data[(preview.end - offset):] + afterPrefix := "<p>" + offset = preview.end - len(afterPrefix) + node.Data = before + nextSibling := node.NextSibling + node.Parent.InsertBefore(&html.Node{ + Type: html.RawNode, + Data: "</p>", + }, nextSibling) + node.Parent.InsertBefore(previewNode, nextSibling) + afterNode := &html.Node{ + Type: html.RawNode, + Data: afterPrefix + after, + } + node.Parent.InsertBefore(afterNode, nextSibling) + node = afterNode + } + + node = node.NextSibling + } +} + +// emojiShortCodeProcessor for rendering text like :smile: into emoji +func emojiShortCodeProcessor(ctx *RenderContext, node *html.Node) { + start := 0 + next := node.NextSibling + for node != nil && node != next && start < len(node.Data) { + m := EmojiShortCodeRegex.FindStringSubmatchIndex(node.Data[start:]) + if m == nil { + return + } + m[0] += start + m[1] += start + + start = m[1] + + alias := node.Data[m[0]:m[1]] + alias = strings.ReplaceAll(alias, ":", "") + converted := emoji.FromAlias(alias) + if converted == nil { + // check if this is a custom reaction + if _, exist := setting.UI.CustomEmojisMap[alias]; exist { + replaceContent(node, m[0], m[1], createCustomEmoji(alias)) + node = node.NextSibling.NextSibling + start = 0 + continue + } + continue + } + + replaceContent(node, m[0], m[1], createEmoji(converted.Emoji, "emoji", converted.Description)) + node = node.NextSibling.NextSibling + start = 0 + } +} + +// emoji processor to match emoji and add emoji class +func emojiProcessor(ctx *RenderContext, node *html.Node) { + start := 0 + next := node.NextSibling + for node != nil && node != next && start < len(node.Data) { + m := emoji.FindEmojiSubmatchIndex(node.Data[start:]) + if m == nil { + return + } + m[0] += start + m[1] += start + + codepoint := node.Data[m[0]:m[1]] + start = m[1] + val := emoji.FromCode(codepoint) + if val != nil { + replaceContent(node, m[0], m[1], createEmoji(codepoint, "emoji", val.Description)) + node = node.NextSibling.NextSibling + start = 0 + } + } +} + +// hashCurrentPatternProcessor renders SHA1 strings to corresponding links that +// are assumed to be in the same repository. +func hashCurrentPatternProcessor(ctx *RenderContext, node *html.Node) { + if ctx.Metas == nil || ctx.Metas["user"] == "" || ctx.Metas["repo"] == "" || ctx.Metas["repoPath"] == "" { + return + } + + start := 0 + next := node.NextSibling + if ctx.ShaExistCache == nil { + ctx.ShaExistCache = make(map[string]bool) + } + for node != nil && node != next && start < len(node.Data) { + m := hashCurrentPattern.FindStringSubmatchIndex(node.Data[start:]) + if m == nil { + return + } + m[2] += start + m[3] += start + + hash := node.Data[m[2]:m[3]] + // The regex does not lie, it matches the hash pattern. + // However, a regex cannot know if a hash actually exists or not. + // We could assume that a SHA1 hash should probably contain alphas AND numerics + // but that is not always the case. + // Although unlikely, deadbeef and 1234567 are valid short forms of SHA1 hash + // as used by git and github for linking and thus we have to do similar. + // Because of this, we check to make sure that a matched hash is actually + // a commit in the repository before making it a link. + + // check cache first + exist, inCache := ctx.ShaExistCache[hash] + if !inCache { + if ctx.GitRepo == nil { + var err error + ctx.GitRepo, err = git.OpenRepository(ctx.Ctx, ctx.Metas["repoPath"]) + if err != nil { + log.Error("unable to open repository: %s Error: %v", ctx.Metas["repoPath"], err) + return + } + ctx.AddCancel(func() { + ctx.GitRepo.Close() + ctx.GitRepo = nil + }) + } + + exist = ctx.GitRepo.IsObjectExist(hash) + ctx.ShaExistCache[hash] = exist + } + + if !exist { + start = m[3] + continue + } + + link := util.URLJoin(ctx.Links.Prefix(), ctx.Metas["user"], ctx.Metas["repo"], "commit", hash) + replaceContent(node, m[2], m[3], createCodeLink(link, base.ShortSha(hash), "commit")) + start = 0 + node = node.NextSibling.NextSibling + } +} + +// emailAddressProcessor replaces raw email addresses with a mailto: link. +func emailAddressProcessor(ctx *RenderContext, node *html.Node) { + next := node.NextSibling + for node != nil && node != next { + m := emailRegex.FindStringSubmatchIndex(node.Data) + if m == nil { + return + } + + mail := node.Data[m[2]:m[3]] + replaceContent(node, m[2], m[3], createLink("mailto:"+mail, mail, "mailto")) + node = node.NextSibling.NextSibling + } +} + +// linkProcessor creates links for any HTTP or HTTPS URL not captured by +// markdown. +func linkProcessor(ctx *RenderContext, node *html.Node) { + next := node.NextSibling + for node != nil && node != next { + m := common.LinkRegex.FindStringIndex(node.Data) + if m == nil { + return + } + + uri := node.Data[m[0]:m[1]] + replaceContent(node, m[0], m[1], createLink(uri, uri, "link")) + node = node.NextSibling.NextSibling + } +} + +func genDefaultLinkProcessor(defaultLink string) processor { + return func(ctx *RenderContext, node *html.Node) { + ch := &html.Node{ + Parent: node, + Type: html.TextNode, + Data: node.Data, + } + + node.Type = html.ElementNode + node.Data = "a" + node.DataAtom = atom.A + node.Attr = []html.Attribute{ + {Key: "href", Val: defaultLink}, + {Key: "class", Val: "default-link muted"}, + } + node.FirstChild, node.LastChild = ch, ch + } +} + +// descriptionLinkProcessor creates links for DescriptionHTML +func descriptionLinkProcessor(ctx *RenderContext, node *html.Node) { + next := node.NextSibling + for node != nil && node != next { + m := common.LinkRegex.FindStringIndex(node.Data) + if m == nil { + return + } + + uri := node.Data[m[0]:m[1]] + replaceContent(node, m[0], m[1], createDescriptionLink(uri, uri)) + node = node.NextSibling.NextSibling + } +} + +func createDescriptionLink(href, content string) *html.Node { + textNode := &html.Node{ + Type: html.TextNode, + Data: content, + } + linkNode := &html.Node{ + FirstChild: textNode, + LastChild: textNode, + Type: html.ElementNode, + Data: "a", + DataAtom: atom.A, + Attr: []html.Attribute{ + {Key: "href", Val: href}, + {Key: "target", Val: "_blank"}, + {Key: "rel", Val: "noopener noreferrer"}, + }, + } + textNode.Parent = linkNode + return linkNode +} diff --git a/modules/markup/html_internal_test.go b/modules/markup/html_internal_test.go new file mode 100644 index 00000000..adc93adb --- /dev/null +++ b/modules/markup/html_internal_test.go @@ -0,0 +1,486 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markup + +import ( + "fmt" + "strconv" + "strings" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + TestAppURL = "http://localhost:3000/" + TestOrgRepo = "gogits/gogs" + TestRepoURL = TestAppURL + TestOrgRepo + "/" +) + +// externalIssueLink an HTML link to an alphanumeric-style issue +func externalIssueLink(baseURL, class, name string) string { + return link(util.URLJoin(baseURL, name), class, name) +} + +// numericLink an HTML to a numeric-style issue +func numericIssueLink(baseURL, class string, index int, marker string) string { + return link(util.URLJoin(baseURL, strconv.Itoa(index)), class, fmt.Sprintf("%s%d", marker, index)) +} + +// link an HTML link +func link(href, class, contents string) string { + if class != "" { + class = " class=\"" + class + "\"" + } + + return fmt.Sprintf("<a href=\"%s\"%s>%s</a>", href, class, contents) +} + +var numericMetas = map[string]string{ + "format": "https://someurl.com/{user}/{repo}/{index}", + "user": "someUser", + "repo": "someRepo", + "style": IssueNameStyleNumeric, +} + +var alphanumericMetas = map[string]string{ + "format": "https://someurl.com/{user}/{repo}/{index}", + "user": "someUser", + "repo": "someRepo", + "style": IssueNameStyleAlphanumeric, +} + +var regexpMetas = map[string]string{ + "format": "https://someurl.com/{user}/{repo}/{index}", + "user": "someUser", + "repo": "someRepo", + "style": IssueNameStyleRegexp, +} + +// these values should match the TestOrgRepo const above +var localMetas = map[string]string{ + "user": "gogits", + "repo": "gogs", +} + +func TestRender_IssueIndexPattern(t *testing.T) { + // numeric: render inputs without valid mentions + test := func(s string) { + testRenderIssueIndexPattern(t, s, s, &RenderContext{ + Ctx: git.DefaultContext, + }) + testRenderIssueIndexPattern(t, s, s, &RenderContext{ + Ctx: git.DefaultContext, + Metas: numericMetas, + }) + } + + // should not render anything when there are no mentions + test("") + test("this is a test") + test("test 123 123 1234") + test("#") + test("# # #") + test("# 123") + test("#abcd") + test("test#1234") + test("#1234test") + test("#abcd") + test("test!1234") + test("!1234test") + test(" test !1234test") + test("/home/gitea/#1234") + test("/home/gitea/!1234") + + // should not render issue mention without leading space + test("test#54321 issue") + + // should not render issue mention without trailing space + test("test #54321issue") +} + +func TestRender_IssueIndexPattern2(t *testing.T) { + setting.AppURL = TestAppURL + + // numeric: render inputs with valid mentions + test := func(s, expectedFmt, marker string, indices ...int) { + var path, prefix string + isExternal := false + if marker == "!" { + path = "pulls" + prefix = "http://localhost:3000/someUser/someRepo/pulls/" + } else { + path = "issues" + prefix = "https://someurl.com/someUser/someRepo/" + isExternal = true + } + + links := make([]any, len(indices)) + for i, index := range indices { + links[i] = numericIssueLink(util.URLJoin(TestRepoURL, path), "ref-issue", index, marker) + } + expectedNil := fmt.Sprintf(expectedFmt, links...) + testRenderIssueIndexPattern(t, s, expectedNil, &RenderContext{ + Ctx: git.DefaultContext, + Metas: localMetas, + }) + + class := "ref-issue" + if isExternal { + class += " ref-external-issue" + } + + for i, index := range indices { + links[i] = numericIssueLink(prefix, class, index, marker) + } + expectedNum := fmt.Sprintf(expectedFmt, links...) + testRenderIssueIndexPattern(t, s, expectedNum, &RenderContext{ + Ctx: git.DefaultContext, + Metas: numericMetas, + }) + } + + // should render freestanding mentions + test("#1234 test", "%s test", "#", 1234) + test("test #8 issue", "test %s issue", "#", 8) + test("!1234 test", "%s test", "!", 1234) + test("test !8 issue", "test %s issue", "!", 8) + test("test issue #1234", "test issue %s", "#", 1234) + test("fixes issue #1234.", "fixes issue %s.", "#", 1234) + + // should render mentions in parentheses / brackets + test("(#54321 issue)", "(%s issue)", "#", 54321) + test("[#54321 issue]", "[%s issue]", "#", 54321) + test("test (#9801 extra) issue", "test (%s extra) issue", "#", 9801) + test("test (!9801 extra) issue", "test (%s extra) issue", "!", 9801) + test("test (#1)", "test (%s)", "#", 1) + + // should render multiple issue mentions in the same line + test("#54321 #1243", "%s %s", "#", 54321, 1243) + test("wow (#54321 #1243)", "wow (%s %s)", "#", 54321, 1243) + test("(#4)(#5)", "(%s)(%s)", "#", 4, 5) + test("#1 (#4321) test", "%s (%s) test", "#", 1, 4321) + + // should render with : + test("#1234: test", "%s: test", "#", 1234) + test("wow (#54321: test)", "wow (%s: test)", "#", 54321) +} + +func TestRender_IssueIndexPattern3(t *testing.T) { + setting.AppURL = TestAppURL + + // alphanumeric: render inputs without valid mentions + test := func(s string) { + testRenderIssueIndexPattern(t, s, s, &RenderContext{ + Ctx: git.DefaultContext, + Metas: alphanumericMetas, + }) + } + test("") + test("this is a test") + test("test 123 123 1234") + test("#") + test("# 123") + test("#abcd") + test("test #123") + test("abc-1234") // issue prefix must be capital + test("ABc-1234") // issue prefix must be _all_ capital + test("ABCDEFGHIJK-1234") // the limit is 10 characters in the prefix + test("ABC1234") // dash is required + test("test ABC- test") // number is required + test("test -1234 test") // prefix is required + test("testABC-123 test") // leading space is required + test("test ABC-123test") // trailing space is required + test("ABC-0123") // no leading zero +} + +func TestRender_IssueIndexPattern4(t *testing.T) { + setting.AppURL = TestAppURL + + // alphanumeric: render inputs with valid mentions + test := func(s, expectedFmt string, names ...string) { + links := make([]any, len(names)) + for i, name := range names { + links[i] = externalIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue ref-external-issue", name) + } + expected := fmt.Sprintf(expectedFmt, links...) + testRenderIssueIndexPattern(t, s, expected, &RenderContext{ + Ctx: git.DefaultContext, + Metas: alphanumericMetas, + }) + } + test("OTT-1234 test", "%s test", "OTT-1234") + test("test T-12 issue", "test %s issue", "T-12") + test("test issue ABCDEFGHIJ-1234567890", "test issue %s", "ABCDEFGHIJ-1234567890") +} + +func TestRender_IssueIndexPattern5(t *testing.T) { + setting.AppURL = TestAppURL + + // regexp: render inputs without valid mentions + test := func(s, expectedFmt, pattern string, ids, names []string) { + metas := regexpMetas + metas["regexp"] = pattern + links := make([]any, len(ids)) + for i, id := range ids { + links[i] = link(util.URLJoin("https://someurl.com/someUser/someRepo/", id), "ref-issue ref-external-issue", names[i]) + } + + expected := fmt.Sprintf(expectedFmt, links...) + testRenderIssueIndexPattern(t, s, expected, &RenderContext{ + Ctx: git.DefaultContext, + Metas: metas, + }) + } + + test("abc ISSUE-123 def", "abc %s def", + "ISSUE-(\\d+)", + []string{"123"}, + []string{"ISSUE-123"}, + ) + + test("abc (ISSUE 123) def", "abc %s def", + "\\(ISSUE (\\d+)\\)", + []string{"123"}, + []string{"(ISSUE 123)"}, + ) + + test("abc ISSUE-123 def", "abc %s def", + "(ISSUE-(\\d+))", + []string{"ISSUE-123"}, + []string{"ISSUE-123"}, + ) + + testRenderIssueIndexPattern(t, "will not match", "will not match", &RenderContext{ + Ctx: git.DefaultContext, + Metas: regexpMetas, + }) +} + +func TestRender_IssueIndexPattern_Document(t *testing.T) { + setting.AppURL = TestAppURL + metas := map[string]string{ + "format": "https://someurl.com/{user}/{repo}/{index}", + "user": "someUser", + "repo": "someRepo", + "style": IssueNameStyleNumeric, + "mode": "document", + } + + testRenderIssueIndexPattern(t, "#1", "#1", &RenderContext{ + Ctx: git.DefaultContext, + Metas: metas, + }) + testRenderIssueIndexPattern(t, "#1312", "#1312", &RenderContext{ + Ctx: git.DefaultContext, + Metas: metas, + }) + testRenderIssueIndexPattern(t, "!1", "!1", &RenderContext{ + Ctx: git.DefaultContext, + Metas: metas, + }) +} + +func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *RenderContext) { + ctx.Links.AbsolutePrefix = true + if ctx.Links.Base == "" { + ctx.Links.Base = TestRepoURL + } + + var buf strings.Builder + err := postProcess(ctx, []processor{issueIndexPatternProcessor}, strings.NewReader(input), &buf) + require.NoError(t, err) + assert.Equal(t, expected, buf.String(), "input=%q", input) +} + +func TestRender_AutoLink(t *testing.T) { + setting.AppURL = TestAppURL + + test := func(input, expected string) { + var buffer strings.Builder + err := PostProcess(&RenderContext{ + Ctx: git.DefaultContext, + Links: Links{ + Base: TestRepoURL, + }, + Metas: localMetas, + }, strings.NewReader(input), &buffer) + require.NoError(t, err, nil) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer.String())) + + buffer.Reset() + err = PostProcess(&RenderContext{ + Ctx: git.DefaultContext, + Links: Links{ + Base: TestRepoURL, + }, + Metas: localMetas, + IsWiki: true, + }, strings.NewReader(input), &buffer) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer.String())) + } + + // render valid issue URLs + test(util.URLJoin(TestRepoURL, "issues", "3333"), + numericIssueLink(util.URLJoin(TestRepoURL, "issues"), "ref-issue", 3333, "#")) + + // render valid commit URLs + tmp := util.URLJoin(TestRepoURL, "commit", "d8a994ef243349f321568f9e36d5c3f444b99cae") + test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24</code></a>") + tmp += "#diff-2" + test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24 (diff-2)</code></a>") + + // render other commit URLs + tmp = "https://external-link.gitea.io/go-gitea/gitea/commit/d8a994ef243349f321568f9e36d5c3f444b99cae#diff-2" + test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24 (diff-2)</code></a>") +} + +func TestRender_IssueIndexPatternRef(t *testing.T) { + setting.AppURL = TestAppURL + + test := func(input, expected string) { + var buf strings.Builder + err := postProcess(&RenderContext{ + Ctx: git.DefaultContext, + Metas: numericMetas, + }, []processor{issueIndexPatternProcessor}, strings.NewReader(input), &buf) + require.NoError(t, err) + assert.Equal(t, expected, buf.String(), "input=%q", input) + } + + test("alan-turin/Enigma-cryptanalysis#1", `<a href="/alan-turin/enigma-cryptanalysis/issues/1" class="ref-issue">alan-turin/Enigma-cryptanalysis#1</a>`) +} + +func TestRender_FullIssueURLs(t *testing.T) { + setting.AppURL = TestAppURL + + test := func(input, expected string) { + var result strings.Builder + err := postProcess(&RenderContext{ + Ctx: git.DefaultContext, + Links: Links{ + Base: TestRepoURL, + }, + Metas: localMetas, + }, []processor{fullIssuePatternProcessor}, strings.NewReader(input), &result) + require.NoError(t, err) + assert.Equal(t, expected, result.String()) + } + test("Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6", + "Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6") + test("Look here http://localhost:3000/person/repo/issues/4", + `Look here <a href="http://localhost:3000/person/repo/issues/4" class="ref-issue">person/repo#4</a>`) + test("http://localhost:3000/person/repo/issues/4#issuecomment-1234", + `<a href="http://localhost:3000/person/repo/issues/4#issuecomment-1234" class="ref-issue">person/repo#4 (comment)</a>`) + test("http://localhost:3000/gogits/gogs/issues/4", + `<a href="http://localhost:3000/gogits/gogs/issues/4" class="ref-issue">#4</a>`) + test("http://localhost:3000/gogits/gogs/issues/4 test", + `<a href="http://localhost:3000/gogits/gogs/issues/4" class="ref-issue">#4</a> test`) + test("http://localhost:3000/gogits/gogs/issues/4?a=1&b=2#comment-123 test", + `<a href="http://localhost:3000/gogits/gogs/issues/4?a=1&b=2#comment-123" class="ref-issue">#4 (comment)</a> test`) + test("http://localhost:3000/testOrg/testOrgRepo/pulls/2/files#issuecomment-24", + "http://localhost:3000/testOrg/testOrgRepo/pulls/2/files#issuecomment-24") + test("http://localhost:3000/testOrg/testOrgRepo/pulls/2/files", + "http://localhost:3000/testOrg/testOrgRepo/pulls/2/files") +} + +func TestRegExp_sha1CurrentPattern(t *testing.T) { + trueTestCases := []string{ + "d8a994ef243349f321568f9e36d5c3f444b99cae", + "abcdefabcdefabcdefabcdefabcdefabcdefabcd", + "(abcdefabcdefabcdefabcdefabcdefabcdefabcd)", + "[abcdefabcdefabcdefabcdefabcdefabcdefabcd]", + "abcdefabcdefabcdefabcdefabcdefabcdefabcd.", + "abcdefabcdefabcdefabcdefabcdefabcdefabcd:", + } + falseTestCases := []string{ + "test", + "abcdefg", + "e59ff077-2d03-4e6b-964d-63fbaea81f", + "abcdefghijklmnopqrstuvwxyzabcdefghijklmn", + "abcdefghijklmnopqrstuvwxyzabcdefghijklmO", + } + + for _, testCase := range trueTestCases { + assert.True(t, hashCurrentPattern.MatchString(testCase)) + } + for _, testCase := range falseTestCases { + assert.False(t, hashCurrentPattern.MatchString(testCase)) + } +} + +func TestRegExp_anySHA1Pattern(t *testing.T) { + testCases := map[string][]string{ + "https://github.com/jquery/jquery/blob/a644101ed04d0beacea864ce805e0c4f86ba1cd1/test/unit/event.js#L2703": { + "a644101ed04d0beacea864ce805e0c4f86ba1cd1", + "/test/unit/event.js", + "", + "#L2703", + }, + "https://github.com/jquery/jquery/blob/a644101ed04d0beacea864ce805e0c4f86ba1cd1/test/unit/event.js": { + "a644101ed04d0beacea864ce805e0c4f86ba1cd1", + "/test/unit/event.js", + "", + "", + }, + "https://github.com/jquery/jquery/commit/0705be475092aede1eddae01319ec931fb9c65fc": { + "0705be475092aede1eddae01319ec931fb9c65fc", + "", + "", + "", + }, + "https://github.com/jquery/jquery/tree/0705be475092aede1eddae01319ec931fb9c65fc/src": { + "0705be475092aede1eddae01319ec931fb9c65fc", + "/src", + "", + "", + }, + "https://try.gogs.io/gogs/gogs/commit/d8a994ef243349f321568f9e36d5c3f444b99cae#diff-2": { + "d8a994ef243349f321568f9e36d5c3f444b99cae", + "", + "", + "#diff-2", + }, + "https://codeberg.org/forgejo/forgejo/src/commit/949ab9a5c4cac742f84ae5a9fa186f8d6eb2cdc0/RELEASE-NOTES.md?display=source&w=1#L7-L9": { + "949ab9a5c4cac742f84ae5a9fa186f8d6eb2cdc0", + "/RELEASE-NOTES.md", + "?display=source&w=1", + "#L7-L9", + }, + } + + for k, v := range testCases { + assert.Equal(t, anyHashPattern.FindStringSubmatch(k)[1:], v) + } +} + +func TestRegExp_shortLinkPattern(t *testing.T) { + trueTestCases := []string{ + "[[stuff]]", + "[[]]", + "[[stuff|title=Difficult name with spaces*!]]", + } + falseTestCases := []string{ + "test", + "abcdefg", + "[[]", + "[[", + "[]", + "]]", + "abcdefghijklmnopqrstuvwxyz", + } + + for _, testCase := range trueTestCases { + assert.True(t, shortLinkPattern.MatchString(testCase)) + } + for _, testCase := range falseTestCases { + assert.False(t, shortLinkPattern.MatchString(testCase)) + } +} diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go new file mode 100644 index 00000000..42ce9990 --- /dev/null +++ b/modules/markup/html_test.go @@ -0,0 +1,997 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markup_test + +import ( + "context" + "io" + "os" + "strings" + "testing" + + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/emoji" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/markup/markdown" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/translation" + "code.gitea.io/gitea/modules/util" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var localMetas = map[string]string{ + "user": "gogits", + "repo": "gogs", + "repoPath": "../../tests/gitea-repositories-meta/user13/repo11.git/", +} + +func TestMain(m *testing.M) { + unittest.InitSettings() + if err := git.InitSimple(context.Background()); err != nil { + log.Fatal("git init failed, err: %v", err) + } + os.Exit(m.Run()) +} + +func TestRender_Commits(t *testing.T) { + setting.AppURL = markup.TestAppURL + test := func(input, expected string) { + buffer, err := markup.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + RelativePath: ".md", + Links: markup.Links{ + AbsolutePrefix: true, + Base: markup.TestRepoURL, + }, + Metas: localMetas, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) + } + + sha := "65f1bf27bc3bf70f64657658635e66094edbcb4d" + repo := markup.TestRepoURL + commit := util.URLJoin(repo, "commit", sha) + tree := util.URLJoin(repo, "tree", sha, "src") + + file := util.URLJoin(repo, "commit", sha, "example.txt") + fileWithExtra := file + ":" + fileWithHash := file + "#L2" + fileWithHasExtra := file + "#L2:" + commitCompare := util.URLJoin(repo, "compare", sha+"..."+sha) + commitCompareWithHash := commitCompare + "#L2" + + test(sha, `<p><a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test(sha[:7], `<p><a href="`+commit[:len(commit)-(40-7)]+`" rel="nofollow"><code>65f1bf2</code></a></p>`) + test(sha[:39], `<p><a href="`+commit[:len(commit)-(40-39)]+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test(commit, `<p><a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test(tree, `<p><a href="`+tree+`" rel="nofollow"><code>65f1bf27bc/src</code></a></p>`) + + test(file, `<p><a href="`+file+`" rel="nofollow"><code>65f1bf27bc/example.txt</code></a></p>`) + test(fileWithExtra, `<p><a href="`+file+`" rel="nofollow"><code>65f1bf27bc/example.txt</code></a>:</p>`) + test(fileWithHash, `<p><a href="`+fileWithHash+`" rel="nofollow"><code>65f1bf27bc/example.txt (L2)</code></a></p>`) + test(fileWithHasExtra, `<p><a href="`+fileWithHash+`" rel="nofollow"><code>65f1bf27bc/example.txt (L2)</code></a>:</p>`) + test(commitCompare, `<p><a href="`+commitCompare+`" rel="nofollow"><code>65f1bf27bc...65f1bf27bc</code></a></p>`) + test(commitCompareWithHash, `<p><a href="`+commitCompareWithHash+`" rel="nofollow"><code>65f1bf27bc...65f1bf27bc (L2)</code></a></p>`) + + test("commit "+sha, `<p>commit <a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`) + test("/home/gitea/"+sha, "<p>/home/gitea/"+sha+"</p>") + test("deadbeef", `<p>deadbeef</p>`) + test("d27ace93", `<p>d27ace93</p>`) + test(sha[:14]+".x", `<p>`+sha[:14]+`.x</p>`) + + expected14 := `<a href="` + commit[:len(commit)-(40-14)] + `" rel="nofollow"><code>` + sha[:10] + `</code></a>` + test(sha[:14]+".", `<p>`+expected14+`.</p>`) + test(sha[:14]+",", `<p>`+expected14+`,</p>`) + test("["+sha[:14]+"]", `<p>[`+expected14+`]</p>`) +} + +func TestRender_CrossReferences(t *testing.T) { + setting.AppURL = markup.TestAppURL + + test := func(input, expected string) { + buffer, err := markup.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + RelativePath: "a.md", + Links: markup.Links{ + AbsolutePrefix: true, + Base: setting.AppSubURL, + }, + Metas: localMetas, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) + } + + test( + "gogits/gogs#12345", + `<p><a href="`+util.URLJoin(markup.TestAppURL, "gogits", "gogs", "issues", "12345")+`" class="ref-issue" rel="nofollow">gogits/gogs#12345</a></p>`) + test( + "go-gitea/gitea#12345", + `<p><a href="`+util.URLJoin(markup.TestAppURL, "go-gitea", "gitea", "issues", "12345")+`" class="ref-issue" rel="nofollow">go-gitea/gitea#12345</a></p>`) + test( + "/home/gitea/go-gitea/gitea#12345", + `<p>/home/gitea/go-gitea/gitea#12345</p>`) + test( + util.URLJoin(markup.TestAppURL, "gogitea", "gitea", "issues", "12345"), + `<p><a href="`+util.URLJoin(markup.TestAppURL, "gogitea", "gitea", "issues", "12345")+`" class="ref-issue" rel="nofollow">gogitea/gitea#12345</a></p>`) + test( + util.URLJoin(markup.TestAppURL, "go-gitea", "gitea", "issues", "12345"), + `<p><a href="`+util.URLJoin(markup.TestAppURL, "go-gitea", "gitea", "issues", "12345")+`" class="ref-issue" rel="nofollow">go-gitea/gitea#12345</a></p>`) + test( + util.URLJoin(markup.TestAppURL, "gogitea", "some-repo-name", "issues", "12345"), + `<p><a href="`+util.URLJoin(markup.TestAppURL, "gogitea", "some-repo-name", "issues", "12345")+`" class="ref-issue" rel="nofollow">gogitea/some-repo-name#12345</a></p>`) + + sha := "65f1bf27bc3bf70f64657658635e66094edbcb4d" + urlWithQuery := util.URLJoin(markup.TestAppURL, "forgejo", "some-repo-name", "commit", sha, "README.md") + "?display=source#L1-L5" + test( + urlWithQuery, + `<p><a href="`+urlWithQuery+`" rel="nofollow"><code>`+sha[:10]+`/README.md (L1-L5)</code></a></p>`) +} + +func TestRender_links(t *testing.T) { + setting.AppURL = markup.TestAppURL + + test := func(input, expected string) { + buffer, err := markup.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + RelativePath: "a.md", + Links: markup.Links{ + Base: markup.TestRepoURL, + }, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) + } + // Text that should be turned into URL + + defaultCustom := setting.Markdown.CustomURLSchemes + setting.Markdown.CustomURLSchemes = []string{"ftp", "magnet"} + markup.InitializeSanitizer() + markup.CustomLinkURLSchemes(setting.Markdown.CustomURLSchemes) + + test( + "https://www.example.com", + `<p><a href="https://www.example.com" rel="nofollow">https://www.example.com</a></p>`) + test( + "http://www.example.com", + `<p><a href="http://www.example.com" rel="nofollow">http://www.example.com</a></p>`) + test( + "https://example.com", + `<p><a href="https://example.com" rel="nofollow">https://example.com</a></p>`) + test( + "http://example.com", + `<p><a href="http://example.com" rel="nofollow">http://example.com</a></p>`) + test( + "http://foo.com/blah_blah", + `<p><a href="http://foo.com/blah_blah" rel="nofollow">http://foo.com/blah_blah</a></p>`) + test( + "http://foo.com/blah_blah/", + `<p><a href="http://foo.com/blah_blah/" rel="nofollow">http://foo.com/blah_blah/</a></p>`) + test( + "http://www.example.com/wpstyle/?p=364", + `<p><a href="http://www.example.com/wpstyle/?p=364" rel="nofollow">http://www.example.com/wpstyle/?p=364</a></p>`) + test( + "https://www.example.com/foo/?bar=baz&inga=42&quux", + `<p><a href="https://www.example.com/foo/?bar=baz&inga=42&quux" rel="nofollow">https://www.example.com/foo/?bar=baz&inga=42&quux</a></p>`) + test( + "http://142.42.1.1/", + `<p><a href="http://142.42.1.1/" rel="nofollow">http://142.42.1.1/</a></p>`) + test( + "https://github.com/go-gitea/gitea/?p=aaa/bbb.html#ccc-ddd", + `<p><a href="https://github.com/go-gitea/gitea/?p=aaa/bbb.html#ccc-ddd" rel="nofollow">https://github.com/go-gitea/gitea/?p=aaa/bbb.html#ccc-ddd</a></p>`) + test( + "https://en.wikipedia.org/wiki/URL_(disambiguation)", + `<p><a href="https://en.wikipedia.org/wiki/URL_(disambiguation)" rel="nofollow">https://en.wikipedia.org/wiki/URL_(disambiguation)</a></p>`) + test( + "https://foo_bar.example.com/", + `<p><a href="https://foo_bar.example.com/" rel="nofollow">https://foo_bar.example.com/</a></p>`) + test( + "https://stackoverflow.com/questions/2896191/what-is-go-used-fore", + `<p><a href="https://stackoverflow.com/questions/2896191/what-is-go-used-fore" rel="nofollow">https://stackoverflow.com/questions/2896191/what-is-go-used-fore</a></p>`) + test( + "https://username:password@gitea.com", + `<p><a href="https://username:password@gitea.com" rel="nofollow">https://username:password@gitea.com</a></p>`) + test( + "ftp://gitea.com/file.txt", + `<p><a href="ftp://gitea.com/file.txt" rel="nofollow">ftp://gitea.com/file.txt</a></p>`) + test( + "magnet:?xt=urn:btih:5dee65101db281ac9c46344cd6b175cdcadabcde&dn=download", + `<p><a href="magnet:?xt=urn:btih:5dee65101db281ac9c46344cd6b175cdcadabcde&dn=download" rel="nofollow">magnet:?xt=urn:btih:5dee65101db281ac9c46344cd6b175cdcadabcde&dn=download</a></p>`) + + // Test that should *not* be turned into URL + test( + "www.example.com", + `<p>www.example.com</p>`) + test( + "example.com", + `<p>example.com</p>`) + test( + "test.example.com", + `<p>test.example.com</p>`) + test( + "http://", + `<p>http://</p>`) + test( + "https://", + `<p>https://</p>`) + test( + "://", + `<p>://</p>`) + test( + "www", + `<p>www</p>`) + test( + "ftps://gitea.com", + `<p>ftps://gitea.com</p>`) + + // Restore previous settings + setting.Markdown.CustomURLSchemes = defaultCustom + markup.InitializeSanitizer() + markup.CustomLinkURLSchemes(setting.Markdown.CustomURLSchemes) +} + +func TestRender_email(t *testing.T) { + setting.AppURL = markup.TestAppURL + + test := func(input, expected string) { + res, err := markup.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + RelativePath: "a.md", + Links: markup.Links{ + Base: markup.TestRepoURL, + }, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res)) + } + // Text that should be turned into email link + + test( + "info@gitea.com", + `<p><a href="mailto:info@gitea.com" rel="nofollow">info@gitea.com</a></p>`) + test( + "(info@gitea.com)", + `<p>(<a href="mailto:info@gitea.com" rel="nofollow">info@gitea.com</a>)</p>`) + test( + "[info@gitea.com]", + `<p>[<a href="mailto:info@gitea.com" rel="nofollow">info@gitea.com</a>]</p>`) + test( + "info@gitea.com.", + `<p><a href="mailto:info@gitea.com" rel="nofollow">info@gitea.com</a>.</p>`) + test( + "firstname+lastname@gitea.com", + `<p><a href="mailto:firstname+lastname@gitea.com" rel="nofollow">firstname+lastname@gitea.com</a></p>`) + test( + "send email to info@gitea.co.uk.", + `<p>send email to <a href="mailto:info@gitea.co.uk" rel="nofollow">info@gitea.co.uk</a>.</p>`) + + test( + `j.doe@example.com, + j.doe@example.com. + j.doe@example.com; + j.doe@example.com? + j.doe@example.com!`, + `<p><a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>,<br/> +<a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>.<br/> +<a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>;<br/> +<a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>?<br/> +<a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>!</p>`) + + // Test that should *not* be turned into email links + test( + "\"info@gitea.com\"", + `<p>"info@gitea.com"</p>`) + test( + "/home/gitea/mailstore/info@gitea/com", + `<p>/home/gitea/mailstore/info@gitea/com</p>`) + test( + "git@try.gitea.io:go-gitea/gitea.git", + `<p>git@try.gitea.io:go-gitea/gitea.git</p>`) + test( + "gitea@3", + `<p>gitea@3</p>`) + test( + "gitea@gmail.c", + `<p>gitea@gmail.c</p>`) + test( + "email@domain@domain.com", + `<p>email@domain@domain.com</p>`) + test( + "email@domain..com", + `<p>email@domain..com</p>`) +} + +func TestRender_emoji(t *testing.T) { + setting.AppURL = markup.TestAppURL + setting.StaticURLPrefix = markup.TestAppURL + + test := func(input, expected string) { + expected = strings.ReplaceAll(expected, "&", "&") + buffer, err := markup.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + RelativePath: "a.md", + Links: markup.Links{ + Base: markup.TestRepoURL, + }, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) + } + + // Make sure we can successfully match every emoji in our dataset with regex + for i := range emoji.GemojiData { + test( + emoji.GemojiData[i].Emoji, + `<p><span class="emoji" aria-label="`+emoji.GemojiData[i].Description+`">`+emoji.GemojiData[i].Emoji+`</span></p>`) + } + for i := range emoji.GemojiData { + test( + ":"+emoji.GemojiData[i].Aliases[0]+":", + `<p><span class="emoji" aria-label="`+emoji.GemojiData[i].Description+`">`+emoji.GemojiData[i].Emoji+`</span></p>`) + } + + // Text that should be turned into or recognized as emoji + test( + ":gitea:", + `<p><span class="emoji" aria-label="gitea"><img alt=":gitea:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/gitea.png"/></span></p>`) + test( + ":custom-emoji:", + `<p>:custom-emoji:</p>`) + setting.UI.CustomEmojisMap["custom-emoji"] = ":custom-emoji:" + test( + ":custom-emoji:", + `<p><span class="emoji" aria-label="custom-emoji"><img alt=":custom-emoji:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/custom-emoji.png"/></span></p>`) + test( + "这是字符:1::+1: some🐊 \U0001f44d:custom-emoji: :gitea:", + `<p>这是字符:1:<span class="emoji" aria-label="thumbs up">👍</span> some<span class="emoji" aria-label="crocodile">🐊</span> `+ + `<span class="emoji" aria-label="thumbs up">👍</span><span class="emoji" aria-label="custom-emoji"><img alt=":custom-emoji:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/custom-emoji.png"/></span> `+ + `<span class="emoji" aria-label="gitea"><img alt=":gitea:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/gitea.png"/></span></p>`) + test( + "Some text with 😄 in the middle", + `<p>Some text with <span class="emoji" aria-label="grinning face with smiling eyes">😄</span> in the middle</p>`) + test( + "Some text with :smile: in the middle", + `<p>Some text with <span class="emoji" aria-label="grinning face with smiling eyes">😄</span> in the middle</p>`) + test( + "Some text with 😄😄 2 emoji next to each other", + `<p>Some text with <span class="emoji" aria-label="grinning face with smiling eyes">😄</span><span class="emoji" aria-label="grinning face with smiling eyes">😄</span> 2 emoji next to each other</p>`) + test( + "😎🤪🔐🤑❓", + `<p><span class="emoji" aria-label="smiling face with sunglasses">😎</span><span class="emoji" aria-label="zany face">🤪</span><span class="emoji" aria-label="locked with key">🔐</span><span class="emoji" aria-label="money-mouth face">🤑</span><span class="emoji" aria-label="red question mark">❓</span></p>`) + + // should match nothing + test( + "2001:0db8:85a3:0000:0000:8a2e:0370:7334", + `<p>2001:0db8:85a3:0000:0000:8a2e:0370:7334</p>`) + test( + ":not exist:", + `<p>:not exist:</p>`) +} + +func TestRender_ShortLinks(t *testing.T) { + setting.AppURL = markup.TestAppURL + tree := util.URLJoin(markup.TestRepoURL, "src", "master") + + test := func(input, expected, expectedWiki string) { + buffer, err := markdown.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: markup.TestRepoURL, + BranchPath: "master", + }, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) + buffer, err = markdown.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: markup.TestRepoURL, + }, + Metas: localMetas, + IsWiki: true, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(string(buffer))) + } + + mediatree := util.URLJoin(markup.TestRepoURL, "media", "master") + url := util.URLJoin(tree, "Link") + otherURL := util.URLJoin(tree, "Other-Link") + encodedURL := util.URLJoin(tree, "Link%3F") + imgurl := util.URLJoin(mediatree, "Link.jpg") + otherImgurl := util.URLJoin(mediatree, "Link+Other.jpg") + encodedImgurl := util.URLJoin(mediatree, "Link+%23.jpg") + notencodedImgurl := util.URLJoin(mediatree, "some", "path", "Link+#.jpg") + urlWiki := util.URLJoin(markup.TestRepoURL, "wiki", "Link") + otherURLWiki := util.URLJoin(markup.TestRepoURL, "wiki", "Other-Link") + encodedURLWiki := util.URLJoin(markup.TestRepoURL, "wiki", "Link%3F") + imgurlWiki := util.URLJoin(markup.TestRepoURL, "wiki", "raw", "Link.jpg") + otherImgurlWiki := util.URLJoin(markup.TestRepoURL, "wiki", "raw", "Link+Other.jpg") + encodedImgurlWiki := util.URLJoin(markup.TestRepoURL, "wiki", "raw", "Link+%23.jpg") + notencodedImgurlWiki := util.URLJoin(markup.TestRepoURL, "wiki", "raw", "some", "path", "Link+#.jpg") + favicon := "http://google.com/favicon.ico" + + test( + "[[Link]]", + `<p><a href="`+url+`" rel="nofollow">Link</a></p>`, + `<p><a href="`+urlWiki+`" rel="nofollow">Link</a></p>`) + test( + "[[Link.jpg]]", + `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" title="Link.jpg" alt="Link.jpg"/></a></p>`, + `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" title="Link.jpg" alt="Link.jpg"/></a></p>`) + test( + "[["+favicon+"]]", + `<p><a href="`+favicon+`" rel="nofollow"><img src="`+favicon+`" title="favicon.ico" alt="`+favicon+`"/></a></p>`, + `<p><a href="`+favicon+`" rel="nofollow"><img src="`+favicon+`" title="favicon.ico" alt="`+favicon+`"/></a></p>`) + test( + "[[Name|Link]]", + `<p><a href="`+url+`" rel="nofollow">Name</a></p>`, + `<p><a href="`+urlWiki+`" rel="nofollow">Name</a></p>`) + test( + "[[Name|Link.jpg]]", + `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" title="Name" alt="Name"/></a></p>`, + `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" title="Name" alt="Name"/></a></p>`) + test( + "[[Name|Link.jpg|alt=AltName]]", + `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" title="AltName" alt="AltName"/></a></p>`, + `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" title="AltName" alt="AltName"/></a></p>`) + test( + "[[Name|Link.jpg|title=Title]]", + `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" title="Title" alt="Title"/></a></p>`, + `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" title="Title" alt="Title"/></a></p>`) + test( + "[[Name|Link.jpg|alt=AltName|title=Title]]", + `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" title="Title" alt="AltName"/></a></p>`, + `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" title="Title" alt="AltName"/></a></p>`) + test( + "[[Name|Link.jpg|alt=\"AltName\"|title='Title']]", + `<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" title="Title" alt="AltName"/></a></p>`, + `<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" title="Title" alt="AltName"/></a></p>`) + test( + "[[Name|Link Other.jpg|alt=\"AltName\"|title='Title']]", + `<p><a href="`+otherImgurl+`" rel="nofollow"><img src="`+otherImgurl+`" title="Title" alt="AltName"/></a></p>`, + `<p><a href="`+otherImgurlWiki+`" rel="nofollow"><img src="`+otherImgurlWiki+`" title="Title" alt="AltName"/></a></p>`) + test( + "[[Link]] [[Other Link]]", + `<p><a href="`+url+`" rel="nofollow">Link</a> <a href="`+otherURL+`" rel="nofollow">Other Link</a></p>`, + `<p><a href="`+urlWiki+`" rel="nofollow">Link</a> <a href="`+otherURLWiki+`" rel="nofollow">Other Link</a></p>`) + test( + "[[Link?]]", + `<p><a href="`+encodedURL+`" rel="nofollow">Link?</a></p>`, + `<p><a href="`+encodedURLWiki+`" rel="nofollow">Link?</a></p>`) + test( + "[[Link]] [[Other Link]] [[Link?]]", + `<p><a href="`+url+`" rel="nofollow">Link</a> <a href="`+otherURL+`" rel="nofollow">Other Link</a> <a href="`+encodedURL+`" rel="nofollow">Link?</a></p>`, + `<p><a href="`+urlWiki+`" rel="nofollow">Link</a> <a href="`+otherURLWiki+`" rel="nofollow">Other Link</a> <a href="`+encodedURLWiki+`" rel="nofollow">Link?</a></p>`) + test( + "[[Link #.jpg]]", + `<p><a href="`+encodedImgurl+`" rel="nofollow"><img src="`+encodedImgurl+`" title="Link #.jpg" alt="Link #.jpg"/></a></p>`, + `<p><a href="`+encodedImgurlWiki+`" rel="nofollow"><img src="`+encodedImgurlWiki+`" title="Link #.jpg" alt="Link #.jpg"/></a></p>`) + test( + "[[Name|Link #.jpg|alt=\"AltName\"|title='Title']]", + `<p><a href="`+encodedImgurl+`" rel="nofollow"><img src="`+encodedImgurl+`" title="Title" alt="AltName"/></a></p>`, + `<p><a href="`+encodedImgurlWiki+`" rel="nofollow"><img src="`+encodedImgurlWiki+`" title="Title" alt="AltName"/></a></p>`) + test( + "[[some/path/Link #.jpg]]", + `<p><a href="`+notencodedImgurl+`" rel="nofollow"><img src="`+notencodedImgurl+`" title="Link #.jpg" alt="some/path/Link #.jpg"/></a></p>`, + `<p><a href="`+notencodedImgurlWiki+`" rel="nofollow"><img src="`+notencodedImgurlWiki+`" title="Link #.jpg" alt="some/path/Link #.jpg"/></a></p>`) + test( + "<p><a href=\"https://example.org\">[[foobar]]</a></p>", + `<p><a href="https://example.org" rel="nofollow">[[foobar]]</a></p>`, + `<p><a href="https://example.org" rel="nofollow">[[foobar]]</a></p>`) +} + +func TestRender_RelativeImages(t *testing.T) { + setting.AppURL = markup.TestAppURL + + test := func(input, expected, expectedWiki string) { + buffer, err := markdown.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: markup.TestRepoURL, + BranchPath: "master", + }, + Metas: localMetas, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) + buffer, err = markdown.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: markup.TestRepoURL, + }, + Metas: localMetas, + IsWiki: true, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(string(buffer))) + } + + rawwiki := util.URLJoin(markup.TestRepoURL, "wiki", "raw") + mediatree := util.URLJoin(markup.TestRepoURL, "media", "master") + + test( + `<img src="Link">`, + `<img src="`+util.URLJoin(mediatree, "Link")+`"/>`, + `<img src="`+util.URLJoin(rawwiki, "Link")+`"/>`) + + test( + `<img src="./icon.png">`, + `<img src="`+util.URLJoin(mediatree, "icon.png")+`"/>`, + `<img src="`+util.URLJoin(rawwiki, "icon.png")+`"/>`) +} + +func Test_ParseClusterFuzz(t *testing.T) { + setting.AppURL = markup.TestAppURL + + localMetas := map[string]string{ + "user": "go-gitea", + "repo": "gitea", + } + + data := "<A><maTH><tr><MN><bodY ÿ><temPlate></template><tH><tr></A><tH><d<bodY " + + var res strings.Builder + err := markup.PostProcess(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: "https://example.com", + }, + Metas: localMetas, + }, strings.NewReader(data), &res) + require.NoError(t, err) + assert.NotContains(t, res.String(), "<html") + + data = "<!DOCTYPE html>\n<A><maTH><tr><MN><bodY ÿ><temPlate></template><tH><tr></A><tH><d<bodY " + + res.Reset() + err = markup.PostProcess(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: "https://example.com", + }, + Metas: localMetas, + }, strings.NewReader(data), &res) + + require.NoError(t, err) + assert.NotContains(t, res.String(), "<html") +} + +func TestPostProcess_RenderDocument(t *testing.T) { + setting.AppURL = markup.TestAppURL + setting.StaticURLPrefix = markup.TestAppURL // can't run standalone + + localMetas := map[string]string{ + "user": "go-gitea", + "repo": "gitea", + "mode": "document", + } + + test := func(input, expected string) { + var res strings.Builder + err := markup.PostProcess(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + AbsolutePrefix: true, + Base: "https://example.com", + }, + Metas: localMetas, + }, strings.NewReader(input), &res) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res.String())) + } + + // Issue index shouldn't be post processing in a document. + test( + "#1", + "#1") + + // But cross-referenced issue index should work. + test( + "go-gitea/gitea#12345", + `<a href="`+util.URLJoin(markup.TestAppURL, "go-gitea", "gitea", "issues", "12345")+`" class="ref-issue">go-gitea/gitea#12345</a>`) + + // Test that other post processing still works. + test( + ":gitea:", + `<span class="emoji" aria-label="gitea"><img alt=":gitea:" src="`+setting.StaticURLPrefix+`/assets/img/emoji/gitea.png"/></span>`) + test( + "Some text with 😄 in the middle", + `Some text with <span class="emoji" aria-label="grinning face with smiling eyes">😄</span> in the middle`) + test("http://localhost:3000/person/repo/issues/4#issuecomment-1234", + `<a href="http://localhost:3000/person/repo/issues/4#issuecomment-1234" class="ref-issue">person/repo#4 (comment)</a>`) +} + +func TestIssue16020(t *testing.T) { + setting.AppURL = markup.TestAppURL + + localMetas := map[string]string{ + "user": "go-gitea", + "repo": "gitea", + } + + data := `<img src="data:image/png;base64,i//V"/>` + + var res strings.Builder + err := markup.PostProcess(&markup.RenderContext{ + Ctx: git.DefaultContext, + Metas: localMetas, + }, strings.NewReader(data), &res) + require.NoError(t, err) + assert.Equal(t, data, res.String()) +} + +func BenchmarkEmojiPostprocess(b *testing.B) { + data := "🥰 " + for len(data) < 1<<16 { + data += data + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + var res strings.Builder + err := markup.PostProcess(&markup.RenderContext{ + Ctx: git.DefaultContext, + Metas: localMetas, + }, strings.NewReader(data), &res) + require.NoError(b, err) + } +} + +func TestFuzz(t *testing.T) { + s := "t/l/issues/8#/../../a" + renderContext := markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: "https://example.com/go-gitea/gitea", + }, + Metas: map[string]string{ + "user": "go-gitea", + "repo": "gitea", + }, + } + + err := markup.PostProcess(&renderContext, strings.NewReader(s), io.Discard) + + require.NoError(t, err) +} + +func TestIssue18471(t *testing.T) { + data := `http://domain/org/repo/compare/783b039...da951ce` + + var res strings.Builder + err := markup.PostProcess(&markup.RenderContext{ + Ctx: git.DefaultContext, + Metas: localMetas, + }, strings.NewReader(data), &res) + + require.NoError(t, err) + assert.Equal(t, "<a href=\"http://domain/org/repo/compare/783b039...da951ce\" class=\"compare\"><code class=\"nohighlight\">783b039...da951ce</code></a>", res.String()) +} + +func TestRender_FilePreview(t *testing.T) { + defer test.MockVariableValue(&setting.StaticRootPath, "../../")() + defer test.MockVariableValue(&setting.Names, []string{"english"})() + defer test.MockVariableValue(&setting.Langs, []string{"en-US"})() + translation.InitLocales(context.Background()) + + setting.AppURL = markup.TestAppURL + markup.Init(&markup.ProcessorHelper{ + GetRepoFileBlob: func(ctx context.Context, ownerName, repoName, commitSha, filePath string, language *string) (*git.Blob, error) { + gitRepo, err := git.OpenRepository(git.DefaultContext, "./tests/repo/repo1_filepreview") + require.NoError(t, err) + defer gitRepo.Close() + + commit, err := gitRepo.GetCommit("HEAD") + require.NoError(t, err) + + blob, err := commit.GetBlobByPath("path/to/file.go") + require.NoError(t, err) + + return blob, nil + }, + }) + + sha := "190d9492934af498c3f669d6a2431dc5459e5b20" + commitFilePreview := util.URLJoin(markup.TestRepoURL, "src", "commit", sha, "path", "to", "file.go") + "#L2-L3" + + testRender := func(input, expected string, metas map[string]string) { + buffer, err := markup.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + RelativePath: ".md", + Metas: metas, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) + } + + t.Run("single", func(t *testing.T) { + testRender( + commitFilePreview, + `<p></p>`+ + `<div class="file-preview-box">`+ + `<div class="header">`+ + `<div>`+ + `<a href="http://localhost:3000/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20/path/to/file.go#L2-L3" class="muted" rel="nofollow">path/to/file.go</a>`+ + `</div>`+ + `<span class="text small grey">`+ + `Lines 2 to 3 in <a href="http://localhost:3000/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20" class="text black" rel="nofollow">190d949</a>`+ + `</span>`+ + `</div>`+ + `<div class="ui table">`+ + `<table class="file-preview">`+ + `<tbody>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="2"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">B</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="3"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">C</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `</tbody>`+ + `</table>`+ + `</div>`+ + `</div>`+ + `<p></p>`, + localMetas, + ) + }) + + t.Run("cross-repo", func(t *testing.T) { + testRender( + commitFilePreview, + `<p></p>`+ + `<div class="file-preview-box">`+ + `<div class="header">`+ + `<div>`+ + `<a href="http://localhost:3000/gogits/gogs/" rel="nofollow">gogits/gogs</a> – `+ + `<a href="http://localhost:3000/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20/path/to/file.go#L2-L3" class="muted" rel="nofollow">path/to/file.go</a>`+ + `</div>`+ + `<span class="text small grey">`+ + `Lines 2 to 3 in <a href="http://localhost:3000/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20" class="text black" rel="nofollow">gogits/gogs@190d949</a>`+ + `</span>`+ + `</div>`+ + `<div class="ui table">`+ + `<table class="file-preview">`+ + `<tbody>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="2"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">B</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="3"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">C</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `</tbody>`+ + `</table>`+ + `</div>`+ + `</div>`+ + `<p></p>`, + map[string]string{ + "user": "gogits", + "repo": "gogs2", + }, + ) + }) + + t.Run("AppSubURL", func(t *testing.T) { + urlWithSub := util.URLJoin(markup.TestAppURL, "sub", markup.TestOrgRepo, "src", "commit", sha, "path", "to", "file.go") + "#L2-L3" + + testRender( + urlWithSub, + `<p><a href="http://localhost:3000/sub/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20/path/to/file.go#L2-L3" rel="nofollow"><code>190d949293/path/to/file.go (L2-L3)</code></a></p>`, + localMetas, + ) + + defer test.MockVariableValue(&setting.AppURL, markup.TestAppURL+"sub/")() + defer test.MockVariableValue(&setting.AppSubURL, "/sub")() + + testRender( + urlWithSub, + `<p></p>`+ + `<div class="file-preview-box">`+ + `<div class="header">`+ + `<div>`+ + `<a href="http://localhost:3000/sub/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20/path/to/file.go#L2-L3" class="muted" rel="nofollow">path/to/file.go</a>`+ + `</div>`+ + `<span class="text small grey">`+ + `Lines 2 to 3 in <a href="http://localhost:3000/sub/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20" class="text black" rel="nofollow">190d949</a>`+ + `</span>`+ + `</div>`+ + `<div class="ui table">`+ + `<table class="file-preview">`+ + `<tbody>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="2"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">B</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="3"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">C</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `</tbody>`+ + `</table>`+ + `</div>`+ + `</div>`+ + `<p></p>`, + localMetas, + ) + + testRender( + "first without sub "+commitFilePreview+" second "+urlWithSub, + `<p>first without sub <a href="http://localhost:3000/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20/path/to/file.go#L2-L3" rel="nofollow"><code>190d949293/path/to/file.go (L2-L3)</code></a> second </p>`+ + `<div class="file-preview-box">`+ + `<div class="header">`+ + `<div>`+ + `<a href="http://localhost:3000/sub/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20/path/to/file.go#L2-L3" class="muted" rel="nofollow">path/to/file.go</a>`+ + `</div>`+ + `<span class="text small grey">`+ + `Lines 2 to 3 in <a href="http://localhost:3000/sub/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20" class="text black" rel="nofollow">190d949</a>`+ + `</span>`+ + `</div>`+ + `<div class="ui table">`+ + `<table class="file-preview">`+ + `<tbody>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="2"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">B</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="3"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">C</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `</tbody>`+ + `</table>`+ + `</div>`+ + `</div>`+ + `<p></p>`, + localMetas, + ) + }) + + t.Run("multiples", func(t *testing.T) { + testRender( + "first "+commitFilePreview+" second "+commitFilePreview, + `<p>first </p>`+ + `<div class="file-preview-box">`+ + `<div class="header">`+ + `<div>`+ + `<a href="http://localhost:3000/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20/path/to/file.go#L2-L3" class="muted" rel="nofollow">path/to/file.go</a>`+ + `</div>`+ + `<span class="text small grey">`+ + `Lines 2 to 3 in <a href="http://localhost:3000/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20" class="text black" rel="nofollow">190d949</a>`+ + `</span>`+ + `</div>`+ + `<div class="ui table">`+ + `<table class="file-preview">`+ + `<tbody>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="2"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">B</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="3"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">C</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `</tbody>`+ + `</table>`+ + `</div>`+ + `</div>`+ + `<p> second </p>`+ + `<div class="file-preview-box">`+ + `<div class="header">`+ + `<div>`+ + `<a href="http://localhost:3000/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20/path/to/file.go#L2-L3" class="muted" rel="nofollow">path/to/file.go</a>`+ + `</div>`+ + `<span class="text small grey">`+ + `Lines 2 to 3 in <a href="http://localhost:3000/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20" class="text black" rel="nofollow">190d949</a>`+ + `</span>`+ + `</div>`+ + `<div class="ui table">`+ + `<table class="file-preview">`+ + `<tbody>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="2"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">B</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="3"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">C</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `</tbody>`+ + `</table>`+ + `</div>`+ + `</div>`+ + `<p></p>`, + localMetas, + ) + + testRender( + "first "+commitFilePreview+" second "+commitFilePreview+" third "+commitFilePreview, + `<p>first </p>`+ + `<div class="file-preview-box">`+ + `<div class="header">`+ + `<div>`+ + `<a href="http://localhost:3000/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20/path/to/file.go#L2-L3" class="muted" rel="nofollow">path/to/file.go</a>`+ + `</div>`+ + `<span class="text small grey">`+ + `Lines 2 to 3 in <a href="http://localhost:3000/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20" class="text black" rel="nofollow">190d949</a>`+ + `</span>`+ + `</div>`+ + `<div class="ui table">`+ + `<table class="file-preview">`+ + `<tbody>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="2"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">B</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="3"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">C</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `</tbody>`+ + `</table>`+ + `</div>`+ + `</div>`+ + `<p> second </p>`+ + `<div class="file-preview-box">`+ + `<div class="header">`+ + `<div>`+ + `<a href="http://localhost:3000/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20/path/to/file.go#L2-L3" class="muted" rel="nofollow">path/to/file.go</a>`+ + `</div>`+ + `<span class="text small grey">`+ + `Lines 2 to 3 in <a href="http://localhost:3000/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20" class="text black" rel="nofollow">190d949</a>`+ + `</span>`+ + `</div>`+ + `<div class="ui table">`+ + `<table class="file-preview">`+ + `<tbody>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="2"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">B</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="3"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">C</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `</tbody>`+ + `</table>`+ + `</div>`+ + `</div>`+ + `<p> third </p>`+ + `<div class="file-preview-box">`+ + `<div class="header">`+ + `<div>`+ + `<a href="http://localhost:3000/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20/path/to/file.go#L2-L3" class="muted" rel="nofollow">path/to/file.go</a>`+ + `</div>`+ + `<span class="text small grey">`+ + `Lines 2 to 3 in <a href="http://localhost:3000/gogits/gogs/src/commit/190d9492934af498c3f669d6a2431dc5459e5b20" class="text black" rel="nofollow">190d949</a>`+ + `</span>`+ + `</div>`+ + `<div class="ui table">`+ + `<table class="file-preview">`+ + `<tbody>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="2"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">B</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `<tr>`+ + `<td class="lines-num"><span data-line-number="3"></span></td>`+ + `<td class="lines-code chroma"><code class="code-inner"><span class="nx">C</span>`+"\n"+`</code></td>`+ + `</tr>`+ + `</tbody>`+ + `</table>`+ + `</div>`+ + `</div>`+ + `<p></p>`, + localMetas, + ) + }) +} diff --git a/modules/markup/markdown/ast.go b/modules/markup/markdown/ast.go new file mode 100644 index 00000000..7f0ac6a9 --- /dev/null +++ b/modules/markup/markdown/ast.go @@ -0,0 +1,176 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "strconv" + + "github.com/yuin/goldmark/ast" +) + +// Details is a block that contains Summary and details +type Details struct { + ast.BaseBlock +} + +// Dump implements Node.Dump . +func (n *Details) Dump(source []byte, level int) { + ast.DumpHelper(n, source, level, nil, nil) +} + +// KindDetails is the NodeKind for Details +var KindDetails = ast.NewNodeKind("Details") + +// Kind implements Node.Kind. +func (n *Details) Kind() ast.NodeKind { + return KindDetails +} + +// NewDetails returns a new Paragraph node. +func NewDetails() *Details { + return &Details{ + BaseBlock: ast.BaseBlock{}, + } +} + +// IsDetails returns true if the given node implements the Details interface, +// otherwise false. +func IsDetails(node ast.Node) bool { + _, ok := node.(*Details) + return ok +} + +// Summary is a block that contains the summary of details block +type Summary struct { + ast.BaseBlock +} + +// Dump implements Node.Dump . +func (n *Summary) Dump(source []byte, level int) { + ast.DumpHelper(n, source, level, nil, nil) +} + +// KindSummary is the NodeKind for Summary +var KindSummary = ast.NewNodeKind("Summary") + +// Kind implements Node.Kind. +func (n *Summary) Kind() ast.NodeKind { + return KindSummary +} + +// NewSummary returns a new Summary node. +func NewSummary() *Summary { + return &Summary{ + BaseBlock: ast.BaseBlock{}, + } +} + +// IsSummary returns true if the given node implements the Summary interface, +// otherwise false. +func IsSummary(node ast.Node) bool { + _, ok := node.(*Summary) + return ok +} + +// TaskCheckBoxListItem is a block that represents a list item of a markdown block with a checkbox +type TaskCheckBoxListItem struct { + *ast.ListItem + IsChecked bool + SourcePosition int +} + +// KindTaskCheckBoxListItem is the NodeKind for TaskCheckBoxListItem +var KindTaskCheckBoxListItem = ast.NewNodeKind("TaskCheckBoxListItem") + +// Dump implements Node.Dump . +func (n *TaskCheckBoxListItem) Dump(source []byte, level int) { + m := map[string]string{} + m["IsChecked"] = strconv.FormatBool(n.IsChecked) + m["SourcePosition"] = strconv.FormatInt(int64(n.SourcePosition), 10) + ast.DumpHelper(n, source, level, m, nil) +} + +// Kind implements Node.Kind. +func (n *TaskCheckBoxListItem) Kind() ast.NodeKind { + return KindTaskCheckBoxListItem +} + +// NewTaskCheckBoxListItem returns a new TaskCheckBoxListItem node. +func NewTaskCheckBoxListItem(listItem *ast.ListItem) *TaskCheckBoxListItem { + return &TaskCheckBoxListItem{ + ListItem: listItem, + } +} + +// IsTaskCheckBoxListItem returns true if the given node implements the TaskCheckBoxListItem interface, +// otherwise false. +func IsTaskCheckBoxListItem(node ast.Node) bool { + _, ok := node.(*TaskCheckBoxListItem) + return ok +} + +// Icon is an inline for a fomantic icon +type Icon struct { + ast.BaseInline + Name []byte +} + +// Dump implements Node.Dump . +func (n *Icon) Dump(source []byte, level int) { + m := map[string]string{} + m["Name"] = string(n.Name) + ast.DumpHelper(n, source, level, m, nil) +} + +// KindIcon is the NodeKind for Icon +var KindIcon = ast.NewNodeKind("Icon") + +// Kind implements Node.Kind. +func (n *Icon) Kind() ast.NodeKind { + return KindIcon +} + +// NewIcon returns a new Paragraph node. +func NewIcon(name string) *Icon { + return &Icon{ + BaseInline: ast.BaseInline{}, + Name: []byte(name), + } +} + +// IsIcon returns true if the given node implements the Icon interface, +// otherwise false. +func IsIcon(node ast.Node) bool { + _, ok := node.(*Icon) + return ok +} + +// ColorPreview is an inline for a color preview +type ColorPreview struct { + ast.BaseInline + Color []byte +} + +// Dump implements Node.Dump. +func (n *ColorPreview) Dump(source []byte, level int) { + m := map[string]string{} + m["Color"] = string(n.Color) + ast.DumpHelper(n, source, level, m, nil) +} + +// KindColorPreview is the NodeKind for ColorPreview +var KindColorPreview = ast.NewNodeKind("ColorPreview") + +// Kind implements Node.Kind. +func (n *ColorPreview) Kind() ast.NodeKind { + return KindColorPreview +} + +// NewColorPreview returns a new Span node. +func NewColorPreview(color []byte) *ColorPreview { + return &ColorPreview{ + BaseInline: ast.BaseInline{}, + Color: color, + } +} diff --git a/modules/markup/markdown/callout/ast.go b/modules/markup/markdown/callout/ast.go new file mode 100644 index 00000000..a5b1bbc2 --- /dev/null +++ b/modules/markup/markdown/callout/ast.go @@ -0,0 +1,37 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package callout + +import ( + "github.com/yuin/goldmark/ast" +) + +// Attention is an inline for an attention +type Attention struct { + ast.BaseInline + AttentionType string +} + +// Dump implements Node.Dump. +func (n *Attention) Dump(source []byte, level int) { + m := map[string]string{} + m["AttentionType"] = n.AttentionType + ast.DumpHelper(n, source, level, m, nil) +} + +// KindAttention is the NodeKind for Attention +var KindAttention = ast.NewNodeKind("Attention") + +// Kind implements Node.Kind. +func (n *Attention) Kind() ast.NodeKind { + return KindAttention +} + +// NewAttention returns a new Attention node. +func NewAttention(attentionType string) *Attention { + return &Attention{ + BaseInline: ast.BaseInline{}, + AttentionType: attentionType, + } +} diff --git a/modules/markup/markdown/callout/github.go b/modules/markup/markdown/callout/github.go new file mode 100644 index 00000000..debad42b --- /dev/null +++ b/modules/markup/markdown/callout/github.go @@ -0,0 +1,142 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2024 The Forgejo Authors c/o Codeberg e.V.. All rights reserved. +// SPDX-License-Identifier: MIT + +package callout + +import ( + "strings" + + "code.gitea.io/gitea/modules/svg" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +type GitHubCalloutTransformer struct{} + +// Transform transforms the given AST tree. +func (g *GitHubCalloutTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) { + supportedAttentionTypes := map[string]bool{ + "note": true, + "tip": true, + "important": true, + "warning": true, + "caution": true, + } + + _ = ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + + switch v := n.(type) { + case *ast.Blockquote: + if v.ChildCount() == 0 { + return ast.WalkContinue, nil + } + + // We only want attention blockquotes when the AST looks like: + // Text: "[" + // Text: "!TYPE" + // Text(SoftLineBreak): "]" + + // grab these nodes and make sure we adhere to the attention blockquote structure + firstParagraph := v.FirstChild() + if firstParagraph.ChildCount() < 3 { + return ast.WalkContinue, nil + } + firstTextNode, ok := firstParagraph.FirstChild().(*ast.Text) + if !ok || string(firstTextNode.Text(reader.Source())) != "[" { + return ast.WalkContinue, nil + } + secondTextNode, ok := firstTextNode.NextSibling().(*ast.Text) + if !ok { + return ast.WalkContinue, nil + } + // If the second node's text isn't one of the supported attention + // types, continue walking. + secondTextNodeText := secondTextNode.Text(reader.Source()) + attentionType := strings.ToLower(strings.TrimPrefix(string(secondTextNodeText), "!")) + if _, has := supportedAttentionTypes[attentionType]; !has { + return ast.WalkContinue, nil + } + + thirdTextNode, ok := secondTextNode.NextSibling().(*ast.Text) + if !ok || string(thirdTextNode.Text(reader.Source())) != "]" { + return ast.WalkContinue, nil + } + + // color the blockquote + v.SetAttributeString("class", []byte("attention-header attention-"+attentionType)) + + // create an emphasis to make it bold + attentionParagraph := ast.NewParagraph() + attentionParagraph.SetAttributeString("class", []byte("attention-title")) + emphasis := ast.NewEmphasis(2) + emphasis.SetAttributeString("class", []byte("attention-"+attentionType)) + firstParagraph.InsertBefore(firstParagraph, firstTextNode, emphasis) + + // capitalize first letter + attentionText := ast.NewString([]byte(strings.ToUpper(string(attentionType[0])) + attentionType[1:])) + + // replace the ![TYPE] with a dedicated paragraph of icon+Type + emphasis.AppendChild(emphasis, attentionText) + attentionParagraph.AppendChild(attentionParagraph, NewAttention(attentionType)) + attentionParagraph.AppendChild(attentionParagraph, emphasis) + firstParagraph.Parent().InsertBefore(firstParagraph.Parent(), firstParagraph, attentionParagraph) + firstParagraph.RemoveChild(firstParagraph, firstTextNode) + firstParagraph.RemoveChild(firstParagraph, secondTextNode) + firstParagraph.RemoveChild(firstParagraph, thirdTextNode) + } + return ast.WalkContinue, nil + }) +} + +type GitHubCalloutHTMLRenderer struct { + html.Config +} + +// RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs. +func (r *GitHubCalloutHTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(KindAttention, r.renderAttention) +} + +// renderAttention renders a quote marked with i.e. "> **Note**" or "> **Warning**" with a corresponding svg +func (r *GitHubCalloutHTMLRenderer) renderAttention(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + n := node.(*Attention) + + var octiconName string + switch n.AttentionType { + case "note": + octiconName = "info" + case "tip": + octiconName = "light-bulb" + case "important": + octiconName = "report" + case "warning": + octiconName = "alert" + case "caution": + octiconName = "stop" + default: + octiconName = "info" + } + _, _ = w.WriteString(string(svg.RenderHTML("octicon-"+octiconName, 16, "attention-icon attention-"+n.AttentionType))) + } + return ast.WalkContinue, nil +} + +func NewGitHubCalloutHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { + r := &GitHubCalloutHTMLRenderer{ + Config: html.NewConfig(), + } + for _, opt := range opts { + opt.SetHTMLOption(&r.Config) + } + return r +} diff --git a/modules/markup/markdown/callout/github_legacy.go b/modules/markup/markdown/callout/github_legacy.go new file mode 100644 index 00000000..eb15e1e6 --- /dev/null +++ b/modules/markup/markdown/callout/github_legacy.go @@ -0,0 +1,71 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2024 The Forgejo Authors c/o Codeberg e.V.. All rights reserved. +// SPDX-License-Identifier: MIT + +package callout + +import ( + "strings" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/text" +) + +// Transformer for GitHub's legacy callout markup. +type GitHubLegacyCalloutTransformer struct{} + +func (g *GitHubLegacyCalloutTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) { + supportedCalloutTypes := map[string]bool{"Note": true, "Warning": true} + + _ = ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + + switch v := n.(type) { + case *ast.Blockquote: + if v.ChildCount() == 0 { + return ast.WalkContinue, nil + } + + // The first paragraph contains the callout type. + firstParagraph := v.FirstChild() + if firstParagraph.ChildCount() < 1 { + return ast.WalkContinue, nil + } + + // In the legacy GitHub callout markup, the first node of the first + // paragraph should be an emphasis. + calloutNode, ok := firstParagraph.FirstChild().(*ast.Emphasis) + if !ok { + return ast.WalkContinue, nil + } + calloutText := string(calloutNode.Text(reader.Source())) + calloutType := strings.ToLower(calloutText) + // We only support "Note" and "Warning" callouts in legacy mode, + // match only those. + if _, has := supportedCalloutTypes[calloutText]; !has { + return ast.WalkContinue, nil + } + + // Set the attention attribute on the emphasis + calloutNode.SetAttributeString("class", []byte("attention-"+calloutType)) + + // color the blockquote + v.SetAttributeString("class", []byte("attention-header attention-"+calloutType)) + + // Create new paragraph. + attentionParagraph := ast.NewParagraph() + attentionParagraph.SetAttributeString("class", []byte("attention-title")) + + // Move the callout node to the paragraph and insert the paragraph. + attentionParagraph.AppendChild(attentionParagraph, NewAttention(calloutType)) + attentionParagraph.AppendChild(attentionParagraph, calloutNode) + firstParagraph.Parent().InsertBefore(firstParagraph.Parent(), firstParagraph, attentionParagraph) + firstParagraph.RemoveChild(firstParagraph, calloutNode) + } + + return ast.WalkContinue, nil + }) +} diff --git a/modules/markup/markdown/color_util.go b/modules/markup/markdown/color_util.go new file mode 100644 index 00000000..355fef3f --- /dev/null +++ b/modules/markup/markdown/color_util.go @@ -0,0 +1,19 @@ +// Copyright 2024 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import "regexp" + +var ( + hexRGB = regexp.MustCompile(`^#([0-9a-f]{3}|[0-9a-f]{6}|[0-9a-f]{8})$`) + hsl = regexp.MustCompile(`^hsl\([ ]*([012]?[0-9]{1,2}|3[0-5][0-9]|360),[ ]*([0-9]{0,2}|100)\%,[ ]*([0-9]{0,2}|100)\%\)$`) + hsla = regexp.MustCompile(`^hsla\(([ ]*[012]?[0-9]{1,2}|3[0-5][0-9]|360),[ ]*([0-9]{0,2}|100)\%,[ ]*([0-9]{0,2}|100)\%,[ ]*(1|1\.0|0|(0\.[0-9]+))\)$`) + rgb = regexp.MustCompile(`^rgb\(([ ]*((([0-9]{1,2}|100)\%)|(([01]?[0-9]{1,2})|(2[0-4][0-9])|(25[0-5]))),){2}([ ]*((([0-9]{1,2}|100)\%)|(([01]?[0-9]{1,2})|(2[0-4][0-9])|(25[0-5]))))\)$`) + rgba = regexp.MustCompile(`^rgba\(([ ]*((([0-9]{1,2}|100)\%)|(([01]?[0-9]{1,2})|(2[0-4][0-9])|(25[0-5]))),){3}[ ]*(1(\.0)?|0|(0\.[0-9]+))\)$`) +) + +// matchColor return if color is in the form of hex RGB, HSL(A) or RGB(A). +func matchColor(color string) bool { + return hexRGB.MatchString(color) || rgb.MatchString(color) || rgba.MatchString(color) || hsl.MatchString(color) || hsla.MatchString(color) +} diff --git a/modules/markup/markdown/color_util_test.go b/modules/markup/markdown/color_util_test.go new file mode 100644 index 00000000..c6e0555a --- /dev/null +++ b/modules/markup/markdown/color_util_test.go @@ -0,0 +1,50 @@ +// Copyright 2024 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMatchColor(t *testing.T) { + testCases := []struct { + input string + expected bool + }{ + {"#ddeeffa0", true}, + {"#ddeefe", true}, + {"#abcdef", true}, + {"#abcdeg", false}, + {"#abcdefg0", false}, + {"black", false}, + {"violet", false}, + {"rgb(255, 255, 255)", true}, + {"rgb(0, 0, 0)", true}, + {"rgb(256, 0, 0)", false}, + {"rgb(0, 256, 0)", false}, + {"rgb(0, 0, 256)", false}, + {"rgb(0, 0, 0, 1)", false}, + {"rgba(0, 0, 0)", false}, + {"rgba(0, 255, 0, 1)", true}, + {"rgba(32, 255, 12, 0.55)", true}, + {"rgba(32, 256, 12, 0.55)", false}, + {"hsl(0, 0%, 0%)", true}, + {"hsl(360, 100%, 100%)", true}, + {"hsl(361, 100%, 50%)", false}, + {"hsl(360, 101%, 50%)", false}, + {"hsl(360, 100%, 101%)", false}, + {"hsl(0, 0%, 0%, 0)", false}, + {"hsla(0, 0%, 0%)", false}, + {"hsla(0, 0%, 0%, 0)", true}, + {"hsla(0, 0%, 0%, 1)", true}, + {"hsla(0, 0%, 0%, 0.5)", true}, + {"hsla(0, 0%, 0%, 1.5)", false}, + } + for _, testCase := range testCases { + actual := matchColor(testCase.input) + assert.Equal(t, testCase.expected, actual) + } +} diff --git a/modules/markup/markdown/convertyaml.go b/modules/markup/markdown/convertyaml.go new file mode 100644 index 00000000..1675b68b --- /dev/null +++ b/modules/markup/markdown/convertyaml.go @@ -0,0 +1,83 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "github.com/yuin/goldmark/ast" + east "github.com/yuin/goldmark/extension/ast" + "gopkg.in/yaml.v3" +) + +func nodeToTable(meta *yaml.Node) ast.Node { + for { + if meta == nil { + return nil + } + switch meta.Kind { + case yaml.DocumentNode: + meta = meta.Content[0] + continue + default: + } + break + } + switch meta.Kind { + case yaml.MappingNode: + return mappingNodeToTable(meta) + case yaml.SequenceNode: + return sequenceNodeToTable(meta) + default: + return ast.NewString([]byte(meta.Value)) + } +} + +func mappingNodeToTable(meta *yaml.Node) ast.Node { + table := east.NewTable() + alignments := make([]east.Alignment, 0, len(meta.Content)/2) + for i := 0; i < len(meta.Content); i += 2 { + alignments = append(alignments, east.AlignNone) + } + + headerRow := east.NewTableRow(alignments) + valueRow := east.NewTableRow(alignments) + for i := 0; i < len(meta.Content); i += 2 { + cell := east.NewTableCell() + + cell.AppendChild(cell, nodeToTable(meta.Content[i])) + headerRow.AppendChild(headerRow, cell) + + if i+1 < len(meta.Content) { + cell = east.NewTableCell() + cell.AppendChild(cell, nodeToTable(meta.Content[i+1])) + valueRow.AppendChild(valueRow, cell) + } + } + + table.AppendChild(table, east.NewTableHeader(headerRow)) + table.AppendChild(table, valueRow) + return table +} + +func sequenceNodeToTable(meta *yaml.Node) ast.Node { + table := east.NewTable() + alignments := []east.Alignment{east.AlignNone} + for _, item := range meta.Content { + row := east.NewTableRow(alignments) + cell := east.NewTableCell() + cell.AppendChild(cell, nodeToTable(item)) + row.AppendChild(row, cell) + table.AppendChild(table, row) + } + return table +} + +func nodeToDetails(meta *yaml.Node, icon string) ast.Node { + details := NewDetails() + summary := NewSummary() + summary.AppendChild(summary, NewIcon(icon)) + details.AppendChild(details, summary) + details.AppendChild(details, nodeToTable(meta)) + + return details +} diff --git a/modules/markup/markdown/goldmark.go b/modules/markup/markdown/goldmark.go new file mode 100644 index 00000000..0290e131 --- /dev/null +++ b/modules/markup/markdown/goldmark.go @@ -0,0 +1,213 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "fmt" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/setting" + + "github.com/yuin/goldmark/ast" + east "github.com/yuin/goldmark/extension/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +var byteMailto = []byte("mailto:") + +// ASTTransformer is a default transformer of the goldmark tree. +type ASTTransformer struct{} + +func (g *ASTTransformer) applyElementDir(n ast.Node) { + if markup.DefaultProcessorHelper.ElementDir != "" { + n.SetAttributeString("dir", []byte(markup.DefaultProcessorHelper.ElementDir)) + } +} + +// Transform transforms the given AST tree. +func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) { + firstChild := node.FirstChild() + tocMode := "" + ctx := pc.Get(renderContextKey).(*markup.RenderContext) + rc := pc.Get(renderConfigKey).(*RenderConfig) + + tocList := make([]markup.Header, 0, 20) + if rc.yamlNode != nil { + metaNode := rc.toMetaNode() + if metaNode != nil { + node.InsertBefore(node, firstChild, metaNode) + } + tocMode = rc.TOC + } + + _ = ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + + switch v := n.(type) { + case *ast.Heading: + g.transformHeading(ctx, v, reader, &tocList) + case *ast.Paragraph: + g.applyElementDir(v) + case *ast.Image: + g.transformImage(ctx, v) + case *ast.Link: + g.transformLink(ctx, v) + case *ast.List: + g.transformList(ctx, v, rc) + case *ast.Text: + if v.SoftLineBreak() && !v.HardLineBreak() { + if ctx.Metas["mode"] != "document" { + v.SetHardLineBreak(setting.Markdown.EnableHardLineBreakInComments) + } else { + v.SetHardLineBreak(setting.Markdown.EnableHardLineBreakInDocuments) + } + } + case *ast.CodeSpan: + g.transformCodeSpan(ctx, v, reader) + } + return ast.WalkContinue, nil + }) + + showTocInMain := tocMode == "true" /* old behavior, in main view */ || tocMode == "main" + showTocInSidebar := !showTocInMain && tocMode != "false" // not hidden, not main, then show it in sidebar + if len(tocList) > 0 && (showTocInMain || showTocInSidebar) { + if showTocInMain { + tocNode := createTOCNode(tocList, rc.Lang, nil) + node.InsertBefore(node, firstChild, tocNode) + } else { + tocNode := createTOCNode(tocList, rc.Lang, map[string]string{"open": "open"}) + ctx.SidebarTocNode = tocNode + } + } + + if len(rc.Lang) > 0 { + node.SetAttributeString("lang", []byte(rc.Lang)) + } +} + +// NewHTMLRenderer creates a HTMLRenderer to render +// in the gitea form. +func NewHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { + r := &HTMLRenderer{ + Config: html.NewConfig(), + } + for _, opt := range opts { + opt.SetHTMLOption(&r.Config) + } + return r +} + +// HTMLRenderer is a renderer.NodeRenderer implementation that +// renders gitea specific features. +type HTMLRenderer struct { + html.Config +} + +// RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs. +func (r *HTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(ast.KindDocument, r.renderDocument) + reg.Register(KindDetails, r.renderDetails) + reg.Register(KindSummary, r.renderSummary) + reg.Register(KindIcon, r.renderIcon) + reg.Register(ast.KindCodeSpan, r.renderCodeSpan) + reg.Register(KindTaskCheckBoxListItem, r.renderTaskCheckBoxListItem) + reg.Register(east.KindTaskCheckBox, r.renderTaskCheckBox) +} + +func (r *HTMLRenderer) renderDocument(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.Document) + + if val, has := n.AttributeString("lang"); has { + var err error + if entering { + _, err = w.WriteString("<div") + if err == nil { + _, err = w.WriteString(fmt.Sprintf(` lang=%q`, val)) + } + if err == nil { + _, err = w.WriteRune('>') + } + } else { + _, err = w.WriteString("</div>") + } + + if err != nil { + return ast.WalkStop, err + } + } + + return ast.WalkContinue, nil +} + +func (r *HTMLRenderer) renderDetails(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + var err error + if entering { + if _, err = w.WriteString("<details"); err != nil { + return ast.WalkStop, err + } + html.RenderAttributes(w, node, nil) + _, err = w.WriteString(">") + } else { + _, err = w.WriteString("</details>") + } + + if err != nil { + return ast.WalkStop, err + } + + return ast.WalkContinue, nil +} + +func (r *HTMLRenderer) renderSummary(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + var err error + if entering { + _, err = w.WriteString("<summary>") + } else { + _, err = w.WriteString("</summary>") + } + + if err != nil { + return ast.WalkStop, err + } + + return ast.WalkContinue, nil +} + +var validNameRE = regexp.MustCompile("^[a-z ]+$") + +func (r *HTMLRenderer) renderIcon(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + + n := node.(*Icon) + + name := strings.TrimSpace(strings.ToLower(string(n.Name))) + + if len(name) == 0 { + // skip this + return ast.WalkContinue, nil + } + + if !validNameRE.MatchString(name) { + // skip this + return ast.WalkContinue, nil + } + + var err error + _, err = w.WriteString(fmt.Sprintf(`<i class="icon %s"></i>`, name)) + if err != nil { + return ast.WalkStop, err + } + + return ast.WalkContinue, nil +} diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go new file mode 100644 index 00000000..77c876df --- /dev/null +++ b/modules/markup/markdown/markdown.go @@ -0,0 +1,305 @@ +// Copyright 2014 The Gogs Authors. All rights reserved. +// Copyright 2018 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "fmt" + "html/template" + "io" + "strings" + "sync" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/markup/common" + "code.gitea.io/gitea/modules/markup/markdown/callout" + "code.gitea.io/gitea/modules/markup/markdown/math" + "code.gitea.io/gitea/modules/setting" + giteautil "code.gitea.io/gitea/modules/util" + + chromahtml "github.com/alecthomas/chroma/v2/formatters/html" + "github.com/yuin/goldmark" + highlighting "github.com/yuin/goldmark-highlighting/v2" + meta "github.com/yuin/goldmark-meta" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/util" +) + +var ( + specMarkdown goldmark.Markdown + specMarkdownOnce sync.Once +) + +var ( + renderContextKey = parser.NewContextKey() + renderConfigKey = parser.NewContextKey() +) + +type limitWriter struct { + w io.Writer + sum int64 + limit int64 +} + +// Write implements the standard Write interface: +func (l *limitWriter) Write(data []byte) (int, error) { + leftToWrite := l.limit - l.sum + if leftToWrite < int64(len(data)) { + n, err := l.w.Write(data[:leftToWrite]) + l.sum += int64(n) + if err != nil { + return n, err + } + return n, fmt.Errorf("rendered content too large - truncating render") + } + n, err := l.w.Write(data) + l.sum += int64(n) + return n, err +} + +// newParserContext creates a parser.Context with the render context set +func newParserContext(ctx *markup.RenderContext) parser.Context { + pc := parser.NewContext(parser.WithIDs(newPrefixedIDs())) + pc.Set(renderContextKey, ctx) + return pc +} + +// SpecializedMarkdown sets up the Gitea specific markdown extensions +func SpecializedMarkdown() goldmark.Markdown { + specMarkdownOnce.Do(func() { + specMarkdown = goldmark.New( + goldmark.WithExtensions( + extension.NewTable( + extension.WithTableCellAlignMethod(extension.TableCellAlignAttribute)), + extension.Strikethrough, + extension.TaskList, + extension.DefinitionList, + common.FootnoteExtension, + highlighting.NewHighlighting( + highlighting.WithFormatOptions( + chromahtml.WithClasses(true), + chromahtml.PreventSurroundingPre(true), + ), + highlighting.WithWrapperRenderer(func(w util.BufWriter, c highlighting.CodeBlockContext, entering bool) { + if entering { + language, _ := c.Language() + if language == nil { + language = []byte("text") + } + + languageStr := string(language) + + preClasses := []string{"code-block"} + if languageStr == "mermaid" || languageStr == "math" { + preClasses = append(preClasses, "is-loading") + } + + _, err := w.WriteString(`<pre class="` + strings.Join(preClasses, " ") + `">`) + if err != nil { + return + } + + // include language-x class as part of commonmark spec + // the "display" class is used by "js/markup/math.js" to render the code element as a block + _, err = w.WriteString(`<code class="chroma language-` + string(language) + ` display">`) + if err != nil { + return + } + } else { + _, err := w.WriteString("</code></pre>") + if err != nil { + return + } + } + }), + ), + math.NewExtension( + math.Enabled(setting.Markdown.EnableMath), + ), + meta.Meta, + ), + goldmark.WithParserOptions( + parser.WithAttribute(), + parser.WithAutoHeadingID(), + parser.WithASTTransformers( + util.Prioritized(&callout.GitHubLegacyCalloutTransformer{}, 8000), + util.Prioritized(&callout.GitHubCalloutTransformer{}, 9000), + util.Prioritized(&ASTTransformer{}, 10000), + ), + ), + goldmark.WithRendererOptions( + html.WithUnsafe(), + ), + ) + + // Override the original Tasklist renderer! + specMarkdown.Renderer().AddOptions( + renderer.WithNodeRenderers( + util.Prioritized(callout.NewGitHubCalloutHTMLRenderer(), 10), + util.Prioritized(NewHTMLRenderer(), 10), + ), + ) + }) + return specMarkdown +} + +// actualRender renders Markdown to HTML without handling special links. +func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + converter := SpecializedMarkdown() + lw := &limitWriter{ + w: output, + limit: setting.UI.MaxDisplayFileSize * 3, + } + + // FIXME: should we include a timeout to abort the renderer if it takes too long? + defer func() { + err := recover() + if err == nil { + return + } + + log.Warn("Unable to render markdown due to panic in goldmark: %v", err) + if log.IsDebug() { + log.Debug("Panic in markdown: %v\n%s", err, log.Stack(2)) + } + }() + + // FIXME: Don't read all to memory, but goldmark doesn't support + pc := newParserContext(ctx) + buf, err := io.ReadAll(input) + if err != nil { + log.Error("Unable to ReadAll: %v", err) + return err + } + buf = giteautil.NormalizeEOL(buf) + + // Preserve original length. + bufWithMetadataLength := len(buf) + + rc := &RenderConfig{ + Meta: renderMetaModeFromString(string(ctx.RenderMetaAs)), + Icon: "table", + Lang: "", + } + buf, _ = ExtractMetadataBytes(buf, rc) + + metaLength := bufWithMetadataLength - len(buf) + if metaLength < 0 { + metaLength = 0 + } + rc.metaLength = metaLength + + pc.Set(renderConfigKey, rc) + + if err := converter.Convert(buf, lw, parser.WithContext(pc)); err != nil { + log.Error("Unable to render: %v", err) + return err + } + + return nil +} + +// Note: The output of this method must get sanitized. +func render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + defer func() { + err := recover() + if err == nil { + return + } + + log.Warn("Unable to render markdown due to panic in goldmark - will return raw bytes") + if log.IsDebug() { + log.Debug("Panic in markdown: %v\n%s", err, log.Stack(2)) + } + _, err = io.Copy(output, input) + if err != nil { + log.Error("io.Copy failed: %v", err) + } + }() + return actualRender(ctx, input, output) +} + +// MarkupName describes markup's name +var MarkupName = "markdown" + +func init() { + markup.RegisterRenderer(Renderer{}) +} + +// Renderer implements markup.Renderer +type Renderer struct{} + +var _ markup.PostProcessRenderer = (*Renderer)(nil) + +// Name implements markup.Renderer +func (Renderer) Name() string { + return MarkupName +} + +// NeedPostProcess implements markup.PostProcessRenderer +func (Renderer) NeedPostProcess() bool { return true } + +// Extensions implements markup.Renderer +func (Renderer) Extensions() []string { + return setting.Markdown.FileExtensions +} + +// SanitizerRules implements markup.Renderer +func (Renderer) SanitizerRules() []setting.MarkupSanitizerRule { + return []setting.MarkupSanitizerRule{} +} + +// Render implements markup.Renderer +func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + return render(ctx, input, output) +} + +// Render renders Markdown to HTML with all specific handling stuff. +func Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + if ctx.Type == "" { + ctx.Type = MarkupName + } + return markup.Render(ctx, input, output) +} + +// RenderString renders Markdown string to HTML with all specific handling stuff and return string +func RenderString(ctx *markup.RenderContext, content string) (template.HTML, error) { + var buf strings.Builder + if err := Render(ctx, strings.NewReader(content), &buf); err != nil { + return "", err + } + return template.HTML(buf.String()), nil +} + +// RenderRaw renders Markdown to HTML without handling special links. +func RenderRaw(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + rd, wr := io.Pipe() + defer func() { + _ = rd.Close() + _ = wr.Close() + }() + + go func() { + if err := render(ctx, input, wr); err != nil { + _ = wr.CloseWithError(err) + return + } + _ = wr.Close() + }() + + return markup.SanitizeReader(rd, "", output) +} + +// RenderRawString renders Markdown to HTML without handling special links and return string +func RenderRawString(ctx *markup.RenderContext, content string) (string, error) { + var buf strings.Builder + if err := RenderRaw(ctx, strings.NewReader(content), &buf); err != nil { + return "", err + } + return buf.String(), nil +} diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go new file mode 100644 index 00000000..32426e0d --- /dev/null +++ b/modules/markup/markdown/markdown_test.go @@ -0,0 +1,1227 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown_test + +import ( + "context" + "html/template" + "os" + "strings" + "testing" + + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/markup/markdown" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/util" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + AppURL = "http://localhost:3000/" + FullURL = AppURL + "gogits/gogs/" +) + +// these values should match the const above +var localMetas = map[string]string{ + "user": "gogits", + "repo": "gogs", + "repoPath": "../../../tests/gitea-repositories-meta/user13/repo11.git/", +} + +func TestMain(m *testing.M) { + unittest.InitSettings() + if err := git.InitSimple(context.Background()); err != nil { + log.Fatal("git init failed, err: %v", err) + } + markup.Init(&markup.ProcessorHelper{ + IsUsernameMentionable: func(ctx context.Context, username string) bool { + return username == "r-lyeh" + }, + }) + os.Exit(m.Run()) +} + +func TestRender_StandardLinks(t *testing.T) { + setting.AppURL = AppURL + + test := func(input, expected, expectedWiki string) { + buffer, err := markdown.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: FullURL, + }, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) + + buffer, err = markdown.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: FullURL, + }, + IsWiki: true, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(string(buffer))) + } + + googleRendered := `<p><a href="https://google.com/" rel="nofollow">https://google.com/</a></p>` + test("<https://google.com/>", googleRendered, googleRendered) + + lnk := util.URLJoin(FullURL, "WikiPage") + lnkWiki := util.URLJoin(FullURL, "wiki", "WikiPage") + test("[WikiPage](WikiPage)", + `<p><a href="`+lnk+`" rel="nofollow">WikiPage</a></p>`, + `<p><a href="`+lnkWiki+`" rel="nofollow">WikiPage</a></p>`) +} + +func TestRender_Images(t *testing.T) { + setting.AppURL = AppURL + + test := func(input, expected string) { + buffer, err := markdown.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: FullURL, + }, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) + } + + url := "../../.images/src/02/train.jpg" + title := "Train" + href := "https://gitea.io" + result := util.URLJoin(FullURL, url) + // hint: With Markdown v2.5.2, there is a new syntax: [link](URL){:target="_blank"} , but we do not support it now + + test( + "!["+title+"]("+url+")", + `<p><a href="`+result+`" target="_blank" rel="nofollow noopener"><img src="`+result+`" alt="`+title+`"/></a></p>`) + + test( + "[["+title+"|"+url+"]]", + `<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" title="`+title+`" alt="`+title+`"/></a></p>`) + test( + "[!["+title+"]("+url+")]("+href+")", + `<p><a href="`+href+`" rel="nofollow"><img src="`+result+`" alt="`+title+`"/></a></p>`) + + test( + "!["+title+"]("+url+")", + `<p><a href="`+result+`" target="_blank" rel="nofollow noopener"><img src="`+result+`" alt="`+title+`"/></a></p>`) + + test( + "[["+title+"|"+url+"]]", + `<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" title="`+title+`" alt="`+title+`"/></a></p>`) + test( + "[!["+title+"]("+url+")]("+href+")", + `<p><a href="`+href+`" rel="nofollow"><img src="`+result+`" alt="`+title+`"/></a></p>`) +} + +func testAnswers(baseURLContent, baseURLImages string) []string { + return []string{ + `<p>Wiki! Enjoy :)</p> +<ul> +<li><a href="` + baseURLContent + `/Links" rel="nofollow">Links, Language bindings, Engine bindings</a></li> +<li><a href="` + baseURLContent + `/Tips" rel="nofollow">Tips</a></li> +</ul> +<p>See commit <a href="/gogits/gogs/commit/65f1bf27bc" rel="nofollow"><code>65f1bf27bc</code></a></p> +<p>Ideas and codes</p> +<ul> +<li>Bezier widget (by <a href="/r-lyeh" rel="nofollow">@r-lyeh</a>) <a href="http://localhost:3000/ocornut/imgui/issues/786" class="ref-issue" rel="nofollow">ocornut/imgui#786</a></li> +<li>Bezier widget (by <a href="/r-lyeh" rel="nofollow">@r-lyeh</a>) <a href="http://localhost:3000/gogits/gogs/issues/786" class="ref-issue" rel="nofollow">#786</a></li> +<li>Node graph editors <a href="https://github.com/ocornut/imgui/issues/306" rel="nofollow">https://github.com/ocornut/imgui/issues/306</a></li> +<li><a href="` + baseURLContent + `/memory_editor_example" rel="nofollow">Memory Editor</a></li> +<li><a href="` + baseURLContent + `/plot_var_example" rel="nofollow">Plot var helper</a></li> +</ul> +`, + `<h2 id="user-content-what-is-wine-staging">What is Wine Staging?</h2> +<p><strong>Wine Staging</strong> on website <a href="http://wine-staging.com" rel="nofollow">wine-staging.com</a>.</p> +<h2 id="user-content-quick-links">Quick Links</h2> +<p>Here are some links to the most important topics. You can find the full list of pages at the sidebar.</p> +<table> +<thead> +<tr> +<th><a href="` + baseURLImages + `/images/icon-install.png" rel="nofollow"><img src="` + baseURLImages + `/images/icon-install.png" title="icon-install.png" alt="images/icon-install.png"/></a></th> +<th><a href="` + baseURLContent + `/Installation" rel="nofollow">Installation</a></th> +</tr> +</thead> +<tbody> +<tr> +<td><a href="` + baseURLImages + `/images/icon-usage.png" rel="nofollow"><img src="` + baseURLImages + `/images/icon-usage.png" title="icon-usage.png" alt="images/icon-usage.png"/></a></td> +<td><a href="` + baseURLContent + `/Usage" rel="nofollow">Usage</a></td> +</tr> +</tbody> +</table> +`, + `<p><a href="http://www.excelsiorjet.com/" rel="nofollow">Excelsior JET</a> allows you to create native executables for Windows, Linux and Mac OS X.</p> +<ol> +<li><a href="https://github.com/libgdx/libgdx/wiki/Gradle-on-the-Commandline#packaging-for-the-desktop" rel="nofollow">Package your libGDX application</a><br/> +<a href="` + baseURLImages + `/images/1.png" rel="nofollow"><img src="` + baseURLImages + `/images/1.png" title="1.png" alt="images/1.png"/></a></li> +<li>Perform a test run by hitting the Run! button.<br/> +<a href="` + baseURLImages + `/images/2.png" rel="nofollow"><img src="` + baseURLImages + `/images/2.png" title="2.png" alt="images/2.png"/></a></li> +</ol> +<h2 id="user-content-custom-id">More tests</h2> +<p>(from <a href="https://www.markdownguide.org/extended-syntax/" rel="nofollow">https://www.markdownguide.org/extended-syntax/</a>)</p> +<h3 id="user-content-checkboxes">Checkboxes</h3> +<ul> +<li class="task-list-item"><input type="checkbox" disabled="" data-source-position="434"/>unchecked</li> +<li class="task-list-item"><input type="checkbox" disabled="" data-source-position="450" checked=""/>checked</li> +<li class="task-list-item"><input type="checkbox" disabled="" data-source-position="464"/>still unchecked</li> +</ul> +<h3 id="user-content-definition-list">Definition list</h3> +<dl> +<dt>First Term</dt> +<dd>This is the definition of the first term.</dd> +<dt>Second Term</dt> +<dd>This is one definition of the second term.</dd> +<dd>This is another definition of the second term.</dd> +</dl> +<h3 id="user-content-footnotes">Footnotes</h3> +<p>Here is a simple footnote,<sup id="fnref:user-content-1"><a href="#fn:user-content-1" rel="nofollow">1</a></sup> and here is a longer one.<sup id="fnref:user-content-bignote"><a href="#fn:user-content-bignote" rel="nofollow">2</a></sup></p> +<div> +<hr/> +<ol> +<li id="fn:user-content-1"> +<p>This is the first footnote. <a href="#fnref:user-content-1" rel="nofollow">↩︎</a></p> +</li> +<li id="fn:user-content-bignote"> +<p>Here is one with multiple paragraphs and code.</p> +<p>Indent paragraphs to include them in the footnote.</p> +<p><code>{ my code }</code></p> +<p>Add as many paragraphs as you like. <a href="#fnref:user-content-bignote" rel="nofollow">↩︎</a></p> +</li> +</ol> +</div> +`, `<ul> +<li class="task-list-item"><input type="checkbox" disabled="" data-source-position="3"/> If you want to rebase/retry this PR, click this checkbox.</li> +</ul> +<hr/> +<p>This PR has been generated by <a href="https://github.com/renovatebot/renovate" rel="nofollow">Renovate Bot</a>.</p> +`, + } +} + +// Test cases without ambiguous links +var sameCases = []string{ + // dear imgui wiki markdown extract: special wiki syntax + `Wiki! Enjoy :) +- [[Links, Language bindings, Engine bindings|Links]] +- [[Tips]] + +See commit 65f1bf27bc + +Ideas and codes + +- Bezier widget (by @r-lyeh) ` + AppURL + `ocornut/imgui/issues/786 +- Bezier widget (by @r-lyeh) ` + AppURL + `gogits/gogs/issues/786 +- Node graph editors https://github.com/ocornut/imgui/issues/306 +- [[Memory Editor|memory_editor_example]] +- [[Plot var helper|plot_var_example]]`, + // wine-staging wiki home extract: tables, special wiki syntax, images + `## What is Wine Staging? +**Wine Staging** on website [wine-staging.com](http://wine-staging.com). + +## Quick Links +Here are some links to the most important topics. You can find the full list of pages at the sidebar. + +| [[images/icon-install.png]] | [[Installation]] | +|--------------------------------|----------------------------------------------------------| +| [[images/icon-usage.png]] | [[Usage]] | +`, + // libgdx wiki page: inline images with special syntax + `[Excelsior JET](http://www.excelsiorjet.com/) allows you to create native executables for Windows, Linux and Mac OS X. + +1. [Package your libGDX application](https://github.com/libgdx/libgdx/wiki/Gradle-on-the-Commandline#packaging-for-the-desktop) +[[images/1.png]] +2. Perform a test run by hitting the Run! button. +[[images/2.png]] + +## More tests {#custom-id} + +(from https://www.markdownguide.org/extended-syntax/) + +### Checkboxes + +- [ ] unchecked +- [x] checked +- [ ] still unchecked + +### Definition list + +First Term +: This is the definition of the first term. + +Second Term +: This is one definition of the second term. +: This is another definition of the second term. + +### Footnotes + +Here is a simple footnote,[^1] and here is a longer one.[^bignote] + +[^1]: This is the first footnote. + +[^bignote]: Here is one with multiple paragraphs and code. + + Indent paragraphs to include them in the footnote. + + ` + "`{ my code }`" + ` + + Add as many paragraphs as you like. +`, + ` +- [ ] <!-- rebase-check --> If you want to rebase/retry this PR, click this checkbox. + +--- + +This PR has been generated by [Renovate Bot](https://github.com/renovatebot/renovate). + +<!-- test-comment -->`, +} + +func TestTotal_RenderWiki(t *testing.T) { + setting.AppURL = AppURL + + answers := testAnswers(util.URLJoin(FullURL, "wiki"), util.URLJoin(FullURL, "wiki", "raw")) + + for i := 0; i < len(sameCases); i++ { + line, err := markdown.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: FullURL, + }, + Metas: localMetas, + IsWiki: true, + }, sameCases[i]) + require.NoError(t, err) + assert.Equal(t, template.HTML(answers[i]), line) + } + + testCases := []string{ + // Guard wiki sidebar: special syntax + `[[Guardfile-DSL / Configuring-Guard|Guardfile-DSL---Configuring-Guard]]`, + // rendered + `<p><a href="` + FullURL + `wiki/Guardfile-DSL---Configuring-Guard" rel="nofollow">Guardfile-DSL / Configuring-Guard</a></p> +`, + // special syntax + `[[Name|Link]]`, + // rendered + `<p><a href="` + FullURL + `wiki/Link" rel="nofollow">Name</a></p> +`, + } + + for i := 0; i < len(testCases); i += 2 { + line, err := markdown.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: FullURL, + }, + IsWiki: true, + }, testCases[i]) + require.NoError(t, err) + assert.Equal(t, template.HTML(testCases[i+1]), line) + } +} + +func TestTotal_RenderString(t *testing.T) { + setting.AppURL = AppURL + + answers := testAnswers(util.URLJoin(FullURL, "src", "master"), util.URLJoin(FullURL, "media", "master")) + + for i := 0; i < len(sameCases); i++ { + line, err := markdown.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: FullURL, + BranchPath: "master", + }, + Metas: localMetas, + }, sameCases[i]) + require.NoError(t, err) + assert.Equal(t, template.HTML(answers[i]), line) + } + + testCases := []string{} + + for i := 0; i < len(testCases); i += 2 { + line, err := markdown.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: FullURL, + }, + }, testCases[i]) + require.NoError(t, err) + assert.Equal(t, template.HTML(testCases[i+1]), line) + } +} + +func TestRender_RenderParagraphs(t *testing.T) { + test := func(t *testing.T, str string, cnt int) { + res, err := markdown.RenderRawString(&markup.RenderContext{Ctx: git.DefaultContext}, str) + require.NoError(t, err) + assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for unix should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res) + + mac := strings.ReplaceAll(str, "\n", "\r") + res, err = markdown.RenderRawString(&markup.RenderContext{Ctx: git.DefaultContext}, mac) + require.NoError(t, err) + assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for mac should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res) + + dos := strings.ReplaceAll(str, "\n", "\r\n") + res, err = markdown.RenderRawString(&markup.RenderContext{Ctx: git.DefaultContext}, dos) + require.NoError(t, err) + assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for windows should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res) + } + + test(t, "\nOne\nTwo\nThree", 1) + test(t, "\n\nOne\nTwo\nThree", 1) + test(t, "\n\nOne\nTwo\nThree\n\n\n", 1) + test(t, "A\n\nB\nC\n", 2) + test(t, "A\n\n\nB\nC\n", 2) +} + +func TestMarkdownRenderRaw(t *testing.T) { + testcases := [][]byte{ + { // clusterfuzz_testcase_minimized_fuzz_markdown_render_raw_6267570554535936 + 0x2a, 0x20, 0x2d, 0x0a, 0x09, 0x20, 0x60, 0x5b, 0x0a, 0x09, 0x20, 0x60, + 0x5b, + }, + { // clusterfuzz_testcase_minimized_fuzz_markdown_render_raw_6278827345051648 + 0x2d, 0x20, 0x2d, 0x0d, 0x09, 0x60, 0x0d, 0x09, 0x60, + }, + { // clusterfuzz_testcase_minimized_fuzz_markdown_render_raw_6016973788020736[] = { + 0x7b, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x3d, 0x35, 0x7d, 0x0a, 0x3d, + }, + } + + for _, testcase := range testcases { + log.Info("Test markdown render error with fuzzy data: %x, the following errors can be recovered", testcase) + _, err := markdown.RenderRawString(&markup.RenderContext{Ctx: git.DefaultContext}, string(testcase)) + require.NoError(t, err) + } +} + +func TestRenderSiblingImages_Issue12925(t *testing.T) { + testcase := ` + +` + expected := `<p><a href="/image1" target="_blank" rel="nofollow noopener"><img src="/image1" alt="image1"></a><br> +<a href="/image2" target="_blank" rel="nofollow noopener"><img src="/image2" alt="image2"></a></p> +` + res, err := markdown.RenderRawString(&markup.RenderContext{Ctx: git.DefaultContext}, testcase) + require.NoError(t, err) + assert.Equal(t, expected, res) +} + +func TestRenderEmojiInLinks_Issue12331(t *testing.T) { + testcase := `[Link with emoji :moon: in text](https://gitea.io)` + expected := `<p><a href="https://gitea.io" rel="nofollow">Link with emoji <span class="emoji" aria-label="waxing gibbous moon">🌔</span> in text</a></p> +` + res, err := markdown.RenderString(&markup.RenderContext{Ctx: git.DefaultContext}, testcase) + require.NoError(t, err) + assert.Equal(t, template.HTML(expected), res) +} + +func TestColorPreview(t *testing.T) { + const nl = "\n" + positiveTests := []struct { + testcase string + expected string + }{ + { // hex + "`#FF0000`", + `<p><code>#FF0000<span class="color-preview" style="background-color: #FF0000"></span></code></p>` + nl, + }, + { // rgb + "`rgb(16, 32, 64)`", + `<p><code>rgb(16, 32, 64)<span class="color-preview" style="background-color: rgb(16, 32, 64)"></span></code></p>` + nl, + }, + { // short hex + "This is the color white `#000`", + `<p>This is the color white <code>#000<span class="color-preview" style="background-color: #000"></span></code></p>` + nl, + }, + { // hsl + "HSL stands for hue, saturation, and lightness. An example: `hsl(0, 100%, 50%)`.", + `<p>HSL stands for hue, saturation, and lightness. An example: <code>hsl(0, 100%, 50%)<span class="color-preview" style="background-color: hsl(0, 100%, 50%)"></span></code>.</p>` + nl, + }, + { // uppercase hsl + "HSL stands for hue, saturation, and lightness. An example: `HSL(0, 100%, 50%)`.", + `<p>HSL stands for hue, saturation, and lightness. An example: <code>HSL(0, 100%, 50%)<span class="color-preview" style="background-color: HSL(0, 100%, 50%)"></span></code>.</p>` + nl, + }, + } + + for _, test := range positiveTests { + res, err := markdown.RenderString(&markup.RenderContext{Ctx: git.DefaultContext}, test.testcase) + require.NoError(t, err, "Unexpected error in testcase: %q", test.testcase) + assert.Equal(t, template.HTML(test.expected), res, "Unexpected result in testcase %q", test.testcase) + } + + negativeTests := []string{ + // not a color code + "`FF0000`", + // inside a code block + "```javascript" + nl + `const red = "#FF0000";` + nl + "```", + // no backticks + "rgb(166, 32, 64)", + // typo + "`hsI(0, 100%, 50%)`", // codespell-ignore + // looks like a color but not really + "`hsl(40, 60, 80)`", + } + + for _, test := range negativeTests { + res, err := markdown.RenderString(&markup.RenderContext{Ctx: git.DefaultContext}, test) + require.NoError(t, err, "Unexpected error in testcase: %q", test) + assert.NotContains(t, res, `<span class="color-preview" style="background-color: `, "Unexpected result in testcase %q", test) + } +} + +func TestMathBlock(t *testing.T) { + const nl = "\n" + testcases := []struct { + testcase string + expected string + }{ + { + "$a$", + `<p><code class="language-math is-loading">a</code></p>` + nl, + }, + { + "$ a $", + `<p><code class="language-math is-loading">a</code></p>` + nl, + }, + { + "$a$ $b$", + `<p><code class="language-math is-loading">a</code> <code class="language-math is-loading">b</code></p>` + nl, + }, + { + `\(a\) \(b\)`, + `<p><code class="language-math is-loading">a</code> <code class="language-math is-loading">b</code></p>` + nl, + }, + { + `$a$.`, + `<p><code class="language-math is-loading">a</code>.</p>` + nl, + }, + { + `.$a$`, + `<p>.$a$</p>` + nl, + }, + { + `$a a$b b$`, + `<p>$a a$b b$</p>` + nl, + }, + { + `a a$b b`, + `<p>a a$b b</p>` + nl, + }, + { + `a$b $a a$b b$`, + `<p>a$b $a a$b b$</p>` + nl, + }, + { + "a$x$", + `<p>a$x$</p>` + nl, + }, + { + "$x$a", + `<p>$x$a</p>` + nl, + }, + { + "$$a$$", + `<pre class="code-block is-loading"><code class="chroma language-math display">a</code></pre>` + nl, + }, + { + `\[a b\]`, + `<pre class="code-block is-loading"><code class="chroma language-math display">a b</code></pre>` + nl, + }, + { + `\[a b]`, + `<p>[a b]</p>` + nl, + }, + { + `$$a`, + `<p>$$a</p>` + nl, + }, + { + "$a$ ($b$) [$c$] {$d$}", + `<p><code class="language-math is-loading">a</code> (<code class="language-math is-loading">b</code>) [$c$] {$d$}</p>` + nl, + }, + } + + for _, test := range testcases { + res, err := markdown.RenderString(&markup.RenderContext{Ctx: git.DefaultContext}, test.testcase) + require.NoError(t, err, "Unexpected error in testcase: %q", test.testcase) + assert.Equal(t, template.HTML(test.expected), res, "Unexpected result in testcase %q", test.testcase) + } +} + +func TestFootnote(t *testing.T) { + testcases := []struct { + testcase string + expected string + }{ + { + `Citation needed[^0]. +[^0]: Source`, + `<p>Citation needed<sup id="fnref:user-content-0"><a href="#fn:user-content-0" rel="nofollow">1</a></sup>.</p> +<div> +<hr/> +<ol> +<li id="fn:user-content-0"> +<p>Source <a href="#fnref:user-content-0" rel="nofollow">↩︎</a></p> +</li> +</ol> +</div> +`, + }, + { + `Citation needed[^0]`, + `<p>Citation needed[^0]</p> +`, + }, + { + `Citation needed[^1], Citation needed twice[^3] +[^3]: Source`, + `<p>Citation needed[^1], Citation needed twice<sup id="fnref:user-content-3"><a href="#fn:user-content-3" rel="nofollow">1</a></sup></p> +<div> +<hr/> +<ol> +<li id="fn:user-content-3"> +<p>Source <a href="#fnref:user-content-3" rel="nofollow">↩︎</a></p> +</li> +</ol> +</div> +`, + }, + { + `Citation needed[^0] +[^1]: Source`, + `<p>Citation needed[^0]</p> +`, + }, + { + `Citation needed[^0] +[^0]: Source 1 +[^0]: Source 2`, + `<p>Citation needed<sup id="fnref:user-content-0"><a href="#fn:user-content-0" rel="nofollow">1</a></sup></p> +<div> +<hr/> +<ol> +<li id="fn:user-content-0"> +<p>Source 1 <a href="#fnref:user-content-0" rel="nofollow">↩︎</a></p> +</li> +</ol> +</div> +`, + }, + { + `Citation needed![^0] +[^0]: Source`, + `<p>Citation needed<sup id="fnref:user-content-0"><a href="#fn:user-content-0" rel="nofollow">1</a></sup></p> +<div> +<hr/> +<ol> +<li id="fn:user-content-0"> +<p>Source <a href="#fnref:user-content-0" rel="nofollow">↩︎</a></p> +</li> +</ol> +</div> +`, + }, + { + `Trigger [^`, + `<p>Trigger [^</p> +`, + }, + { + `Trigger 2 [^0`, + `<p>Trigger 2 [^0</p> +`, + }, + { + `Citation needed[^0] +[^0]: Source with citation needed[^1] +[^1]: Source`, + `<p>Citation needed<sup id="fnref:user-content-0"><a href="#fn:user-content-0" rel="nofollow">1</a></sup></p> +<div> +<hr/> +<ol> +<li id="fn:user-content-0"> +<p>Source with citation needed<sup id="fnref:user-content-1"><a href="#fn:user-content-1" rel="nofollow">2</a></sup> <a href="#fnref:user-content-0" rel="nofollow">↩︎</a></p> +</li> +<li id="fn:user-content-1"> +<p>Source <a href="#fnref:user-content-1" rel="nofollow">↩︎</a></p> +</li> +</ol> +</div> +`, + }, + { + `Citation needed[^#] +[^#]: Source`, + `<p>Citation needed<sup id="fnref:user-content-1"><a href="#fn:user-content-1" rel="nofollow">1</a></sup></p> +<div> +<hr/> +<ol> +<li id="fn:user-content-1"> +<p>Source <a href="#fnref:user-content-1" rel="nofollow">↩︎</a></p> +</li> +</ol> +</div> +`, + }, + { + `Citation needed[^0] + [^0]: Source`, + `<p>Citation needed[^0]<br/> +[^0]: Source</p> +`, + }, + { + `[^0]: Source + +Citation needed[^0].`, + `<p>Citation needed<sup id="fnref:user-content-0"><a href="#fn:user-content-0" rel="nofollow">1</a></sup>.</p> +<div> +<hr/> +<ol> +<li id="fn:user-content-0"> +<p>Source <a href="#fnref:user-content-0" rel="nofollow">↩︎</a></p> +</li> +</ol> +</div> +`, + }, + { + `Citation needed[^] +[^]: Source`, + `<p>Citation needed[^]<br/> +[^]: Source</p> +`, + }, + { + `Citation needed[^0] +[^0] Source`, + `<p>Citation needed[^0]<br/> +[^0] Source</p> +`, + }, + { + `Citation needed[^0] +[^0 Source`, + `<p>Citation needed[^0]<br/> +[^0 Source</p> +`, + }, + { + `Citation needed[^0] [^0]: Source`, + `<p>Citation needed[^0] [^0]: Source</p> +`, + }, + { + `Citation needed[^Source here 0 # 9-3] +[^Source here 0 # 9-3]: Source`, + `<p>Citation needed<sup id="fnref:user-content-source-here-0-9-3"><a href="#fn:user-content-source-here-0-9-3" rel="nofollow">1</a></sup></p> +<div> +<hr/> +<ol> +<li id="fn:user-content-source-here-0-9-3"> +<p>Source <a href="#fnref:user-content-source-here-0-9-3" rel="nofollow">↩︎</a></p> +</li> +</ol> +</div> +`, + }, + { + `Citation needed[^0] +[^0]:`, + `<p>Citation needed<sup id="fnref:user-content-0"><a href="#fn:user-content-0" rel="nofollow">1</a></sup></p> +<div> +<hr/> +<ol> +<li id="fn:user-content-0"> + <a href="#fnref:user-content-0" rel="nofollow">↩︎</a></li> +</ol> +</div> +`, + }, + } + for _, test := range testcases { + res, err := markdown.RenderString(&markup.RenderContext{Ctx: git.DefaultContext}, test.testcase) + require.NoError(t, err, "Unexpected error in testcase: %q", test.testcase) + assert.Equal(t, test.expected, string(res), "Unexpected result in testcase %q", test.testcase) + } +} + +func TestTaskList(t *testing.T) { + testcases := []struct { + testcase string + expected string + }{ + { + // data-source-position should take into account YAML frontmatter. + `--- +foo: bar +--- +- [ ] task 1`, + `<details><summary><i class="icon table"></i></summary><table> +<thead> +<tr> +<th>foo</th> +</tr> +</thead> +<tbody> +<tr> +<td>bar</td> +</tr> +</tbody> +</table> +</details><ul> +<li class="task-list-item"><input type="checkbox" disabled="" data-source-position="19"/>task 1</li> +</ul> +`, + }, + } + + for _, test := range testcases { + res, err := markdown.RenderString(&markup.RenderContext{Ctx: git.DefaultContext}, test.testcase) + require.NoError(t, err, "Unexpected error in testcase: %q", test.testcase) + assert.Equal(t, template.HTML(test.expected), res, "Unexpected result in testcase %q", test.testcase) + } +} + +func TestRenderLinks(t *testing.T) { + input := ` space @mention-user${SPACE}${SPACE} +/just/a/path.bin +https://example.com/file.bin +[local link](file.bin) +[remote link](https://example.com) +[[local link|file.bin]] +[[remote link|https://example.com]] + + + + +[[local image|image.jpg]] +[[remote link|https://example.com/image.jpg]] +https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare +https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit +:+1: +mail@domain.com +@mention-user test +#123 + space${SPACE}${SPACE} +` + input = strings.ReplaceAll(input, "${SPACE}", " ") // replace ${SPACE} with " ", to avoid some editor's auto-trimming + cases := []struct { + Links markup.Links + IsWiki bool + Expected string + }{ + { // 0 + Links: markup.Links{}, + IsWiki: false, + Expected: `<p>space @mention-user<br/> +/just/a/path.bin<br/> +<a href="https://example.com/file.bin" rel="nofollow">https://example.com/file.bin</a><br/> +<a href="/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/src/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/image.jpg" target="_blank" rel="nofollow noopener"><img src="/image.jpg" alt="local image"/></a><br/> +<a href="/path/file" target="_blank" rel="nofollow noopener"><img src="/path/file" alt="local image"/></a><br/> +<a href="/path/file" target="_blank" rel="nofollow noopener"><img src="/path/file" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/> +<a href="/image.jpg" rel="nofollow"><img src="/image.jpg" title="local image" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/> +<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" rel="nofollow">https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare<br/> +<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" rel="nofollow">https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit<br/> +<span class="emoji" aria-label="thumbs up">👍</span><br/> +<a href="mailto:mail@domain.com" rel="nofollow">mail@domain.com</a><br/> +@mention-user test<br/> +#123<br/> +space</p> +`, + }, + { // 1 + Links: markup.Links{}, + IsWiki: true, + Expected: `<p>space @mention-user<br/> +/just/a/path.bin<br/> +<a href="https://example.com/file.bin" rel="nofollow">https://example.com/file.bin</a><br/> +<a href="/wiki/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/wiki/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/wiki/raw/image.jpg" target="_blank" rel="nofollow noopener"><img src="/wiki/raw/image.jpg" alt="local image"/></a><br/> +<a href="/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/wiki/raw/path/file" alt="local image"/></a><br/> +<a href="/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/wiki/raw/path/file" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/> +<a href="/wiki/raw/image.jpg" rel="nofollow"><img src="/wiki/raw/image.jpg" title="local image" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/> +<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" rel="nofollow">https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare<br/> +<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" rel="nofollow">https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit<br/> +<span class="emoji" aria-label="thumbs up">👍</span><br/> +<a href="mailto:mail@domain.com" rel="nofollow">mail@domain.com</a><br/> +@mention-user test<br/> +#123<br/> +space</p> +`, + }, + { // 2 + Links: markup.Links{ + Base: "https://gitea.io/", + }, + IsWiki: false, + Expected: `<p>space @mention-user<br/> +/just/a/path.bin<br/> +<a href="https://example.com/file.bin" rel="nofollow">https://example.com/file.bin</a><br/> +<a href="https://gitea.io/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="https://gitea.io/src/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="https://gitea.io/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://gitea.io/image.jpg" alt="local image"/></a><br/> +<a href="https://gitea.io/path/file" target="_blank" rel="nofollow noopener"><img src="https://gitea.io/path/file" alt="local image"/></a><br/> +<a href="https://gitea.io/path/file" target="_blank" rel="nofollow noopener"><img src="https://gitea.io/path/file" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/> +<a href="https://gitea.io/image.jpg" rel="nofollow"><img src="https://gitea.io/image.jpg" title="local image" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/> +<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" rel="nofollow">https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare<br/> +<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" rel="nofollow">https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit<br/> +<span class="emoji" aria-label="thumbs up">👍</span><br/> +<a href="mailto:mail@domain.com" rel="nofollow">mail@domain.com</a><br/> +@mention-user test<br/> +#123<br/> +space</p> +`, + }, + { // 3 + Links: markup.Links{ + Base: "https://gitea.io/", + }, + IsWiki: true, + Expected: `<p>space @mention-user<br/> +/just/a/path.bin<br/> +<a href="https://example.com/file.bin" rel="nofollow">https://example.com/file.bin</a><br/> +<a href="https://gitea.io/wiki/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="https://gitea.io/wiki/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="https://gitea.io/wiki/raw/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://gitea.io/wiki/raw/image.jpg" alt="local image"/></a><br/> +<a href="https://gitea.io/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="https://gitea.io/wiki/raw/path/file" alt="local image"/></a><br/> +<a href="https://gitea.io/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="https://gitea.io/wiki/raw/path/file" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/> +<a href="https://gitea.io/wiki/raw/image.jpg" rel="nofollow"><img src="https://gitea.io/wiki/raw/image.jpg" title="local image" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/> +<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" rel="nofollow">https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare<br/> +<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" rel="nofollow">https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit<br/> +<span class="emoji" aria-label="thumbs up">👍</span><br/> +<a href="mailto:mail@domain.com" rel="nofollow">mail@domain.com</a><br/> +@mention-user test<br/> +#123<br/> +space</p> +`, + }, + { // 4 + Links: markup.Links{ + Base: "/relative/path", + }, + IsWiki: false, + Expected: `<p>space @mention-user<br/> +/just/a/path.bin<br/> +<a href="https://example.com/file.bin" rel="nofollow">https://example.com/file.bin</a><br/> +<a href="/relative/path/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/relative/path/src/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/relative/path/image.jpg" target="_blank" rel="nofollow noopener"><img src="/relative/path/image.jpg" alt="local image"/></a><br/> +<a href="/relative/path/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/path/file" alt="local image"/></a><br/> +<a href="/relative/path/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/path/file" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/> +<a href="/relative/path/image.jpg" rel="nofollow"><img src="/relative/path/image.jpg" title="local image" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/> +<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" rel="nofollow">https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare<br/> +<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" rel="nofollow">https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit<br/> +<span class="emoji" aria-label="thumbs up">👍</span><br/> +<a href="mailto:mail@domain.com" rel="nofollow">mail@domain.com</a><br/> +@mention-user test<br/> +#123<br/> +space</p> +`, + }, + { // 5 + Links: markup.Links{ + Base: "/relative/path", + }, + IsWiki: true, + Expected: `<p>space @mention-user<br/> +/just/a/path.bin<br/> +<a href="https://example.com/file.bin" rel="nofollow">https://example.com/file.bin</a><br/> +<a href="/relative/path/wiki/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/relative/path/wiki/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/relative/path/wiki/raw/image.jpg" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/image.jpg" alt="local image"/></a><br/> +<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/> +<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/> +<a href="/relative/path/wiki/raw/image.jpg" rel="nofollow"><img src="/relative/path/wiki/raw/image.jpg" title="local image" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/> +<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" rel="nofollow">https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare<br/> +<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" rel="nofollow">https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit<br/> +<span class="emoji" aria-label="thumbs up">👍</span><br/> +<a href="mailto:mail@domain.com" rel="nofollow">mail@domain.com</a><br/> +@mention-user test<br/> +#123<br/> +space</p> +`, + }, + { // 6 + Links: markup.Links{ + Base: "/user/repo", + BranchPath: "branch/main", + }, + IsWiki: false, + Expected: `<p>space @mention-user<br/> +/just/a/path.bin<br/> +<a href="https://example.com/file.bin" rel="nofollow">https://example.com/file.bin</a><br/> +<a href="/user/repo/src/branch/main/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/user/repo/src/branch/main/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/user/repo/media/branch/main/image.jpg" target="_blank" rel="nofollow noopener"><img src="/user/repo/media/branch/main/image.jpg" alt="local image"/></a><br/> +<a href="/user/repo/media/branch/main/path/file" target="_blank" rel="nofollow noopener"><img src="/user/repo/media/branch/main/path/file" alt="local image"/></a><br/> +<a href="/user/repo/media/branch/main/path/file" target="_blank" rel="nofollow noopener"><img src="/user/repo/media/branch/main/path/file" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/> +<a href="/user/repo/media/branch/main/image.jpg" rel="nofollow"><img src="/user/repo/media/branch/main/image.jpg" title="local image" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/> +<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" rel="nofollow">https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare<br/> +<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" rel="nofollow">https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit<br/> +<span class="emoji" aria-label="thumbs up">👍</span><br/> +<a href="mailto:mail@domain.com" rel="nofollow">mail@domain.com</a><br/> +@mention-user test<br/> +#123<br/> +space</p> +`, + }, + { // 7 + Links: markup.Links{ + Base: "/relative/path", + BranchPath: "branch/main", + }, + IsWiki: true, + Expected: `<p>space @mention-user<br/> +/just/a/path.bin<br/> +<a href="https://example.com/file.bin" rel="nofollow">https://example.com/file.bin</a><br/> +<a href="/relative/path/wiki/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/relative/path/wiki/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/relative/path/wiki/raw/image.jpg" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/image.jpg" alt="local image"/></a><br/> +<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/> +<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/> +<a href="/relative/path/wiki/raw/image.jpg" rel="nofollow"><img src="/relative/path/wiki/raw/image.jpg" title="local image" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/> +<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" rel="nofollow">https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare<br/> +<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" rel="nofollow">https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit<br/> +<span class="emoji" aria-label="thumbs up">👍</span><br/> +<a href="mailto:mail@domain.com" rel="nofollow">mail@domain.com</a><br/> +@mention-user test<br/> +#123<br/> +space</p> +`, + }, + { // 8 + Links: markup.Links{ + Base: "/user/repo", + TreePath: "sub/folder", + }, + IsWiki: false, + Expected: `<p>space @mention-user<br/> +/just/a/path.bin<br/> +<a href="https://example.com/file.bin" rel="nofollow">https://example.com/file.bin</a><br/> +<a href="/user/repo/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/user/repo/src/sub/folder/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/user/repo/image.jpg" target="_blank" rel="nofollow noopener"><img src="/user/repo/image.jpg" alt="local image"/></a><br/> +<a href="/user/repo/path/file" target="_blank" rel="nofollow noopener"><img src="/user/repo/path/file" alt="local image"/></a><br/> +<a href="/user/repo/path/file" target="_blank" rel="nofollow noopener"><img src="/user/repo/path/file" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/> +<a href="/user/repo/image.jpg" rel="nofollow"><img src="/user/repo/image.jpg" title="local image" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/> +<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" rel="nofollow">https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare<br/> +<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" rel="nofollow">https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit<br/> +<span class="emoji" aria-label="thumbs up">👍</span><br/> +<a href="mailto:mail@domain.com" rel="nofollow">mail@domain.com</a><br/> +@mention-user test<br/> +#123<br/> +space</p> +`, + }, + { // 9 + Links: markup.Links{ + Base: "/relative/path", + TreePath: "sub/folder", + }, + IsWiki: true, + Expected: `<p>space @mention-user<br/> +/just/a/path.bin<br/> +<a href="https://example.com/file.bin" rel="nofollow">https://example.com/file.bin</a><br/> +<a href="/relative/path/wiki/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/relative/path/wiki/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/relative/path/wiki/raw/image.jpg" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/image.jpg" alt="local image"/></a><br/> +<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/> +<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/> +<a href="/relative/path/wiki/raw/image.jpg" rel="nofollow"><img src="/relative/path/wiki/raw/image.jpg" title="local image" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/> +<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" rel="nofollow">https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare<br/> +<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" rel="nofollow">https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit<br/> +<span class="emoji" aria-label="thumbs up">👍</span><br/> +<a href="mailto:mail@domain.com" rel="nofollow">mail@domain.com</a><br/> +@mention-user test<br/> +#123<br/> +space</p> +`, + }, + { // 10 + Links: markup.Links{ + Base: "/user/repo", + BranchPath: "branch/main", + TreePath: "sub/folder", + }, + IsWiki: false, + Expected: `<p>space @mention-user<br/> +/just/a/path.bin<br/> +<a href="https://example.com/file.bin" rel="nofollow">https://example.com/file.bin</a><br/> +<a href="/user/repo/src/branch/main/sub/folder/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/user/repo/src/branch/main/sub/folder/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/user/repo/media/branch/main/sub/folder/image.jpg" target="_blank" rel="nofollow noopener"><img src="/user/repo/media/branch/main/sub/folder/image.jpg" alt="local image"/></a><br/> +<a href="/user/repo/media/branch/main/sub/folder/path/file" target="_blank" rel="nofollow noopener"><img src="/user/repo/media/branch/main/sub/folder/path/file" alt="local image"/></a><br/> +<a href="/user/repo/media/branch/main/sub/folder/path/file" target="_blank" rel="nofollow noopener"><img src="/user/repo/media/branch/main/sub/folder/path/file" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/> +<a href="/user/repo/media/branch/main/sub/folder/image.jpg" rel="nofollow"><img src="/user/repo/media/branch/main/sub/folder/image.jpg" title="local image" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/> +<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" rel="nofollow">https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare<br/> +<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" rel="nofollow">https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit<br/> +<span class="emoji" aria-label="thumbs up">👍</span><br/> +<a href="mailto:mail@domain.com" rel="nofollow">mail@domain.com</a><br/> +@mention-user test<br/> +#123<br/> +space</p> +`, + }, + { // 11 + Links: markup.Links{ + Base: "/relative/path", + BranchPath: "branch/main", + TreePath: "sub/folder", + }, + IsWiki: true, + Expected: `<p>space @mention-user<br/> +/just/a/path.bin<br/> +<a href="https://example.com/file.bin" rel="nofollow">https://example.com/file.bin</a><br/> +<a href="/relative/path/wiki/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/relative/path/wiki/file.bin" rel="nofollow">local link</a><br/> +<a href="https://example.com" rel="nofollow">remote link</a><br/> +<a href="/relative/path/wiki/raw/image.jpg" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/image.jpg" alt="local image"/></a><br/> +<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/> +<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/> +<a href="/relative/path/wiki/raw/image.jpg" rel="nofollow"><img src="/relative/path/wiki/raw/image.jpg" title="local image" alt="local image"/></a><br/> +<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/> +<a href="https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash" rel="nofollow">https://example.com/user/repo/compare/88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb#hash</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb...12fc37a3c0a4dda553bdcfc80c178a58247f42fb pare<br/> +<a href="https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb" rel="nofollow">https://example.com/user/repo/commit/88fc37a3c0a4dda553bdcfc80c178a58247f42fb</a><br/> +com 88fc37a3c0a4dda553bdcfc80c178a58247f42fb mit<br/> +<span class="emoji" aria-label="thumbs up">👍</span><br/> +<a href="mailto:mail@domain.com" rel="nofollow">mail@domain.com</a><br/> +@mention-user test<br/> +#123<br/> +space</p> +`, + }, + } + + for i, c := range cases { + result, err := markdown.RenderString(&markup.RenderContext{Ctx: context.Background(), Links: c.Links, IsWiki: c.IsWiki}, input) + require.NoError(t, err, "Unexpected error in testcase: %v", i) + assert.Equal(t, template.HTML(c.Expected), result, "Unexpected result in testcase %v", i) + } +} + +func TestCustomMarkdownURL(t *testing.T) { + defer test.MockVariableValue(&setting.Markdown.CustomURLSchemes, []string{"abp"})() + setting.AppURL = AppURL + + test := func(input, expected string) { + buffer, err := markdown.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: FullURL, + BranchPath: "branch/main", + }, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) + } + + test("[test](abp:subscribe?location=https://codeberg.org/filters.txt&title=joy)", + `<p><a href="abp:subscribe?location=https://codeberg.org/filters.txt&title=joy" rel="nofollow">test</a></p>`) + + // Ensure that the schema itself without `:` is still made absolute. + test("[test](abp)", + `<p><a href="http://localhost:3000/gogits/gogs/src/branch/main/abp" rel="nofollow">test</a></p>`) +} + +func TestCallout(t *testing.T) { + setting.AppURL = AppURL + + test := func(input, expected string) { + buffer, err := markdown.RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) + } + + test(">\n0", "<blockquote>\n</blockquote>\n<p>0</p>") +} diff --git a/modules/markup/markdown/math/block_node.go b/modules/markup/markdown/math/block_node.go new file mode 100644 index 00000000..10d17ff8 --- /dev/null +++ b/modules/markup/markdown/math/block_node.go @@ -0,0 +1,41 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package math + +import "github.com/yuin/goldmark/ast" + +// Block represents a display math block e.g. $$...$$ or \[...\] +type Block struct { + ast.BaseBlock + Dollars bool + Indent int + Closed bool +} + +// KindBlock is the node kind for math blocks +var KindBlock = ast.NewNodeKind("MathBlock") + +// NewBlock creates a new math Block +func NewBlock(dollars bool, indent int) *Block { + return &Block{ + Dollars: dollars, + Indent: indent, + } +} + +// Dump dumps the block to a string +func (n *Block) Dump(source []byte, level int) { + m := map[string]string{} + ast.DumpHelper(n, source, level, m, nil) +} + +// Kind returns KindBlock for math Blocks +func (n *Block) Kind() ast.NodeKind { + return KindBlock +} + +// IsRaw returns true as this block should not be processed further +func (n *Block) IsRaw() bool { + return true +} diff --git a/modules/markup/markdown/math/block_parser.go b/modules/markup/markdown/math/block_parser.go new file mode 100644 index 00000000..f3262c82 --- /dev/null +++ b/modules/markup/markdown/math/block_parser.go @@ -0,0 +1,119 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package math + +import ( + "bytes" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +type blockParser struct { + parseDollars bool +} + +// NewBlockParser creates a new math BlockParser +func NewBlockParser(parseDollarBlocks bool) parser.BlockParser { + return &blockParser{ + parseDollars: parseDollarBlocks, + } +} + +// Open parses the current line and returns a result of parsing. +func (b *blockParser) Open(parent ast.Node, reader text.Reader, pc parser.Context) (ast.Node, parser.State) { + line, segment := reader.PeekLine() + pos := pc.BlockOffset() + if pos == -1 || len(line[pos:]) < 2 { + return nil, parser.NoChildren + } + + dollars := false + if b.parseDollars && line[pos] == '$' && line[pos+1] == '$' { + dollars = true + } else if line[pos] != '\\' || line[pos+1] != '[' { + return nil, parser.NoChildren + } + + node := NewBlock(dollars, pos) + + // Now we need to check if the ending block is on the segment... + endBytes := []byte{'\\', ']'} + if dollars { + endBytes = []byte{'$', '$'} + } + idx := bytes.Index(line[pos+2:], endBytes) + if idx >= 0 { + segment.Stop = segment.Start + idx + 2 + reader.Advance(segment.Len() - 1) + segment.Start += 2 + node.Lines().Append(segment) + node.Closed = true + return node, parser.Close | parser.NoChildren + } + + return nil, parser.NoChildren +} + +// Continue parses the current line and returns a result of parsing. +func (b *blockParser) Continue(node ast.Node, reader text.Reader, pc parser.Context) parser.State { + block := node.(*Block) + if block.Closed { + return parser.Close + } + + line, segment := reader.PeekLine() + w, pos := util.IndentWidth(line, 0) + if w < 4 { + if block.Dollars { + i := pos + for ; i < len(line) && line[i] == '$'; i++ { + } + length := i - pos + if length >= 2 && util.IsBlank(line[i:]) { + reader.Advance(segment.Stop - segment.Start - segment.Padding) + block.Closed = true + return parser.Close + } + } else if len(line[pos:]) > 1 && line[pos] == '\\' && line[pos+1] == ']' && util.IsBlank(line[pos+2:]) { + reader.Advance(segment.Stop - segment.Start - segment.Padding) + block.Closed = true + return parser.Close + } + } + + pos, padding := util.IndentPosition(line, 0, block.Indent) + seg := text.NewSegmentPadding(segment.Start+pos, segment.Stop, padding) + node.Lines().Append(seg) + reader.AdvanceAndSetPadding(segment.Stop-segment.Start-pos-1, padding) + return parser.Continue | parser.NoChildren +} + +// Close will be called when the parser returns Close. +func (b *blockParser) Close(node ast.Node, reader text.Reader, pc parser.Context) { + // noop +} + +// CanInterruptParagraph returns true if the parser can interrupt paragraphs, +// otherwise false. +func (b *blockParser) CanInterruptParagraph() bool { + return true +} + +// CanAcceptIndentedLine returns true if the parser can open new node when +// the given line is being indented more than 3 spaces. +func (b *blockParser) CanAcceptIndentedLine() bool { + return false +} + +// Trigger returns a list of characters that triggers Parse method of +// this parser. +// If Trigger returns a nil, Open will be called with any lines. +// +// We leave this as nil as our parse method is quick enough +func (b *blockParser) Trigger() []byte { + return nil +} diff --git a/modules/markup/markdown/math/block_renderer.go b/modules/markup/markdown/math/block_renderer.go new file mode 100644 index 00000000..84817ef1 --- /dev/null +++ b/modules/markup/markdown/math/block_renderer.go @@ -0,0 +1,42 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package math + +import ( + gast "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/util" +) + +// BlockRenderer represents a renderer for math Blocks +type BlockRenderer struct{} + +// NewBlockRenderer creates a new renderer for math Blocks +func NewBlockRenderer() renderer.NodeRenderer { + return &BlockRenderer{} +} + +// RegisterFuncs registers the renderer for math Blocks +func (r *BlockRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(KindBlock, r.renderBlock) +} + +func (r *BlockRenderer) writeLines(w util.BufWriter, source []byte, n gast.Node) { + l := n.Lines().Len() + for i := 0; i < l; i++ { + line := n.Lines().At(i) + _, _ = w.Write(util.EscapeHTML(line.Value(source))) + } +} + +func (r *BlockRenderer) renderBlock(w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) { + n := node.(*Block) + if entering { + _, _ = w.WriteString(`<pre class="code-block is-loading"><code class="chroma language-math display">`) + r.writeLines(w, source, n) + } else { + _, _ = w.WriteString(`</code></pre>` + "\n") + } + return gast.WalkContinue, nil +} diff --git a/modules/markup/markdown/math/inline_node.go b/modules/markup/markdown/math/inline_node.go new file mode 100644 index 00000000..2221a251 --- /dev/null +++ b/modules/markup/markdown/math/inline_node.go @@ -0,0 +1,48 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package math + +import ( + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/util" +) + +// Inline represents inline math e.g. $...$ or \(...\) +type Inline struct { + ast.BaseInline +} + +// Inline implements Inline.Inline. +func (n *Inline) Inline() {} + +// IsBlank returns if this inline node is empty +func (n *Inline) IsBlank(source []byte) bool { + for c := n.FirstChild(); c != nil; c = c.NextSibling() { + text := c.(*ast.Text).Segment + if !util.IsBlank(text.Value(source)) { + return false + } + } + return true +} + +// Dump renders this inline math as debug +func (n *Inline) Dump(source []byte, level int) { + ast.DumpHelper(n, source, level, nil, nil) +} + +// KindInline is the kind for math inline +var KindInline = ast.NewNodeKind("MathInline") + +// Kind returns KindInline +func (n *Inline) Kind() ast.NodeKind { + return KindInline +} + +// NewInline creates a new ast math inline node +func NewInline() *Inline { + return &Inline{ + BaseInline: ast.BaseInline{}, + } +} diff --git a/modules/markup/markdown/math/inline_parser.go b/modules/markup/markdown/math/inline_parser.go new file mode 100644 index 00000000..614cf329 --- /dev/null +++ b/modules/markup/markdown/math/inline_parser.go @@ -0,0 +1,131 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package math + +import ( + "bytes" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/text" +) + +type inlineParser struct { + start []byte + end []byte +} + +var defaultInlineDollarParser = &inlineParser{ + start: []byte{'$'}, + end: []byte{'$'}, +} + +// NewInlineDollarParser returns a new inline parser +func NewInlineDollarParser() parser.InlineParser { + return defaultInlineDollarParser +} + +var defaultInlineBracketParser = &inlineParser{ + start: []byte{'\\', '('}, + end: []byte{'\\', ')'}, +} + +// NewInlineDollarParser returns a new inline parser +func NewInlineBracketParser() parser.InlineParser { + return defaultInlineBracketParser +} + +// Trigger triggers this parser on $ or \ +func (parser *inlineParser) Trigger() []byte { + return parser.start[0:1] +} + +func isPunctuation(b byte) bool { + return b == '.' || b == '!' || b == '?' || b == ',' || b == ';' || b == ':' +} + +func isBracket(b byte) bool { + return b == ')' +} + +func isAlphanumeric(b byte) bool { + return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') +} + +// Parse parses the current line and returns a result of parsing. +func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser.Context) ast.Node { + line, _ := block.PeekLine() + + if !bytes.HasPrefix(line, parser.start) { + // We'll catch this one on the next time round + return nil + } + + precedingCharacter := block.PrecendingCharacter() + if precedingCharacter < 256 && (isAlphanumeric(byte(precedingCharacter)) || isPunctuation(byte(precedingCharacter))) { + // need to exclude things like `a$` from being considered a start + return nil + } + + // move the opener marker point at the start of the text + opener := len(parser.start) + + // Now look for an ending line + ender := opener + for { + pos := bytes.Index(line[ender:], parser.end) + if pos < 0 { + return nil + } + + ender += pos + + // Now we want to check the character at the end of our parser section + // that is ender + len(parser.end) and check if char before ender is '\' + pos = ender + len(parser.end) + if len(line) <= pos { + break + } + suceedingCharacter := line[pos] + if !isPunctuation(suceedingCharacter) && !(suceedingCharacter == ' ') && !isBracket(suceedingCharacter) { + return nil + } + if line[ender-1] != '\\' { + break + } + + // move the pointer onwards + ender += len(parser.end) + } + + block.Advance(opener) + _, pos := block.Position() + node := NewInline() + segment := pos.WithStop(pos.Start + ender - opener) + node.AppendChild(node, ast.NewRawTextSegment(segment)) + block.Advance(ender - opener + len(parser.end)) + + trimBlock(node, block) + return node +} + +func trimBlock(node *Inline, block text.Reader) { + if node.IsBlank(block.Source()) { + return + } + + // trim first space and last space + first := node.FirstChild().(*ast.Text) + if !(!first.Segment.IsEmpty() && block.Source()[first.Segment.Start] == ' ') { + return + } + + last := node.LastChild().(*ast.Text) + if !(!last.Segment.IsEmpty() && block.Source()[last.Segment.Stop-1] == ' ') { + return + } + + first.Segment = first.Segment.WithStart(first.Segment.Start + 1) + last.Segment = last.Segment.WithStop(last.Segment.Stop - 1) +} diff --git a/modules/markup/markdown/math/inline_renderer.go b/modules/markup/markdown/math/inline_renderer.go new file mode 100644 index 00000000..b4e9ade0 --- /dev/null +++ b/modules/markup/markdown/math/inline_renderer.go @@ -0,0 +1,46 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package math + +import ( + "bytes" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/util" +) + +// InlineRenderer is an inline renderer +type InlineRenderer struct{} + +// NewInlineRenderer returns a new renderer for inline math +func NewInlineRenderer() renderer.NodeRenderer { + return &InlineRenderer{} +} + +func (r *InlineRenderer) renderInline(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + _, _ = w.WriteString(`<code class="language-math is-loading">`) + for c := n.FirstChild(); c != nil; c = c.NextSibling() { + segment := c.(*ast.Text).Segment + value := util.EscapeHTML(segment.Value(source)) + if bytes.HasSuffix(value, []byte("\n")) { + _, _ = w.Write(value[:len(value)-1]) + if c != n.LastChild() { + _, _ = w.Write([]byte(" ")) + } + } else { + _, _ = w.Write(value) + } + } + return ast.WalkSkipChildren, nil + } + _, _ = w.WriteString(`</code>`) + return ast.WalkContinue, nil +} + +// RegisterFuncs registers the renderer for inline math nodes +func (r *InlineRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(KindInline, r.renderInline) +} diff --git a/modules/markup/markdown/math/math.go b/modules/markup/markdown/math/math.go new file mode 100644 index 00000000..8a507535 --- /dev/null +++ b/modules/markup/markdown/math/math.go @@ -0,0 +1,107 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package math + +import ( + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/util" +) + +// Extension is a math extension +type Extension struct { + enabled bool + parseDollarInline bool + parseDollarBlock bool +} + +// Option is the interface Options should implement +type Option interface { + SetOption(e *Extension) +} + +type extensionFunc func(e *Extension) + +func (fn extensionFunc) SetOption(e *Extension) { + fn(e) +} + +// Enabled enables or disables this extension +func Enabled(enable ...bool) Option { + value := true + if len(enable) > 0 { + value = enable[0] + } + return extensionFunc(func(e *Extension) { + e.enabled = value + }) +} + +// WithInlineDollarParser enables or disables the parsing of $...$ +func WithInlineDollarParser(enable ...bool) Option { + value := true + if len(enable) > 0 { + value = enable[0] + } + return extensionFunc(func(e *Extension) { + e.parseDollarInline = value + }) +} + +// WithBlockDollarParser enables or disables the parsing of $$...$$ +func WithBlockDollarParser(enable ...bool) Option { + value := true + if len(enable) > 0 { + value = enable[0] + } + return extensionFunc(func(e *Extension) { + e.parseDollarBlock = value + }) +} + +// Math represents a math extension with default rendered delimiters +var Math = &Extension{ + enabled: true, + parseDollarBlock: true, + parseDollarInline: true, +} + +// NewExtension creates a new math extension with the provided options +func NewExtension(opts ...Option) *Extension { + r := &Extension{ + enabled: true, + parseDollarBlock: true, + parseDollarInline: true, + } + + for _, o := range opts { + o.SetOption(r) + } + return r +} + +// Extend extends goldmark with our parsers and renderers +func (e *Extension) Extend(m goldmark.Markdown) { + if !e.enabled { + return + } + + m.Parser().AddOptions(parser.WithBlockParsers( + util.Prioritized(NewBlockParser(e.parseDollarBlock), 701), + )) + + inlines := []util.PrioritizedValue{ + util.Prioritized(NewInlineBracketParser(), 501), + } + if e.parseDollarInline { + inlines = append(inlines, util.Prioritized(NewInlineDollarParser(), 501)) + } + m.Parser().AddOptions(parser.WithInlineParsers(inlines...)) + + m.Renderer().AddOptions(renderer.WithNodeRenderers( + util.Prioritized(NewBlockRenderer(), 501), + util.Prioritized(NewInlineRenderer(), 502), + )) +} diff --git a/modules/markup/markdown/meta.go b/modules/markup/markdown/meta.go new file mode 100644 index 00000000..e76b253e --- /dev/null +++ b/modules/markup/markdown/meta.go @@ -0,0 +1,103 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "bytes" + "errors" + "unicode" + "unicode/utf8" + + "gopkg.in/yaml.v3" +) + +func isYAMLSeparator(line []byte) bool { + idx := 0 + for ; idx < len(line); idx++ { + if line[idx] >= utf8.RuneSelf { + r, sz := utf8.DecodeRune(line[idx:]) + if !unicode.IsSpace(r) { + return false + } + idx += sz + continue + } + if line[idx] != ' ' { + break + } + } + dashCount := 0 + for ; idx < len(line); idx++ { + if line[idx] != '-' { + break + } + dashCount++ + } + if dashCount < 3 { + return false + } + for ; idx < len(line); idx++ { + if line[idx] >= utf8.RuneSelf { + r, sz := utf8.DecodeRune(line[idx:]) + if !unicode.IsSpace(r) { + return false + } + idx += sz + continue + } + if line[idx] != ' ' { + return false + } + } + return true +} + +// ExtractMetadata consumes a markdown file, parses YAML frontmatter, +// and returns the frontmatter metadata separated from the markdown content +func ExtractMetadata(contents string, out any) (string, error) { + body, err := ExtractMetadataBytes([]byte(contents), out) + return string(body), err +} + +// ExtractMetadata consumes a markdown file, parses YAML frontmatter, +// and returns the frontmatter metadata separated from the markdown content +func ExtractMetadataBytes(contents []byte, out any) ([]byte, error) { + var front, body []byte + + start, end := 0, len(contents) + idx := bytes.IndexByte(contents[start:], '\n') + if idx >= 0 { + end = start + idx + } + line := contents[start:end] + + if !isYAMLSeparator(line) { + return contents, errors.New("frontmatter must start with a separator line") + } + frontMatterStart := end + 1 + for start = frontMatterStart; start < len(contents); start = end + 1 { + end = len(contents) + idx := bytes.IndexByte(contents[start:], '\n') + if idx >= 0 { + end = start + idx + } + line := contents[start:end] + if isYAMLSeparator(line) { + front = contents[frontMatterStart:start] + if end+1 < len(contents) { + body = contents[end+1:] + } + break + } + } + + if len(front) == 0 { + return contents, errors.New("could not determine metadata") + } + + if err := yaml.Unmarshal(front, out); err != nil { + return contents, err + } + return body, nil +} diff --git a/modules/markup/markdown/meta_test.go b/modules/markup/markdown/meta_test.go new file mode 100644 index 00000000..d341ae43 --- /dev/null +++ b/modules/markup/markdown/meta_test.go @@ -0,0 +1,110 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +/* +IssueTemplate is a legacy to keep the unit tests working. +Copied from structs.IssueTemplate, the original type has been changed a lot to support yaml template. +*/ +type IssueTemplate struct { + Name string `json:"name" yaml:"name"` + Title string `json:"title" yaml:"title"` + About string `json:"about" yaml:"about"` + Labels []string `json:"labels" yaml:"labels"` + Ref string `json:"ref" yaml:"ref"` +} + +func (it *IssueTemplate) Valid() bool { + return strings.TrimSpace(it.Name) != "" && strings.TrimSpace(it.About) != "" +} + +func TestExtractMetadata(t *testing.T) { + t.Run("ValidFrontAndBody", func(t *testing.T) { + var meta IssueTemplate + body, err := ExtractMetadata(fmt.Sprintf("%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest), &meta) + require.NoError(t, err) + assert.Equal(t, bodyTest, body) + assert.Equal(t, metaTest, meta) + assert.True(t, meta.Valid()) + }) + + t.Run("NoFirstSeparator", func(t *testing.T) { + var meta IssueTemplate + _, err := ExtractMetadata(fmt.Sprintf("%s\n%s\n%s", frontTest, sepTest, bodyTest), &meta) + require.Error(t, err) + }) + + t.Run("NoLastSeparator", func(t *testing.T) { + var meta IssueTemplate + _, err := ExtractMetadata(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, bodyTest), &meta) + require.Error(t, err) + }) + + t.Run("NoBody", func(t *testing.T) { + var meta IssueTemplate + body, err := ExtractMetadata(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest), &meta) + require.NoError(t, err) + assert.Equal(t, "", body) + assert.Equal(t, metaTest, meta) + assert.True(t, meta.Valid()) + }) +} + +func TestExtractMetadataBytes(t *testing.T) { + t.Run("ValidFrontAndBody", func(t *testing.T) { + var meta IssueTemplate + body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest)), &meta) + require.NoError(t, err) + assert.Equal(t, bodyTest, string(body)) + assert.Equal(t, metaTest, meta) + assert.True(t, meta.Valid()) + }) + + t.Run("NoFirstSeparator", func(t *testing.T) { + var meta IssueTemplate + _, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", frontTest, sepTest, bodyTest)), &meta) + require.Error(t, err) + }) + + t.Run("NoLastSeparator", func(t *testing.T) { + var meta IssueTemplate + _, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, bodyTest)), &meta) + require.Error(t, err) + }) + + t.Run("NoBody", func(t *testing.T) { + var meta IssueTemplate + body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest)), &meta) + require.NoError(t, err) + assert.Equal(t, "", string(body)) + assert.Equal(t, metaTest, meta) + assert.True(t, meta.Valid()) + }) +} + +var ( + sepTest = "-----" + frontTest = `name: Test +about: "A Test" +title: "Test Title" +labels: + - bug + - "test label"` + bodyTest = "This is the body" + metaTest = IssueTemplate{ + Name: "Test", + About: "A Test", + Title: "Test Title", + Labels: []string{"bug", "test label"}, + } +) diff --git a/modules/markup/markdown/prefixed_id.go b/modules/markup/markdown/prefixed_id.go new file mode 100644 index 00000000..63d7fadc --- /dev/null +++ b/modules/markup/markdown/prefixed_id.go @@ -0,0 +1,59 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "bytes" + "fmt" + + "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/markup/common" + "code.gitea.io/gitea/modules/util" + + "github.com/yuin/goldmark/ast" +) + +type prefixedIDs struct { + values container.Set[string] +} + +// Generate generates a new element id. +func (p *prefixedIDs) Generate(value []byte, kind ast.NodeKind) []byte { + dft := []byte("id") + if kind == ast.KindHeading { + dft = []byte("heading") + } + return p.GenerateWithDefault(value, dft) +} + +// GenerateWithDefault generates a new element id. +func (p *prefixedIDs) GenerateWithDefault(value, dft []byte) []byte { + result := common.CleanValue(value) + if len(result) == 0 { + result = dft + } + if !bytes.HasPrefix(result, []byte("user-content-")) { + result = append([]byte("user-content-"), result...) + } + if p.values.Add(util.UnsafeBytesToString(result)) { + return result + } + for i := 1; ; i++ { + newResult := fmt.Sprintf("%s-%d", result, i) + if p.values.Add(newResult) { + return []byte(newResult) + } + } +} + +// Put puts a given element id to the used ids table. +func (p *prefixedIDs) Put(value []byte) { + p.values.Add(util.UnsafeBytesToString(value)) +} + +func newPrefixedIDs() *prefixedIDs { + return &prefixedIDs{ + values: make(container.Set[string]), + } +} diff --git a/modules/markup/markdown/renderconfig.go b/modules/markup/markdown/renderconfig.go new file mode 100644 index 00000000..f4c48d1b --- /dev/null +++ b/modules/markup/markdown/renderconfig.go @@ -0,0 +1,126 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "fmt" + "strings" + + "code.gitea.io/gitea/modules/markup" + + "github.com/yuin/goldmark/ast" + "gopkg.in/yaml.v3" +) + +// RenderConfig represents rendering configuration for this file +type RenderConfig struct { + Meta markup.RenderMetaMode + Icon string + TOC string // "false": hide, "side"/empty: in sidebar, "main"/"true": in main view + Lang string + yamlNode *yaml.Node + + // Used internally. Cannot be controlled by frontmatter. + metaLength int +} + +func renderMetaModeFromString(s string) markup.RenderMetaMode { + switch strings.TrimSpace(strings.ToLower(s)) { + case "none": + return markup.RenderMetaAsNone + case "table": + return markup.RenderMetaAsTable + default: // "details" + return markup.RenderMetaAsDetails + } +} + +// UnmarshalYAML implement yaml.v3 UnmarshalYAML +func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { + if rc == nil { + return nil + } + + rc.yamlNode = value + + type commonRenderConfig struct { + TOC string `yaml:"include_toc"` + Lang string `yaml:"lang"` + } + var basic commonRenderConfig + if err := value.Decode(&basic); err != nil { + return fmt.Errorf("unable to decode into commonRenderConfig %w", err) + } + + if basic.Lang != "" { + rc.Lang = basic.Lang + } + + rc.TOC = basic.TOC + + type controlStringRenderConfig struct { + Gitea string `yaml:"gitea"` + } + + var stringBasic controlStringRenderConfig + + if err := value.Decode(&stringBasic); err == nil { + if stringBasic.Gitea != "" { + rc.Meta = renderMetaModeFromString(stringBasic.Gitea) + } + return nil + } + + type yamlRenderConfig struct { + Meta *string `yaml:"meta"` + Icon *string `yaml:"details_icon"` + TOC *string `yaml:"include_toc"` + Lang *string `yaml:"lang"` + } + + type yamlRenderConfigWrapper struct { + Gitea *yamlRenderConfig `yaml:"gitea"` + } + + var cfg yamlRenderConfigWrapper + if err := value.Decode(&cfg); err != nil { + return fmt.Errorf("unable to decode into yamlRenderConfigWrapper %w", err) + } + + if cfg.Gitea == nil { + return nil + } + + if cfg.Gitea.Meta != nil { + rc.Meta = renderMetaModeFromString(*cfg.Gitea.Meta) + } + + if cfg.Gitea.Icon != nil { + rc.Icon = strings.TrimSpace(strings.ToLower(*cfg.Gitea.Icon)) + } + + if cfg.Gitea.Lang != nil && *cfg.Gitea.Lang != "" { + rc.Lang = *cfg.Gitea.Lang + } + + if cfg.Gitea.TOC != nil { + rc.TOC = *cfg.Gitea.TOC + } + + return nil +} + +func (rc *RenderConfig) toMetaNode() ast.Node { + if rc.yamlNode == nil { + return nil + } + switch rc.Meta { + case markup.RenderMetaAsTable: + return nodeToTable(rc.yamlNode) + case markup.RenderMetaAsDetails: + return nodeToDetails(rc.yamlNode, rc.Icon) + default: + return nil + } +} diff --git a/modules/markup/markdown/renderconfig_test.go b/modules/markup/markdown/renderconfig_test.go new file mode 100644 index 00000000..c53acdc7 --- /dev/null +++ b/modules/markup/markdown/renderconfig_test.go @@ -0,0 +1,162 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestRenderConfig_UnmarshalYAML(t *testing.T) { + tests := []struct { + name string + expected *RenderConfig + args string + }{ + { + "empty", &RenderConfig{ + Meta: "table", + Icon: "table", + Lang: "", + }, "", + }, + { + "lang", &RenderConfig{ + Meta: "table", + Icon: "table", + Lang: "test", + }, "lang: test", + }, + { + "metatable", &RenderConfig{ + Meta: "table", + Icon: "table", + Lang: "", + }, "gitea: table", + }, + { + "metanone", &RenderConfig{ + Meta: "none", + Icon: "table", + Lang: "", + }, "gitea: none", + }, + { + "metadetails", &RenderConfig{ + Meta: "details", + Icon: "table", + Lang: "", + }, "gitea: details", + }, + { + "metawrong", &RenderConfig{ + Meta: "details", + Icon: "table", + Lang: "", + }, "gitea: wrong", + }, + { + "toc", &RenderConfig{ + TOC: "true", + Meta: "table", + Icon: "table", + Lang: "", + }, "include_toc: true", + }, + { + "tocfalse", &RenderConfig{ + TOC: "false", + Meta: "table", + Icon: "table", + Lang: "", + }, "include_toc: false", + }, + { + "toclang", &RenderConfig{ + Meta: "table", + Icon: "table", + TOC: "true", + Lang: "testlang", + }, ` + include_toc: true + lang: testlang + `, + }, + { + "complexlang", &RenderConfig{ + Meta: "table", + Icon: "table", + Lang: "testlang", + }, ` + gitea: + lang: testlang + `, + }, + { + "complexlang2", &RenderConfig{ + Meta: "table", + Icon: "table", + Lang: "testlang", + }, ` + lang: notright + gitea: + lang: testlang +`, + }, + { + "complexlang", &RenderConfig{ + Meta: "table", + Icon: "table", + Lang: "testlang", + }, ` + gitea: + lang: testlang +`, + }, + { + "complex2", &RenderConfig{ + Lang: "two", + Meta: "table", + TOC: "true", + Icon: "smiley", + }, ` + lang: one + include_toc: true + gitea: + details_icon: smiley + meta: table + include_toc: true + lang: two +`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := &RenderConfig{ + Meta: "table", + Icon: "table", + Lang: "", + } + if err := yaml.Unmarshal([]byte(strings.ReplaceAll(tt.args, "\t", " ")), got); err != nil { + t.Errorf("RenderConfig.UnmarshalYAML() error = %v\n%q", err, tt.args) + return + } + + if got.Meta != tt.expected.Meta { + t.Errorf("Meta Expected %s Got %s", tt.expected.Meta, got.Meta) + } + if got.Icon != tt.expected.Icon { + t.Errorf("Icon Expected %s Got %s", tt.expected.Icon, got.Icon) + } + if got.Lang != tt.expected.Lang { + t.Errorf("Lang Expected %s Got %s", tt.expected.Lang, got.Lang) + } + if got.TOC != tt.expected.TOC { + t.Errorf("TOC Expected %q Got %q", tt.expected.TOC, got.TOC) + } + }) + } +} diff --git a/modules/markup/markdown/toc.go b/modules/markup/markdown/toc.go new file mode 100644 index 00000000..38f744a2 --- /dev/null +++ b/modules/markup/markdown/toc.go @@ -0,0 +1,54 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "fmt" + "net/url" + + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/translation" + + "github.com/yuin/goldmark/ast" +) + +func createTOCNode(toc []markup.Header, lang string, detailsAttrs map[string]string) ast.Node { + details := NewDetails() + summary := NewSummary() + + for k, v := range detailsAttrs { + details.SetAttributeString(k, []byte(v)) + } + + summary.AppendChild(summary, ast.NewString([]byte(translation.NewLocale(lang).TrString("toc")))) + details.AppendChild(details, summary) + ul := ast.NewList('-') + details.AppendChild(details, ul) + currentLevel := 6 + for _, header := range toc { + if header.Level < currentLevel { + currentLevel = header.Level + } + } + for _, header := range toc { + for currentLevel > header.Level { + ul = ul.Parent().(*ast.List) + currentLevel-- + } + for currentLevel < header.Level { + newL := ast.NewList('-') + ul.AppendChild(ul, newL) + currentLevel++ + ul = newL + } + li := ast.NewListItem(currentLevel * 2) + a := ast.NewLink() + a.Destination = []byte(fmt.Sprintf("#%s", url.QueryEscape(header.ID))) + a.AppendChild(a, ast.NewString([]byte(header.Text))) + li.AppendChild(li, a) + ul.AppendChild(ul, li) + } + + return details +} diff --git a/modules/markup/markdown/transform_codespan.go b/modules/markup/markdown/transform_codespan.go new file mode 100644 index 00000000..a2cd4fb5 --- /dev/null +++ b/modules/markup/markdown/transform_codespan.go @@ -0,0 +1,56 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "bytes" + "fmt" + "strings" + + "code.gitea.io/gitea/modules/markup" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +// renderCodeSpan renders CodeSpan elements (like goldmark upstream does) but also renders ColorPreview elements. +// See #21474 for reference +func (r *HTMLRenderer) renderCodeSpan(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + if n.Attributes() != nil { + _, _ = w.WriteString("<code") + html.RenderAttributes(w, n, html.CodeAttributeFilter) + _ = w.WriteByte('>') + } else { + _, _ = w.WriteString("<code>") + } + for c := n.FirstChild(); c != nil; c = c.NextSibling() { + switch v := c.(type) { + case *ast.Text: + segment := v.Segment + value := segment.Value(source) + if bytes.HasSuffix(value, []byte("\n")) { + r.Writer.RawWrite(w, value[:len(value)-1]) + r.Writer.RawWrite(w, []byte(" ")) + } else { + r.Writer.RawWrite(w, value) + } + case *ColorPreview: + _, _ = w.WriteString(fmt.Sprintf(`<span class="color-preview" style="background-color: %v"></span>`, string(v.Color))) + } + } + return ast.WalkSkipChildren, nil + } + _, _ = w.WriteString("</code>") + return ast.WalkContinue, nil +} + +func (g *ASTTransformer) transformCodeSpan(_ *markup.RenderContext, v *ast.CodeSpan, reader text.Reader) { + colorContent := v.Text(reader.Source()) + if matchColor(strings.ToLower(string(colorContent))) { + v.AppendChild(v, NewColorPreview(colorContent)) + } +} diff --git a/modules/markup/markdown/transform_heading.go b/modules/markup/markdown/transform_heading.go new file mode 100644 index 00000000..6d48f34d --- /dev/null +++ b/modules/markup/markdown/transform_heading.go @@ -0,0 +1,32 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "fmt" + + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/util" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" +) + +func (g *ASTTransformer) transformHeading(_ *markup.RenderContext, v *ast.Heading, reader text.Reader, tocList *[]markup.Header) { + for _, attr := range v.Attributes() { + if _, ok := attr.Value.([]byte); !ok { + v.SetAttribute(attr.Name, []byte(fmt.Sprintf("%v", attr.Value))) + } + } + txt := v.Text(reader.Source()) + header := markup.Header{ + Text: util.UnsafeBytesToString(txt), + Level: v.Level, + } + if id, found := v.AttributeString("id"); found { + header.ID = util.UnsafeBytesToString(id.([]byte)) + } + *tocList = append(*tocList, header) + g.applyElementDir(v) +} diff --git a/modules/markup/markdown/transform_image.go b/modules/markup/markdown/transform_image.go new file mode 100644 index 00000000..b34a710f --- /dev/null +++ b/modules/markup/markdown/transform_image.go @@ -0,0 +1,65 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "strings" + + "code.gitea.io/gitea/modules/markup" + giteautil "code.gitea.io/gitea/modules/util" + + "github.com/yuin/goldmark/ast" +) + +func (g *ASTTransformer) transformImage(ctx *markup.RenderContext, v *ast.Image) { + // Images need two things: + // + // 1. Their src needs to munged to be a real value + // 2. If they're not wrapped with a link they need a link wrapper + + // Check if the destination is a real link + if len(v.Destination) > 0 && !markup.IsLink(v.Destination) { + v.Destination = []byte(giteautil.URLJoin( + ctx.Links.ResolveMediaLink(ctx.IsWiki), + strings.TrimLeft(string(v.Destination), "/"), + )) + } + + parent := v.Parent() + // Create a link around image only if parent is not already a link + if _, ok := parent.(*ast.Link); !ok && parent != nil { + next := v.NextSibling() + + // Create a link wrapper + wrap := ast.NewLink() + wrap.Destination = v.Destination + wrap.Title = v.Title + wrap.SetAttributeString("target", []byte("_blank")) + + // Duplicate the current image node + image := ast.NewImage(ast.NewLink()) + image.Destination = v.Destination + image.Title = v.Title + for _, attr := range v.Attributes() { + image.SetAttribute(attr.Name, attr.Value) + } + for child := v.FirstChild(); child != nil; { + next := child.NextSibling() + image.AppendChild(image, child) + child = next + } + + // Append our duplicate image to the wrapper link + wrap.AppendChild(wrap, image) + + // Wire in the next sibling + wrap.SetNextSibling(next) + + // Replace the current node with the wrapper link + parent.ReplaceChild(parent, v, wrap) + + // But most importantly ensure the next sibling is still on the old image too + v.SetNextSibling(next) + } +} diff --git a/modules/markup/markdown/transform_link.go b/modules/markup/markdown/transform_link.go new file mode 100644 index 00000000..e6f38364 --- /dev/null +++ b/modules/markup/markdown/transform_link.go @@ -0,0 +1,46 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "bytes" + "slices" + + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/setting" + giteautil "code.gitea.io/gitea/modules/util" + + "github.com/yuin/goldmark/ast" +) + +func (g *ASTTransformer) transformLink(ctx *markup.RenderContext, v *ast.Link) { + // Links need their href to munged to be a real value + link := v.Destination + + // Do not process the link if it's not a link, starts with an hashtag + // (indicating it's an anchor link), starts with `mailto:` or any of the + // custom markdown URLs. + processLink := len(link) > 0 && !markup.IsLink(link) && + link[0] != '#' && !bytes.HasPrefix(link, byteMailto) && + !slices.ContainsFunc(setting.Markdown.CustomURLSchemes, func(s string) bool { + return bytes.HasPrefix(link, []byte(s+":")) + }) + + if processLink { + var base string + if ctx.IsWiki { + base = ctx.Links.WikiLink() + } else if ctx.Links.HasBranchInfo() { + base = ctx.Links.SrcLink() + } else { + base = ctx.Links.Base + } + + link = []byte(giteautil.URLJoin(base, string(link))) + } + if len(link) > 0 && link[0] == '#' { + link = []byte("#user-content-" + string(link)[1:]) + } + v.Destination = link +} diff --git a/modules/markup/markdown/transform_list.go b/modules/markup/markdown/transform_list.go new file mode 100644 index 00000000..b982fd4a --- /dev/null +++ b/modules/markup/markdown/transform_list.go @@ -0,0 +1,85 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markdown + +import ( + "fmt" + + "code.gitea.io/gitea/modules/markup" + + "github.com/yuin/goldmark/ast" + east "github.com/yuin/goldmark/extension/ast" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/util" +) + +func (r *HTMLRenderer) renderTaskCheckBoxListItem(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*TaskCheckBoxListItem) + if entering { + if n.Attributes() != nil { + _, _ = w.WriteString("<li") + html.RenderAttributes(w, n, html.ListItemAttributeFilter) + _ = w.WriteByte('>') + } else { + _, _ = w.WriteString("<li>") + } + fmt.Fprintf(w, `<input type="checkbox" disabled="" data-source-position="%d"`, n.SourcePosition) + if n.IsChecked { + _, _ = w.WriteString(` checked=""`) + } + if r.XHTML { + _, _ = w.WriteString(` />`) + } else { + _ = w.WriteByte('>') + } + fc := n.FirstChild() + if fc != nil { + if _, ok := fc.(*ast.TextBlock); !ok { + _ = w.WriteByte('\n') + } + } + } else { + _, _ = w.WriteString("</li>\n") + } + return ast.WalkContinue, nil +} + +func (r *HTMLRenderer) renderTaskCheckBox(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + return ast.WalkContinue, nil +} + +func (g *ASTTransformer) transformList(_ *markup.RenderContext, v *ast.List, rc *RenderConfig) { + if v.HasChildren() { + children := make([]ast.Node, 0, v.ChildCount()) + child := v.FirstChild() + for child != nil { + children = append(children, child) + child = child.NextSibling() + } + v.RemoveChildren(v) + + for _, child := range children { + listItem := child.(*ast.ListItem) + if !child.HasChildren() || !child.FirstChild().HasChildren() { + v.AppendChild(v, child) + continue + } + taskCheckBox, ok := child.FirstChild().FirstChild().(*east.TaskCheckBox) + if !ok { + v.AppendChild(v, child) + continue + } + newChild := NewTaskCheckBoxListItem(listItem) + newChild.IsChecked = taskCheckBox.IsChecked + newChild.SetAttributeString("class", []byte("task-list-item")) + segments := newChild.FirstChild().Lines() + if segments.Len() > 0 { + segment := segments.At(0) + newChild.SourcePosition = rc.metaLength + segment.Start + } + v.AppendChild(v, newChild) + } + } + g.applyElementDir(v) +} diff --git a/modules/markup/mdstripper/mdstripper.go b/modules/markup/mdstripper/mdstripper.go new file mode 100644 index 00000000..2a69d952 --- /dev/null +++ b/modules/markup/mdstripper/mdstripper.go @@ -0,0 +1,199 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package mdstripper + +import ( + "bytes" + "io" + "net/url" + "strings" + "sync" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup/common" + "code.gitea.io/gitea/modules/setting" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" +) + +var ( + giteaHostInit sync.Once + giteaHost *url.URL +) + +type stripRenderer struct { + localhost *url.URL + links []string + empty bool +} + +func (r *stripRenderer) Render(w io.Writer, source []byte, doc ast.Node) error { + return ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + switch v := n.(type) { + case *ast.Text: + if !v.IsRaw() { + _, prevSibIsText := n.PreviousSibling().(*ast.Text) + coalesce := prevSibIsText + r.processString( + w, + v.Text(source), + coalesce) + if v.SoftLineBreak() { + r.doubleSpace(w) + } + } + return ast.WalkContinue, nil + case *ast.Link: + r.processLink(v.Destination) + return ast.WalkSkipChildren, nil + case *ast.AutoLink: + // This could be a reference to an issue or pull - if so convert it + r.processAutoLink(w, v.URL(source)) + return ast.WalkSkipChildren, nil + } + return ast.WalkContinue, nil + }) +} + +func (r *stripRenderer) doubleSpace(w io.Writer) { + if !r.empty { + _, _ = w.Write([]byte{'\n'}) + } +} + +func (r *stripRenderer) processString(w io.Writer, text []byte, coalesce bool) { + // Always break-up words + if !coalesce { + r.doubleSpace(w) + } + _, _ = w.Write(text) + r.empty = false +} + +// ProcessAutoLinks to detect and handle links to issues and pulls +func (r *stripRenderer) processAutoLink(w io.Writer, link []byte) { + linkStr := string(link) + u, err := url.Parse(linkStr) + if err != nil { + // Process out of band + r.links = append(r.links, linkStr) + return + } + + // Note: we're not attempting to match the URL scheme (http/https) + host := strings.ToLower(u.Host) + if host != "" && host != strings.ToLower(r.localhost.Host) { + // Process out of band + r.links = append(r.links, linkStr) + return + } + + // We want: /user/repo/issues/3 + parts := strings.Split(strings.TrimPrefix(u.EscapedPath(), r.localhost.EscapedPath()), "/") + if len(parts) != 5 || parts[0] != "" { + // Process out of band + r.links = append(r.links, linkStr) + return + } + + var sep string + if parts[3] == "issues" { + sep = "#" + } else if parts[3] == "pulls" { + sep = "!" + } else { + // Process out of band + r.links = append(r.links, linkStr) + return + } + + _, _ = w.Write([]byte(parts[1])) + _, _ = w.Write([]byte("/")) + _, _ = w.Write([]byte(parts[2])) + _, _ = w.Write([]byte(sep)) + _, _ = w.Write([]byte(parts[4])) +} + +func (r *stripRenderer) processLink(link []byte) { + // Links are processed out of band + r.links = append(r.links, string(link)) +} + +// GetLinks returns the list of link data collected while parsing +func (r *stripRenderer) GetLinks() []string { + return r.links +} + +// AddOptions adds given option to this renderer. +func (r *stripRenderer) AddOptions(...renderer.Option) { + // no-op +} + +// StripMarkdown parses markdown content by removing all markup and code blocks +// in order to extract links and other references +func StripMarkdown(rawBytes []byte) (string, []string) { + buf, links := StripMarkdownBytes(rawBytes) + return string(buf), links +} + +var ( + stripParser parser.Parser + once = sync.Once{} +) + +// StripMarkdownBytes parses markdown content by removing all markup and code blocks +// in order to extract links and other references +func StripMarkdownBytes(rawBytes []byte) ([]byte, []string) { + once.Do(func() { + gdMarkdown := goldmark.New( + goldmark.WithExtensions(extension.Table, + extension.Strikethrough, + extension.TaskList, + extension.DefinitionList, + common.FootnoteExtension, + common.Linkify, + ), + goldmark.WithParserOptions( + parser.WithAttribute(), + parser.WithAutoHeadingID(), + ), + goldmark.WithRendererOptions( + html.WithUnsafe(), + ), + ) + stripParser = gdMarkdown.Parser() + }) + stripper := &stripRenderer{ + localhost: getGiteaHost(), + links: make([]string, 0, 10), + empty: true, + } + reader := text.NewReader(rawBytes) + doc := stripParser.Parse(reader) + var buf bytes.Buffer + if err := stripper.Render(&buf, rawBytes, doc); err != nil { + log.Error("Unable to strip: %v", err) + } + return buf.Bytes(), stripper.GetLinks() +} + +// getGiteaHostName returns a normalized string with the local host name, with no scheme or port information +func getGiteaHost() *url.URL { + giteaHostInit.Do(func() { + var err error + if giteaHost, err = url.Parse(setting.AppURL); err != nil { + giteaHost = &url.URL{} + } + }) + return giteaHost +} diff --git a/modules/markup/mdstripper/mdstripper_test.go b/modules/markup/mdstripper/mdstripper_test.go new file mode 100644 index 00000000..ea34df0a --- /dev/null +++ b/modules/markup/mdstripper/mdstripper_test.go @@ -0,0 +1,85 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package mdstripper + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMarkdownStripper(t *testing.T) { + type testItem struct { + markdown string + expectedText []string + expectedLinks []string + } + + list := []testItem{ + { + ` +## This is a title + +This is [one](link) to paradise. +This **is emphasized**. +This: should coalesce. + +` + "```" + ` +This is a code block. +This should not appear in the output at all. +` + "```" + ` + +* Bullet 1 +* Bullet 2 + +A HIDDEN ` + "`" + `GHOST` + "`" + ` IN THIS LINE. + `, + []string{ + "This is a title", + "This is", + "to paradise.", + "This", + "is emphasized", + ".", + "This: should coalesce.", + "Bullet 1", + "Bullet 2", + "A HIDDEN", + "IN THIS LINE.", + }, + []string{ + "link", + }, + }, + { + "Simply closes: #29 yes", + []string{ + "Simply closes: #29 yes", + }, + []string{}, + }, + { + "Simply closes: !29 yes", + []string{ + "Simply closes: !29 yes", + }, + []string{}, + }, + } + + for _, test := range list { + text, links := StripMarkdown([]byte(test.markdown)) + rawlines := strings.Split(text, "\n") + lines := make([]string, 0, len(rawlines)) + for _, line := range rawlines { + line := strings.TrimSpace(line) + if line != "" { + lines = append(lines, line) + } + } + assert.EqualValues(t, test.expectedText, lines) + assert.EqualValues(t, test.expectedLinks, links) + } +} diff --git a/modules/markup/orgmode/orgmode.go b/modules/markup/orgmode/orgmode.go new file mode 100644 index 00000000..391ee6c1 --- /dev/null +++ b/modules/markup/orgmode/orgmode.go @@ -0,0 +1,196 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markup + +import ( + "fmt" + "html" + "io" + "strings" + + "code.gitea.io/gitea/modules/highlight" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + + "github.com/alecthomas/chroma/v2" + "github.com/alecthomas/chroma/v2/lexers" + "github.com/niklasfasching/go-org/org" +) + +func init() { + markup.RegisterRenderer(Renderer{}) +} + +// Renderer implements markup.Renderer for orgmode +type Renderer struct{} + +var _ markup.PostProcessRenderer = (*Renderer)(nil) + +// Name implements markup.Renderer +func (Renderer) Name() string { + return "orgmode" +} + +// NeedPostProcess implements markup.PostProcessRenderer +func (Renderer) NeedPostProcess() bool { return true } + +// Extensions implements markup.Renderer +func (Renderer) Extensions() []string { + return []string{".org"} +} + +// SanitizerRules implements markup.Renderer +func (Renderer) SanitizerRules() []setting.MarkupSanitizerRule { + return []setting.MarkupSanitizerRule{} +} + +// Render renders orgmode rawbytes to HTML +func Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + htmlWriter := org.NewHTMLWriter() + htmlWriter.HighlightCodeBlock = func(source, lang string, inline bool, params map[string]string) string { + defer func() { + if err := recover(); err != nil { + log.Error("Panic in HighlightCodeBlock: %v\n%s", err, log.Stack(2)) + panic(err) + } + }() + var w strings.Builder + if _, err := w.WriteString(`<pre>`); err != nil { + return "" + } + + lexer := lexers.Get(lang) + if lexer == nil && lang == "" { + lexer = lexers.Analyse(source) + if lexer == nil { + lexer = lexers.Fallback + } + lang = strings.ToLower(lexer.Config().Name) + } + + if lexer == nil { + // include language-x class as part of commonmark spec + if _, err := w.WriteString(`<code class="chroma language-` + lang + `">`); err != nil { + return "" + } + if _, err := w.WriteString(html.EscapeString(source)); err != nil { + return "" + } + } else { + // include language-x class as part of commonmark spec + if _, err := w.WriteString(`<code class="chroma language-` + lang + `">`); err != nil { + return "" + } + lexer = chroma.Coalesce(lexer) + + if _, err := w.WriteString(string(highlight.CodeFromLexer(lexer, source))); err != nil { + return "" + } + } + + if _, err := w.WriteString("</code></pre>"); err != nil { + return "" + } + + return w.String() + } + + w := &Writer{ + HTMLWriter: htmlWriter, + Ctx: ctx, + } + + htmlWriter.ExtendingWriter = w + + res, err := org.New().Silent().Parse(input, "").Write(w) + if err != nil { + return fmt.Errorf("orgmode.Render failed: %w", err) + } + _, err = io.Copy(output, strings.NewReader(res)) + return err +} + +// RenderString renders orgmode string to HTML string +func RenderString(ctx *markup.RenderContext, content string) (string, error) { + var buf strings.Builder + if err := Render(ctx, strings.NewReader(content), &buf); err != nil { + return "", err + } + return buf.String(), nil +} + +// Render renders orgmode string to HTML string +func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + return Render(ctx, input, output) +} + +// Writer implements org.Writer +type Writer struct { + *org.HTMLWriter + Ctx *markup.RenderContext +} + +const mailto = "mailto:" + +func (r *Writer) resolveLink(node org.Node) string { + l, ok := node.(org.RegularLink) + if !ok { + l = org.RegularLink{URL: strings.TrimPrefix(org.String(node), "file:")} + } + + link := html.EscapeString(l.URL) + if l.Protocol == "file" { + link = link[len("file:"):] + } + if len(link) > 0 && !markup.IsLinkStr(link) && + link[0] != '#' && !strings.HasPrefix(link, mailto) { + var base string + if r.Ctx.IsWiki { + base = r.Ctx.Links.WikiLink() + } else if r.Ctx.Links.HasBranchInfo() { + base = r.Ctx.Links.SrcLink() + } else { + base = r.Ctx.Links.Base + } + + switch l.Kind() { + case "image", "video": + base = r.Ctx.Links.ResolveMediaLink(r.Ctx.IsWiki) + } + + link = util.URLJoin(base, link) + } + return link +} + +// WriteRegularLink renders images, links or videos +func (r *Writer) WriteRegularLink(l org.RegularLink) { + link := r.resolveLink(l) + + // Inspired by https://github.com/niklasfasching/go-org/blob/6eb20dbda93cb88c3503f7508dc78cbbc639378f/org/html_writer.go#L406-L427 + switch l.Kind() { + case "image": + if l.Description == nil { + fmt.Fprintf(r, `<img src="%s" alt="%s" />`, link, link) + } else { + imageSrc := r.resolveLink(l.Description[0]) + fmt.Fprintf(r, `<a href="%s"><img src="%s" alt="%s" /></a>`, link, imageSrc, imageSrc) + } + case "video": + if l.Description == nil { + fmt.Fprintf(r, `<video src="%s">%s</video>`, link, link) + } else { + videoSrc := r.resolveLink(l.Description[0]) + fmt.Fprintf(r, `<a href="%s"><video src="%s">%s</video></a>`, link, videoSrc, videoSrc) + } + default: + description := link + if l.Description != nil { + description = r.WriteNodesAsString(l.Description...) + } + fmt.Fprintf(r, `<a href="%s">%s</a>`, link, description) + } +} diff --git a/modules/markup/orgmode/orgmode_test.go b/modules/markup/orgmode/orgmode_test.go new file mode 100644 index 00000000..f41d86a8 --- /dev/null +++ b/modules/markup/orgmode/orgmode_test.go @@ -0,0 +1,160 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markup + +import ( + "strings" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + AppURL = "http://localhost:3000/" + Repo = "gogits/gogs" + AppSubURL = AppURL + Repo + "/" +) + +func TestRender_StandardLinks(t *testing.T) { + setting.AppURL = AppURL + setting.AppSubURL = AppSubURL + + test := func(input, expected string) { + buffer, err := RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: setting.AppSubURL, + }, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) + } + + // No BranchPath or TreePath set. + test("[[file:comfy][comfy]]", + `<p><a href="http://localhost:3000/gogits/gogs/comfy">comfy</a></p>`) + + test("[[https://google.com/]]", + `<p><a href="https://google.com/">https://google.com/</a></p>`) + + lnk := util.URLJoin(AppSubURL, "WikiPage") + test("[[WikiPage][WikiPage]]", + `<p><a href="`+lnk+`">WikiPage</a></p>`) +} + +func TestRender_BaseLinks(t *testing.T) { + setting.AppURL = AppURL + setting.AppSubURL = AppSubURL + + testBranch := func(input, expected string) { + buffer, err := RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: setting.AppSubURL, + BranchPath: "branch/main", + }, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) + } + + testBranchTree := func(input, expected string) { + buffer, err := RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: setting.AppSubURL, + BranchPath: "branch/main", + TreePath: "deep/nested/folder", + }, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) + } + + testBranch("[[file:comfy][comfy]]", + `<p><a href="http://localhost:3000/gogits/gogs/src/branch/main/comfy">comfy</a></p>`) + testBranchTree("[[file:comfy][comfy]]", + `<p><a href="http://localhost:3000/gogits/gogs/src/branch/main/deep/nested/folder/comfy">comfy</a></p>`) + + testBranch("[[file:./src][./src/]]", + `<p><a href="http://localhost:3000/gogits/gogs/src/branch/main/src">./src/</a></p>`) + testBranchTree("[[file:./src][./src/]]", + `<p><a href="http://localhost:3000/gogits/gogs/src/branch/main/deep/nested/folder/src">./src/</a></p>`) +} + +func TestRender_Media(t *testing.T) { + setting.AppURL = AppURL + setting.AppSubURL = AppSubURL + + test := func(input, expected string) { + buffer, err := RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + Links: markup.Links{ + Base: setting.AppSubURL, + }, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) + } + + url := "../../.images/src/02/train.jpg" + result := util.URLJoin(AppSubURL, url) + + test("[[file:"+url+"]]", + `<p><img src="`+result+`" alt="`+result+`" /></p>`) + + // With description. + test("[[https://example.com][https://example.com/example.svg]]", + `<p><a href="https://example.com"><img src="https://example.com/example.svg" alt="https://example.com/example.svg" /></a></p>`) + test("[[https://example.com][pre https://example.com/example.svg post]]", + `<p><a href="https://example.com">pre <img src="https://example.com/example.svg" alt="https://example.com/example.svg" /> post</a></p>`) + test("[[https://example.com][https://example.com/example.mp4]]", + `<p><a href="https://example.com"><video src="https://example.com/example.mp4">https://example.com/example.mp4</video></a></p>`) + test("[[https://example.com][pre https://example.com/example.mp4 post]]", + `<p><a href="https://example.com">pre <video src="https://example.com/example.mp4">https://example.com/example.mp4</video> post</a></p>`) + + // Without description. + test("[[https://example.com/example.svg]]", + `<p><img src="https://example.com/example.svg" alt="https://example.com/example.svg" /></p>`) + test("[[https://example.com/example.mp4]]", + `<p><video src="https://example.com/example.mp4">https://example.com/example.mp4</video></p>`) + + // Text description. + test("[[file:./lem-post.png][file:./lem-post.png]]", + `<p><a href="http://localhost:3000/gogits/gogs/lem-post.png"><img src="http://localhost:3000/gogits/gogs/lem-post.png" alt="http://localhost:3000/gogits/gogs/lem-post.png" /></a></p>`) + test("[[file:./lem-post.mp4][file:./lem-post.mp4]]", + `<p><a href="http://localhost:3000/gogits/gogs/lem-post.mp4"><video src="http://localhost:3000/gogits/gogs/lem-post.mp4">http://localhost:3000/gogits/gogs/lem-post.mp4</video></a></p>`) +} + +func TestRender_Source(t *testing.T) { + setting.AppURL = AppURL + setting.AppSubURL = AppSubURL + + test := func(input, expected string) { + buffer, err := RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, + }, input) + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) + } + + test(`#+begin_src go +// HelloWorld prints "Hello World" +func HelloWorld() { + fmt.Println("Hello World") +} +#+end_src +`, `<div class="src src-go"> +<pre><code class="chroma language-go"><span class="c1">// HelloWorld prints "Hello World" +</span><span class="c1"></span><span class="kd">func</span> <span class="nf">HelloWorld</span><span class="p">()</span> <span class="p">{</span> + <span class="nx">fmt</span><span class="p">.</span><span class="nf">Println</span><span class="p">(</span><span class="s">"Hello World"</span><span class="p">)</span> +<span class="p">}</span></code></pre> +</div>`) +} diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go new file mode 100644 index 00000000..f1beee96 --- /dev/null +++ b/modules/markup/renderer.go @@ -0,0 +1,394 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markup + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/url" + "path/filepath" + "strings" + "sync" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + + "github.com/yuin/goldmark/ast" +) + +type RenderMetaMode string + +const ( + RenderMetaAsDetails RenderMetaMode = "details" // default + RenderMetaAsNone RenderMetaMode = "none" + RenderMetaAsTable RenderMetaMode = "table" +) + +type ProcessorHelper struct { + IsUsernameMentionable func(ctx context.Context, username string) bool + GetRepoFileBlob func(ctx context.Context, ownerName, repoName, commitSha, filePath string, language *string) (*git.Blob, error) + + ElementDir string // the direction of the elements, eg: "ltr", "rtl", "auto", default to no direction attribute +} + +var DefaultProcessorHelper ProcessorHelper + +// Init initialize regexps for markdown parsing +func Init(ph *ProcessorHelper) { + if ph != nil { + DefaultProcessorHelper = *ph + } + + NewSanitizer() + if len(setting.Markdown.CustomURLSchemes) > 0 { + CustomLinkURLSchemes(setting.Markdown.CustomURLSchemes) + } + + // since setting maybe changed extensions, this will reload all renderer extensions mapping + extRenderers = make(map[string]Renderer) + for _, renderer := range renderers { + for _, ext := range renderer.Extensions() { + extRenderers[strings.ToLower(ext)] = renderer + } + } +} + +// Header holds the data about a header. +type Header struct { + Level int + Text string + ID string +} + +// RenderContext represents a render context +type RenderContext struct { + Ctx context.Context + RelativePath string // relative path from tree root of the branch + Type string + IsWiki bool + Links Links + Metas map[string]string + DefaultLink string + GitRepo *git.Repository + ShaExistCache map[string]bool + cancelFn func() + SidebarTocNode ast.Node + RenderMetaAs RenderMetaMode + InStandalonePage bool // used by external render. the router "/org/repo/render/..." will output the rendered content in a standalone page +} + +type Links struct { + AbsolutePrefix bool + Base string + BranchPath string + TreePath string +} + +func (l *Links) Prefix() string { + if l.AbsolutePrefix { + return setting.AppURL + } + return setting.AppSubURL +} + +func (l *Links) HasBranchInfo() bool { + return l.BranchPath != "" +} + +func (l *Links) SrcLink() string { + return util.URLJoin(l.Base, "src", l.BranchPath, l.TreePath) +} + +func (l *Links) MediaLink() string { + return util.URLJoin(l.Base, "media", l.BranchPath, l.TreePath) +} + +func (l *Links) RawLink() string { + return util.URLJoin(l.Base, "raw", l.BranchPath, l.TreePath) +} + +func (l *Links) WikiLink() string { + return util.URLJoin(l.Base, "wiki") +} + +func (l *Links) WikiRawLink() string { + return util.URLJoin(l.Base, "wiki/raw") +} + +func (l *Links) ResolveMediaLink(isWiki bool) string { + if isWiki { + return l.WikiRawLink() + } else if l.HasBranchInfo() { + return l.MediaLink() + } + return l.Base +} + +// Cancel runs any cleanup functions that have been registered for this Ctx +func (ctx *RenderContext) Cancel() { + if ctx == nil { + return + } + ctx.ShaExistCache = map[string]bool{} + if ctx.cancelFn == nil { + return + } + ctx.cancelFn() +} + +// AddCancel adds the provided fn as a Cleanup for this Ctx +func (ctx *RenderContext) AddCancel(fn func()) { + if ctx == nil { + return + } + oldCancelFn := ctx.cancelFn + if oldCancelFn == nil { + ctx.cancelFn = fn + return + } + ctx.cancelFn = func() { + defer oldCancelFn() + fn() + } +} + +// Renderer defines an interface for rendering markup file to HTML +type Renderer interface { + Name() string // markup format name + Extensions() []string + SanitizerRules() []setting.MarkupSanitizerRule + Render(ctx *RenderContext, input io.Reader, output io.Writer) error +} + +// PostProcessRenderer defines an interface for renderers who need post process +type PostProcessRenderer interface { + NeedPostProcess() bool +} + +// PostProcessRenderer defines an interface for external renderers +type ExternalRenderer interface { + // SanitizerDisabled disabled sanitize if return true + SanitizerDisabled() bool + + // DisplayInIFrame represents whether render the content with an iframe + DisplayInIFrame() bool +} + +// RendererContentDetector detects if the content can be rendered +// by specified renderer +type RendererContentDetector interface { + CanRender(filename string, input io.Reader) bool +} + +var ( + extRenderers = make(map[string]Renderer) + renderers = make(map[string]Renderer) +) + +// RegisterRenderer registers a new markup file renderer +func RegisterRenderer(renderer Renderer) { + renderers[renderer.Name()] = renderer + for _, ext := range renderer.Extensions() { + extRenderers[strings.ToLower(ext)] = renderer + } +} + +// GetRendererByFileName get renderer by filename +func GetRendererByFileName(filename string) Renderer { + extension := strings.ToLower(filepath.Ext(filename)) + return extRenderers[extension] +} + +// GetRendererByType returns a renderer according type +func GetRendererByType(tp string) Renderer { + return renderers[tp] +} + +// DetectRendererType detects the markup type of the content +func DetectRendererType(filename string, input io.Reader) string { + buf, err := io.ReadAll(input) + if err != nil { + return "" + } + for _, renderer := range renderers { + if detector, ok := renderer.(RendererContentDetector); ok && detector.CanRender(filename, bytes.NewReader(buf)) { + return renderer.Name() + } + } + return "" +} + +// Render renders markup file to HTML with all specific handling stuff. +func Render(ctx *RenderContext, input io.Reader, output io.Writer) error { + if ctx.Type != "" { + return renderByType(ctx, input, output) + } else if ctx.RelativePath != "" { + return renderFile(ctx, input, output) + } + return errors.New("Render options both filename and type missing") +} + +// RenderString renders Markup string to HTML with all specific handling stuff and return string +func RenderString(ctx *RenderContext, content string) (string, error) { + var buf strings.Builder + if err := Render(ctx, strings.NewReader(content), &buf); err != nil { + return "", err + } + return buf.String(), nil +} + +type nopCloser struct { + io.Writer +} + +func (nopCloser) Close() error { return nil } + +func renderIFrame(ctx *RenderContext, output io.Writer) error { + // set height="0" ahead, otherwise the scrollHeight would be max(150, realHeight) + // at the moment, only "allow-scripts" is allowed for sandbox mode. + // "allow-same-origin" should never be used, it leads to XSS attack, and it makes the JS in iframe can access parent window's config and CSRF token + // TODO: when using dark theme, if the rendered content doesn't have proper style, the default text color is black, which is not easy to read + _, err := io.WriteString(output, fmt.Sprintf(` +<iframe src="%s/%s/%s/render/%s/%s" +name="giteaExternalRender" +onload="this.height=giteaExternalRender.document.documentElement.scrollHeight" +width="100%%" height="0" scrolling="no" frameborder="0" style="overflow: hidden" +sandbox="allow-scripts" +></iframe>`, + setting.AppSubURL, + url.PathEscape(ctx.Metas["user"]), + url.PathEscape(ctx.Metas["repo"]), + ctx.Metas["BranchNameSubURL"], + url.PathEscape(ctx.RelativePath), + )) + return err +} + +func render(ctx *RenderContext, renderer Renderer, input io.Reader, output io.Writer) error { + var wg sync.WaitGroup + var err error + pr, pw := io.Pipe() + defer func() { + _ = pr.Close() + _ = pw.Close() + }() + + var pr2 io.ReadCloser + var pw2 io.WriteCloser + + var sanitizerDisabled bool + if r, ok := renderer.(ExternalRenderer); ok { + sanitizerDisabled = r.SanitizerDisabled() + } + + if !sanitizerDisabled { + pr2, pw2 = io.Pipe() + defer func() { + _ = pr2.Close() + _ = pw2.Close() + }() + + wg.Add(1) + go func() { + err = SanitizeReader(pr2, renderer.Name(), output) + _ = pr2.Close() + wg.Done() + }() + } else { + pw2 = nopCloser{output} + } + + wg.Add(1) + go func() { + if r, ok := renderer.(PostProcessRenderer); ok && r.NeedPostProcess() { + err = PostProcess(ctx, pr, pw2) + } else { + _, err = io.Copy(pw2, pr) + } + _ = pr.Close() + _ = pw2.Close() + wg.Done() + }() + + if err1 := renderer.Render(ctx, input, pw); err1 != nil { + return err1 + } + _ = pw.Close() + + wg.Wait() + return err +} + +// ErrUnsupportedRenderType represents +type ErrUnsupportedRenderType struct { + Type string +} + +func (err ErrUnsupportedRenderType) Error() string { + return fmt.Sprintf("Unsupported render type: %s", err.Type) +} + +func renderByType(ctx *RenderContext, input io.Reader, output io.Writer) error { + if renderer, ok := renderers[ctx.Type]; ok { + return render(ctx, renderer, input, output) + } + return ErrUnsupportedRenderType{ctx.Type} +} + +// ErrUnsupportedRenderExtension represents the error when extension doesn't supported to render +type ErrUnsupportedRenderExtension struct { + Extension string +} + +func IsErrUnsupportedRenderExtension(err error) bool { + _, ok := err.(ErrUnsupportedRenderExtension) + return ok +} + +func (err ErrUnsupportedRenderExtension) Error() string { + return fmt.Sprintf("Unsupported render extension: %s", err.Extension) +} + +func renderFile(ctx *RenderContext, input io.Reader, output io.Writer) error { + extension := strings.ToLower(filepath.Ext(ctx.RelativePath)) + if renderer, ok := extRenderers[extension]; ok { + if r, ok := renderer.(ExternalRenderer); ok && r.DisplayInIFrame() { + if !ctx.InStandalonePage { + // for an external render, it could only output its content in a standalone page + // otherwise, a <iframe> should be outputted to embed the external rendered page + return renderIFrame(ctx, output) + } + } + return render(ctx, renderer, input, output) + } + return ErrUnsupportedRenderExtension{extension} +} + +// Type returns if markup format via the filename +func Type(filename string) string { + if parser := GetRendererByFileName(filename); parser != nil { + return parser.Name() + } + return "" +} + +// IsMarkupFile reports whether file is a markup type file +func IsMarkupFile(name, markup string) bool { + if parser := GetRendererByFileName(name); parser != nil { + return parser.Name() == markup + } + return false +} + +func PreviewableExtensions() []string { + extensions := make([]string, 0, len(extRenderers)) + for extension := range extRenderers { + extensions = append(extensions, extension) + } + return extensions +} diff --git a/modules/markup/renderer_test.go b/modules/markup/renderer_test.go new file mode 100644 index 00000000..0791081f --- /dev/null +++ b/modules/markup/renderer_test.go @@ -0,0 +1,4 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markup_test diff --git a/modules/markup/sanitizer.go b/modules/markup/sanitizer.go new file mode 100644 index 00000000..d07bba30 --- /dev/null +++ b/modules/markup/sanitizer.go @@ -0,0 +1,235 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Copyright 2017 The Gogs Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markup + +import ( + "io" + "net/url" + "regexp" + "sync" + + "code.gitea.io/gitea/modules/setting" + + "github.com/microcosm-cc/bluemonday" +) + +// Sanitizer is a protection wrapper of *bluemonday.Policy which does not allow +// any modification to the underlying policies once it's been created. +type Sanitizer struct { + defaultPolicy *bluemonday.Policy + descriptionPolicy *bluemonday.Policy + rendererPolicies map[string]*bluemonday.Policy + init sync.Once +} + +var ( + sanitizer = &Sanitizer{} + allowAllRegex = regexp.MustCompile(".+") +) + +// NewSanitizer initializes sanitizer with allowed attributes based on settings. +// Multiple calls to this function will only create one instance of Sanitizer during +// entire application lifecycle. +func NewSanitizer() { + sanitizer.init.Do(func() { + InitializeSanitizer() + }) +} + +// InitializeSanitizer (re)initializes the current sanitizer to account for changes in settings +func InitializeSanitizer() { + sanitizer.rendererPolicies = map[string]*bluemonday.Policy{} + sanitizer.defaultPolicy = createDefaultPolicy() + sanitizer.descriptionPolicy = createRepoDescriptionPolicy() + + for name, renderer := range renderers { + sanitizerRules := renderer.SanitizerRules() + if len(sanitizerRules) > 0 { + policy := createDefaultPolicy() + addSanitizerRules(policy, sanitizerRules) + sanitizer.rendererPolicies[name] = policy + } + } +} + +func createDefaultPolicy() *bluemonday.Policy { + policy := bluemonday.UGCPolicy() + + // For JS code copy and Mermaid loading state + policy.AllowAttrs("class").Matching(regexp.MustCompile(`^code-block( is-loading)?$`)).OnElements("pre") + + // For color preview + policy.AllowAttrs("class").Matching(regexp.MustCompile(`^color-preview$`)).OnElements("span") + + // For attention + policy.AllowAttrs("class").Matching(regexp.MustCompile(`^attention-title$`)).OnElements("p") + policy.AllowAttrs("class").Matching(regexp.MustCompile(`^attention-header attention-\w+$`)).OnElements("blockquote") + policy.AllowAttrs("class").Matching(regexp.MustCompile(`^attention-\w+$`)).OnElements("strong") + policy.AllowAttrs("class").Matching(regexp.MustCompile(`^attention-icon attention-\w+ svg octicon-[\w-]+$`)).OnElements("svg") + policy.AllowAttrs("viewBox", "width", "height", "aria-hidden").OnElements("svg") + policy.AllowAttrs("fill-rule", "d").OnElements("path") + + // For Chroma markdown plugin + policy.AllowAttrs("class").Matching(regexp.MustCompile(`^(chroma )?language-[\w-]+( display)?( is-loading)?$`)).OnElements("code") + + // Checkboxes + policy.AllowAttrs("type").Matching(regexp.MustCompile(`^checkbox$`)).OnElements("input") + policy.AllowAttrs("checked", "disabled", "data-source-position").OnElements("input") + + // Custom URL-Schemes + if len(setting.Markdown.CustomURLSchemes) > 0 { + policy.AllowURLSchemes(setting.Markdown.CustomURLSchemes...) + } else { + policy.AllowURLSchemesMatching(allowAllRegex) + + // Even if every scheme is allowed, these three are blocked for security reasons + disallowScheme := func(*url.URL) bool { + return false + } + policy.AllowURLSchemeWithCustomPolicy("javascript", disallowScheme) + policy.AllowURLSchemeWithCustomPolicy("vbscript", disallowScheme) + policy.AllowURLSchemeWithCustomPolicy("data", disallowScheme) + } + + // Allow classes for anchors + policy.AllowAttrs("class").Matching(regexp.MustCompile(`ref-issue( ref-external-issue)?`)).OnElements("a") + + // Allow classes for task lists + policy.AllowAttrs("class").Matching(regexp.MustCompile(`task-list-item`)).OnElements("li") + + // Allow classes for org mode list item status. + policy.AllowAttrs("class").Matching(regexp.MustCompile(`^(unchecked|checked|indeterminate)$`)).OnElements("li") + + // Allow icons + policy.AllowAttrs("class").Matching(regexp.MustCompile(`^icon(\s+[\p{L}\p{N}_-]+)+$`)).OnElements("i") + + // Allow classes for emojis + policy.AllowAttrs("class").Matching(regexp.MustCompile(`emoji`)).OnElements("img") + + // Allow icons, emojis, chroma syntax and keyword markup on span + policy.AllowAttrs("class").Matching(regexp.MustCompile(`^((icon(\s+[\p{L}\p{N}_-]+)+)|(emoji)|(language-math display)|(language-math inline))$|^([a-z][a-z0-9]{0,2})$|^` + keywordClass + `$`)).OnElements("span") + + // Allow 'color' and 'background-color' properties for the style attribute on text elements. + policy.AllowStyles("color", "background-color").OnElements("span", "p") + + // Allow classes for file preview links... + policy.AllowAttrs("class").Matching(regexp.MustCompile("^(lines-num|lines-code chroma)$")).OnElements("td") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^code-inner$")).OnElements("code") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^file-preview-box$")).OnElements("div") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^ui table$")).OnElements("div") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^header$")).OnElements("div") + policy.AllowAttrs("data-line-number").Matching(regexp.MustCompile("^[0-9]+$")).OnElements("span") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^text small grey$")).OnElements("span") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^file-preview*")).OnElements("table") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^lines-escape$")).OnElements("td") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^toggle-escape-button btn interact-bg$")).OnElements("button") + policy.AllowAttrs("title").OnElements("button") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^ambiguous-code-point$")).OnElements("span") + policy.AllowAttrs("data-tooltip-content").OnElements("span") + policy.AllowAttrs("class").Matching(regexp.MustCompile("muted|(text black)")).OnElements("a") + policy.AllowAttrs("class").Matching(regexp.MustCompile("^ui warning message tw-text-left$")).OnElements("div") + + // Allow generally safe attributes + generalSafeAttrs := []string{ + "abbr", "accept", "accept-charset", + "accesskey", "action", "align", "alt", + "aria-describedby", "aria-hidden", "aria-label", "aria-labelledby", + "axis", "border", "cellpadding", "cellspacing", "char", + "charoff", "charset", "checked", + "clear", "cols", "colspan", "color", + "compact", "coords", "datetime", "dir", + "disabled", "enctype", "for", "frame", + "headers", "height", "hreflang", + "hspace", "ismap", "label", "lang", + "maxlength", "media", "method", + "multiple", "name", "nohref", "noshade", + "nowrap", "open", "prompt", "readonly", "rel", "rev", + "rows", "rowspan", "rules", "scope", + "selected", "shape", "size", "span", + "start", "summary", "tabindex", "target", + "title", "type", "usemap", "valign", "value", + "vspace", "width", "itemprop", + } + + generalSafeElements := []string{ + "h1", "h2", "h3", "h4", "h5", "h6", "h7", "h8", "br", "b", "i", "strong", "em", "a", "pre", "code", "img", "tt", + "div", "ins", "del", "sup", "sub", "p", "ol", "ul", "table", "thead", "tbody", "tfoot", "blockquote", "label", + "dl", "dt", "dd", "kbd", "q", "samp", "var", "hr", "ruby", "rt", "rp", "li", "tr", "td", "th", "s", "strike", "summary", + "details", "caption", "figure", "figcaption", + "abbr", "bdo", "cite", "dfn", "mark", "small", "span", "time", "video", "wbr", + } + + policy.AllowAttrs(generalSafeAttrs...).OnElements(generalSafeElements...) + + policy.AllowAttrs("src", "autoplay", "controls").OnElements("video") + + policy.AllowAttrs("itemscope", "itemtype").OnElements("div") + + // FIXME: Need to handle longdesc in img but there is no easy way to do it + + // Custom keyword markup + addSanitizerRules(policy, setting.ExternalSanitizerRules) + + return policy +} + +// createRepoDescriptionPolicy returns a minimal more strict policy that is used for +// repository descriptions. +func createRepoDescriptionPolicy() *bluemonday.Policy { + policy := bluemonday.NewPolicy() + policy.AllowStandardURLs() + + // Allow italics and bold. + policy.AllowElements("i", "b", "em", "strong") + + // Allow code. + policy.AllowElements("code") + + // Allow links + policy.AllowAttrs("href", "target", "rel").OnElements("a") + + // Allow classes for emojis + policy.AllowAttrs("class").Matching(regexp.MustCompile(`^emoji$`)).OnElements("img", "span") + policy.AllowAttrs("aria-label").OnElements("span") + + return policy +} + +func addSanitizerRules(policy *bluemonday.Policy, rules []setting.MarkupSanitizerRule) { + for _, rule := range rules { + if rule.AllowDataURIImages { + policy.AllowDataURIImages() + } + if rule.Element != "" { + if rule.Regexp != nil { + policy.AllowAttrs(rule.AllowAttr).Matching(rule.Regexp).OnElements(rule.Element) + } else { + policy.AllowAttrs(rule.AllowAttr).OnElements(rule.Element) + } + } + } +} + +// SanitizeDescription sanitizes the HTML generated for a repository description. +func SanitizeDescription(s string) string { + NewSanitizer() + return sanitizer.descriptionPolicy.Sanitize(s) +} + +// Sanitize takes a string that contains a HTML fragment or document and applies policy whitelist. +func Sanitize(s string) string { + NewSanitizer() + return sanitizer.defaultPolicy.Sanitize(s) +} + +// SanitizeReader sanitizes a Reader +func SanitizeReader(r io.Reader, renderer string, w io.Writer) error { + NewSanitizer() + policy, exist := sanitizer.rendererPolicies[renderer] + if !exist { + policy = sanitizer.defaultPolicy + } + return policy.SanitizeReaderToWriter(r, w) +} diff --git a/modules/markup/sanitizer_test.go b/modules/markup/sanitizer_test.go new file mode 100644 index 00000000..163620c2 --- /dev/null +++ b/modules/markup/sanitizer_test.go @@ -0,0 +1,108 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Copyright 2017 The Gogs Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package markup + +import ( + "html/template" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_Sanitizer(t *testing.T) { + NewSanitizer() + testCases := []string{ + // Regular + `<a onblur="alert(secret)" href="http://www.google.com">Google</a>`, `<a href="http://www.google.com" rel="nofollow">Google</a>`, + + // Code highlighting class + `<code class="random string"></code>`, `<code></code>`, + `<code class="language-random ui tab active menu attached animating sidebar following bar center"></code>`, `<code></code>`, + `<code class="language-go"></code>`, `<code class="language-go"></code>`, + + // Input checkbox + `<input type="hidden">`, ``, + `<input type="checkbox">`, `<input type="checkbox">`, + `<input checked disabled autofocus>`, `<input checked="" disabled="">`, + + // Code highlight injection + `<code class="language-random ui tab active menu attached animating sidebar following bar center"></code>`, `<code></code>`, + `<code class="language-lol ui tab active menu attached animating sidebar following bar center"> +<code class="language-lol ui container input huge basic segment center"> </code> +<img src="https://try.gogs.io/img/favicon.png" width="200" height="200"> +<code class="language-lol ui container input massive basic segment">Hello there! Something has gone wrong, we are working on it.</code> +<code class="language-lol ui container input huge basic segment">In the meantime, play a game with us at <a href="http://example.com/">example.com</a>.</code> +</code>`, "<code>\n<code>\u00a0</code>\n<img src=\"https://try.gogs.io/img/favicon.png\" width=\"200\" height=\"200\">\n<code>Hello there! Something has gone wrong, we are working on it.</code>\n<code>In the meantime, play a game with us at\u00a0<a href=\"http://example.com/\" rel=\"nofollow\">example.com</a>.</code>\n</code>", + + // <kbd> tags + `<kbd>Ctrl + C</kbd>`, `<kbd>Ctrl + C</kbd>`, + `<i class="dropdown icon">NAUGHTY</i>`, `<i>NAUGHTY</i>`, + `<i class="icon dropdown"></i>`, `<i class="icon dropdown"></i>`, + `<input type="checkbox" disabled=""/>unchecked`, `<input type="checkbox" disabled=""/>unchecked`, + `<span class="emoji dropdown">NAUGHTY</span>`, `<span>NAUGHTY</span>`, + `<span class="emoji">contents</span>`, `<span class="emoji">contents</span>`, + + // Color property + `<span style="color: red">Hello World</span>`, `<span style="color: red">Hello World</span>`, + `<p style="color: red">Hello World</p>`, `<p style="color: red">Hello World</p>`, + `<code style="color: red">Hello World</code>`, `<code>Hello World</code>`, + `<span style="bad-color: red">Hello World</span>`, `<span>Hello World</span>`, + `<p style="bad-color: red">Hello World</p>`, `<p>Hello World</p>`, + `<code style="bad-color: red">Hello World</code>`, `<code>Hello World</code>`, + + // Org mode status of list items. + `<li class="checked"></li>`, `<li class="checked"></li>`, + `<li class="unchecked"></li>`, `<li class="unchecked"></li>`, + `<li class="indeterminate"></li>`, `<li class="indeterminate"></li>`, + + // URLs + `<a href="cbthunderlink://somebase64string)">my custom URL scheme</a>`, `<a href="cbthunderlink://somebase64string)" rel="nofollow">my custom URL scheme</a>`, + `<a href="matrix:roomid/psumPMeAfzgAeQpXMG:feneas.org?action=join">my custom URL scheme</a>`, `<a href="matrix:roomid/psumPMeAfzgAeQpXMG:feneas.org?action=join" rel="nofollow">my custom URL scheme</a>`, + + // Disallow dangerous url schemes + `<a href="javascript:alert('xss')">bad</a>`, `bad`, + `<a href="vbscript:no">bad</a>`, `bad`, + `<a href="data:1234">bad</a>`, `bad`, + } + + for i := 0; i < len(testCases); i += 2 { + assert.Equal(t, testCases[i+1], Sanitize(testCases[i])) + } +} + +func TestDescriptionSanitizer(t *testing.T) { + NewSanitizer() + + testCases := []string{ + `<h1>Title</h1>`, `Title`, + `<img src='img.png' alt='image'>`, ``, + `<span class="emoji" aria-label="thumbs up">THUMBS UP</span>`, `<span class="emoji" aria-label="thumbs up">THUMBS UP</span>`, + `<span style="color: red">Hello World</span>`, `<span>Hello World</span>`, + `<br>`, ``, + `<a href="https://example.com" target="_blank" rel="noopener noreferrer">https://example.com</a>`, `<a href="https://example.com" target="_blank" rel="noopener noreferrer nofollow">https://example.com</a>`, + `<mark>Important!</mark>`, `Important!`, + `<details>Click me! <summary>Nothing to see here.</summary></details>`, `Click me! Nothing to see here.`, + `<input type="hidden">`, ``, + `<b>I</b> have a <i>strong</i> <strong>opinion</strong> about <em>this</em>.`, `<b>I</b> have a <i>strong</i> <strong>opinion</strong> about <em>this</em>.`, + `Provides alternative <code>wg(8)</code> tool`, `Provides alternative <code>wg(8)</code> tool`, + `<a href="javascript:alert('xss')">Click me</a>.`, `Click me.`, + `<a href="data:text/html,<script>alert('xss')</script>">Click me</a>.`, `Click me.`, + `<a href="vbscript:msgbox("xss")">Click me</a>.`, `Click me.`, + } + + for i := 0; i < len(testCases); i += 2 { + assert.Equal(t, testCases[i+1], SanitizeDescription(testCases[i])) + } +} + +func TestSanitizeNonEscape(t *testing.T) { + descStr := "<scrİpt><script>alert(document.domain)</script></scrİpt>" + + output := template.HTML(Sanitize(descStr)) + if strings.Contains(string(output), "<script>") { + t.Errorf("un-escaped <script> in output: %q", output) + } +} diff --git a/modules/markup/tests/repo/repo1_filepreview/HEAD b/modules/markup/tests/repo/repo1_filepreview/HEAD new file mode 100644 index 00000000..cb089cd8 --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/modules/markup/tests/repo/repo1_filepreview/config b/modules/markup/tests/repo/repo1_filepreview/config new file mode 100644 index 00000000..42cc799c --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/config @@ -0,0 +1,6 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = true +[remote "origin"] + url = /home/mai/projects/codeark/forgejo/forgejo/modules/markup/tests/repo/repo1_filepreview/../../__test_repo diff --git a/modules/markup/tests/repo/repo1_filepreview/description b/modules/markup/tests/repo/repo1_filepreview/description new file mode 100644 index 00000000..498b267a --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/description @@ -0,0 +1 @@ +Unnamed repository; edit this file 'description' to name the repository. diff --git a/modules/markup/tests/repo/repo1_filepreview/info/exclude b/modules/markup/tests/repo/repo1_filepreview/info/exclude new file mode 100644 index 00000000..a5196d1b --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/info/exclude @@ -0,0 +1,6 @@ +# git ls-files --others --exclude-from=.git/info/exclude +# Lines that start with '#' are comments. +# For a project mostly in C, the following would be a good set of +# exclude patterns (uncomment them if you want to use them): +# *.[oa] +# *~ diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/19/0d9492934af498c3f669d6a2431dc5459e5b20 b/modules/markup/tests/repo/repo1_filepreview/objects/19/0d9492934af498c3f669d6a2431dc5459e5b20 Binary files differnew file mode 100644 index 00000000..161d0baf --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/objects/19/0d9492934af498c3f669d6a2431dc5459e5b20 diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 b/modules/markup/tests/repo/repo1_filepreview/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 Binary files differnew file mode 100644 index 00000000..adf64119 --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/83/57a737d04385bb7f2ab59ff184be94756e7972 b/modules/markup/tests/repo/repo1_filepreview/objects/83/57a737d04385bb7f2ab59ff184be94756e7972 Binary files differnew file mode 100644 index 00000000..1b87aa8b --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/objects/83/57a737d04385bb7f2ab59ff184be94756e7972 diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/84/22d40f12717e1ebd5cef2449f6c09d1f775969 b/modules/markup/tests/repo/repo1_filepreview/objects/84/22d40f12717e1ebd5cef2449f6c09d1f775969 Binary files differnew file mode 100644 index 00000000..d38170a5 --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/objects/84/22d40f12717e1ebd5cef2449f6c09d1f775969 diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/d4/490327def9658be036d6a52c4417d84e74dd4c b/modules/markup/tests/repo/repo1_filepreview/objects/d4/490327def9658be036d6a52c4417d84e74dd4c Binary files differnew file mode 100644 index 00000000..fe37c115 --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/objects/d4/490327def9658be036d6a52c4417d84e74dd4c diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/ee/2b1253d9cf407796e2e724926cbe3a974b214d b/modules/markup/tests/repo/repo1_filepreview/objects/ee/2b1253d9cf407796e2e724926cbe3a974b214d new file mode 100644 index 00000000..e13ca647 --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/objects/ee/2b1253d9cf407796e2e724926cbe3a974b214d @@ -0,0 +1 @@ +x+)JMU06e040031QHIKghQ/TX'7潊s#3
\ No newline at end of file diff --git a/modules/markup/tests/repo/repo1_filepreview/refs/heads/master b/modules/markup/tests/repo/repo1_filepreview/refs/heads/master new file mode 100644 index 00000000..49c348b4 --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/refs/heads/master @@ -0,0 +1 @@ +190d9492934af498c3f669d6a2431dc5459e5b20 |