diff options
Diffstat (limited to 'dependencies/pkg/mod/github.com/goccy/go-yaml@v1.9.6/lexer/lexer.go')
-rw-r--r-- | dependencies/pkg/mod/github.com/goccy/go-yaml@v1.9.6/lexer/lexer.go | 23 |
1 files changed, 23 insertions, 0 deletions
diff --git a/dependencies/pkg/mod/github.com/goccy/go-yaml@v1.9.6/lexer/lexer.go b/dependencies/pkg/mod/github.com/goccy/go-yaml@v1.9.6/lexer/lexer.go new file mode 100644 index 0000000..3207f4f --- /dev/null +++ b/dependencies/pkg/mod/github.com/goccy/go-yaml@v1.9.6/lexer/lexer.go @@ -0,0 +1,23 @@ +package lexer + +import ( + "io" + + "github.com/goccy/go-yaml/scanner" + "github.com/goccy/go-yaml/token" +) + +// Tokenize split to token instances from string +func Tokenize(src string) token.Tokens { + var s scanner.Scanner + s.Init(src) + var tokens token.Tokens + for { + subTokens, err := s.Scan() + if err == io.EOF { + break + } + tokens.Add(subTokens...) + } + return tokens +} |