blob: 3207f4f22734b4ea39b49e12c5e3345bb3d74aa1 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
|
package lexer
import (
"io"
"github.com/goccy/go-yaml/scanner"
"github.com/goccy/go-yaml/token"
)
// Tokenize split to token instances from string
func Tokenize(src string) token.Tokens {
var s scanner.Scanner
s.Init(src)
var tokens token.Tokens
for {
subTokens, err := s.Scan()
if err == io.EOF {
break
}
tokens.Add(subTokens...)
}
return tokens
}
|
