added view collection type

tools/tokenizer/tokenizer.go

@@ -34,8 +34,9 @@ func NewFromBytes(b []byte) *Tokenizer {
 // New creates new Tokenizer from the provided reader with DefaultSeparators.
 func New(r io.Reader) *Tokenizer {
 	return &Tokenizer{
-		r:          bufio.NewReader(r),
-		separators: DefaultSeparators,
+		r:             bufio.NewReader(r),
+		separators:    DefaultSeparators,
+		keepSeparator: false,
 	}
 }
 
@@ -44,14 +45,21 @@ func New(r io.Reader) *Tokenizer {
 type Tokenizer struct {
 	r *bufio.Reader
 
-	separators []rune
+	separators    []rune
+	keepSeparator bool
 }
 
-// SetSeparators specifies the provided separatos of the current Tokenizer.
-func (s *Tokenizer) SetSeparators(separators ...rune) {
+// Separators defines the provided separatos of the current Tokenizer.
+func (s *Tokenizer) Separators(separators ...rune) {
 	s.separators = separators
 }
 
+// KeepSeparator defines whether to keep the separator rune as part
+// of the token (default to false).
+func (s *Tokenizer) KeepSeparator(state bool) {
+	s.keepSeparator = state
+}
+
 // Scan reads and returns the next available token from the Tokenizer's buffer (trimmed).
 //
 // Returns [io.EOF] error when there are no more tokens to scan.
@@ -128,6 +136,9 @@ func (s *Tokenizer) readToken() (string, error) {
 		}
 
 		if s.isSeperatorRune(ch) && parenthesis == 0 && quoteCh == eof {
+			if s.keepSeparator {
+				buf.WriteRune(ch)
+			}
 			break
 		}
 

tools/tokenizer/tokenizer_test.go

@@ -34,6 +34,10 @@ func TestFactories(t *testing.T) {
 			t.Fatalf("[%s] Expected reader with content %q, got %q", s.name, expectedContent, content)
 		}
 
+		if s.tk.keepSeparator != false {
+			t.Fatalf("[%s] Expected false, got true", s.name)
+		}
+
 		if len(s.tk.separators) != len(DefaultSeparators) {
 			t.Fatalf("[%s] Expected \n%v, \ngot \n%v", s.name, DefaultSeparators, s.tk.separators)
 		}
@@ -81,23 +85,26 @@ func TestScan(t *testing.T) {
 
 func TestScanAll(t *testing.T) {
 	scenarios := []struct {
-		name         string
-		content      string
-		separators   []rune
-		expectError  bool
-		expectTokens []string
+		name          string
+		content       string
+		separators    []rune
+		keepSeparator bool
+		expectError   bool
+		expectTokens  []string
 	}{
 		{
 			"empty string",
 			"",
 			DefaultSeparators,
 			false,
+			false,
 			nil,
 		},
 		{
 			"unbalanced parenthesis",
 			`(a,b() c`,
 			DefaultSeparators,
+			false,
 			true,
 			[]string{},
 		},
@@ -105,6 +112,7 @@ func TestScanAll(t *testing.T) {
 			"unmatching quotes",
 			`'asd"`,
 			DefaultSeparators,
+			false,
 			true,
 			[]string{},
 		},
@@ -113,6 +121,7 @@ func TestScanAll(t *testing.T) {
 			`a, b, c, d, e 123, "abc"`,
 			nil,
 			false,
+			false,
 			[]string{
 				`a, b, c, d, e 123, "abc"`,
 			},
@@ -122,6 +131,7 @@ func TestScanAll(t *testing.T) {
 			`a, b, c, d e, "a,b,  c  ", (123, 456)`,
 			DefaultSeparators,
 			false,
+			false,
 			[]string{
 				"a",
 				"b",
@@ -131,6 +141,21 @@ func TestScanAll(t *testing.T) {
 				`(123, 456)`,
 			},
 		},
+		{
+			"default separators (with preserve)",
+			`a, b, c, d e, "a,b,  c  ", (123, 456)`,
+			DefaultSeparators,
+			true,
+			false,
+			[]string{
+				"a,",
+				"b,",
+				"c,",
+				"d e,",
+				`"a,b,  c  ",`,
+				`(123, 456)`,
+			},
+		},
 		{
 			"custom separators",
 			`   a   , 123.456, b, c d, (
@@ -138,6 +163,7 @@ func TestScanAll(t *testing.T) {
 			),"(abc d", "abc) d", "(abc) d \" " 'abc "'`,
 			[]rune{',', ' ', '\t', '\n'},
 			false,
+			false,
 			[]string{
 				"a",
 				"123.456",
@@ -151,12 +177,34 @@ func TestScanAll(t *testing.T) {
 				`'abc "'`,
 			},
 		},
+		{
+			"custom separators (with preserve)",
+			`   a   , 123.456, b, c d, (
+				test (a,b,c) " 123 "
+			),"(abc d", "abc) d", "(abc) d \" " 'abc "'`,
+			[]rune{',', ' ', '\t', '\n'},
+			true,
+			false,
+			[]string{
+				"a ",
+				"123.456,",
+				"b,",
+				"c ",
+				"d,",
+				"(\n\t\t\t\ttest (a,b,c) \" 123 \"\n\t\t\t),",
+				`"(abc d",`,
+				`"abc) d",`,
+				`"(abc) d \" " `,
+				`'abc "'`,
+			},
+		},
 	}
 
 	for _, s := range scenarios {
 		tk := NewFromString(s.content)
 
-		tk.SetSeparators(s.separators...)
+		tk.Separators(s.separators...)
+		tk.KeepSeparator(s.keepSeparator)
 
 		tokens, err := tk.ScanAll()