merge v0.23.0-rc changes

tools/tokenizer/tokenizer.go

@@ -168,22 +168,6 @@ func (t *Tokenizer) readToken() (string, error) {
 	return strings.Trim(buf.String(), t.trimCutset), nil
 }
 
-// readWhiteSpaces consumes all contiguous whitespace runes.
-func (t *Tokenizer) readWhiteSpaces() {
-	for {
-		ch := t.read()
-
-		if ch == eof {
-			break
-		}
-
-		if !t.isWhitespaceRune(ch) {
-			t.unread()
-			break
-		}
-	}
-}
-
 // read reads the next rune from the buffered reader.
 // Returns the `rune(0)` if an error or `io.EOF` occurs.
 func (t *Tokenizer) read() rune {
@@ -225,17 +209,6 @@ func (t *Tokenizer) isSeperatorRune(ch rune) bool {
 	return false
 }
 
-// isWhitespaceRune checks if a rune is a space character (eg. space, tab, new line).
-func (t *Tokenizer) isWhitespaceRune(ch rune) bool {
-	for _, c := range whitespaceChars {
-		if c == ch {
-			return true
-		}
-	}
-
-	return false
-}
-
 // isQuoteRune checks if a rune is a quote.
 func (t *Tokenizer) isQuoteRune(ch rune) bool {
 	return ch == '\'' || ch == '"' || ch == '`'

tools/tokenizer/tokenizer_test.go

@@ -159,8 +159,8 @@ func TestScanAll(t *testing.T) {
 			},
 		},
 		{
-			name: "keep separators",
-			content: `a, b, c, d  e, "a,b,  c  ",	(123, 456)`,
+			name:              "keep separators",
+			content:           `a, b, c, d  e, "a,b,  c  ",	(123, 456)`,
 			separators:        []rune{',', ' '}, // the space should be removed from the cutset
 			keepSeparator:     true,
 			keepEmptyTokens:   true,