From b36cc498071d89a3187bd98779b8a03e089fbef1 Mon Sep 17 00:00:00 2001
From: Hidetatz Yaginuma <hidetatz@gmail.com>
Date: Sat, 9 Sep 2023 10:32:24 +0900
Subject: [PATCH] refactor

---
 tokenizer.go | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/tokenizer.go b/tokenizer.go
index 2b929db..f8200fe 100644
--- a/tokenizer.go
+++ b/tokenizer.go
@@ -1,5 +1,6 @@
 package main
 
+// tokenizer reads and manages tokens read by tokenreader.
 type tokenizer struct {
 	tr     *tokenreader
 	tokens []*token
@@ -10,14 +11,6 @@ func newtokenizer(mod *module) *tokenizer {
 	return &tokenizer{tr: newtokenreader(mod)}
 }
 
-func (t *tokenizer) mark() int {
-	return t.pos
-}
-
-func (t *tokenizer) reset(pos int) {
-	t.pos = pos
-}
-
 func (t *tokenizer) gettoken() (*token, error) {
 	tk, err := t.peektoken()
 	if err != nil {
@@ -40,3 +33,11 @@ func (t *tokenizer) peektoken() (*token, error) {
 
 	return t.tokens[t.pos], nil
 }
+
+func (t *tokenizer) mark() int {
+	return t.pos
+}
+
+func (t *tokenizer) reset(pos int) {
+	t.pos = pos
+}