scanner_test.go

package jstream

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"sync/atomic"
	"testing"
)

var (
	smallInput  = make([]byte, 1024*12)       // 12K
	mediumInput = make([]byte, 1024*1024*12)  // 12MB
	largeInput  = make([]byte, 1024*1024*128) // 128MB
)

func TestScanner(t *testing.T) {
	data := []byte("abcdefghijklmnopqrstuvwxyz0123456789")

	var i int
	r := bytes.NewReader(data)
	scanner := newScanner(r)
	for scanner.pos < atomic.LoadInt64(&scanner.end) {
		c := scanner.next()
		if scanner.readerErr != nil {
			t.Fatal(scanner.readerErr)
		}
		if c != data[i] {
			t.Fatalf("expected %s, got %s", string(data[i]), string(c))
		}
		t.Logf("pos=%d remaining=%d (%s)", i, r.Len(), string(c))
		i++
	}
}

type mockReader struct {
	pos       int
	mockData  byte
	failAfter int
}

func newMockReader(failAfter int, data byte) *mockReader {
	return &mockReader{0, data, failAfter}
}

func (r *mockReader) Read(p []byte) (n int, err error) {
	if r.pos >= r.failAfter {
		return 0, fmt.Errorf("intentionally unexpected reader error")
	}
	r.pos++
	p[0] = r.mockData
	return 1, nil
}

func TestScannerFailure(t *testing.T) {
	var (
		i         int
		failAfter = 900
		mockData  = byte(32)
	)

	r := newMockReader(failAfter, mockData)
	scanner := newScanner(r)

	for i < 1000 {
		c := scanner.next()
		if c == byte(0) {
			break
		}
		if c != mockData {
			t.Fatalf("expected \"%s\", got \"%s\"", string(mockData), string(c))
		}
		i++
	}
	c := scanner.next()
	if scanner.readerErr == nil {
		t.Fatalf("failed to receive expected error after %d bytes", failAfter)
	}
	if c != byte(0) {
		t.Fatalf("expected null byte, got %v", c)
	}
}

func BenchmarkBufioScanner(b *testing.B) {
	b.Run("small", func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			benchmarkBufioScanner(smallInput)
		}
	})
	b.Run("medium", func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			benchmarkBufioScanner(mediumInput)
		}
	})
	b.Run("large", func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			benchmarkBufioScanner(largeInput)
		}
	})
}

func benchmarkBufioScanner(b []byte) {
	s := bufio.NewScanner(bytes.NewReader(b))
	s.Split(bufio.ScanBytes)
	for s.Scan() {
		s.Bytes()
	}
}

func BenchmarkBufioReader(b *testing.B) {
	b.Run("small", func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			benchmarkBufioReader(smallInput)
		}
	})
	b.Run("medium", func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			benchmarkBufioReader(mediumInput)
		}
	})
	b.Run("large", func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			benchmarkBufioReader(largeInput)
		}
	})
}

func benchmarkBufioReader(b []byte) {
	br := bufio.NewReader(bytes.NewReader(b))
loop:
	for {
		_, err := br.ReadByte()
		switch err {
		case nil:
			continue loop
		case io.EOF:
			break loop
		default:
			panic(err)
		}
	}
}

func BenchmarkScanner(b *testing.B) {
	b.Run("small", func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			benchmarkScanner(smallInput)
		}
	})
	b.Run("medium", func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			benchmarkScanner(mediumInput)
		}
	})
	b.Run("large", func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			benchmarkScanner(largeInput)
		}
	})
}

func benchmarkScanner(b []byte) {
	r := bytes.NewReader(b)

	scanner := newScanner(r)
	for scanner.remaining() > 0 {
		scanner.next()
	}
}