pax_global_header00006660000000000000000000000064134135205130014507gustar00rootroot0000000000000052 comment=25c428535bd3f55a16f149a9daebd3fa4c5a562b pelletier-go-buffruneio-25c4285/000077500000000000000000000000001341352051300165175ustar00rootroot00000000000000pelletier-go-buffruneio-25c4285/.travis.yml000066400000000000000000000001271341352051300206300ustar00rootroot00000000000000language: go sudo: false go: - 1.6.4 - 1.7.6 - 1.8.5 - 1.9.2 - tip pelletier-go-buffruneio-25c4285/LICENSE000066400000000000000000000020611341352051300175230ustar00rootroot00000000000000MIT License Copyright (c) 2018 Thomas Pelletier Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. pelletier-go-buffruneio-25c4285/README.md000066400000000000000000000042131341352051300177760ustar00rootroot00000000000000# buffruneio [![Tests Status](https://travis-ci.org/pelletier/go-buffruneio.svg?branch=master)](https://travis-ci.org/pelletier/go-buffruneio) [![GoDoc](https://godoc.org/github.com/pelletier/go-buffruneio?status.svg)](https://godoc.org/github.com/pelletier/go-buffruneio) Buffruneio provides rune-based buffered input. ```go import "github.com/pelletier/go-buffruneio" ``` ## Examples ```go import ( "fmt" "github.com/pelletier/go-buffruneio" "strings" ) reader := buffruneio.NewReader(strings.NewReader("abcd")) fmt.Println(reader.ReadRune()) // 'a' fmt.Println(reader.ReadRune()) // 'b' fmt.Println(reader.ReadRune()) // 'c' reader.UnreadRune() reader.UnreadRune() fmt.Println(reader.ReadRune()) // 'b' fmt.Println(reader.ReadRune()) // 'c' ``` ## Documentation The documentation and additional examples are available at [godoc.org](http://godoc.org/github.com/pelletier/go-buffruneio). ## Contribute Feel free to report bugs and patches using GitHub's pull requests system on [pelletier/go-buffruneio](https://github.com/pelletier/go-buffruneio). Any feedback is much appreciated! ## LICENSE Copyright (c) 2016 - 2018 Thomas Pelletier Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. pelletier-go-buffruneio-25c4285/buffruneio.go000066400000000000000000000072271341352051300212220ustar00rootroot00000000000000// Package buffruneio provides rune-based buffered input. package buffruneio import ( "bufio" "errors" "io" "unicode/utf8" ) // EOF is a rune value indicating end-of-file. const EOF = -1 // ErrNoRuneToUnread is the error returned when UnreadRune is called with nothing to unread. var ErrNoRuneToUnread = errors.New("no rune to unwind") // A Reader implements rune-based input for an underlying byte stream. type Reader struct { buffer []rune current int input *bufio.Reader } // NewReader returns a new Reader reading the given input. func NewReader(input io.Reader) *Reader { return &Reader{ input: bufio.NewReader(input), } } // The rune buffer stores -2 to represent RuneError of length 1 (UTF-8 decoding errors). const badRune = -2 // feedBuffer adds a rune to the buffer. // If EOF is reached, it adds EOF to the buffer and returns nil. // If a different error is encountered, it returns the error without // adding to the buffer. func (rd *Reader) feedBuffer() error { if rd.buffer == nil { rd.buffer = make([]rune, 0, 256) } r, size, err := rd.input.ReadRune() if err != nil { if err != io.EOF { return err } r = EOF } if r == utf8.RuneError && size == 1 { r = badRune } rd.buffer = append(rd.buffer, r) return nil } // ReadRune reads and returns the next rune from the input. // The rune is also saved in an internal buffer, in case UnreadRune is called. // To avoid unbounded buffer growth, the caller must call Forget at appropriate intervals. // // At end of file, ReadRune returns EOF, 0, nil. // On read errors other than io.EOF, ReadRune returns EOF, 0, err. func (rd *Reader) ReadRune() (rune, int, error) { if rd.current >= len(rd.buffer) { if err := rd.feedBuffer(); err != nil { return EOF, 0, err } } r := rd.buffer[rd.current] rd.current++ if r == badRune { return utf8.RuneError, 1, nil } if r == EOF { return EOF, 0, nil } return r, utf8.RuneLen(r), nil } // UnreadRune rewinds the input by one rune, undoing the effect of a single ReadRune call. // UnreadRune may be called multiple times to rewind a sequence of ReadRune calls, // up to the last time Forget was called or the beginning of the input. // // If there are no ReadRune calls left to undo, UnreadRune returns ErrNoRuneToUnread. func (rd *Reader) UnreadRune() error { if rd.current == 0 { return ErrNoRuneToUnread } rd.current-- return nil } // Forget discards buffered runes before the current input position. // Calling Forget makes it impossible to UnreadRune earlier than the current input position // but is necessary to avoid unbounded buffer growth. func (rd *Reader) Forget() { n := copy(rd.buffer, rd.buffer[rd.current:]) rd.current = 0 rd.buffer = rd.buffer[:n] } // PeekRunes returns the next n runes in the input, // without advancing the current input position. // // If the input has fewer than n runes and then returns // an io.EOF error, PeekRune returns a slice containing // the available runes followed by EOF. // On other hand, if the input ends early with a non-io.EOF error, // PeekRune returns a slice containing only the available runes, // with no terminating EOF. func (rd *Reader) PeekRunes(n int) []rune { for len(rd.buffer)-rd.current < n && !rd.haveEOF() { if err := rd.feedBuffer(); err != nil { break } } res := make([]rune, 0, n) for i := 0; i < n; i++ { if rd.current + i >= len(rd.buffer) { // reached end of buffer before reading as much as we wanted break } r := rd.buffer[rd.current+i] if r == badRune { r = utf8.RuneError } res = append(res, r) if r == EOF { break } } return res } func (rd *Reader) haveEOF() bool { return rd.current < len(rd.buffer) && rd.buffer[len(rd.buffer)-1] == EOF } pelletier-go-buffruneio-25c4285/buffruneio_test.go000066400000000000000000000166701341352051300222630ustar00rootroot00000000000000package buffruneio import ( "reflect" "runtime/debug" "strings" "testing" "unicode/utf8" "io" "fmt" ) func assertNoError(t *testing.T, err error) { if err != nil { t.Log("unexpected error", err) debug.PrintStack() t.FailNow() } } func assumeRunesArray(t *testing.T, expected []rune, got []rune) { if len(expected) != len(got) { t.Fatal("expected", len(expected), "runes, but got", len(got)) } for i := 0; i < len(got); i++ { if expected[i] != got[i] { t.Fatal("expected rune", expected[i], "at index", i, "but got", got[i]) } } } func assumeRune(t *testing.T, rd *Reader, r rune) { gotRune, size, err := rd.ReadRune() wantSize := utf8.RuneLen(r) if wantSize < 0 { wantSize = 0 } if gotRune != r || size != wantSize || err != nil { t.Fatalf("ReadRune() = %q, %d, %v, wanted %q, %d, nil", gotRune, size, err, r, wantSize) } } func assumeBadRune(t *testing.T, rd *Reader) { gotRune, size, err := rd.ReadRune() if gotRune != utf8.RuneError || size != 1 || err != nil { t.Fatalf("ReadRune() = %q, %d, %v, wanted %q, 1, nil", gotRune, size, err, utf8.RuneError) } } func TestReadString(t *testing.T) { s := "hello" rd := NewReader(strings.NewReader(s)) assumeRune(t, rd, 'h') assumeRune(t, rd, 'e') assumeRune(t, rd, 'l') assumeRune(t, rd, 'l') assumeRune(t, rd, 'o') assumeRune(t, rd, EOF) } func TestMultipleEOF(t *testing.T) { s := "" rd := NewReader(strings.NewReader(s)) assumeRune(t, rd, EOF) assumeRune(t, rd, EOF) } func TestBadRunes(t *testing.T) { s := "ab\xff\ufffd\xffcd" rd := NewReader(strings.NewReader(s)) assumeRune(t, rd, 'a') assumeRune(t, rd, 'b') assumeBadRune(t, rd) assumeRune(t, rd, utf8.RuneError) assumeBadRune(t, rd) assumeRune(t, rd, 'c') assumeRune(t, rd, 'd') for i := 0; i < 6; i++ { assertNoError(t, rd.UnreadRune()) } assumeRune(t, rd, 'b') assumeBadRune(t, rd) assumeRune(t, rd, utf8.RuneError) assumeBadRune(t, rd) assumeRune(t, rd, 'c') assumeRune(t, rd, 'd') } func TestUnread(t *testing.T) { s := "ab" rd := NewReader(strings.NewReader(s)) assumeRune(t, rd, 'a') assumeRune(t, rd, 'b') assertNoError(t, rd.UnreadRune()) assumeRune(t, rd, 'b') assumeRune(t, rd, EOF) } func TestUnreadEOF(t *testing.T) { s := "x" rd := NewReader(strings.NewReader(s)) _ = rd.UnreadRune() assumeRune(t, rd, 'x') assumeRune(t, rd, EOF) assumeRune(t, rd, EOF) assertNoError(t, rd.UnreadRune()) assumeRune(t, rd, EOF) assertNoError(t, rd.UnreadRune()) assertNoError(t, rd.UnreadRune()) assumeRune(t, rd, EOF) assumeRune(t, rd, EOF) assertNoError(t, rd.UnreadRune()) assertNoError(t, rd.UnreadRune()) assertNoError(t, rd.UnreadRune()) assumeRune(t, rd, 'x') assumeRune(t, rd, EOF) assumeRune(t, rd, EOF) } func TestForget(t *testing.T) { s := "helio" rd := NewReader(strings.NewReader(s)) assumeRune(t, rd, 'h') assumeRune(t, rd, 'e') assumeRune(t, rd, 'l') assumeRune(t, rd, 'i') rd.Forget() if rd.UnreadRune() != ErrNoRuneToUnread { t.Fatal("no rune should be available") } assumeRune(t, rd, 'o') } func TestForgetAfterUnread(t *testing.T) { s := "helio" rd := NewReader(strings.NewReader(s)) assumeRune(t, rd, 'h') assumeRune(t, rd, 'e') assumeRune(t, rd, 'l') assumeRune(t, rd, 'i') assertNoError(t, rd.UnreadRune()) rd.Forget() if rd.UnreadRune() != ErrNoRuneToUnread { t.Fatal("no rune should be available") } assumeRune(t, rd, 'i') assumeRune(t, rd, 'o') } func TestForgetEmpty(t *testing.T) { s := "" rd := NewReader(strings.NewReader(s)) rd.Forget() assumeRune(t, rd, EOF) rd.Forget() } func TestPeekEmpty(t *testing.T) { s := "" rd := NewReader(strings.NewReader(s)) runes := rd.PeekRunes(1) if len(runes) != 1 { t.Fatal("incorrect number of runes", len(runes)) } if runes[0] != EOF { t.Fatal("incorrect rune", runes[0]) } } func TestPeek(t *testing.T) { s := "a" rd := NewReader(strings.NewReader(s)) runes := rd.PeekRunes(1) assumeRunesArray(t, []rune{'a'}, runes) runes = rd.PeekRunes(1) assumeRunesArray(t, []rune{'a'}, runes) assumeRune(t, rd, 'a') runes = rd.PeekRunes(1) assumeRunesArray(t, []rune{EOF}, runes) assumeRune(t, rd, EOF) } func TestPeekLarge(t *testing.T) { s := "abcdefg☺\xff☹" rd := NewReader(strings.NewReader(s)) runes := rd.PeekRunes(100) want := []rune{'a', 'b', 'c', 'd', 'e', 'f', 'g', '☺', utf8.RuneError, '☹', EOF} if !reflect.DeepEqual(runes, want) { t.Fatalf("PeekRunes(100) = %q, want %q", runes, want) } } var bigString = strings.Repeat("abcdefghi☺\xff☹", 1024) // 16 kB const bigStringRunes = 12 * 1024 // 12k runes func BenchmarkRead16K(b *testing.B) { // Read 16K with no unread, no forget. benchmarkRead(b, 1, false) } func BenchmarkReadForget16K(b *testing.B) { // Read 16K, forgetting every 128 runes. benchmarkRead(b, 1, true) } func BenchmarkReadRewind16K(b *testing.B) { // Read 16K, unread all, read that 16K again. benchmarkRead(b, 2, false) } func benchmarkRead(b *testing.B, count int, forget bool) { if len(bigString) != 16*1024 { b.Fatal("wrong length for bigString") } sr0 := strings.NewReader(bigString) sr := new(strings.Reader) b.SetBytes(int64(len(bigString))) b.ReportAllocs() for i := 0; i < b.N; i++ { *sr = *sr0 rd := NewReader(sr) for repeat := 0; repeat < count; repeat++ { for j := 0; j < bigStringRunes; j++ { r, _, err := rd.ReadRune() if err != nil { b.Fatal(err) } if r == EOF { b.Fatal("unexpected EOF") } if forget && j%128 == 127 { rd.Forget() } } r, _, err := rd.ReadRune() if err != nil { b.Fatal(err) } if r != EOF { b.Fatalf("missing EOF - %q", r) } if repeat == count-1 { break } for rd.UnreadRune() == nil { // keep unreading } } } } // test reader that will fail reading after a given number of reads type failingReader struct { r io.Reader // underlying reader failAfter int // start failing after that number of reads readCount int // number of reads already done } func newFailingReaderFromString(s string, failAfter int) *failingReader { return &failingReader{ r: strings.NewReader(s), failAfter: failAfter, readCount: 0, } } func (r *failingReader) Read(b []byte) (n int, err error) { if r.readCount < r.failAfter { n, err = r.r.Read(b) r.readCount++ return } return 0, fmt.Errorf("expected read failure") } func TestReadFails(t *testing.T) { size := 4097 // needs to be more than bufio.defaultBufSize, which is 4096 s := make([]byte, size) for i := 0; i < size; i++ { s[i] = 'a' } rd := NewReader(newFailingReaderFromString(string(s), 1)) runes := rd.PeekRunes(256) // first read, ok runes = rd.PeekRunes(1) // rune already loaded, ok runes = rd.PeekRunes(4097) // forces a new read, fails if len(runes) != 4096 { t.Fatalf("expected %d runes. got %d", 4096, len(runes)) } if runes[4095] != 'a' { t.Fatalf("expected last rune to be 'a'. got '%c'", runes[4095]) } rd = NewReader(newFailingReaderFromString(string(s), 1)) for i := 0; i < size - 1; i++ { r, size, err := rd.ReadRune() // read all the runes but last if err != nil { t.Fatalf("no error expeceted at that point, got %s", err) } if size != 1 { t.Fatalf("reading runes that should have size 1, got size %d", size) } if r != 'a' { t.Fatalf("reading a string of 'a', got %c", r) } } // EOF, 0, err r, n, err := rd.ReadRune() // should error if r != EOF { t.Fatalf("expected EOF, got %c", r) } if n != 0 { t.Fatalf("expected size 0, got %d", n) } if err.Error() != "expected read failure" { t.Fatalf("incorrect error: %s", err.Error()) } }