pax_global_header00006660000000000000000000000064151215313060014507gustar00rootroot0000000000000052 comment=ef756c453c7b491eb91ef5bffbe228920b7a76de golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/000077500000000000000000000000001512153130600232555ustar00rootroot00000000000000golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/LICENSE000066400000000000000000000020761512153130600242670ustar00rootroot00000000000000MIT License Copyright (c) 2018 Максим Федосеев Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/README.md000066400000000000000000000014661512153130600245430ustar00rootroot00000000000000# concurrent [![GoDoc](https://godoc.org/github.com/codesenberg/concurrent?status.svg)](http://godoc.org/github.com/codesenberg/concurrent) *concurrent* is a set of collections and various other things to be used in concurrent environment, such as: * [histogram](http://godoc.org/github.com/codesenberg/concurrent/generic/histogram). Packages in _generic_ subfolder are meant to be used with [gengen](https://github.com/joeshaw/gengen). Consult its [README](https://github.com/joeshaw/gengen#how-to-use-it) for examples, but usually it goes something like this: ```bash gengen github.com/codesenberg/concurrent/generic/histogram float64 ``` After this you might want to fill out appropriate functions in generated files marked with TODO commentaries and remove build tag to get autogenerated test suite.golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/doc.go000066400000000000000000000003011512153130600243430ustar00rootroot00000000000000/*Package concurrent provides concurrent collections and various other utilities. "generic" subdirectory contains generic container implementations to be used with gengen.*/ package concurrent golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/float64/000077500000000000000000000000001512153130600245345ustar00rootroot00000000000000golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/float64/hash/000077500000000000000000000000001512153130600254575ustar00rootroot00000000000000golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/float64/hash/hash.go000066400000000000000000000004711512153130600267330ustar00rootroot00000000000000package hash import "math" // Float64 is a simple hash function for float64 values. func Float64(f float64) uint32 { result := math.Float64bits(f) return uint32(result ^ (result >> 32)) } // Float32 is a simple hash function for float32 values. func Float32(f float32) uint32 { return math.Float32bits(f) } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/float64/histogram/000077500000000000000000000000001512153130600265315ustar00rootroot00000000000000golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/float64/histogram/decls_test.go000066400000000000000000000005431512153130600312130ustar00rootroot00000000000000package histogram func testValues() []float64 { return []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} } func testHistogram() *Histogram { return Default() } const distributionSize = 10000 var benchValueSets = []struct { name string data []float64 }{ {"uniform", uniform(0, 10000, distributionSize)}, {"normal", normal(5000, 500, distributionSize)}, } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/float64/histogram/default.go000066400000000000000000000007051512153130600305060ustar00rootroot00000000000000package histogram import "github.com/codesenberg/concurrent/float64/hash" // WithDefaultHash creates histogram with specified shardCount and // reasonable sharding function. func WithDefaultHash(shardsCount uint32) (*Histogram, error) { return New(shardsCount, hash.Float64) } // Default creates histogram with reasonable defaults. func Default() *Histogram { // We can safely ignore the error in this case h, _ := WithDefaultHash(32) return h } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/float64/histogram/generate.go000066400000000000000000000001441512153130600306510ustar00rootroot00000000000000package histogram //go:generate gengen github.com/codesenberg/concurrent/generic/histogram float64 golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/float64/histogram/histogram.go000066400000000000000000000051121512153130600310540ustar00rootroot00000000000000package histogram import ( "errors" "sync" ) // Errors returned by New. var ( ErrNoShardFn = errors.New("no shard function provided in constructor") ErrZeroShardCount = errors.New("can't create histogram with zero shards") ) // Histogram is a generic histogram implementation intended for use // in concurrent environment. All methods are goroutine-safe. // Zero value and nil are not valid Histograms. type Histogram struct { shards []*histogramShard shardFn func(float64) uint32 } type histogramShard struct { sync.RWMutex counters map[float64]uint64 } // New creates a new histogram. It can panic // if the shardsCount is more than runtime can handle // (i.e. slice size limit). func New( shardsCount uint32, shardFn func(float64) uint32, ) (*Histogram, error) { if shardFn == nil { return nil, ErrNoShardFn } if shardsCount == 0 { return nil, ErrZeroShardCount } shards := make([]*histogramShard, shardsCount) for i := range shards { shards[i] = &histogramShard{ counters: make(map[float64]uint64), } } return &Histogram{ shards: shards, shardFn: shardFn, }, nil } func (h *Histogram) getShardFor(key float64) *histogramShard { shardNum := h.shardFn(key) % uint32(len(h.shards)) return h.shards[shardNum] } // Increment increments counter associated with the specified key by 1. func (h *Histogram) Increment(key float64) { h.Add(key, 1) } // Add increments counter associated with the specified key by // the amount specified. func (h *Histogram) Add(key float64, amount uint64) { shard := h.getShardFor(key) shard.Lock() shard.counters[key] += amount shard.Unlock() } // Get returns current value for the given key. func (h *Histogram) Get(key float64) uint64 { shard := h.getShardFor(key) shard.RLock() result := shard.counters[key] shard.RUnlock() return result } // Count calculates the number of elements in histogram in a more // efficient way that can be implemented with VisitAll func (h *Histogram) Count() uint64 { for _, sh := range h.shards { sh.RLock() } count := uint64(0) for _, sh := range h.shards { count += uint64(len(sh.counters)) } for _, sh := range h.shards { sh.RUnlock() } return count } // VisitAll function applies function to each key-value pair in histogram. // If function returns false iteration stops. Locks the entire histogram. func (h *Histogram) VisitAll(fn func(float64, uint64) bool) { for _, sh := range h.shards { sh.RLock() } defer func() { for _, sh := range h.shards { sh.RUnlock() } }() for _, sh := range h.shards { for k, v := range sh.counters { if !fn(k, v) { return } } } } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/float64/histogram/histogram_test.go000066400000000000000000000050761512153130600321240ustar00rootroot00000000000000package histogram import ( "fmt" "sync" "sync/atomic" "testing" "time" ) func TestHistogramIncrement(t *testing.T) { h := testHistogram() key := testValues()[0] h.Increment(key) if c := h.Get(key); c != 1 { t.Errorf("expected to get 1, but got %v", c) } } func TestHistogramCount(t *testing.T) { h := testHistogram() keys := testValues() for _, k := range keys { h.Increment(k) } if a, e := h.Count(), uint64(len(keys)); e != a { t.Errorf("expected %v entries, but found %v", e, a) } } func TestHistogramVisitAll(t *testing.T) { h := testHistogram() keys := testValues() keysMap := make(map[float64]struct{}) for _, k := range keys { h.Increment(k) keysMap[k] = struct{}{} } foundKeys := make(map[float64]struct{}) h.VisitAll(func(key float64, c uint64) bool { if c != 1 { t.Errorf("expected %v for key %v, but got %v", 1, key, c) return false } foundKeys[key] = struct{}{} return true }) for k := range foundKeys { delete(keysMap, k) } if len(keysMap) != 0 { t.Log("following keys where not found in histogram") for k := range keysMap { t.Log(k) } t.Fail() } } func TestHistogramConcurrentIncrement(t *testing.T) { keys := testValues() counters := make(map[float64]*uint64) for _, k := range keys { counters[k] = new(uint64) } goroutinesCount := 64 doneCh := make(chan struct{}) var wg sync.WaitGroup wg.Add(goroutinesCount) h := testHistogram() for i := 0; i < goroutinesCount; i++ { go func() { defer wg.Done() idx := 0 for { select { case <-doneCh: return default: key := keys[idx] h.Increment(key) atomic.AddUint64(counters[key], 1) idx = (idx + 1) % len(keys) } } }() } time.Sleep(100 * time.Millisecond) close(doneCh) wg.Wait() for k, exp := range counters { act := h.Get(k) if *exp != act { t.Errorf("for %v - got %v, want %v", k, act, exp) break } } } func BenchmarkHistogramPerformance(b *testing.B) { parallelismLevels := []int{1, 2, 4, 8, 16, 32} for _, p := range parallelismLevels { for _, set := range benchValueSets { b.Run( fmt.Sprintf( "goroutines=GOMAXPROCS*%v,set=%v", p, set.name, ), benchmarkHistogramPerformance(p, set.data), ) } } } func benchmarkHistogramPerformance(p int, data []float64) func(*testing.B) { return func(b *testing.B) { h := testHistogram() b.SetParallelism(p) dlen := uint64(len(data)) b.RunParallel(func(pb *testing.PB) { idx := uint64(0) key := data[idx] for pb.Next() { h.Increment(key) idx = atomic.AddUint64(&idx, 1) % dlen key = data[idx] } }) } } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/float64/histogram/utils_for_test.go000066400000000000000000000006551512153130600321330ustar00rootroot00000000000000package histogram import "math/rand" func uniform(min, max, size int) []float64 { result := make([]float64, size) diff := float64(max - min) for i := range result { result[i] = float64(min) + rand.Float64()*diff } return result } func normal(mean, stdev, size int) []float64 { result := make([]float64, size) for i := range result { result[i] = rand.NormFloat64()*float64(stdev) + float64(mean) } return result } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/generic/000077500000000000000000000000001512153130600246715ustar00rootroot00000000000000golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/generic/histogram/000077500000000000000000000000001512153130600266665ustar00rootroot00000000000000golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/generic/histogram/decls_test.go000066400000000000000000000011601512153130600313440ustar00rootroot00000000000000// +build never // TODO(you): uncomment build tag when decls_test.go is filled package histogram import "github.com/joeshaw/gengen/generic" // TODO(you): fill in this function as described below. func testValues() []generic.T { panic("fill this function to provide one or more distinct values") } // TODO(you): replace this with a function that returns some // histogram to use in tests. func testHistogram() *Histogram { panic("fill in with appropriate code") } var benchValueSets = []struct { name string data []generic.T }{ // TODO(you)[optional]: fill those in with some value sets to use // in benchmark. } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/generic/histogram/histogram.go000066400000000000000000000052021512153130600312110ustar00rootroot00000000000000package histogram import ( "errors" "sync" "github.com/joeshaw/gengen/generic" ) // Errors returned by New. var ( ErrNoShardFn = errors.New("no shard function provided in constructor") ErrZeroShardCount = errors.New("can't create histogram with zero shards") ) // Histogram is a generic histogram implementation intended for use // in concurrent environment. All methods are goroutine-safe. // Zero value and nil are not valid Histograms. type Histogram struct { shards []*histogramShard shardFn func(generic.T) uint32 } type histogramShard struct { sync.RWMutex counters map[generic.T]uint64 } // New creates a new histogram. It can panic // if the shardsCount is more than runtime can handle // (i.e. slice size limit). func New( shardsCount uint32, shardFn func(generic.T) uint32, ) (*Histogram, error) { if shardFn == nil { return nil, ErrNoShardFn } if shardsCount == 0 { return nil, ErrZeroShardCount } shards := make([]*histogramShard, shardsCount) for i := range shards { shards[i] = &histogramShard{ counters: make(map[generic.T]uint64), } } return &Histogram{ shards: shards, shardFn: shardFn, }, nil } func (h *Histogram) getShardFor(key generic.T) *histogramShard { shardNum := h.shardFn(key) % uint32(len(h.shards)) return h.shards[shardNum] } // Increment increments counter associated with the specified key by 1. func (h *Histogram) Increment(key generic.T) { h.Add(key, 1) } // Add increments counter associated with the specified key by // the amount specified. func (h *Histogram) Add(key generic.T, amount uint64) { shard := h.getShardFor(key) shard.Lock() shard.counters[key] += amount shard.Unlock() } // Get returns current value for the given key. func (h *Histogram) Get(key generic.T) uint64 { shard := h.getShardFor(key) shard.RLock() result := shard.counters[key] shard.RUnlock() return result } // Count calculates the number of elements in histogram in a more // efficient way that can be implemented with VisitAll func (h *Histogram) Count() uint64 { for _, sh := range h.shards { sh.RLock() } count := uint64(0) for _, sh := range h.shards { count += uint64(len(sh.counters)) } for _, sh := range h.shards { sh.RUnlock() } return count } // VisitAll function applies function to each key-value pair in histogram. // If function returns false iteration stops. Locks the entire histogram. func (h *Histogram) VisitAll(fn func(generic.T, uint64) bool) { for _, sh := range h.shards { sh.RLock() } defer func() { for _, sh := range h.shards { sh.RUnlock() } }() for _, sh := range h.shards { for k, v := range sh.counters { if !fn(k, v) { return } } } } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/generic/histogram/histogram_test.go000066400000000000000000000052771512153130600322640ustar00rootroot00000000000000// +build never // TODO(you): uncomment build tag when decls_test.go is filled package histogram import ( "fmt" "sync" "sync/atomic" "testing" "time" "github.com/joeshaw/gengen/generic" ) func TestHistogramIncrement(t *testing.T) { h := testHistogram() key := testValues()[0] h.Increment(key) if c := h.Get(key); c != 1 { t.Errorf("expected to get 1, but got %v", c) } } func TestHistogramCount(t *testing.T) { h := testHistogram() keys := testValues() for _, k := range keys { h.Increment(k) } if a, e := h.Count(), uint64(len(keys)); e != a { t.Errorf("expected %v entries, but found %v", e, a) } } func TestHistogramVisitAll(t *testing.T) { h := testHistogram() keys := testValues() keysMap := make(map[generic.T]struct{}) for _, k := range keys { h.Increment(k) keysMap[k] = struct{}{} } foundKeys := make(map[generic.T]struct{}) h.VisitAll(func(key generic.T, c uint64) bool { if c != 1 { t.Errorf("expected %v for key %v, but got %v", 1, key, c) return false } foundKeys[key] = struct{}{} return true }) for k := range foundKeys { delete(keysMap, k) } if len(keysMap) != 0 { t.Log("following keys where not found in histogram") for k := range keysMap { t.Log(k) } t.Fail() } } func TestHistogramConcurrentIncrement(t *testing.T) { keys := testValues() counters := make(map[generic.T]*uint64) for _, k := range keys { counters[k] = new(uint64) } goroutinesCount := 64 doneCh := make(chan struct{}) var wg sync.WaitGroup wg.Add(goroutinesCount) h := testHistogram() for i := 0; i < goroutinesCount; i++ { go func() { defer wg.Done() idx := 0 for { select { case <-doneCh: return default: key := keys[idx] h.Increment(key) atomic.AddUint64(counters[key], 1) idx = (idx + 1) % len(keys) } } }() } time.Sleep(100 * time.Millisecond) close(doneCh) wg.Wait() for k, exp := range counters { act := h.Get(k) if *exp != act { t.Errorf("for %v - got %v, want %v", k, act, exp) break } } } func BenchmarkHistogramPerformance(b *testing.B) { parallelismLevels := []int{1, 2, 4, 8, 16, 32} for _, p := range parallelismLevels { for _, set := range benchValueSets { b.Run( fmt.Sprintf( "goroutines=GOMAXPROCS*%v,set=%v", p, set.name, ), benchmarkHistogramPerformance(p, set.data), ) } } } func benchmarkHistogramPerformance(p int, data []generic.T) func(*testing.B) { return func(b *testing.B) { h := testHistogram() b.SetParallelism(p) dlen := uint64(len(data)) b.RunParallel(func(pb *testing.PB) { idx := uint64(0) key := data[idx] for pb.Next() { h.Increment(key) idx = atomic.AddUint64(&idx, 1) % dlen key = data[idx] } }) } } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/string/000077500000000000000000000000001512153130600245635ustar00rootroot00000000000000golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/string/histogram/000077500000000000000000000000001512153130600265605ustar00rootroot00000000000000golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/string/histogram/decls_test.go000066400000000000000000000012711512153130600312410ustar00rootroot00000000000000package histogram func testValues() []string { return []string{"a", "b", "c", "aa", "ab", "ac", "ba", "bb", "bc", "ca", "cb", "cc"} } func testHistogram() *Histogram { return Default() } var benchValueSets = []struct { name string data []string }{ {"small-set-of-strings", testValues()}, {"few-of-small-strings", generateStrings(172831233, 100, 10)}, {"few-of-medium-strings", generateStrings(172831233, 100, 100)}, {"few-of-big-strings", generateStrings(172831233, 100, 1000)}, {"lots-of-small-strings", generateStrings(172831233, 10000, 10)}, {"lots-of-medium-strings", generateStrings(172831233, 10000, 100)}, {"lots-of-big-strings", generateStrings(172831233, 10000, 1000)}, } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/string/histogram/default.go000066400000000000000000000011121512153130600305260ustar00rootroot00000000000000package histogram func hashString(s string) uint32 { // fnv32 hash const prime32 = 16777619 hash := uint32(2166136261) for _, c := range []byte(s) { hash *= prime32 hash ^= uint32(c) } return hash } // WithDefaultHash creates histogram with specified shardCount and // reasonable sharding function. func WithDefaultHash(shardsCount uint32) (*Histogram, error) { return New(shardsCount, hashString) } // Default creates histogram with reasonable defaults. func Default() *Histogram { // We can safely ignore the error in this case h, _ := WithDefaultHash(32) return h } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/string/histogram/generate.go000066400000000000000000000001431512153130600306770ustar00rootroot00000000000000package histogram //go:generate gengen github.com/codesenberg/concurrent/generic/histogram string golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/string/histogram/histogram.go000066400000000000000000000051011512153130600311010ustar00rootroot00000000000000package histogram import ( "errors" "sync" ) // Errors returned by New. var ( ErrNoShardFn = errors.New("no shard function provided in constructor") ErrZeroShardCount = errors.New("can't create histogram with zero shards") ) // Histogram is a generic histogram implementation intended for use // in concurrent environment. All methods are goroutine-safe. // Zero value and nil are not valid Histograms. type Histogram struct { shards []*histogramShard shardFn func(string) uint32 } type histogramShard struct { sync.RWMutex counters map[string]uint64 } // New creates a new histogram. It can panic // if the shardsCount is more than runtime can handle // (i.e. slice size limit). func New( shardsCount uint32, shardFn func(string) uint32, ) (*Histogram, error) { if shardFn == nil { return nil, ErrNoShardFn } if shardsCount == 0 { return nil, ErrZeroShardCount } shards := make([]*histogramShard, shardsCount) for i := range shards { shards[i] = &histogramShard{ counters: make(map[string]uint64), } } return &Histogram{ shards: shards, shardFn: shardFn, }, nil } func (h *Histogram) getShardFor(key string) *histogramShard { shardNum := h.shardFn(key) % uint32(len(h.shards)) return h.shards[shardNum] } // Increment increments counter associated with the specified key by 1. func (h *Histogram) Increment(key string) { h.Add(key, 1) } // Add increments counter associated with the specified key by // the amount specified. func (h *Histogram) Add(key string, amount uint64) { shard := h.getShardFor(key) shard.Lock() shard.counters[key] += amount shard.Unlock() } // Get returns current value for the given key. func (h *Histogram) Get(key string) uint64 { shard := h.getShardFor(key) shard.RLock() result := shard.counters[key] shard.RUnlock() return result } // Count calculates the number of elements in histogram in a more // efficient way that can be implemented with VisitAll func (h *Histogram) Count() uint64 { for _, sh := range h.shards { sh.RLock() } count := uint64(0) for _, sh := range h.shards { count += uint64(len(sh.counters)) } for _, sh := range h.shards { sh.RUnlock() } return count } // VisitAll function applies function to each key-value pair in histogram. // If function returns false iteration stops. Locks the entire histogram. func (h *Histogram) VisitAll(fn func(string, uint64) bool) { for _, sh := range h.shards { sh.RLock() } defer func() { for _, sh := range h.shards { sh.RUnlock() } }() for _, sh := range h.shards { for k, v := range sh.counters { if !fn(k, v) { return } } } } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/string/histogram/histogram_test.go000066400000000000000000000050711512153130600321460ustar00rootroot00000000000000package histogram import ( "fmt" "sync" "sync/atomic" "testing" "time" ) func TestHistogramIncrement(t *testing.T) { h := testHistogram() key := testValues()[0] h.Increment(key) if c := h.Get(key); c != 1 { t.Errorf("expected to get 1, but got %v", c) } } func TestHistogramCount(t *testing.T) { h := testHistogram() keys := testValues() for _, k := range keys { h.Increment(k) } if a, e := h.Count(), uint64(len(keys)); e != a { t.Errorf("expected %v entries, but found %v", e, a) } } func TestHistogramVisitAll(t *testing.T) { h := testHistogram() keys := testValues() keysMap := make(map[string]struct{}) for _, k := range keys { h.Increment(k) keysMap[k] = struct{}{} } foundKeys := make(map[string]struct{}) h.VisitAll(func(key string, c uint64) bool { if c != 1 { t.Errorf("expected %v for key %v, but got %v", 1, key, c) return false } foundKeys[key] = struct{}{} return true }) for k := range foundKeys { delete(keysMap, k) } if len(keysMap) != 0 { t.Log("following keys where not found in histogram") for k := range keysMap { t.Log(k) } t.Fail() } } func TestHistogramConcurrentIncrement(t *testing.T) { keys := testValues() counters := make(map[string]*uint64) for _, k := range keys { counters[k] = new(uint64) } goroutinesCount := 64 doneCh := make(chan struct{}) var wg sync.WaitGroup wg.Add(goroutinesCount) h := testHistogram() for i := 0; i < goroutinesCount; i++ { go func() { defer wg.Done() idx := 0 for { select { case <-doneCh: return default: key := keys[idx] h.Increment(key) atomic.AddUint64(counters[key], 1) idx = (idx + 1) % len(keys) } } }() } time.Sleep(100 * time.Millisecond) close(doneCh) wg.Wait() for k, exp := range counters { act := h.Get(k) if *exp != act { t.Errorf("for %v - got %v, want %v", k, act, exp) break } } } func BenchmarkHistogramPerformance(b *testing.B) { parallelismLevels := []int{1, 2, 4, 8, 16, 32} for _, p := range parallelismLevels { for _, set := range benchValueSets { b.Run( fmt.Sprintf( "goroutines=GOMAXPROCS*%v,set=%v", p, set.name, ), benchmarkHistogramPerformance(p, set.data), ) } } } func benchmarkHistogramPerformance(p int, data []string) func(*testing.B) { return func(b *testing.B) { h := testHistogram() b.SetParallelism(p) dlen := uint64(len(data)) b.RunParallel(func(pb *testing.PB) { idx := uint64(0) key := data[idx] for pb.Next() { h.Increment(key) idx = atomic.AddUint64(&idx, 1) % dlen key = data[idx] } }) } } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/string/histogram/utils_for_test.go000066400000000000000000000007221512153130600321550ustar00rootroot00000000000000package histogram import "math/rand" func generateStrings(seed int64, count, size int) []string { rand.Seed(seed) result := make([]string, count) randomBytes := make([]byte, size) for i := range result { _, err := rand.Read(randomBytes) if err != nil { panic(err) } // Make it a string from some random letters. for j := range randomBytes { randomBytes[j] = 'a' + (randomBytes[j] % 28) } result[i] = string(randomBytes) } return result } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/uint64/000077500000000000000000000000001512153130600244065ustar00rootroot00000000000000golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/uint64/hash/000077500000000000000000000000001512153130600253315ustar00rootroot00000000000000golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/uint64/hash/hash.go000066400000000000000000000002071512153130600266020ustar00rootroot00000000000000package hash // Uint64 is a simple hash function for uint64 values func Uint64(u uint64) uint32 { return uint32(u) ^ uint32(u>>32) } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/uint64/histogram/000077500000000000000000000000001512153130600264035ustar00rootroot00000000000000golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/uint64/histogram/decls_test.go000066400000000000000000000004571512153130600310710ustar00rootroot00000000000000package histogram func testValues() []uint64 { return []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} } func testHistogram() *Histogram { return Default() } var benchValueSets = []struct { name string data []uint64 }{ {"1-10", urange(1, 10)}, {"1-100", urange(1, 100)}, {"1-1000", urange(1, 1000)}, } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/uint64/histogram/default.go000066400000000000000000000007031512153130600303560ustar00rootroot00000000000000package histogram import "github.com/codesenberg/concurrent/uint64/hash" // WithDefaultHash creates histogram with specified shardCount and // reasonable sharding function. func WithDefaultHash(shardsCount uint32) (*Histogram, error) { return New(shardsCount, hash.Uint64) } // Default creates histogram with reasonable defaults. func Default() *Histogram { // We can safely ignore the error in this case h, _ := WithDefaultHash(32) return h } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/uint64/histogram/generate.go000066400000000000000000000001431512153130600305220ustar00rootroot00000000000000package histogram //go:generate gengen github.com/codesenberg/concurrent/generic/histogram uint64 golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/uint64/histogram/histogram.go000066400000000000000000000051011512153130600307240ustar00rootroot00000000000000package histogram import ( "errors" "sync" ) // Errors returned by New. var ( ErrNoShardFn = errors.New("no shard function provided in constructor") ErrZeroShardCount = errors.New("can't create histogram with zero shards") ) // Histogram is a generic histogram implementation intended for use // in concurrent environment. All methods are goroutine-safe. // Zero value and nil are not valid Histograms. type Histogram struct { shards []*histogramShard shardFn func(uint64) uint32 } type histogramShard struct { sync.RWMutex counters map[uint64]uint64 } // New creates a new histogram. It can panic // if the shardsCount is more than runtime can handle // (i.e. slice size limit). func New( shardsCount uint32, shardFn func(uint64) uint32, ) (*Histogram, error) { if shardFn == nil { return nil, ErrNoShardFn } if shardsCount == 0 { return nil, ErrZeroShardCount } shards := make([]*histogramShard, shardsCount) for i := range shards { shards[i] = &histogramShard{ counters: make(map[uint64]uint64), } } return &Histogram{ shards: shards, shardFn: shardFn, }, nil } func (h *Histogram) getShardFor(key uint64) *histogramShard { shardNum := h.shardFn(key) % uint32(len(h.shards)) return h.shards[shardNum] } // Increment increments counter associated with the specified key by 1. func (h *Histogram) Increment(key uint64) { h.Add(key, 1) } // Add increments counter associated with the specified key by // the amount specified. func (h *Histogram) Add(key uint64, amount uint64) { shard := h.getShardFor(key) shard.Lock() shard.counters[key] += amount shard.Unlock() } // Get returns current value for the given key. func (h *Histogram) Get(key uint64) uint64 { shard := h.getShardFor(key) shard.RLock() result := shard.counters[key] shard.RUnlock() return result } // Count calculates the number of elements in histogram in a more // efficient way that can be implemented with VisitAll func (h *Histogram) Count() uint64 { for _, sh := range h.shards { sh.RLock() } count := uint64(0) for _, sh := range h.shards { count += uint64(len(sh.counters)) } for _, sh := range h.shards { sh.RUnlock() } return count } // VisitAll function applies function to each key-value pair in histogram. // If function returns false iteration stops. Locks the entire histogram. func (h *Histogram) VisitAll(fn func(uint64, uint64) bool) { for _, sh := range h.shards { sh.RLock() } defer func() { for _, sh := range h.shards { sh.RUnlock() } }() for _, sh := range h.shards { for k, v := range sh.counters { if !fn(k, v) { return } } } } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/uint64/histogram/histogram_test.go000066400000000000000000000050711512153130600317710ustar00rootroot00000000000000package histogram import ( "fmt" "sync" "sync/atomic" "testing" "time" ) func TestHistogramIncrement(t *testing.T) { h := testHistogram() key := testValues()[0] h.Increment(key) if c := h.Get(key); c != 1 { t.Errorf("expected to get 1, but got %v", c) } } func TestHistogramCount(t *testing.T) { h := testHistogram() keys := testValues() for _, k := range keys { h.Increment(k) } if a, e := h.Count(), uint64(len(keys)); e != a { t.Errorf("expected %v entries, but found %v", e, a) } } func TestHistogramVisitAll(t *testing.T) { h := testHistogram() keys := testValues() keysMap := make(map[uint64]struct{}) for _, k := range keys { h.Increment(k) keysMap[k] = struct{}{} } foundKeys := make(map[uint64]struct{}) h.VisitAll(func(key uint64, c uint64) bool { if c != 1 { t.Errorf("expected %v for key %v, but got %v", 1, key, c) return false } foundKeys[key] = struct{}{} return true }) for k := range foundKeys { delete(keysMap, k) } if len(keysMap) != 0 { t.Log("following keys where not found in histogram") for k := range keysMap { t.Log(k) } t.Fail() } } func TestHistogramConcurrentIncrement(t *testing.T) { keys := testValues() counters := make(map[uint64]*uint64) for _, k := range keys { counters[k] = new(uint64) } goroutinesCount := 64 doneCh := make(chan struct{}) var wg sync.WaitGroup wg.Add(goroutinesCount) h := testHistogram() for i := 0; i < goroutinesCount; i++ { go func() { defer wg.Done() idx := 0 for { select { case <-doneCh: return default: key := keys[idx] h.Increment(key) atomic.AddUint64(counters[key], 1) idx = (idx + 1) % len(keys) } } }() } time.Sleep(100 * time.Millisecond) close(doneCh) wg.Wait() for k, exp := range counters { act := h.Get(k) if *exp != act { t.Errorf("for %v - got %v, want %v", k, act, exp) break } } } func BenchmarkHistogramPerformance(b *testing.B) { parallelismLevels := []int{1, 2, 4, 8, 16, 32} for _, p := range parallelismLevels { for _, set := range benchValueSets { b.Run( fmt.Sprintf( "goroutines=GOMAXPROCS*%v,set=%v", p, set.name, ), benchmarkHistogramPerformance(p, set.data), ) } } } func benchmarkHistogramPerformance(p int, data []uint64) func(*testing.B) { return func(b *testing.B) { h := testHistogram() b.SetParallelism(p) dlen := uint64(len(data)) b.RunParallel(func(pb *testing.PB) { idx := uint64(0) key := data[idx] for pb.Next() { h.Increment(key) idx = atomic.AddUint64(&idx, 1) % dlen key = data[idx] } }) } } golang-github-codesenberg-concurrent-0.0~git20180531.64560cf/uint64/histogram/utils_for_test.go000066400000000000000000000003401512153130600317740ustar00rootroot00000000000000package histogram func urange(from, to uint64) []uint64 { if to < from { panic("to must be less than from") } result := make([]uint64, to-from+1) for i := from; i <= to; i++ { result[i-from] = i } return result }