profile
viewpoint
If you are wondering where the data of this site comes from, please visit https://api.github.com/users/replay/events. GitMemory does not store any data, but only uses NGINX to cache data for a period of time. The idea behind GitMemory is simply to give users a better reading experience.
Mauro Stettler replay Grafana Labs (@grafana) Asuncion, Paraguay

graphite-project/graphite-web 5326

A highly scalable real-time graphing system

grafana/carbon-relay-ng 440

Fast carbon relay+aggregator with admin interfaces for making changes online - production ready

replay/ngx_http_consistent_hash 122

a module which enables the nginx to use the same consistent hashing distribution for memcache servers as the php memcache module

replay/ngx_http_lower_upper_case 29

simple module to uppercase/lowercase strings in the nginx config

replay/go-generic-object-store 6

go-generic-object-store is a small, fast, light-weight, in-memory, off-heap library designed specifically for use with string interning and other similar redundancy reduction concepts.

replay/ngx_http_generate_secure_download_links 5

nginx module to generate secured download links on the fly

DanCech/graphite-web 1

A highly scalable real-time graphing system

grafana/graphite-web 1

A highly scalable real-time graphing system

replay/graphiteDataCollector 1

a tool to collect data via plugins and send it to the graphite carbon server

replay/kausalco_public 1

Kausal's monorepo for Open Source projects

PullRequestReviewEvent

Pull request review commentgrafana/dskit

In-memory implementation of etcd client for testing

 package etcd  import (-	"flag"+	"bytes"+	"context" 	"fmt" 	"io"-	"io/ioutil"-	"net/url"-	"time"+	"sync"  	"github.com/go-kit/kit/log"-	"go.etcd.io/etcd/server/v3/embed"-	"go.etcd.io/etcd/server/v3/etcdserver/api/v3client"+	"go.etcd.io/etcd/api/v3/etcdserverpb"+	"go.etcd.io/etcd/api/v3/mvccpb"+	clientv3 "go.etcd.io/etcd/client/v3" -	"github.com/grafana/dskit/closer" 	"github.com/grafana/dskit/flagext" 	"github.com/grafana/dskit/kv/codec" ) -const etcdStartTimeout = 30 * time.Second+// channelBufferSize is the size of the channels used to send events from Put, Delete,+// and transactions as well as the channel used to send filtered events to watchers.+const channelBufferSize = 10++// NewInMemoryClient creates an Etcd Client implementation that uses an in-memory+// version of the underlying Etcd client.+func NewInMemoryClient(codec codec.Codec, logger log.Logger) (*Client, io.Closer) {+	// Make sure to set default values for the config including number of retries,+	// otherwise the client won't even attempt a CAS operation+	cfg := Config{}+	flagext.DefaultValues(&cfg)++	kv := newMockKV()+	client := &Client{+		cfg:    cfg,+		codec:  codec,+		cli:    kv,+		logger: logger,+	}++	return client, kv+}++// newMockKV creates an in-memory implementation of an etcd client+func newMockKV() *mockKV {+	kv := &mockKV{+		values:    make(map[string]mvccpb.KeyValue),+		valuesMtx: sync.Mutex{},+		close:     make(chan struct{}),+		events:    make(map[chan clientv3.Event]struct{}),+		eventsMtx: sync.Mutex{},+	}++	return kv+}++// mockKV is an in-memory implementation of an Etcd client.+//+// This implementation has many limitations compared to the real client since it+// only exists to be used by the Etcd kv.Client implementation during unit tests. As+// such some behavior may be missing or incorrect compared to the real client. This+// is determined to be an acceptable tradeoff to avoid needing to depend on an entire+// Etcd server for unit tests.+//+// Known limitations:+//+// * Compact is not implemented and will panic+// * RequestProgress is not implemented and will panic+// * Only exact and prefix matching is supported for Get, Put, and Delete+// * There may be inconsistencies with how various version numbers are adjusted+//   but none that are exposed by kv.Client unit tests+type mockKV struct {+	// Key-value pairs created by put calls or transactions+	values    map[string]mvccpb.KeyValue+	valuesMtx sync.Mutex++	// Channel for stopping all running watch goroutines and closing+	// and cleaning up all channels used for sending events to watchers+	close chan struct{}++	// Channels that should receive events in response to Put or Delete+	// calls. These channels are in turn read by goroutines that apply+	// filtering before sending watch responses to their callers.+	events    map[chan clientv3.Event]struct{}+	eventsMtx sync.Mutex+}++// Watch implements the Clientv3Facade interface+func (m *mockKV) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan {+	watcher := make(chan clientv3.WatchResponse, channelBufferSize)+	consumer := m.createEventConsumer(channelBufferSize)++	go func() {+		defer func() {+			// When this goroutine ends, remove and close the channel written to by the+			// Put and Delete methods as well as closing the channel read by the caller+			// of the Watch method+			m.destroyEventConsumer(consumer)++			// non-blocking send+			select {+			case watcher <- clientv3.WatchResponse{Canceled: true}:+			default:+			}++			close(watcher)+		}()++		for {+			select {+			case <-ctx.Done():+				// Context cancelled for this watcher, run cleanup logic and exit+				return+			case <-m.close:+				// Close method called for all watchers, run cleanup logic and exit+				return+			case e := <-consumer:+				op := clientv3.OpGet(key, opts...)+				match := m.isMatch(op, *e.Kv)++				if match {+					// non-blocking send+					select {+					case watcher <- clientv3.WatchResponse{Events: []*clientv3.Event{&e}}:+					default:+					}+				}+			}+		}+	}()++	return watcher+}++// createEventConsumer creates and returns a new channel that is registered to receive+// events for Puts and Deletes.+func (m *mockKV) createEventConsumer(bufSz int) chan clientv3.Event {+	ch := make(chan clientv3.Event, bufSz)+	m.eventsMtx.Lock()+	m.events[ch] = struct{}{}+	m.eventsMtx.Unlock()+	return ch+}++// destroyEventConsumer removes the given channel from the list of channels that events+// should be sent to and closes it.+func (m *mockKV) destroyEventConsumer(ch chan clientv3.Event) {+	m.eventsMtx.Lock()+	delete(m.events, ch)+	m.eventsMtx.Unlock()+	close(ch)+}++// sendEvent writes an event to all currently registered events. The consumer+// channels are each read by a goroutine that filters the event and sends it to+// the caller of the Watch method.+func (m *mockKV) sendEvent(e clientv3.Event) {+	m.eventsMtx.Lock()+	for ch := range m.events {+		// non-blocking send+		select {+		case ch <- e:+		default:+		}+	}+	m.eventsMtx.Unlock()+}++// RequestProgress implements the Clientv3Facade interface+func (m *mockKV) RequestProgress(context.Context) error {+	panic("RequestProgress unimplemented")+}++// Close implements the Clientv3Facade interface+func (m *mockKV) Close() error {+	close(m.close)+	return nil+}++// Get implements the Clientv3Facade interface+func (m *mockKV) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {+	op := clientv3.OpGet(key, opts...)+	res, err := m.Do(ctx, op) -// Mock returns a Mock Etcd client.-// Inspired by https://github.com/ligato/cn-infra/blob/master/db/keyval/etcd/mocks/embeded_etcd.go.-func Mock(codec codec.Codec, logger log.Logger) (*Client, io.Closer, error) {-	dir, err := ioutil.TempDir("", "etcd") 	if err != nil {-		return nil, nil, err+		return nil, err 	} -	cfg := embed.NewConfig()-	cfg.Logger = "zap"-	cfg.Dir = dir-	lpurl, _ := url.Parse("http://localhost:0")-	lcurl, _ := url.Parse("http://localhost:0")-	cfg.LPUrls = []url.URL{*lpurl}-	cfg.LCUrls = []url.URL{*lcurl}+	return res.Get(), nil+}++// Delete implements the Clientv3Facade interface+func (m *mockKV) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) {+	op := clientv3.OpDelete(key, opts...)+	res, err := m.Do(ctx, op) -	etcd, err := embed.StartEtcd(cfg) 	if err != nil {-		return nil, nil, err+		return nil, err 	} -	select {-	case <-etcd.Server.ReadyNotify():-	case <-time.After(etcdStartTimeout):-		etcd.Server.Stop() // trigger a shutdown-		return nil, nil, fmt.Errorf("server took too long to start")+	return res.Del(), nil+}++// Put implements the Clientv3Facade interface+func (m *mockKV) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) {+	op := clientv3.OpPut(key, val, opts...)+	res, err := m.Do(ctx, op)++	if err != nil {+		return nil, err 	} -	closer := closer.Func(func() error {-		etcd.Server.Stop()-		return nil+	return res.Put(), nil+}++// Txn implements the Clientv3Facade interface+func (m *mockKV) Txn(ctx context.Context) clientv3.Txn {+	return &mockTxn{+		ctx: ctx,+		kv:  m,+	}+}++// Compact implements the Clientv3Facade interface+func (m *mockKV) Compact(context.Context, int64, ...clientv3.CompactOption) (*clientv3.CompactResponse, error) {+	panic("Compact unimplemented")+}++// Do implements the Clientv3Facade interface+func (m *mockKV) Do(_ context.Context, op clientv3.Op) (clientv3.OpResponse, error) {+	m.valuesMtx.Lock()+	defer m.valuesMtx.Unlock()+	return m.doInternal(op)+}++func (m *mockKV) doInternal(op clientv3.Op) (clientv3.OpResponse, error) {+	if op.IsGet() {+		return m.doGet(op)+	} else if op.IsPut() {+		return m.doPut(op)+	} else if op.IsDelete() {+		return m.doDelete(op)+	} else if op.IsTxn() {+		return m.doTxn(op)+	} else {+		panic(fmt.Sprintf("unsupported operation: %+v", op))+	}+}++func (m *mockKV) doGet(op clientv3.Op) (clientv3.OpResponse, error) {+	kvs := make([]*mvccpb.KeyValue, 0)+	matching := m.matchingKeys(op, m.values)++	for _, k := range matching {+		kv := m.values[k]+		kvs = append(kvs, &kv)+	}++	res := clientv3.GetResponse{+		Kvs:   kvs,+		Count: int64(len(kvs)),+	}++	return res.OpResponse(), nil+}++func (m *mockKV) doDelete(op clientv3.Op) (clientv3.OpResponse, error) {+	matching := m.matchingKeys(op, m.values)++	for _, k := range matching {+		kv := m.values[k]+		kv.ModRevision = kv.Version++		m.sendEvent(clientv3.Event{+			Type: mvccpb.DELETE,+			Kv:   &kv,+		})++		delete(m.values, k)+	}++	res := clientv3.DeleteResponse{Deleted: int64(len(matching))}+	return res.OpResponse(), nil+}++func (m *mockKV) doPut(op clientv3.Op) (clientv3.OpResponse, error) {+	keyBytes := op.KeyBytes()+	valBytes := op.ValueBytes()+	key := string(keyBytes)++	var newVal mvccpb.KeyValue+	oldVal, ok := m.values[key]++	if ok {+		newVal = oldVal+		newVal.Version = newVal.Version + 1+		newVal.ModRevision = newVal.ModRevision + 1+		newVal.Value = valBytes+	} else {+		newVal = mvccpb.KeyValue{+			Key:            keyBytes,+			Value:          valBytes,+			Version:        1,+			CreateRevision: 1,+			ModRevision:    1,+		}+	}++	m.values[key] = newVal+	m.sendEvent(clientv3.Event{+		Type: mvccpb.PUT,+		Kv:   &newVal, 	}) -	var config Config-	flagext.DefaultValues(&config)+	res := clientv3.PutResponse{}+	return res.OpResponse(), nil+} -	client := &Client{-		cfg:    config,-		codec:  codec,-		cli:    v3client.New(etcd.Server),-		logger: logger,+func (m *mockKV) doTxn(op clientv3.Op) (clientv3.OpResponse, error) {+	cmps, thens, elses := op.Txn()+	succeeded := m.evalCmps(cmps)++	var toRun []clientv3.Op+	if succeeded {+		toRun = thens+	} else {+		toRun = elses+	}++	responses := make([]*etcdserverpb.ResponseOp, len(toRun))+	for _, o := range toRun {+		_, err := m.doInternal(o)+		responses = append(responses, &etcdserverpb.ResponseOp{Response: nil})

ok, nvm

56quarters

comment created time in 21 hours

PullRequestReviewEvent

Pull request review commentgrafana/dskit

In-memory implementation of etcd client for testing

 package etcd  import (-	"flag"+	"bytes"+	"context" 	"fmt" 	"io"-	"io/ioutil"-	"net/url"-	"time"+	"sync"  	"github.com/go-kit/kit/log"-	"go.etcd.io/etcd/server/v3/embed"-	"go.etcd.io/etcd/server/v3/etcdserver/api/v3client"+	"go.etcd.io/etcd/api/v3/etcdserverpb"+	"go.etcd.io/etcd/api/v3/mvccpb"+	clientv3 "go.etcd.io/etcd/client/v3" -	"github.com/grafana/dskit/closer" 	"github.com/grafana/dskit/flagext" 	"github.com/grafana/dskit/kv/codec" ) -const etcdStartTimeout = 30 * time.Second+// channelBufferSize is the size of the channels used to send events from Put, Delete,+// and transactions as well as the channel used to send filtered events to watchers.+const channelBufferSize = 10++// NewInMemoryClient creates an Etcd Client implementation that uses an in-memory+// version of the underlying Etcd client.+func NewInMemoryClient(codec codec.Codec, logger log.Logger) (*Client, io.Closer) {+	// Make sure to set default values for the config including number of retries,+	// otherwise the client won't even attempt a CAS operation+	cfg := Config{}+	flagext.DefaultValues(&cfg)++	kv := newMockKV()+	client := &Client{+		cfg:    cfg,+		codec:  codec,+		cli:    kv,+		logger: logger,+	}++	return client, kv+}++// newMockKV creates an in-memory implementation of an etcd client+func newMockKV() *mockKV {+	kv := &mockKV{+		values:    make(map[string]mvccpb.KeyValue),+		valuesMtx: sync.Mutex{},+		close:     make(chan struct{}),+		events:    make(map[chan clientv3.Event]struct{}),+		eventsMtx: sync.Mutex{},+	}++	return kv+}++// mockKV is an in-memory implementation of an Etcd client.+//+// This implementation has many limitations compared to the real client since it+// only exists to be used by the Etcd kv.Client implementation during unit tests. As+// such some behavior may be missing or incorrect compared to the real client. This+// is determined to be an acceptable tradeoff to avoid needing to depend on an entire+// Etcd server for unit tests.+//+// Known limitations:+//+// * Compact is not implemented and will panic+// * RequestProgress is not implemented and will panic+// * Only exact and prefix matching is supported for Get, Put, and Delete+// * There may be inconsistencies with how various version numbers are adjusted+//   but none that are exposed by kv.Client unit tests+type mockKV struct {+	// Key-value pairs created by put calls or transactions+	values    map[string]mvccpb.KeyValue+	valuesMtx sync.Mutex++	// Channel for stopping all running watch goroutines and closing+	// and cleaning up all channels used for sending events to watchers+	close chan struct{}++	// Channels that should receive events in response to Put or Delete+	// calls. These channels are in turn read by goroutines that apply+	// filtering before sending watch responses to their callers.+	events    map[chan clientv3.Event]struct{}+	eventsMtx sync.Mutex+}++// Watch implements the Clientv3Facade interface+func (m *mockKV) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan {+	watcher := make(chan clientv3.WatchResponse, channelBufferSize)+	consumer := m.createEventConsumer(channelBufferSize)++	go func() {+		defer func() {+			// When this goroutine ends, remove and close the channel written to by the+			// Put and Delete methods as well as closing the channel read by the caller+			// of the Watch method+			m.destroyEventConsumer(consumer)++			// non-blocking send+			select {+			case watcher <- clientv3.WatchResponse{Canceled: true}:+			default:+			}++			close(watcher)+		}()++		for {+			select {+			case <-ctx.Done():+				// Context cancelled for this watcher, run cleanup logic and exit+				return+			case <-m.close:+				// Close method called for all watchers, run cleanup logic and exit+				return+			case e := <-consumer:+				op := clientv3.OpGet(key, opts...)+				match := m.isMatch(op, *e.Kv)++				if match {+					// non-blocking send+					select {+					case watcher <- clientv3.WatchResponse{Events: []*clientv3.Event{&e}}:+					default:+					}+				}+			}+		}+	}()++	return watcher+}++// createEventConsumer creates and returns a new channel that is registered to receive+// events for Puts and Deletes.+func (m *mockKV) createEventConsumer(bufSz int) chan clientv3.Event {+	ch := make(chan clientv3.Event, bufSz)+	m.eventsMtx.Lock()+	m.events[ch] = struct{}{}+	m.eventsMtx.Unlock()+	return ch+}++// destroyEventConsumer removes the given channel from the list of channels that events+// should be sent to and closes it.+func (m *mockKV) destroyEventConsumer(ch chan clientv3.Event) {+	m.eventsMtx.Lock()+	delete(m.events, ch)+	m.eventsMtx.Unlock()+	close(ch)+}++// sendEvent writes an event to all currently registered events. The consumer+// channels are each read by a goroutine that filters the event and sends it to+// the caller of the Watch method.+func (m *mockKV) sendEvent(e clientv3.Event) {+	m.eventsMtx.Lock()+	for ch := range m.events {+		// non-blocking send+		select {+		case ch <- e:+		default:+		}+	}+	m.eventsMtx.Unlock()+}++// RequestProgress implements the Clientv3Facade interface+func (m *mockKV) RequestProgress(context.Context) error {+	panic("RequestProgress unimplemented")+}++// Close implements the Clientv3Facade interface+func (m *mockKV) Close() error {+	close(m.close)+	return nil+}++// Get implements the Clientv3Facade interface+func (m *mockKV) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {+	op := clientv3.OpGet(key, opts...)+	res, err := m.Do(ctx, op) -// Mock returns a Mock Etcd client.-// Inspired by https://github.com/ligato/cn-infra/blob/master/db/keyval/etcd/mocks/embeded_etcd.go.-func Mock(codec codec.Codec, logger log.Logger) (*Client, io.Closer, error) {-	dir, err := ioutil.TempDir("", "etcd") 	if err != nil {-		return nil, nil, err+		return nil, err 	} -	cfg := embed.NewConfig()-	cfg.Logger = "zap"-	cfg.Dir = dir-	lpurl, _ := url.Parse("http://localhost:0")-	lcurl, _ := url.Parse("http://localhost:0")-	cfg.LPUrls = []url.URL{*lpurl}-	cfg.LCUrls = []url.URL{*lcurl}+	return res.Get(), nil+}++// Delete implements the Clientv3Facade interface+func (m *mockKV) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) {+	op := clientv3.OpDelete(key, opts...)+	res, err := m.Do(ctx, op) -	etcd, err := embed.StartEtcd(cfg) 	if err != nil {-		return nil, nil, err+		return nil, err 	} -	select {-	case <-etcd.Server.ReadyNotify():-	case <-time.After(etcdStartTimeout):-		etcd.Server.Stop() // trigger a shutdown-		return nil, nil, fmt.Errorf("server took too long to start")+	return res.Del(), nil+}++// Put implements the Clientv3Facade interface+func (m *mockKV) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) {+	op := clientv3.OpPut(key, val, opts...)+	res, err := m.Do(ctx, op)++	if err != nil {+		return nil, err 	} -	closer := closer.Func(func() error {-		etcd.Server.Stop()-		return nil+	return res.Put(), nil+}++// Txn implements the Clientv3Facade interface+func (m *mockKV) Txn(ctx context.Context) clientv3.Txn {+	return &mockTxn{+		ctx: ctx,+		kv:  m,+	}+}++// Compact implements the Clientv3Facade interface+func (m *mockKV) Compact(context.Context, int64, ...clientv3.CompactOption) (*clientv3.CompactResponse, error) {+	panic("Compact unimplemented")+}++// Do implements the Clientv3Facade interface+func (m *mockKV) Do(_ context.Context, op clientv3.Op) (clientv3.OpResponse, error) {+	m.valuesMtx.Lock()+	defer m.valuesMtx.Unlock()+	return m.doInternal(op)+}++func (m *mockKV) doInternal(op clientv3.Op) (clientv3.OpResponse, error) {+	if op.IsGet() {+		return m.doGet(op)+	} else if op.IsPut() {+		return m.doPut(op)+	} else if op.IsDelete() {+		return m.doDelete(op)+	} else if op.IsTxn() {+		return m.doTxn(op)+	} else {+		panic(fmt.Sprintf("unsupported operation: %+v", op))+	}+}++func (m *mockKV) doGet(op clientv3.Op) (clientv3.OpResponse, error) {+	kvs := make([]*mvccpb.KeyValue, 0)+	matching := m.matchingKeys(op, m.values)++	for _, k := range matching {+		kv := m.values[k]+		kvs = append(kvs, &kv)+	}++	res := clientv3.GetResponse{+		Kvs:   kvs,+		Count: int64(len(kvs)),+	}++	return res.OpResponse(), nil+}++func (m *mockKV) doDelete(op clientv3.Op) (clientv3.OpResponse, error) {+	matching := m.matchingKeys(op, m.values)++	for _, k := range matching {+		kv := m.values[k]+		kv.ModRevision = kv.Version++		m.sendEvent(clientv3.Event{+			Type: mvccpb.DELETE,+			Kv:   &kv,+		})++		delete(m.values, k)+	}++	res := clientv3.DeleteResponse{Deleted: int64(len(matching))}+	return res.OpResponse(), nil+}++func (m *mockKV) doPut(op clientv3.Op) (clientv3.OpResponse, error) {+	keyBytes := op.KeyBytes()+	valBytes := op.ValueBytes()+	key := string(keyBytes)++	var newVal mvccpb.KeyValue+	oldVal, ok := m.values[key]++	if ok {+		newVal = oldVal+		newVal.Version = newVal.Version + 1+		newVal.ModRevision = newVal.ModRevision + 1+		newVal.Value = valBytes+	} else {+		newVal = mvccpb.KeyValue{+			Key:            keyBytes,+			Value:          valBytes,+			Version:        1,+			CreateRevision: 1,+			ModRevision:    1,+		}+	}++	m.values[key] = newVal+	m.sendEvent(clientv3.Event{+		Type: mvccpb.PUT,+		Kv:   &newVal, 	}) -	var config Config-	flagext.DefaultValues(&config)+	res := clientv3.PutResponse{}+	return res.OpResponse(), nil+} -	client := &Client{-		cfg:    config,-		codec:  codec,-		cli:    v3client.New(etcd.Server),-		logger: logger,+func (m *mockKV) doTxn(op clientv3.Op) (clientv3.OpResponse, error) {+	cmps, thens, elses := op.Txn()+	succeeded := m.evalCmps(cmps)++	var toRun []clientv3.Op+	if succeeded {+		toRun = thens+	} else {+		toRun = elses+	}++	responses := make([]*etcdserverpb.ResponseOp, len(toRun))+	for _, o := range toRun {+		_, err := m.doInternal(o)+		responses = append(responses, &etcdserverpb.ResponseOp{Response: nil})+		if err != nil {

personally i'd put the error check directly after the call-site which potentially returned an error... but in this specific case I think it makes no difference.

56quarters

comment created time in a day

PullRequestReviewEvent
PullRequestReviewEvent

Pull request review commentgrafana/dskit

In-memory implementation of etcd client for testing

 package etcd  import (-	"flag"+	"bytes"+	"context" 	"fmt" 	"io"-	"io/ioutil"-	"net/url"-	"time"+	"sync"  	"github.com/go-kit/kit/log"-	"go.etcd.io/etcd/server/v3/embed"-	"go.etcd.io/etcd/server/v3/etcdserver/api/v3client"+	"go.etcd.io/etcd/api/v3/etcdserverpb"+	"go.etcd.io/etcd/api/v3/mvccpb"+	clientv3 "go.etcd.io/etcd/client/v3" -	"github.com/grafana/dskit/closer" 	"github.com/grafana/dskit/flagext" 	"github.com/grafana/dskit/kv/codec" ) -const etcdStartTimeout = 30 * time.Second+// channelBufferSize is the size of the channels used to send events from Put, Delete,+// and transactions as well as the channel used to send filtered events to watchers.+const channelBufferSize = 10++// NewInMemoryClient creates an Etcd Client implementation that uses an in-memory+// version of the underlying Etcd client.+func NewInMemoryClient(codec codec.Codec, logger log.Logger) (*Client, io.Closer) {+	// Make sure to set default values for the config including number of retries,+	// otherwise the client won't even attempt a CAS operation+	cfg := Config{}+	flagext.DefaultValues(&cfg)++	kv := newMockKV()+	client := &Client{+		cfg:    cfg,+		codec:  codec,+		cli:    kv,+		logger: logger,+	}++	return client, kv+}++// newMockKV creates an in-memory implementation of an etcd client+func newMockKV() *mockKV {+	kv := &mockKV{+		values:    make(map[string]mvccpb.KeyValue),+		valuesMtx: sync.Mutex{},+		close:     make(chan struct{}),+		events:    make(map[chan clientv3.Event]struct{}),+		eventsMtx: sync.Mutex{},+	}++	return kv+}++// mockKV is an in-memory implementation of an Etcd client.+//+// This implementation has many limitations compared to the real client since it+// only exists to be used by the Etcd kv.Client implementation during unit tests. As+// such some behavior may be missing or incorrect compared to the real client. This+// is determined to be an acceptable tradeoff to avoid needing to depend on an entire+// Etcd server for unit tests.+//+// Known limitations:+//+// * Compact is not implemented and will panic+// * RequestProgress is not implemented and will panic+// * Only exact and prefix matching is supported for Get, Put, and Delete+// * There may be inconsistencies with how various version numbers are adjusted+//   but none that are exposed by kv.Client unit tests+type mockKV struct {+	// Key-value pairs created by put calls or transactions+	values    map[string]mvccpb.KeyValue+	valuesMtx sync.Mutex++	// Channel for stopping all running watch goroutines and closing+	// and cleaning up all channels used for sending events to watchers+	close chan struct{}++	// Channels that should receive events in response to Put or Delete+	// calls. These channels are in turn read by goroutines that apply+	// filtering before sending watch responses to their callers.+	events    map[chan clientv3.Event]struct{}+	eventsMtx sync.Mutex+}++// Watch implements the Clientv3Facade interface+func (m *mockKV) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan {+	watcher := make(chan clientv3.WatchResponse, channelBufferSize)+	consumer := m.createEventConsumer(channelBufferSize)++	go func() {+		defer func() {+			// When this goroutine ends, remove and close the channel written to by the+			// Put and Delete methods as well as closing the channel read by the caller+			// of the Watch method+			m.destroyEventConsumer(consumer)++			// non-blocking send+			select {+			case watcher <- clientv3.WatchResponse{Canceled: true}:+			default:+			}++			close(watcher)+		}()++		for {+			select {+			case <-ctx.Done():+				// Context cancelled for this watcher, run cleanup logic and exit+				return+			case <-m.close:+				// Close method called for all watchers, run cleanup logic and exit+				return+			case e := <-consumer:+				op := clientv3.OpGet(key, opts...)+				match := m.isMatch(op, *e.Kv)++				if match {+					// non-blocking send+					select {+					case watcher <- clientv3.WatchResponse{Events: []*clientv3.Event{&e}}:+					default:+					}+				}+			}+		}+	}()++	return watcher+}++// createEventConsumer creates and returns a new channel that is registered to receive+// events for Puts and Deletes.+func (m *mockKV) createEventConsumer(bufSz int) chan clientv3.Event {+	ch := make(chan clientv3.Event, bufSz)+	m.eventsMtx.Lock()+	m.events[ch] = struct{}{}+	m.eventsMtx.Unlock()+	return ch+}++// destroyEventConsumer removes the given channel from the list of channels that events+// should be sent to and closes it.+func (m *mockKV) destroyEventConsumer(ch chan clientv3.Event) {+	m.eventsMtx.Lock()+	delete(m.events, ch)+	m.eventsMtx.Unlock()+	close(ch)+}++// sendEvent writes an event to all currently registered events. The consumer+// channels are each read by a goroutine that filters the event and sends it to+// the caller of the Watch method.+func (m *mockKV) sendEvent(e clientv3.Event) {+	m.eventsMtx.Lock()+	for ch := range m.events {+		// non-blocking send+		select {+		case ch <- e:+		default:+		}+	}+	m.eventsMtx.Unlock()+}++// RequestProgress implements the Clientv3Facade interface+func (m *mockKV) RequestProgress(context.Context) error {+	panic("RequestProgress unimplemented")+}++// Close implements the Clientv3Facade interface+func (m *mockKV) Close() error {+	close(m.close)+	return nil+}++// Get implements the Clientv3Facade interface+func (m *mockKV) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {+	op := clientv3.OpGet(key, opts...)+	res, err := m.Do(ctx, op) -// Mock returns a Mock Etcd client.-// Inspired by https://github.com/ligato/cn-infra/blob/master/db/keyval/etcd/mocks/embeded_etcd.go.-func Mock(codec codec.Codec, logger log.Logger) (*Client, io.Closer, error) {-	dir, err := ioutil.TempDir("", "etcd") 	if err != nil {-		return nil, nil, err+		return nil, err 	} -	cfg := embed.NewConfig()-	cfg.Logger = "zap"-	cfg.Dir = dir-	lpurl, _ := url.Parse("http://localhost:0")-	lcurl, _ := url.Parse("http://localhost:0")-	cfg.LPUrls = []url.URL{*lpurl}-	cfg.LCUrls = []url.URL{*lcurl}+	return res.Get(), nil+}++// Delete implements the Clientv3Facade interface+func (m *mockKV) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) {+	op := clientv3.OpDelete(key, opts...)+	res, err := m.Do(ctx, op) -	etcd, err := embed.StartEtcd(cfg) 	if err != nil {-		return nil, nil, err+		return nil, err 	} -	select {-	case <-etcd.Server.ReadyNotify():-	case <-time.After(etcdStartTimeout):-		etcd.Server.Stop() // trigger a shutdown-		return nil, nil, fmt.Errorf("server took too long to start")+	return res.Del(), nil+}++// Put implements the Clientv3Facade interface+func (m *mockKV) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) {+	op := clientv3.OpPut(key, val, opts...)+	res, err := m.Do(ctx, op)++	if err != nil {+		return nil, err 	} -	closer := closer.Func(func() error {-		etcd.Server.Stop()-		return nil+	return res.Put(), nil+}++// Txn implements the Clientv3Facade interface+func (m *mockKV) Txn(ctx context.Context) clientv3.Txn {+	return &mockTxn{+		ctx: ctx,+		kv:  m,+	}+}++// Compact implements the Clientv3Facade interface+func (m *mockKV) Compact(context.Context, int64, ...clientv3.CompactOption) (*clientv3.CompactResponse, error) {+	panic("Compact unimplemented")+}++// Do implements the Clientv3Facade interface+func (m *mockKV) Do(_ context.Context, op clientv3.Op) (clientv3.OpResponse, error) {+	m.valuesMtx.Lock()+	defer m.valuesMtx.Unlock()+	return m.doInternal(op)+}++func (m *mockKV) doInternal(op clientv3.Op) (clientv3.OpResponse, error) {+	if op.IsGet() {+		return m.doGet(op)+	} else if op.IsPut() {+		return m.doPut(op)+	} else if op.IsDelete() {+		return m.doDelete(op)+	} else if op.IsTxn() {+		return m.doTxn(op)+	} else {+		panic(fmt.Sprintf("unsupported operation: %+v", op))+	}+}++func (m *mockKV) doGet(op clientv3.Op) (clientv3.OpResponse, error) {+	kvs := make([]*mvccpb.KeyValue, 0)+	matching := m.matchingKeys(op, m.values)++	for _, k := range matching {+		kv := m.values[k]+		kvs = append(kvs, &kv)+	}++	res := clientv3.GetResponse{+		Kvs:   kvs,+		Count: int64(len(kvs)),+	}++	return res.OpResponse(), nil+}++func (m *mockKV) doDelete(op clientv3.Op) (clientv3.OpResponse, error) {+	matching := m.matchingKeys(op, m.values)++	for _, k := range matching {+		kv := m.values[k]+		kv.ModRevision = kv.Version++		m.sendEvent(clientv3.Event{+			Type: mvccpb.DELETE,+			Kv:   &kv,+		})++		delete(m.values, k)+	}++	res := clientv3.DeleteResponse{Deleted: int64(len(matching))}+	return res.OpResponse(), nil+}++func (m *mockKV) doPut(op clientv3.Op) (clientv3.OpResponse, error) {+	keyBytes := op.KeyBytes()+	valBytes := op.ValueBytes()+	key := string(keyBytes)++	var newVal mvccpb.KeyValue+	oldVal, ok := m.values[key]++	if ok {+		newVal = oldVal+		newVal.Version = newVal.Version + 1+		newVal.ModRevision = newVal.ModRevision + 1+		newVal.Value = valBytes+	} else {+		newVal = mvccpb.KeyValue{+			Key:            keyBytes,+			Value:          valBytes,+			Version:        1,+			CreateRevision: 1,+			ModRevision:    1,+		}+	}++	m.values[key] = newVal+	m.sendEvent(clientv3.Event{+		Type: mvccpb.PUT,+		Kv:   &newVal, 	}) -	var config Config-	flagext.DefaultValues(&config)+	res := clientv3.PutResponse{}+	return res.OpResponse(), nil+} -	client := &Client{-		cfg:    config,-		codec:  codec,-		cli:    v3client.New(etcd.Server),-		logger: logger,+func (m *mockKV) doTxn(op clientv3.Op) (clientv3.OpResponse, error) {+	cmps, thens, elses := op.Txn()+	succeeded := m.evalCmps(cmps)++	var toRun []clientv3.Op+	if succeeded {+		toRun = thens+	} else {+		toRun = elses+	}++	responses := make([]*etcdserverpb.ResponseOp, len(toRun))+	for _, o := range toRun {+		_, err := m.doInternal(o)+		responses = append(responses, &etcdserverpb.ResponseOp{Response: nil})

should this append the response returned by m.doInternal()?

56quarters

comment created time in a day

PullRequestReviewEvent

Pull request review commentgrafana/dskit

In-memory implementation of etcd client for testing

 package etcd  import (-	"flag"+	"bytes"+	"context" 	"fmt" 	"io"-	"io/ioutil"-	"net/url"-	"time"+	"sync"  	"github.com/go-kit/kit/log"-	"go.etcd.io/etcd/server/v3/embed"-	"go.etcd.io/etcd/server/v3/etcdserver/api/v3client"+	"go.etcd.io/etcd/api/v3/etcdserverpb"+	"go.etcd.io/etcd/api/v3/mvccpb"+	clientv3 "go.etcd.io/etcd/client/v3" -	"github.com/grafana/dskit/closer" 	"github.com/grafana/dskit/flagext" 	"github.com/grafana/dskit/kv/codec" ) -const etcdStartTimeout = 30 * time.Second+// channelBufferSize is the size of the channels used to send events from Put, Delete,+// and transactions as well as the channel used to send filtered events to watchers.+const channelBufferSize = 10++// NewInMemoryClient creates an Etcd Client implementation that uses an in-memory+// version of the underlying Etcd client.+func NewInMemoryClient(codec codec.Codec, logger log.Logger) (*Client, io.Closer) {+	// Make sure to set default values for the config including number of retries,+	// otherwise the client won't even attempt a CAS operation+	cfg := Config{}+	flagext.DefaultValues(&cfg)++	kv := newMockKV()+	client := &Client{+		cfg:    cfg,+		codec:  codec,+		cli:    kv,+		logger: logger,+	}++	return client, kv+}++// newMockKV creates an in-memory implementation of an etcd client+func newMockKV() *mockKV {+	kv := &mockKV{+		values:    make(map[string]mvccpb.KeyValue),+		valuesMtx: sync.Mutex{},+		close:     make(chan struct{}),+		events:    make(map[chan clientv3.Event]struct{}),+		eventsMtx: sync.Mutex{},+	}++	return kv+}++// mockKV is an in-memory implementation of an Etcd client.+//+// This implementation has many limitations compared to the real client since it+// only exists to be used by the Etcd kv.Client implementation during unit tests. As+// such some behavior may be missing or incorrect compared to the real client. This+// is determined to be an acceptable tradeoff to avoid needing to depend on an entire+// Etcd server for unit tests.+//+// Known limitations:+//+// * Compact is not implemented and will panic+// * RequestProgress is not implemented and will panic+// * Only exact and prefix matching is supported for Get, Put, and Delete+// * There may be inconsistencies with how various version numbers are adjusted+//   but none that are exposed by kv.Client unit tests+type mockKV struct {+	// Key-value pairs created by put calls or transactions+	values    map[string]mvccpb.KeyValue+	valuesMtx sync.Mutex++	// Channel for stopping all running watch goroutines and closing+	// and cleaning up all channels used for sending events to watchers+	close chan struct{}++	// Channels that should receive events in response to Put or Delete+	// calls. These channels are in turn read by goroutines that apply+	// filtering before sending watch responses to their callers.+	events    map[chan clientv3.Event]struct{}+	eventsMtx sync.Mutex+}++// Watch implements the Clientv3Facade interface+func (m *mockKV) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan {+	watcher := make(chan clientv3.WatchResponse, channelBufferSize)+	consumer := m.createEventConsumer(channelBufferSize)++	go func() {+		defer func() {+			// When this goroutine ends, remove and close the channel written to by the+			// Put and Delete methods as well as closing the channel read by the caller+			// of the Watch method+			m.destroyEventConsumer(consumer)++			// non-blocking send+			select {+			case watcher <- clientv3.WatchResponse{Canceled: true}:+			default:+			}++			close(watcher)+		}()++		for {+			select {+			case <-ctx.Done():+				// Context cancelled for this watcher, run cleanup logic and exit+				return+			case <-m.close:+				// Close method called for all watchers, run cleanup logic and exit+				return+			case e := <-consumer:+				op := clientv3.OpGet(key, opts...)+				match := m.isMatch(op, *e.Kv)++				if match {+					// non-blocking send+					select {+					case watcher <- clientv3.WatchResponse{Events: []*clientv3.Event{&e}}:+					default:+					}+				}+			}+		}+	}()++	return watcher+}++// createEventConsumer creates and returns a new channel that is registered to receive+// events for Puts and Deletes.+func (m *mockKV) createEventConsumer(bufSz int) chan clientv3.Event {+	ch := make(chan clientv3.Event, bufSz)+	m.eventsMtx.Lock()+	m.events[ch] = struct{}{}+	m.eventsMtx.Unlock()+	return ch+}++// destroyEventConsumer removes the given channel from the list of channels that events+// should be sent to and closes it.+func (m *mockKV) destroyEventConsumer(ch chan clientv3.Event) {+	m.eventsMtx.Lock()+	delete(m.events, ch)+	m.eventsMtx.Unlock()+	close(ch)+}++// sendEvent writes an event to all currently registered events. The consumer+// channels are each read by a goroutine that filters the event and sends it to+// the caller of the Watch method.+func (m *mockKV) sendEvent(e clientv3.Event) {+	m.eventsMtx.Lock()+	for ch := range m.events {+		// non-blocking send+		select {+		case ch <- e:+		default:+		}+	}+	m.eventsMtx.Unlock()+}++// RequestProgress implements the Clientv3Facade interface+func (m *mockKV) RequestProgress(context.Context) error {+	panic("RequestProgress unimplemented")+}++// Close implements the Clientv3Facade interface+func (m *mockKV) Close() error {+	close(m.close)+	return nil+}++// Get implements the Clientv3Facade interface+func (m *mockKV) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {+	op := clientv3.OpGet(key, opts...)+	res, err := m.Do(ctx, op) -// Mock returns a Mock Etcd client.-// Inspired by https://github.com/ligato/cn-infra/blob/master/db/keyval/etcd/mocks/embeded_etcd.go.-func Mock(codec codec.Codec, logger log.Logger) (*Client, io.Closer, error) {-	dir, err := ioutil.TempDir("", "etcd") 	if err != nil {-		return nil, nil, err+		return nil, err 	} -	cfg := embed.NewConfig()-	cfg.Logger = "zap"-	cfg.Dir = dir-	lpurl, _ := url.Parse("http://localhost:0")-	lcurl, _ := url.Parse("http://localhost:0")-	cfg.LPUrls = []url.URL{*lpurl}-	cfg.LCUrls = []url.URL{*lcurl}+	return res.Get(), nil+}++// Delete implements the Clientv3Facade interface+func (m *mockKV) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) {+	op := clientv3.OpDelete(key, opts...)+	res, err := m.Do(ctx, op) -	etcd, err := embed.StartEtcd(cfg) 	if err != nil {-		return nil, nil, err+		return nil, err 	} -	select {-	case <-etcd.Server.ReadyNotify():-	case <-time.After(etcdStartTimeout):-		etcd.Server.Stop() // trigger a shutdown-		return nil, nil, fmt.Errorf("server took too long to start")+	return res.Del(), nil+}++// Put implements the Clientv3Facade interface+func (m *mockKV) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) {+	op := clientv3.OpPut(key, val, opts...)+	res, err := m.Do(ctx, op)++	if err != nil {+		return nil, err 	} -	closer := closer.Func(func() error {-		etcd.Server.Stop()-		return nil+	return res.Put(), nil+}++// Txn implements the Clientv3Facade interface+func (m *mockKV) Txn(ctx context.Context) clientv3.Txn {+	return &mockTxn{+		ctx: ctx,+		kv:  m,+	}+}++// Compact implements the Clientv3Facade interface+func (m *mockKV) Compact(context.Context, int64, ...clientv3.CompactOption) (*clientv3.CompactResponse, error) {+	panic("Compact unimplemented")+}++// Do implements the Clientv3Facade interface+func (m *mockKV) Do(_ context.Context, op clientv3.Op) (clientv3.OpResponse, error) {+	m.valuesMtx.Lock()+	defer m.valuesMtx.Unlock()+	return m.doInternal(op)+}++func (m *mockKV) doInternal(op clientv3.Op) (clientv3.OpResponse, error) {+	if op.IsGet() {+		return m.doGet(op)+	} else if op.IsPut() {+		return m.doPut(op)+	} else if op.IsDelete() {+		return m.doDelete(op)+	} else if op.IsTxn() {+		return m.doTxn(op)+	} else {+		panic(fmt.Sprintf("unsupported operation: %+v", op))+	}+}++func (m *mockKV) doGet(op clientv3.Op) (clientv3.OpResponse, error) {+	kvs := make([]*mvccpb.KeyValue, 0)+	matching := m.matchingKeys(op, m.values)++	for _, k := range matching {+		kv := m.values[k]+		kvs = append(kvs, &kv)+	}++	res := clientv3.GetResponse{+		Kvs:   kvs,+		Count: int64(len(kvs)),+	}++	return res.OpResponse(), nil+}++func (m *mockKV) doDelete(op clientv3.Op) (clientv3.OpResponse, error) {+	matching := m.matchingKeys(op, m.values)++	for _, k := range matching {+		kv := m.values[k]+		kv.ModRevision = kv.Version++		m.sendEvent(clientv3.Event{+			Type: mvccpb.DELETE,+			Kv:   &kv,+		})++		delete(m.values, k)+	}++	res := clientv3.DeleteResponse{Deleted: int64(len(matching))}+	return res.OpResponse(), nil+}++func (m *mockKV) doPut(op clientv3.Op) (clientv3.OpResponse, error) {+	keyBytes := op.KeyBytes()+	valBytes := op.ValueBytes()+	key := string(keyBytes)++	var newVal mvccpb.KeyValue+	oldVal, ok := m.values[key]++	if ok {+		newVal = oldVal+		newVal.Version = newVal.Version + 1+		newVal.ModRevision = newVal.ModRevision + 1+		newVal.Value = valBytes+	} else {+		newVal = mvccpb.KeyValue{+			Key:            keyBytes,+			Value:          valBytes,+			Version:        1,+			CreateRevision: 1,+			ModRevision:    1,+		}+	}++	m.values[key] = newVal+	m.sendEvent(clientv3.Event{+		Type: mvccpb.PUT,+		Kv:   &newVal, 	}) -	var config Config-	flagext.DefaultValues(&config)+	res := clientv3.PutResponse{}+	return res.OpResponse(), nil+} -	client := &Client{-		cfg:    config,-		codec:  codec,-		cli:    v3client.New(etcd.Server),-		logger: logger,+func (m *mockKV) doTxn(op clientv3.Op) (clientv3.OpResponse, error) {+	cmps, thens, elses := op.Txn()+	succeeded := m.evalCmps(cmps)++	var toRun []clientv3.Op+	if succeeded {+		toRun = thens+	} else {+		toRun = elses+	}++	responses := make([]*etcdserverpb.ResponseOp, len(toRun))
	responses := make([]*etcdserverpb.ResponseOp, 0, len(toRun))

Otherwise this will allocate the slice with the given length, not only the given capacity. Further down we append to this slice, the appended values would then end up behind len(toRun) empty values.

56quarters

comment created time in a day

pull request commentcoreos/go-oidc

Add support for MS ADFS

I really don't want to add provider specific code, particularly not for buggy providers. Can you please open a ticket or bug with ADFS?

I understand. I believe this is a known issue with ADFS because I've seen various other discussions about it. I'm not a customer of Microsoft myself and I can't see how I could raise this as an issue to them (I also don't think ADFS is OSS on GitHub).

Thanks for your suggested solution, this is an interesting idea and I think it might work. I'll give that a try.

replay

comment created time in a day

Pull request review commentgrafana/dskit

In-memory implementation of etcd client for testing

 package etcd  import (-	"flag"+	"bytes"+	"context" 	"fmt" 	"io"-	"io/ioutil"-	"net/url"-	"time"+	"sync"  	"github.com/go-kit/kit/log"-	"go.etcd.io/etcd/server/v3/embed"-	"go.etcd.io/etcd/server/v3/etcdserver/api/v3client"+	"go.etcd.io/etcd/api/v3/etcdserverpb"+	"go.etcd.io/etcd/api/v3/mvccpb"+	clientv3 "go.etcd.io/etcd/client/v3" -	"github.com/grafana/dskit/closer" 	"github.com/grafana/dskit/flagext" 	"github.com/grafana/dskit/kv/codec" ) -const etcdStartTimeout = 30 * time.Second+// NewInMemoryClient creates an Etcd Client implementation that uses an in-memory+// version of the underlying Etcd client.+func NewInMemoryClient(codec codec.Codec, logger log.Logger) (*Client, io.Closer) {+	// Make sure to set default values for the config including number of retries,+	// otherwise the client won't even attempt a CAS operation+	cfg := Config{}+	flagext.DefaultValues(&cfg)++	kv := newMockKV()+	client := &Client{+		cfg:    cfg,+		codec:  codec,+		cli:    kv,+		logger: logger,+	}++	return client, kv+}++// newMockKV creates an in-memory implementation of an etcd client+func newMockKV() *mockKV {+	kv := &mockKV{+		values:       make(map[string]mvccpb.KeyValue),+		valuesMtx:    sync.Mutex{},+		close:        make(chan struct{}),+		consumers:    make(map[chan clientv3.Event]struct{}),+		consumersMtx: sync.Mutex{},+	}++	return kv+}++// mockKV is an in-memory implementation of an Etcd client.+//+// This implementation has many limitations compared to the real client since it+// only exist to be used by the Etcd kv.Client implementation during unit tests. As+// such some behavior may be missing or incorrect compared to the real client. This+// is determined to be an acceptable tradeoff to avoid needing to depend on an entire+// Etcd server for unit tests.+//+// Known limitations:+//+// * Compact is not implemented and will panic+// * RequestProgress is not implemented and will panic+// * Only exact and prefix matching is supported for Get, Put, and Delete+// * There may be inconsistencies with how various version numbers are adjusted+//   but none that are exposed by kv.Client unit tests+type mockKV struct {+	// Key-value pairs created by put calls or transactions+	values    map[string]mvccpb.KeyValue+	valuesMtx sync.Mutex++	// Channel for stopping all running watch goroutines and closing+	// and cleaning up all channels used for sending events to watchers+	close chan struct{}++	// Channels that should receive events in response to Put or Delete+	// calls. These channels are in turn read by goroutines that apply+	// filtering before sending watch responses to their callers.+	consumers    map[chan clientv3.Event]struct{}+	consumersMtx sync.Mutex+}++// Watch implements the Clientv3Facade interface+func (m *mockKV) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan {+	out := make(chan clientv3.WatchResponse)+	consumer := m.createConsumer()++	go func() {+		defer func() {+			// When this goroutine ends, remove and close the channel written to by the+			// Put and Delete methods as well as closing the channel read by the caller+			// of the Watch method+			m.destroyConsumer(consumer)+			out <- clientv3.WatchResponse{Canceled: true}+			close(out)+		}()++		for {+			select {+			case <-ctx.Done():+				// Context cancelled for this watcher, run cleanup logic and exit+				return+			case <-m.close:+				// Close method called for all watchers, run cleanup logic and exit+				return+			case e := <-consumer:+				op := clientv3.OpGet(key, opts...)+				match := m.isMatch(op, *e.Kv)++				if match {+					out <- clientv3.WatchResponse{Events: []*clientv3.Event{&e}}+				}+			}+		}+	}()++	return out+}++// createConsumer creates and returns a new channel that is registered to receive+// events for Puts and Deletes.+func (m *mockKV) createConsumer() chan clientv3.Event {+	ch := make(chan clientv3.Event)+	m.consumersMtx.Lock()+	m.consumers[ch] = struct{}{}+	m.consumersMtx.Unlock()+	return ch+}++// destroyConsumer removes the given channel from the list of channels that events+// should be sent to and closes it.+func (m *mockKV) destroyConsumer(ch chan clientv3.Event) {+	m.consumersMtx.Lock()+	delete(m.consumers, ch)+	m.consumersMtx.Unlock()+	close(ch)+}++// sendEvent writes an event to all currently registered consumers. The consumer+// channels are each read by a goroutine that filters the event and sends it to+// the caller of the Watch method.+func (m *mockKV) sendEvent(e clientv3.Event) {+	m.consumersMtx.Lock()+	for ch := range m.consumers {+		ch <- e

I think it's probably a good idea to use channels with a non-zero buffer as well. WDYT?

yes, definitely

56quarters

comment created time in 2 days

PullRequestReviewEvent

Pull request review commentgrafana/dskit

In-memory implementation of etcd client for testing

 package etcd  import (-	"flag"+	"bytes"+	"context" 	"fmt" 	"io"-	"io/ioutil"-	"net/url"-	"time"+	"sync"  	"github.com/go-kit/kit/log"-	"go.etcd.io/etcd/server/v3/embed"-	"go.etcd.io/etcd/server/v3/etcdserver/api/v3client"+	"go.etcd.io/etcd/api/v3/etcdserverpb"+	"go.etcd.io/etcd/api/v3/mvccpb"+	clientv3 "go.etcd.io/etcd/client/v3" -	"github.com/grafana/dskit/closer" 	"github.com/grafana/dskit/flagext" 	"github.com/grafana/dskit/kv/codec" ) -const etcdStartTimeout = 30 * time.Second+// NewInMemoryClient creates an Etcd Client implementation that uses an in-memory+// version of the underlying Etcd client.+func NewInMemoryClient(codec codec.Codec, logger log.Logger) (*Client, io.Closer) {+	// Make sure to set default values for the config including number of retries,+	// otherwise the client won't even attempt a CAS operation+	cfg := Config{}+	flagext.DefaultValues(&cfg)++	kv := newMockKV()+	client := &Client{+		cfg:    cfg,+		codec:  codec,+		cli:    kv,+		logger: logger,+	}++	return client, kv+}++// newMockKV creates an in-memory implementation of an etcd client+func newMockKV() *mockKV {+	kv := &mockKV{+		values:       make(map[string]mvccpb.KeyValue),+		valuesMtx:    sync.Mutex{},+		close:        make(chan struct{}),+		consumers:    make(map[chan clientv3.Event]struct{}),+		consumersMtx: sync.Mutex{},+	}++	return kv+}++// mockKV is an in-memory implementation of an Etcd client.+//+// This implementation has many limitations compared to the real client since it+// only exist to be used by the Etcd kv.Client implementation during unit tests. As+// such some behavior may be missing or incorrect compared to the real client. This+// is determined to be an acceptable tradeoff to avoid needing to depend on an entire+// Etcd server for unit tests.+//+// Known limitations:+//+// * Compact is not implemented and will panic+// * RequestProgress is not implemented and will panic+// * Only exact and prefix matching is supported for Get, Put, and Delete+// * There may be inconsistencies with how various version numbers are adjusted+//   but none that are exposed by kv.Client unit tests+type mockKV struct {+	// Key-value pairs created by put calls or transactions+	values    map[string]mvccpb.KeyValue+	valuesMtx sync.Mutex++	// Channel for stopping all running watch goroutines and closing+	// and cleaning up all channels used for sending events to watchers+	close chan struct{}++	// Channels that should receive events in response to Put or Delete+	// calls. These channels are in turn read by goroutines that apply+	// filtering before sending watch responses to their callers.+	consumers    map[chan clientv3.Event]struct{}+	consumersMtx sync.Mutex+}++// Watch implements the Clientv3Facade interface+func (m *mockKV) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan {+	out := make(chan clientv3.WatchResponse)+	consumer := m.createConsumer()++	go func() {+		defer func() {+			// When this goroutine ends, remove and close the channel written to by the+			// Put and Delete methods as well as closing the channel read by the caller+			// of the Watch method+			m.destroyConsumer(consumer)+			out <- clientv3.WatchResponse{Canceled: true}+			close(out)+		}()++		for {+			select {+			case <-ctx.Done():+				// Context cancelled for this watcher, run cleanup logic and exit+				return+			case <-m.close:+				// Close method called for all watchers, run cleanup logic and exit+				return+			case e := <-consumer:+				op := clientv3.OpGet(key, opts...)+				match := m.isMatch(op, *e.Kv)++				if match {+					out <- clientv3.WatchResponse{Events: []*clientv3.Event{&e}}+				}+			}+		}+	}()++	return out+}++// createConsumer creates and returns a new channel that is registered to receive+// events for Puts and Deletes.+func (m *mockKV) createConsumer() chan clientv3.Event {+	ch := make(chan clientv3.Event)+	m.consumersMtx.Lock()+	m.consumers[ch] = struct{}{}+	m.consumersMtx.Unlock()+	return ch+}++// destroyConsumer removes the given channel from the list of channels that events+// should be sent to and closes it.+func (m *mockKV) destroyConsumer(ch chan clientv3.Event) {+	m.consumersMtx.Lock()+	delete(m.consumers, ch)+	m.consumersMtx.Unlock()+	close(ch)+}++// sendEvent writes an event to all currently registered consumers. The consumer+// channels are each read by a goroutine that filters the event and sends it to+// the caller of the Watch method.+func (m *mockKV) sendEvent(e clientv3.Event) {+	m.consumersMtx.Lock()+	for ch := range m.consumers {+		ch <- e

should this be a non-blocking write? the channels created in createConsumer have a buffer of 0, so a bad consumer could block this.

56quarters

comment created time in 2 days

PullRequestReviewEvent

PR opened coreos/go-oidc

Add support for MS ADFS

Unfortunately ADFS is relying on a non-standard attribute in its OIDC implementation, specifically the access_token_issuer. Details can be found here:

https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-oidce/586de7dd-3385-47c7-93a2-935d9e90441c

This PR adds support for ADFS if the config flag AdfsCompatibility is enabled.

+69 -35

0 comment

3 changed files

pr created time in 3 days

created taggrafana/go-oidc

tagv3.0.2

A Go OpenID Connect client.

created time in 4 days

delete branch grafana/go-oidc

delete branch : implement_adfs_support

delete time in 4 days

push eventgrafana/go-oidc

Mauro Stettler

commit sha 98cf41470c3a0269f59b8b3bdbd064f979c786bd

implement adfs support

view details

Mauro Stettler

commit sha 9440c61a6e2fba9b207e554a524f801b998eba6c

add unit test for ADFS access token issuer

view details

Mauro Stettler

commit sha 52e37ce8cb99086cace5e64e2dcc2f078d5b3eca

better wording

view details

Mauro Stettler

commit sha 032813905e748d536267925beec9806e12dd09bd

Merge pull request #3 from grafana/implement_adfs_support implement adfs support

view details

push time in 4 days

PR merged grafana/go-oidc

implement adfs support

Implementing optional support for ADFS

+69 -35

0 comment

3 changed files

replay

pr closed time in 4 days

push eventgrafana/go-oidc

Mauro Stettler

commit sha 52e37ce8cb99086cace5e64e2dcc2f078d5b3eca

better wording

view details

push time in 5 days

push eventgrafana/go-oidc

Mauro Stettler

commit sha 9440c61a6e2fba9b207e554a524f801b998eba6c

add unit test for ADFS access token issuer

view details

push time in 5 days

PR opened grafana/go-oidc

implement adfs support

I still have to add test cases, but i think this is what we need to do

+54 -35

0 comment

3 changed files

pr created time in 5 days

push eventgrafana/go-oidc

Mauro Stettler

commit sha 98cf41470c3a0269f59b8b3bdbd064f979c786bd

implement adfs support

view details

push time in 5 days

create barnchgrafana/go-oidc

branch : implement_adfs_support

created branch time in 5 days

push eventgrafana/go-oidc

push time in 5 days

push eventgrafana/go-oidc

Mauro Stettler

commit sha f845c63571fe4913c5d29728da97b987a2100e3f

implement adfs support

view details

push time in 5 days

push eventgrafana/go-oidc

Mauro Stettler

commit sha cb547974a2e4009070dd61133a8712d489e52244

Revert "add adfs compatibility"

view details

Mauro Stettler

commit sha e20715e7e971dfdffdf9c897d7a92ce638989509

Merge pull request #2 from grafana/revert-1-support_adfs Revert "add adfs compatibility"

view details

push time in 5 days

PR merged grafana/go-oidc

Revert "add adfs compatibility"

Reverts grafana/go-oidc#1

+7 -54

0 comment

2 changed files

replay

pr closed time in 5 days

PR opened grafana/go-oidc

Revert "add adfs compatibility"

Reverts grafana/go-oidc#1

+7 -54

0 comment

2 changed files

pr created time in 5 days

create barnchgrafana/go-oidc

branch : revert-1-support_adfs

created branch time in 5 days