<commit_before>/**
 * Copyright 2015 Comcast Cable Communications Management, LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package jtl

import (
	"encoding/json"
	"sync"

	. "github.com/Comcast/eel/util"
)

// handleEvent processes an event (usually from the work queue) by selecting the correct handlers, applying the appropriate transformations and then sending off the tranformed event via appropriate publisher(s).
func handleEvent(ctx Context, stats *ServiceStats, event *JDoc, raw string, debug bool, syncExec bool) interface{} {
	debuginfo := make([]interface{}, 0)
	ctx.AddLogValue("destination", "unknown")
	handlers := GetHandlerFactory(ctx).GetHandlersForEvent(ctx, event)
	if len(handlers) == 0 {
		ctx.Log().Info("action", "no_matching_handlers")
		ctx.Log().Debug("debug_action", "no_matching_handlers", "payload", event.GetOriginalObject())
	}
	initialCtx := ctx
	ctx = ctx.SubContext()
	var wg sync.WaitGroup
	for _, handler := range handlers {
		ctx.AddLogValue("topic", handler.Topic)
		ctx.AddLogValue("tenant", handler.TenantId)
		ctx.AddLogValue("handler", handler.Name)
		publishers, err := handler.ProcessEvent(initialCtx.SubContext(), event)
		if err != nil {
			ctx.Log().Error("error_type", "transformation", "cause", "bad_transformation", "handler", handler.Name, "tenant", handler.TenantId, "trace.in.data", event.GetOriginalObject(), "error", err.Error())
			ctx.Log().Metric("bad_transformation", M_Namespace, "xrs", M_Metric, "bad_transformation", M_Unit, "Count", M_Dims, "app="+AppId+"&env="+EnvName+"&instance="+InstanceName+"&destination="+ctx.LogValue("destination").(string), M_Val, 1.0)
			stats.IncErrors()
			continue
		}
		for _, publisher := range publishers {
			dc := ctx.Value(EelDuplicateChecker).(DuplicateChecker)
			if dc.GetTtl() > 0 && dc.IsDuplicate(ctx, []byte(publisher.GetUrl()+"\n"+publisher.GetPayload())) {
				ctx.Log().Info("action", "dropping_duplicate", "handler", handler.Name, "tenant", handler.TenantId)
				ctx.Log().Metric("dropping_duplicate", M_Namespace, "xrs", M_Metric, "dropping_duplicate", M_Unit, "Count", M_Dims, "app="+AppId+"&env="+EnvName+"&instance="+InstanceName+"&destination="+ctx.LogValue("destination").(string), M_Val, 1.0)
				continue
			}
			// trace header
			traceHeaderKey := GetConfig(ctx).HttpTransactionHeader
			if publisher.GetHeaders() == nil {
				publisher.SetHeaders(make(map[string]string, 0))
			}
			if publisher.GetHeaders()[traceHeaderKey] == "" && ctx.LogValue("tx.traceId") != nil {
				publisher.GetHeaders()[traceHeaderKey] = ctx.LogValue("tx.traceId").(string)
			}
			ctx.AddLogValue("tx.traceId", publisher.GetHeaders()[traceHeaderKey])
			ctx.AddValue("tx.traceId", publisher.GetHeaders()[traceHeaderKey])
			// other log params
			ctx.AddLogValue("trace.out.url", publisher.GetUrl())
			//ctx.AddLogValue("trace.in.data", event.GetOriginalObject())
			//ctx.AddLogValue("trace.out.data", publisher.GetPayload())
			ctx.AddLogValue("trace.out.protocol", publisher.GetProtocol())
			//ctx.AddLogValue("trace.out.path", publisher.GetPath())
			ctx.AddLogValue("trace.out.headers", publisher.GetHeaders())
			ctx.AddLogValue("trace.out.protocol", publisher.GetProtocol())
			//ctx.AddLogValue("trace.out.endpoint", publisher.GetEndpoint())
			ctx.AddLogValue("trace.out.verb", publisher.GetVerb())
			ctx.AddLogValue("trace.out.url", publisher.GetUrl())
			if syncExec {
				// no need to call out to endpoint in sync mode
				debuginfo = append(debuginfo, publisher.GetPayloadParsed().GetOriginalObject())
			} else if debug {
				// sequential execution to collect debug info
				_, err := publisher.Publish()
				AddLatencyLog(ctx, stats, "stat.eel.time")
				ctx.AddLogValue("trace.out.endpoint", publisher.GetEndpoint())
				ctx.AddLogValue("trace.out.url", publisher.GetUrl())
				if err != nil {
					ctx.Log().Error("error_type", "publish_event", "error", err.Error(), "cause", "publish_event")
					ctx.Log().Metric("publish_failed", M_Namespace, "xrs", M_Metric, "publish_failed", M_Unit, "Count", M_Dims, "app="+AppId+"&env="+EnvName+"&instance="+InstanceName+"&destination="+ctx.LogValue("destination").(string), M_Val, 1.0)
					stats.IncErrors()
				} else {
					ctx.Log().Info("action", "published_event")
					ctx.Log().Metric("published_event", M_Namespace, "xrs", M_Metric, "published_event", M_Unit, "Count", M_Dims, "app="+AppId+"&env="+EnvName+"&instance="+InstanceName+"&destination="+ctx.LogValue("destination").(string), M_Val, 1.0)
					stats.IncOutCount()
				}
				de := make(map[string]interface{}, 0)
				de["trace.out.endpoint"] = publisher.GetEndpoint()
				de["trace.out.path"] = publisher.GetPath()
				de["trace.out.headers"] = publisher.GetHeaders()
				de["trace.out.protocol"] = publisher.GetProtocol()
				de["trace.out.verb"] = publisher.GetVerb()
				de["trace.out.url"] = publisher.GetUrl()
				if publisher.GetPayload() != "" {
					data := make(map[string]interface{})
					err := json.Unmarshal([]byte(publisher.GetPayload()), &data)
					if err == nil {
						de["trace.out.data"] = data
					} else {
						de["trace.out.data"] = publisher.GetPayload()
					}
				} else {
					de["trace.out.data"] = ""
				}
				de["trace.in.data"] = event.GetOriginalObject()
				de["tenant.id"] = handler.TenantId
				de["handler"] = handler.Name
				de["api"] = publisher.GetApi()
				de["tx.id"] = ctx.Id()
				de["tx.traceId"] = publisher.GetHeaders()[traceHeaderKey]
				if errs := publisher.GetErrors(); errs != nil {
					de["tx.errors"] = errs
				}
				debuginfo = append(debuginfo, de)
			} else {
				//c := ctx
				//p := publisher
				wg.Add(1)
				go func(c Context, p EventPublisher) {
					defer wg.Done()
					defer c.HandlePanic()
					_, err := p.Publish()
					AddLatencyLog(c, stats, "stat.eel.time")
					//c.AddLogValue("trace.out.endpoint", p.GetEndpoint())
					c.AddLogValue("trace.out.url", p.GetUrl())
					if err != nil {
						c.Log().Error("error_type", "publish_event", "error", err.Error(), "cause", "publish_event")
						c.Log().Metric("publish_failed", M_Namespace, "xrs", M_Metric, "publish_failed", M_Unit, "Count", M_Dims, "app="+AppId+"&env="+EnvName+"&instance="+InstanceName+"&destination="+ctx.LogValue("destination").(string), M_Val, 1.0)
						stats.IncErrors()
					} else {
						//if p.GetProtocol() != "null" {
						c.Log().Info("action", "published_event")
						c.Log().Metric("published_event", M_Namespace, "xrs", M_Metric, "published_event", M_Unit, "Count", M_Dims, "app="+AppId+"&env="+EnvName+"&instance="+InstanceName+"&destination="+ctx.LogValue("destination").(string), M_Val, 1.0)
						stats.IncOutCount()
						//}
					}
				}(ctx.SubContext(), publisher)
			}
		}
	}
	wg.Wait()
	return debuginfo
}
<commit_msg>XRULES-8923: erroneous metric<commit_after>/**
 * Copyright 2015 Comcast Cable Communications Management, LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package jtl

import (
	"encoding/json"
	"sync"

	. "github.com/Comcast/eel/util"
)

// handleEvent processes an event (usually from the work queue) by selecting the correct handlers, applying the appropriate transformations and then sending off the tranformed event via appropriate publisher(s).
func handleEvent(ctx Context, stats *ServiceStats, event *JDoc, raw string, debug bool, syncExec bool) interface{} {
	debuginfo := make([]interface{}, 0)
	ctx.AddLogValue("destination", "unknown")
	handlers := GetHandlerFactory(ctx).GetHandlersForEvent(ctx, event)
	if len(handlers) == 0 {
		ctx.Log().Info("action", "no_matching_handlers")
		ctx.Log().Debug("debug_action", "no_matching_handlers", "payload", event.GetOriginalObject())
	}
	initialCtx := ctx
	ctx = ctx.SubContext()
	var wg sync.WaitGroup
	for _, handler := range handlers {
		ctx.AddLogValue("topic", handler.Topic)
		ctx.AddLogValue("tenant", handler.TenantId)
		ctx.AddLogValue("handler", handler.Name)
		publishers, err := handler.ProcessEvent(initialCtx.SubContext(), event)
		if err != nil {
			ctx.Log().Error("error_type", "transformation", "cause", "bad_transformation", "handler", handler.Name, "tenant", handler.TenantId, "trace.in.data", event.GetOriginalObject(), "error", err.Error())
			ctx.Log().Metric("bad_transformation", M_Namespace, "xrs", M_Metric, "bad_transformation", M_Unit, "Count", M_Dims, "app="+AppId+"&env="+EnvName+"&instance="+InstanceName+"&destination="+ctx.LogValue("destination").(string), M_Val, 1.0)
			stats.IncErrors()
			continue
		}
		for _, publisher := range publishers {
			dc := ctx.Value(EelDuplicateChecker).(DuplicateChecker)
			if dc.GetTtl() > 0 && dc.IsDuplicate(ctx, []byte(publisher.GetUrl()+"\n"+publisher.GetPayload())) {
				ctx.Log().Info("action", "dropping_duplicate", "handler", handler.Name, "tenant", handler.TenantId)
				ctx.Log().Metric("dropping_duplicate", M_Namespace, "xrs", M_Metric, "dropping_duplicate", M_Unit, "Count", M_Dims, "app="+AppId+"&env="+EnvName+"&instance="+InstanceName+"&destination="+ctx.LogValue("destination").(string), M_Val, 1.0)
				continue
			}
			// trace header
			traceHeaderKey := GetConfig(ctx).HttpTransactionHeader
			if publisher.GetHeaders() == nil {
				publisher.SetHeaders(make(map[string]string, 0))
			}
			if publisher.GetHeaders()[traceHeaderKey] == "" && ctx.LogValue("tx.traceId") != nil {
				publisher.GetHeaders()[traceHeaderKey] = ctx.LogValue("tx.traceId").(string)
			}
			ctx.AddLogValue("tx.traceId", publisher.GetHeaders()[traceHeaderKey])
			ctx.AddValue("tx.traceId", publisher.GetHeaders()[traceHeaderKey])
			// other log params
			ctx.AddLogValue("trace.out.url", publisher.GetUrl())
			//ctx.AddLogValue("trace.in.data", event.GetOriginalObject())
			//ctx.AddLogValue("trace.out.data", publisher.GetPayload())
			ctx.AddLogValue("trace.out.protocol", publisher.GetProtocol())
			//ctx.AddLogValue("trace.out.path", publisher.GetPath())
			ctx.AddLogValue("trace.out.headers", publisher.GetHeaders())
			ctx.AddLogValue("trace.out.protocol", publisher.GetProtocol())
			//ctx.AddLogValue("trace.out.endpoint", publisher.GetEndpoint())
			ctx.AddLogValue("trace.out.verb", publisher.GetVerb())
			ctx.AddLogValue("trace.out.url", publisher.GetUrl())
			if syncExec {
				// no need to call out to endpoint in sync mode
				debuginfo = append(debuginfo, publisher.GetPayloadParsed().GetOriginalObject())
			} else if debug {
				// sequential execution to collect debug info
				_, err := publisher.Publish()
				AddLatencyLog(ctx, stats, "stat.eel.time")
				ctx.AddLogValue("trace.out.endpoint", publisher.GetEndpoint())
				ctx.AddLogValue("trace.out.url", publisher.GetUrl())
				if err != nil {
					ctx.Log().Error("error_type", "publish_event", "error", err.Error(), "cause", "publish_event")
					ctx.Log().Metric("publish_failed", M_Namespace, "xrs", M_Metric, "publish_failed", M_Unit, "Count", M_Dims, "app="+AppId+"&env="+EnvName+"&instance="+InstanceName+"&destination="+ctx.LogValue("destination").(string), M_Val, 1.0)
					stats.IncErrors()
				} else {
					ctx.Log().Info("action", "published_event")
					ctx.Log().Metric("published_event", M_Namespace, "xrs", M_Metric, "published_event", M_Unit, "Count", M_Dims, "app="+AppId+"&env="+EnvName+"&instance="+InstanceName+"&destination="+ctx.LogValue("destination").(string), M_Val, 1.0)
					stats.IncOutCount()
				}
				de := make(map[string]interface{}, 0)
				de["trace.out.endpoint"] = publisher.GetEndpoint()
				de["trace.out.path"] = publisher.GetPath()
				de["trace.out.headers"] = publisher.GetHeaders()
				de["trace.out.protocol"] = publisher.GetProtocol()
				de["trace.out.verb"] = publisher.GetVerb()
				de["trace.out.url"] = publisher.GetUrl()
				if publisher.GetPayload() != "" {
					data := make(map[string]interface{})
					err := json.Unmarshal([]byte(publisher.GetPayload()), &data)
					if err == nil {
						de["trace.out.data"] = data
					} else {
						de["trace.out.data"] = publisher.GetPayload()
					}
				} else {
					de["trace.out.data"] = ""
				}
				de["trace.in.data"] = event.GetOriginalObject()
				de["tenant.id"] = handler.TenantId
				de["handler"] = handler.Name
				de["api"] = publisher.GetApi()
				de["tx.id"] = ctx.Id()
				de["tx.traceId"] = publisher.GetHeaders()[traceHeaderKey]
				if errs := publisher.GetErrors(); errs != nil {
					de["tx.errors"] = errs
				}
				debuginfo = append(debuginfo, de)
			} else {
				//c := ctx
				//p := publisher
				wg.Add(1)
				go func(c Context, p EventPublisher) {
					defer wg.Done()
					defer c.HandlePanic()
					_, err := p.Publish()
					AddLatencyLog(c, stats, "stat.eel.time")
					//c.AddLogValue("trace.out.endpoint", p.GetEndpoint())
					c.AddLogValue("trace.out.url", p.GetUrl())
					if err != nil {
						c.Log().Error("error_type", "publish_event", "error", err.Error(), "cause", "publish_event")
						c.Log().Metric("publish_failed", M_Namespace, "xrs", M_Metric, "publish_failed", M_Unit, "Count", M_Dims, "app="+AppId+"&env="+EnvName+"&instance="+InstanceName+"&destination="+ctx.LogValue("destination").(string), M_Val, 1.0)
						stats.IncErrors()
					} else {
						if p.GetProtocol() != "null" {
							c.Log().Info("action", "published_event")
							c.Log().Metric("published_event", M_Namespace, "xrs", M_Metric, "published_event", M_Unit, "Count", M_Dims, "app="+AppId+"&env="+EnvName+"&instance="+InstanceName+"&destination="+ctx.LogValue("destination").(string), M_Val, 1.0)
							stats.IncOutCount()
						}
					}
				}(ctx.SubContext(), publisher)
			}
		}
	}
	wg.Wait()
	return debuginfo
}
<|endoftext|>


<commit_before>package filestore

import (
	"bytes"
	"context"
	"io/ioutil"
	"math/rand"
	"testing"

	"github.com/ipfs/go-ipfs/blocks/blockstore"
	dag "github.com/ipfs/go-ipfs/merkledag"
	posinfo "github.com/ipfs/go-ipfs/thirdparty/posinfo"

	ds "gx/ipfs/QmRWDav6mzWseLWeYfVd5fvUKiVe9xNH29YfMF438fG364/go-datastore"
	cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid"
)

func newTestFilestore(t *testing.T) (string, *Filestore) {
	mds := ds.NewMapDatastore()

	testdir, err := ioutil.TempDir("", "filestore-test")
	if err != nil {
		t.Fatal(err)
	}
	fm := NewFileManager(mds, testdir)

	bs := blockstore.NewBlockstore(mds)
	fstore := NewFilestore(bs, fm)
	return testdir, fstore
}

func makeFile(dir string, data []byte) (string, error) {
	f, err := ioutil.TempFile(dir, "file")
	if err != nil {
		return "", err
	}

	_, err = f.Write(data)
	if err != nil {
		return "", err
	}

	return f.Name(), nil
}

func TestBasicFilestore(t *testing.T) {
	dir, fs := newTestFilestore(t)

	buf := make([]byte, 1000)
	rand.Read(buf)

	fname, err := makeFile(dir, buf)
	if err != nil {
		t.Fatal(err)
	}

	var cids []*cid.Cid
	for i := 0; i < 100; i++ {
		n := &posinfo.FilestoreNode{
			PosInfo: &posinfo.PosInfo{
				FullPath: fname,
				Offset:   uint64(i * 10),
			},
			Node: dag.NewRawNode(buf[i*10 : (i+1)*10]),
		}

		err := fs.Put(n)
		if err != nil {
			t.Fatal(err)
		}
		cids = append(cids, n.Node.Cid())
	}

	for i, c := range cids {
		blk, err := fs.Get(c)
		if err != nil {
			t.Fatal(err)
		}

		if !bytes.Equal(blk.RawData(), buf[i*10:(i+1)*10]) {
			t.Fatal("data didnt match on the way out")
		}
	}

	kch, err := fs.AllKeysChan(context.Background())
	if err != nil {
		t.Fatal(err)
	}

	out := make(map[string]struct{})
	for c := range kch {
		out[c.KeyString()] = struct{}{}
	}

	if len(out) != len(cids) {
		t.Fatal("mismatch in number of entries")
	}

	for _, c := range cids {
		if _, ok := out[c.KeyString()]; !ok {
			t.Fatal("missing cid: ", c)
		}
	}
}
<commit_msg>add test for deletes<commit_after>package filestore

import (
	"bytes"
	"context"
	"io/ioutil"
	"math/rand"
	"testing"

	"github.com/ipfs/go-ipfs/blocks/blockstore"
	dag "github.com/ipfs/go-ipfs/merkledag"
	posinfo "github.com/ipfs/go-ipfs/thirdparty/posinfo"

	ds "gx/ipfs/QmRWDav6mzWseLWeYfVd5fvUKiVe9xNH29YfMF438fG364/go-datastore"
	cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid"
)

func newTestFilestore(t *testing.T) (string, *Filestore) {
	mds := ds.NewMapDatastore()

	testdir, err := ioutil.TempDir("", "filestore-test")
	if err != nil {
		t.Fatal(err)
	}
	fm := NewFileManager(mds, testdir)

	bs := blockstore.NewBlockstore(mds)
	fstore := NewFilestore(bs, fm)
	return testdir, fstore
}

func makeFile(dir string, data []byte) (string, error) {
	f, err := ioutil.TempFile(dir, "file")
	if err != nil {
		return "", err
	}

	_, err = f.Write(data)
	if err != nil {
		return "", err
	}

	return f.Name(), nil
}

func TestBasicFilestore(t *testing.T) {
	dir, fs := newTestFilestore(t)

	buf := make([]byte, 1000)
	rand.Read(buf)

	fname, err := makeFile(dir, buf)
	if err != nil {
		t.Fatal(err)
	}

	var cids []*cid.Cid
	for i := 0; i < 100; i++ {
		n := &posinfo.FilestoreNode{
			PosInfo: &posinfo.PosInfo{
				FullPath: fname,
				Offset:   uint64(i * 10),
			},
			Node: dag.NewRawNode(buf[i*10 : (i+1)*10]),
		}

		err := fs.Put(n)
		if err != nil {
			t.Fatal(err)
		}
		cids = append(cids, n.Node.Cid())
	}

	for i, c := range cids {
		blk, err := fs.Get(c)
		if err != nil {
			t.Fatal(err)
		}

		if !bytes.Equal(blk.RawData(), buf[i*10:(i+1)*10]) {
			t.Fatal("data didnt match on the way out")
		}
	}

	kch, err := fs.AllKeysChan(context.Background())
	if err != nil {
		t.Fatal(err)
	}

	out := make(map[string]struct{})
	for c := range kch {
		out[c.KeyString()] = struct{}{}
	}

	if len(out) != len(cids) {
		t.Fatal("mismatch in number of entries")
	}

	for _, c := range cids {
		if _, ok := out[c.KeyString()]; !ok {
			t.Fatal("missing cid: ", c)
		}
	}
}

func randomFileAdd(t *testing.T, fs *Filestore, dir string, size int) (string, []*cid.Cid) {
	buf := make([]byte, size)
	rand.Read(buf)

	fname, err := makeFile(dir, buf)
	if err != nil {
		t.Fatal(err)
	}

	var out []*cid.Cid
	for i := 0; i < size/10; i++ {
		n := &posinfo.FilestoreNode{
			PosInfo: &posinfo.PosInfo{
				FullPath: fname,
				Offset:   uint64(i * 10),
			},
			Node: dag.NewRawNode(buf[i*10 : (i+1)*10]),
		}
		err := fs.Put(n)
		if err != nil {
			t.Fatal(err)
		}
		out = append(out, n.Cid())
	}

	return fname, out
}

func TestDeletes(t *testing.T) {
	dir, fs := newTestFilestore(t)
	_, cids := randomFileAdd(t, fs, dir, 100)
	todelete := cids[:4]
	for _, c := range todelete {
		err := fs.DeleteBlock(c)
		if err != nil {
			t.Fatal(err)
		}
	}

	deleted := make(map[string]bool)
	for _, c := range todelete {
		_, err := fs.Get(c)
		if err != blockstore.ErrNotFound {
			t.Fatal("expected blockstore not found error")
		}
		deleted[c.KeyString()] = true
	}

	keys, err := fs.AllKeysChan(context.Background())
	if err != nil {
		t.Fatal(err)
	}

	for c := range keys {
		if deleted[c.KeyString()] {
			t.Fatal("shouldnt have reference to this key anymore")
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// +genclient
// +genclient:nonNamespaced
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// PriorityClass defines mapping from a priority class name to the priority
// integer value. The value can be any valid integer.
type PriorityClass struct {
	metav1.TypeMeta `json:",inline"`
	// Standard object's metadata.
	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`

	// The value of this priority class. This is the actual priority that pods
	// receive when they have the name of this class in their pod spec.
	Value int32 `json:"value" protobuf:"bytes,2,opt,name=value"`

	// globalDefault specifies whether this PriorityClass should be considered as
	// the default priority for pods that do not have any priority class.
	// +optional
	GlobalDefault bool `json:"globalDefault,omitempty" protobuf:"bytes,3,opt,name=globalDefault"`

	// description is an arbitrary string that usually provides guidelines on
	// when this priority class should be used.
	// +optional
	Description string `json:"description,omitempty" protobuf:"bytes,4,opt,name=description"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// PriorityClassList is a collection of priority classes.
type PriorityClassList struct {
	metav1.TypeMeta `json:",inline"`
	// Standard list metadata
	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
	// +optional
	metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`

	// items is the list of PriorityClasses
	Items []PriorityClass `json:"items" protobuf:"bytes,2,rep,name=items"`
}
<commit_msg>Add API docs for multiple PriorityClasses marked as globalDefault<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// +genclient
// +genclient:nonNamespaced
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// PriorityClass defines mapping from a priority class name to the priority
// integer value. The value can be any valid integer.
type PriorityClass struct {
	metav1.TypeMeta `json:",inline"`
	// Standard object's metadata.
	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`

	// The value of this priority class. This is the actual priority that pods
	// receive when they have the name of this class in their pod spec.
	Value int32 `json:"value" protobuf:"bytes,2,opt,name=value"`

	// globalDefault specifies whether this PriorityClass should be considered as
	// the default priority for pods that do not have any priority class.
	// Only one PriorityClass with can be marked as `globalDefault`. Due to race conditions, more than
	// one PriorityClasses might be created with their `globalDefault` field set to true. In that case,
	// the smallest value of such global default PriorityClasses will be used as the default priority.
	// +optional
	GlobalDefault bool `json:"globalDefault,omitempty" protobuf:"bytes,3,opt,name=globalDefault"`

	// description is an arbitrary string that usually provides guidelines on
	// when this priority class should be used.
	// +optional
	Description string `json:"description,omitempty" protobuf:"bytes,4,opt,name=description"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// PriorityClassList is a collection of priority classes.
type PriorityClassList struct {
	metav1.TypeMeta `json:",inline"`
	// Standard list metadata
	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
	// +optional
	metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`

	// items is the list of PriorityClasses
	Items []PriorityClass `json:"items" protobuf:"bytes,2,rep,name=items"`
}
<|endoftext|>


<commit_before>package common

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/hex"
	"errors"
	"fmt"
	"hash"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"runtime"
)

// DownloadConfig is the configuration given to instantiate a new
// download instance. Once a configuration is used to instantiate
// a download client, it must not be modified.
type DownloadConfig struct {
	// The source URL in the form of a string.
	Url string

	// This is the path to download the file to.
	TargetPath string

	// DownloaderMap maps a schema to a Download.
	DownloaderMap map[string]Downloader

	// If true, this will copy even a local file to the target
	// location. If false, then it will "download" the file by just
	// returning the local path to the file.
	CopyFile bool

	// The hashing implementation to use to checksum the downloaded file.
	Hash hash.Hash

	// The checksum for the downloaded file. The hash implementation configuration
	// for the downloader will be used to verify with this checksum after
	// it is downloaded.
	Checksum []byte

	// What to use for the user agent for HTTP requests. If set to "", use the
	// default user agent provided by Go.
	UserAgent string
}

// A DownloadClient helps download, verify checksums, etc.
type DownloadClient struct {
	config     *DownloadConfig
	downloader Downloader
}

// HashForType returns the Hash implementation for the given string
// type, or nil if the type is not supported.
func HashForType(t string) hash.Hash {
	switch t {
	case "md5":
		return md5.New()
	case "sha1":
		return sha1.New()
	case "sha256":
		return sha256.New()
	case "sha512":
		return sha512.New()
	default:
		return nil
	}
}

// NewDownloadClient returns a new DownloadClient for the given
// configuration.
func NewDownloadClient(c *DownloadConfig) *DownloadClient {
	if c.DownloaderMap == nil {
		c.DownloaderMap = map[string]Downloader{
			"http":  &HTTPDownloader{userAgent: c.UserAgent},
			"https": &HTTPDownloader{userAgent: c.UserAgent},
		}
	}

	return &DownloadClient{config: c}
}

// A downloader is responsible for actually taking a remote URL and
// downloading it.
type Downloader interface {
	Cancel()
	Download(*os.File, *url.URL) error
	Progress() uint
	Total() uint
}

func (d *DownloadClient) Cancel() {
	// TODO(mitchellh): Implement
}

func (d *DownloadClient) Get() (string, error) {
	// If we already have the file and it matches, then just return the target path.
	if verify, _ := d.VerifyChecksum(d.config.TargetPath); verify {
		log.Println("[DEBUG] Initial checksum matched, no download needed.")
		return d.config.TargetPath, nil
	}

	u, err := url.Parse(d.config.Url)
	if err != nil {
		return "", err
	}

	log.Printf("Parsed URL: %#v", u)

	// Files when we don't copy the file are special cased.
	var f *os.File
	var finalPath string
	sourcePath := ""
	if u.Scheme == "file" && !d.config.CopyFile {
		// This is special case for relative path in this case user specify
		// file:../ and after parse destination goes to Opaque
		if u.Path != "" {
			// If url.Path is set just use this
			finalPath = u.Path
		} else if u.Opaque != "" {
			// otherwise try url.Opaque
			finalPath = u.Opaque
		}
		// This is a special case where we use a source file that already exists
		// locally and we don't make a copy. Normally we would copy or download.
		log.Printf("[DEBUG] Using local file: %s", finalPath)

		// Keep track of the source so we can make sure not to delete this later
		sourcePath = finalPath
		if _, err = os.Stat(finalPath); err != nil {
			return "", err
		}
	} else {
		finalPath = d.config.TargetPath

		var ok bool
		d.downloader, ok = d.config.DownloaderMap[u.Scheme]
		if !ok {
			return "", fmt.Errorf("No downloader for scheme: %s", u.Scheme)
		}

		// Otherwise, download using the downloader.
		f, err = os.OpenFile(finalPath, os.O_RDWR|os.O_CREATE, os.FileMode(0666))
		if err != nil {
			return "", err
		}

		log.Printf("[DEBUG] Downloading: %s", u.String())
		err = d.downloader.Download(f, u)
		f.Close()
		if err != nil {
			return "", err
		}
	}

	if d.config.Hash != nil {
		var verify bool
		verify, err = d.VerifyChecksum(finalPath)
		if err == nil && !verify {
			// Only delete the file if we made a copy or downloaded it
			if sourcePath != finalPath {
				os.Remove(finalPath)
			}

			err = fmt.Errorf(
				"checksums didn't match expected: %s",
				hex.EncodeToString(d.config.Checksum))
		}
	}

	return finalPath, err
}

// PercentProgress returns the download progress as a percentage.
func (d *DownloadClient) PercentProgress() int {
	if d.downloader == nil {
		return -1
	}

	return int((float64(d.downloader.Progress()) / float64(d.downloader.Total())) * 100)
}

// VerifyChecksum tests that the path matches the checksum for the
// download.
func (d *DownloadClient) VerifyChecksum(path string) (bool, error) {
	if d.config.Checksum == nil || d.config.Hash == nil {
		return false, errors.New("Checksum or Hash isn't set on download.")
	}

	f, err := os.Open(path)
	if err != nil {
		return false, err
	}
	defer f.Close()

	log.Printf("Verifying checksum of %s", path)
	d.config.Hash.Reset()
	io.Copy(d.config.Hash, f)
	return bytes.Equal(d.config.Hash.Sum(nil), d.config.Checksum), nil
}

// HTTPDownloader is an implementation of Downloader that downloads
// files over HTTP.
type HTTPDownloader struct {
	progress  uint
	total     uint
	userAgent string
}

func (*HTTPDownloader) Cancel() {
	// TODO(mitchellh): Implement
}

func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error {
	log.Printf("Starting download: %s", src.String())

	// Seek to the beginning by default
	if _, err := dst.Seek(0, 0); err != nil {
		return err
	}

	// Reset our progress
	d.progress = 0

	// Make the request. We first make a HEAD request so we can check
	// if the server supports range queries. If the server/URL doesn't
	// support HEAD requests, we just fall back to GET.
	req, err := http.NewRequest("HEAD", src.String(), nil)
	if err != nil {
		return err
	}

	if d.userAgent != "" {
		req.Header.Set("User-Agent", d.userAgent)
	}

	httpClient := &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
		},
	}

	resp, err := httpClient.Do(req)
	if err == nil && (resp.StatusCode >= 200 && resp.StatusCode < 300) {
		// If the HEAD request succeeded, then attempt to set the range
		// query if we can.
		if resp.Header.Get("Accept-Ranges") == "bytes" {
			if fi, err := dst.Stat(); err == nil {
				if _, err = dst.Seek(0, os.SEEK_END); err == nil {
					req.Header.Set("Range", fmt.Sprintf("bytes=%d-", fi.Size()))
					d.progress = uint(fi.Size())
				}
			}
		}
	}

	// Set the request to GET now, and redo the query to download
	req.Method = "GET"

	resp, err = httpClient.Do(req)
	if err != nil {
		return err
	}

	d.total = d.progress + uint(resp.ContentLength)
	var buffer [4096]byte
	for {
		n, err := resp.Body.Read(buffer[:])
		if err != nil && err != io.EOF {
			return err
		}

		d.progress += uint(n)

		if _, werr := dst.Write(buffer[:n]); werr != nil {
			return werr
		}

		if err == io.EOF {
			break
		}
	}

	return nil
}

func (d *HTTPDownloader) Progress() uint {
	return d.progress
}

func (d *HTTPDownloader) Total() uint {
	return d.total
}
<commit_msg>runtime imported but not used<commit_after>package common

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/hex"
	"errors"
	"fmt"
	"hash"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
)

// DownloadConfig is the configuration given to instantiate a new
// download instance. Once a configuration is used to instantiate
// a download client, it must not be modified.
type DownloadConfig struct {
	// The source URL in the form of a string.
	Url string

	// This is the path to download the file to.
	TargetPath string

	// DownloaderMap maps a schema to a Download.
	DownloaderMap map[string]Downloader

	// If true, this will copy even a local file to the target
	// location. If false, then it will "download" the file by just
	// returning the local path to the file.
	CopyFile bool

	// The hashing implementation to use to checksum the downloaded file.
	Hash hash.Hash

	// The checksum for the downloaded file. The hash implementation configuration
	// for the downloader will be used to verify with this checksum after
	// it is downloaded.
	Checksum []byte

	// What to use for the user agent for HTTP requests. If set to "", use the
	// default user agent provided by Go.
	UserAgent string
}

// A DownloadClient helps download, verify checksums, etc.
type DownloadClient struct {
	config     *DownloadConfig
	downloader Downloader
}

// HashForType returns the Hash implementation for the given string
// type, or nil if the type is not supported.
func HashForType(t string) hash.Hash {
	switch t {
	case "md5":
		return md5.New()
	case "sha1":
		return sha1.New()
	case "sha256":
		return sha256.New()
	case "sha512":
		return sha512.New()
	default:
		return nil
	}
}

// NewDownloadClient returns a new DownloadClient for the given
// configuration.
func NewDownloadClient(c *DownloadConfig) *DownloadClient {
	if c.DownloaderMap == nil {
		c.DownloaderMap = map[string]Downloader{
			"http":  &HTTPDownloader{userAgent: c.UserAgent},
			"https": &HTTPDownloader{userAgent: c.UserAgent},
		}
	}

	return &DownloadClient{config: c}
}

// A downloader is responsible for actually taking a remote URL and
// downloading it.
type Downloader interface {
	Cancel()
	Download(*os.File, *url.URL) error
	Progress() uint
	Total() uint
}

func (d *DownloadClient) Cancel() {
	// TODO(mitchellh): Implement
}

func (d *DownloadClient) Get() (string, error) {
	// If we already have the file and it matches, then just return the target path.
	if verify, _ := d.VerifyChecksum(d.config.TargetPath); verify {
		log.Println("[DEBUG] Initial checksum matched, no download needed.")
		return d.config.TargetPath, nil
	}

	u, err := url.Parse(d.config.Url)
	if err != nil {
		return "", err
	}

	log.Printf("Parsed URL: %#v", u)

	// Files when we don't copy the file are special cased.
	var f *os.File
	var finalPath string
	sourcePath := ""
	if u.Scheme == "file" && !d.config.CopyFile {
		// This is special case for relative path in this case user specify
		// file:../ and after parse destination goes to Opaque
		if u.Path != "" {
			// If url.Path is set just use this
			finalPath = u.Path
		} else if u.Opaque != "" {
			// otherwise try url.Opaque
			finalPath = u.Opaque
		}
		// This is a special case where we use a source file that already exists
		// locally and we don't make a copy. Normally we would copy or download.
		log.Printf("[DEBUG] Using local file: %s", finalPath)

		// Keep track of the source so we can make sure not to delete this later
		sourcePath = finalPath
		if _, err = os.Stat(finalPath); err != nil {
			return "", err
		}
	} else {
		finalPath = d.config.TargetPath

		var ok bool
		d.downloader, ok = d.config.DownloaderMap[u.Scheme]
		if !ok {
			return "", fmt.Errorf("No downloader for scheme: %s", u.Scheme)
		}

		// Otherwise, download using the downloader.
		f, err = os.OpenFile(finalPath, os.O_RDWR|os.O_CREATE, os.FileMode(0666))
		if err != nil {
			return "", err
		}

		log.Printf("[DEBUG] Downloading: %s", u.String())
		err = d.downloader.Download(f, u)
		f.Close()
		if err != nil {
			return "", err
		}
	}

	if d.config.Hash != nil {
		var verify bool
		verify, err = d.VerifyChecksum(finalPath)
		if err == nil && !verify {
			// Only delete the file if we made a copy or downloaded it
			if sourcePath != finalPath {
				os.Remove(finalPath)
			}

			err = fmt.Errorf(
				"checksums didn't match expected: %s",
				hex.EncodeToString(d.config.Checksum))
		}
	}

	return finalPath, err
}

// PercentProgress returns the download progress as a percentage.
func (d *DownloadClient) PercentProgress() int {
	if d.downloader == nil {
		return -1
	}

	return int((float64(d.downloader.Progress()) / float64(d.downloader.Total())) * 100)
}

// VerifyChecksum tests that the path matches the checksum for the
// download.
func (d *DownloadClient) VerifyChecksum(path string) (bool, error) {
	if d.config.Checksum == nil || d.config.Hash == nil {
		return false, errors.New("Checksum or Hash isn't set on download.")
	}

	f, err := os.Open(path)
	if err != nil {
		return false, err
	}
	defer f.Close()

	log.Printf("Verifying checksum of %s", path)
	d.config.Hash.Reset()
	io.Copy(d.config.Hash, f)
	return bytes.Equal(d.config.Hash.Sum(nil), d.config.Checksum), nil
}

// HTTPDownloader is an implementation of Downloader that downloads
// files over HTTP.
type HTTPDownloader struct {
	progress  uint
	total     uint
	userAgent string
}

func (*HTTPDownloader) Cancel() {
	// TODO(mitchellh): Implement
}

func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error {
	log.Printf("Starting download: %s", src.String())

	// Seek to the beginning by default
	if _, err := dst.Seek(0, 0); err != nil {
		return err
	}

	// Reset our progress
	d.progress = 0

	// Make the request. We first make a HEAD request so we can check
	// if the server supports range queries. If the server/URL doesn't
	// support HEAD requests, we just fall back to GET.
	req, err := http.NewRequest("HEAD", src.String(), nil)
	if err != nil {
		return err
	}

	if d.userAgent != "" {
		req.Header.Set("User-Agent", d.userAgent)
	}

	httpClient := &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
		},
	}

	resp, err := httpClient.Do(req)
	if err == nil && (resp.StatusCode >= 200 && resp.StatusCode < 300) {
		// If the HEAD request succeeded, then attempt to set the range
		// query if we can.
		if resp.Header.Get("Accept-Ranges") == "bytes" {
			if fi, err := dst.Stat(); err == nil {
				if _, err = dst.Seek(0, os.SEEK_END); err == nil {
					req.Header.Set("Range", fmt.Sprintf("bytes=%d-", fi.Size()))
					d.progress = uint(fi.Size())
				}
			}
		}
	}

	// Set the request to GET now, and redo the query to download
	req.Method = "GET"

	resp, err = httpClient.Do(req)
	if err != nil {
		return err
	}

	d.total = d.progress + uint(resp.ContentLength)
	var buffer [4096]byte
	for {
		n, err := resp.Body.Read(buffer[:])
		if err != nil && err != io.EOF {
			return err
		}

		d.progress += uint(n)

		if _, werr := dst.Write(buffer[:n]); werr != nil {
			return werr
		}

		if err == io.EOF {
			break
		}
	}

	return nil
}

func (d *HTTPDownloader) Progress() uint {
	return d.progress
}

func (d *HTTPDownloader) Total() uint {
	return d.total
}
<|endoftext|>


<commit_before>package aiven

import (
	"encoding/json"
	"errors"
	"fmt"
)

type (
	KafkaTopic struct {
		CleanupPolicy         string       `json:"cleanup_policy"`
		MinimumInSyncReplicas int          `json:"min_insync_replicas"`
		Partitions            []*Partition `json:"partitions"`
		Replication           int          `json:"replication"`
		RetentionBytes        int          `json:"retention_bytes"`
		RetentionHours        int          `json:"retention_hours"`
		State                 string       `json:"state"`
		TopicName             string       `json:"topic_name"`
	}

	Partition struct {
		ConsumerGroups []*ConsumerGroup `json:"consumer_groups"`
		EarliestOffset int              `json:"earliest_offset"`
		ISR            int              `json:"isr"`
		LatestOffset   int              `json:"latest_offset"`
		Partition      int              `json:"partition"`
		Size           int              `json:"size"`
	}

	ConsumerGroup struct {
		GroupName string `json:"group_name"`
		Offset    int    `json:"offset"`
	}

	KafkaTopicsHandler struct {
		client *Client
	}

	CreateKafkaTopicRequest struct {
		CleanupPolicy         *string `json:"cleanup_policy,omitempty"`
		MinimumInSyncReplicas *int    `json:"min_insync_replicas,omitempty"`
		Partitions            *int    `json:"partitions,omitempty"`
		Replication           *int    `json:"replication,omitempty"`
		RetentionBytes        *int    `json:"retention_bytes,omitempty"`
		RetentionHours        *int    `json:"retention_hours,omitempty"`
		TopicName             string  `json:"topic_name"`
	}

	UpdateKafkaTopicRequest struct {
		MinimumInSyncReplicas string `json:"min_insync_replicas"`
		Partitions            *int   `json:"partitions,omitempty"`
		RetentionBytes        *int   `json:"retention_bytes,omitempty"`
		RetentionHours        *int   `json:"retention_hours,omitempty"`
	}

	KafkaTopicResponse struct {
		APIResponse
		Topic *KafkaTopic `json:"topic"`
	}
)

func (h *KafkaTopicsHandler) Create(project, service string, req CreateKafkaTopicRequest) error {
	bts, err := h.client.doPostRequest(fmt.Sprintf("/project/%s/service/%s/topic", project, service), req)
	if err != nil {
		return err
	}

	var rsp *APIResponse
	if err := json.Unmarshal(bts, &rsp); err != nil {
		return err
	}

	if rsp == nil {
		return ErrNoResponseData
	}

	if rsp.Errors != nil && len(rsp.Errors) != 0 {
		return errors.New(rsp.Message)
	}

	return nil
}

func (h *KafkaTopicsHandler) Get(project, service, topic string) (*KafkaTopic, error) {
	rsp, err := h.client.doGetRequest(fmt.Sprintf("/project/%s/service/%s/topic/%s", project, service, topic), nil)
	if err != nil {
		return nil, err
	}

	var response *KafkaTopicResponse
	if err := json.Unmarshal(rsp, &response); err != nil {
		return nil, err
	}

	if len(response.Errors) != 0 {
		return nil, errors.New(response.Message)
	}

	return response.Topic, nil
}

func (h *KafkaTopicsHandler) Update(project, service, topic string, req UpdateKafkaTopicRequest) error {
	bts, err := h.client.doPutRequest(fmt.Sprintf("/project/%s/service/%s/topic/%s", project, service, topic), req)
	if err != nil {
		return err
	}

	var rsp *APIResponse
	if err := json.Unmarshal(bts, &rsp); err != nil {
		return err
	}

	if rsp == nil {
		return ErrNoResponseData
	}

	if rsp.Errors != nil && len(rsp.Errors) != 0 {
		return errors.New(rsp.Message)
	}

	return nil
}

func (h *KafkaTopicsHandler) Delete(project, service, topic string) error {
	bts, err := h.client.doDeleteRequest(fmt.Sprintf("/project/%s/service/%s/topic/%s", project, service, topic), nil)
	if err != nil {
		return err
	}

	return handleDeleteResponse(bts)
}
<commit_msg>adds list method<commit_after>package aiven

import (
	"encoding/json"
	"errors"
	"fmt"
)

type (
	KafkaTopic struct {
		CleanupPolicy         string       `json:"cleanup_policy"`
		MinimumInSyncReplicas int          `json:"min_insync_replicas"`
		Partitions            []*Partition `json:"partitions"`
		Replication           int          `json:"replication"`
		RetentionBytes        int          `json:"retention_bytes"`
		RetentionHours        int          `json:"retention_hours"`
		State                 string       `json:"state"`
		TopicName             string       `json:"topic_name"`
	}

	Partition struct {
		ConsumerGroups []*ConsumerGroup `json:"consumer_groups"`
		EarliestOffset int              `json:"earliest_offset"`
		ISR            int              `json:"isr"`
		LatestOffset   int              `json:"latest_offset"`
		Partition      int              `json:"partition"`
		Size           int              `json:"size"`
	}

	ConsumerGroup struct {
		GroupName string `json:"group_name"`
		Offset    int    `json:"offset"`
	}

	KafkaTopicsHandler struct {
		client *Client
	}

	CreateKafkaTopicRequest struct {
		CleanupPolicy         *string `json:"cleanup_policy,omitempty"`
		MinimumInSyncReplicas *int    `json:"min_insync_replicas,omitempty"`
		Partitions            *int    `json:"partitions,omitempty"`
		Replication           *int    `json:"replication,omitempty"`
		RetentionBytes        *int    `json:"retention_bytes,omitempty"`
		RetentionHours        *int    `json:"retention_hours,omitempty"`
		TopicName             string  `json:"topic_name"`
	}

	UpdateKafkaTopicRequest struct {
		MinimumInSyncReplicas string `json:"min_insync_replicas"`
		Partitions            *int   `json:"partitions,omitempty"`
		RetentionBytes        *int   `json:"retention_bytes,omitempty"`
		RetentionHours        *int   `json:"retention_hours,omitempty"`
	}

	KafkaTopicResponse struct {
		APIResponse
		Topic *KafkaTopic `json:"topic"`
	}

	KafkaTopicsResponse struct {
		APIResponse
		Topics []*KafkaTopic `json:"topics"`
	}
)

func (h *KafkaTopicsHandler) Create(project, service string, req CreateKafkaTopicRequest) error {
	bts, err := h.client.doPostRequest(fmt.Sprintf("/project/%s/service/%s/topic", project, service), req)
	if err != nil {
		return err
	}

	var rsp *APIResponse
	if err := json.Unmarshal(bts, &rsp); err != nil {
		return err
	}

	if rsp == nil {
		return ErrNoResponseData
	}

	if rsp.Errors != nil && len(rsp.Errors) != 0 {
		return errors.New(rsp.Message)
	}

	return nil
}

func (h *KafkaTopicsHandler) Get(project, service, topic string) (*KafkaTopic, error) {
	rsp, err := h.client.doGetRequest(fmt.Sprintf("/project/%s/service/%s/topic/%s", project, service, topic), nil)
	if err != nil {
		return nil, err
	}

	var response *KafkaTopicResponse
	if err := json.Unmarshal(rsp, &response); err != nil {
		return nil, err
	}

	if len(response.Errors) != 0 {
		return nil, errors.New(response.Message)
	}

	return response.Topic, nil
}

func (h *KafkaTopicsHandler) List(project, service string) ([]*KafkaTopic, error) {
	rsp, err := h.client.doGetRequest(fmt.Sprintf("/project/%s/service/%s/topic", project, service), nil)
	if err != nil {
		return nil, err
	}

	var response *KafkaTopicsResponse
	if err := json.Unmarshal(rsp, &response); err != nil {
		return nil, err
	}

	if len(response.Errors) != 0 {
		return nil, errors.New(response.Message)
	}

	return response.Topics, nil
}

func (h *KafkaTopicsHandler) Update(project, service, topic string, req UpdateKafkaTopicRequest) error {
	bts, err := h.client.doPutRequest(fmt.Sprintf("/project/%s/service/%s/topic/%s", project, service, topic), req)
	if err != nil {
		return err
	}

	var rsp *APIResponse
	if err := json.Unmarshal(bts, &rsp); err != nil {
		return err
	}

	if rsp == nil {
		return ErrNoResponseData
	}

	if rsp.Errors != nil && len(rsp.Errors) != 0 {
		return errors.New(rsp.Message)
	}

	return nil
}

func (h *KafkaTopicsHandler) Delete(project, service, topic string) error {
	bts, err := h.client.doDeleteRequest(fmt.Sprintf("/project/%s/service/%s/topic/%s", project, service, topic), nil)
	if err != nil {
		return err
	}

	return handleDeleteResponse(bts)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/watch"
	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/kubernetes/federation/apis/federation"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/genericapiserver/registry/rest"
)

// Registry is an interface implemented by things that know how to store Cluster objects.
type Registry interface {
	ListClusters(ctx genericapirequest.Context, options *api.ListOptions) (*federation.ClusterList, error)
	WatchCluster(ctx genericapirequest.Context, options *api.ListOptions) (watch.Interface, error)
	GetCluster(ctx genericapirequest.Context, name string, options *metav1.GetOptions) (*federation.Cluster, error)
	CreateCluster(ctx genericapirequest.Context, cluster *federation.Cluster) error
	UpdateCluster(ctx genericapirequest.Context, cluster *federation.Cluster) error
	DeleteCluster(ctx genericapirequest.Context, name string) error
}

// storage puts strong typing around storage calls
type storage struct {
	rest.StandardStorage
}

// NewRegistry returns a new Registry interface for the given Storage. Any mismatched
// types will panic.
func NewRegistry(s rest.StandardStorage) Registry {
	return &storage{s}
}

func (s *storage) ListClusters(ctx genericapirequest.Context, options *api.ListOptions) (*federation.ClusterList, error) {
	obj, err := s.List(ctx, options)
	if err != nil {
		return nil, err
	}
	return obj.(*federation.ClusterList), nil
}

func (s *storage) WatchCluster(ctx genericapirequest.Context, options *api.ListOptions) (watch.Interface, error) {
	return s.Watch(ctx, options)
}

func (s *storage) GetCluster(ctx genericapirequest.Context, name string, options *metav1.GetOptions) (*federation.Cluster, error) {
	obj, err := s.Get(ctx, name, options)
	if err != nil {
		return nil, err
	}
	return obj.(*federation.Cluster), nil
}

func (s *storage) CreateCluster(ctx genericapirequest.Context, cluster *federation.Cluster) error {
	_, err := s.Create(ctx, cluster)
	return err
}

func (s *storage) UpdateCluster(ctx genericapirequest.Context, cluster *federation.Cluster) error {
	_, _, err := s.Update(ctx, cluster.Name, rest.DefaultUpdatedObjectInfo(cluster, api.Scheme))
	return err
}

func (s *storage) DeleteCluster(ctx genericapirequest.Context, name string) error {
	_, err := s.Delete(ctx, name, nil)
	return err
}
<commit_msg>Update registries to use metainternalversion list options<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/watch"
	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/kubernetes/federation/apis/federation"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/genericapiserver/registry/rest"
)

// Registry is an interface implemented by things that know how to store Cluster objects.
type Registry interface {
	ListClusters(ctx genericapirequest.Context, options *metainternalversion.ListOptions) (*federation.ClusterList, error)
	WatchCluster(ctx genericapirequest.Context, options *metainternalversion.ListOptions) (watch.Interface, error)
	GetCluster(ctx genericapirequest.Context, name string, options *metav1.GetOptions) (*federation.Cluster, error)
	CreateCluster(ctx genericapirequest.Context, cluster *federation.Cluster) error
	UpdateCluster(ctx genericapirequest.Context, cluster *federation.Cluster) error
	DeleteCluster(ctx genericapirequest.Context, name string) error
}

// storage puts strong typing around storage calls
type storage struct {
	rest.StandardStorage
}

// NewRegistry returns a new Registry interface for the given Storage. Any mismatched
// types will panic.
func NewRegistry(s rest.StandardStorage) Registry {
	return &storage{s}
}

func (s *storage) ListClusters(ctx genericapirequest.Context, options *metainternalversion.ListOptions) (*federation.ClusterList, error) {
	obj, err := s.List(ctx, options)
	if err != nil {
		return nil, err
	}
	return obj.(*federation.ClusterList), nil
}

func (s *storage) WatchCluster(ctx genericapirequest.Context, options *metainternalversion.ListOptions) (watch.Interface, error) {
	return s.Watch(ctx, options)
}

func (s *storage) GetCluster(ctx genericapirequest.Context, name string, options *metav1.GetOptions) (*federation.Cluster, error) {
	obj, err := s.Get(ctx, name, options)
	if err != nil {
		return nil, err
	}
	return obj.(*federation.Cluster), nil
}

func (s *storage) CreateCluster(ctx genericapirequest.Context, cluster *federation.Cluster) error {
	_, err := s.Create(ctx, cluster)
	return err
}

func (s *storage) UpdateCluster(ctx genericapirequest.Context, cluster *federation.Cluster) error {
	_, _, err := s.Update(ctx, cluster.Name, rest.DefaultUpdatedObjectInfo(cluster, api.Scheme))
	return err
}

func (s *storage) DeleteCluster(ctx genericapirequest.Context, name string) error {
	_, err := s.Delete(ctx, name, nil)
	return err
}
<|endoftext|>


<commit_before>package openapi_test

import (
	"reflect"
	"testing"

	openapi "github.com/nasa9084/go-openapi"
)

func TestComponents(t *testing.T) {
	candidates := []candidate{
		{"empty", openapi.Components{}, false},
	}
	testValidater(t, candidates)
}

func TestComponentsValidateKeys(t *testing.T) {
	candidates := []struct {
		label  string
		in     openapi.Components
		hasErr bool
	}{
		{"empty", openapi.Components{}, false},
	}
	for _, c := range candidates {
		if err := openapi.ValidateComponentKeys(c.in); (err != nil) != c.hasErr {
			t.Log(c.label)
			if c.hasErr {
				t.Error("error should be occurred, but not")
				continue
			}
			t.Errorf("error should not be occurred: %s", err)
		}
	}
}
func TestReduceComponentKeys(t *testing.T) {
	candidates := []struct {
		label    string
		in       openapi.Components
		expected []string
	}{
		{"empty", openapi.Components{}, []string{}},
	}
	for _, c := range candidates {
		keys := openapi.ReduceComponentKeys(c.in)
		if !reflect.DeepEqual(keys, c.expected) {
			t.Log(c.label)
			t.Errorf("%+v != %+v", keys, c.expected)
		}
	}
}

func TestReduceComponentObjects(t *testing.T) {
	candidates := []struct {
		label    string
		in       openapi.Components
		expected []openapi.Validater
	}{
		{"empty", openapi.Components{}, []openapi.Validater{}},
	}
	for _, c := range candidates {
		objects := openapi.ReduceComponentObjects(c.in)
		if !reflect.DeepEqual(objects, c.expected) {
			t.Log(c.label)
			t.Errorf("%+v != %+v", objects, c.expected)
		}
	}
}
<commit_msg>add test case for components<commit_after>package openapi_test

import (
	"reflect"
	"testing"

	openapi "github.com/nasa9084/go-openapi"
)

func TestComponents(t *testing.T) {
	candidates := []candidate{
		{"empty", openapi.Components{}, false},
	}
	testValidater(t, candidates)
}

func TestComponentsValidateKeys(t *testing.T) {
	candidates := []struct {
		label  string
		in     openapi.Components
		hasErr bool
	}{
		{"empty", openapi.Components{}, false},
		{"invalidKey", openapi.Components{Parameters: map[string]*openapi.Parameter{"@": &openapi.Parameter{}}}, true},
		{"validKey", openapi.Components{Parameters: map[string]*openapi.Parameter{"foo": &openapi.Parameter{}}}, false},
	}
	for _, c := range candidates {
		if err := openapi.ValidateComponentKeys(c.in); (err != nil) != c.hasErr {
			t.Log(c.label)
			if c.hasErr {
				t.Error("error should be occurred, but not")
				continue
			}
			t.Errorf("error should not be occurred: %s", err)
		}
	}
}
func TestReduceComponentKeys(t *testing.T) {
	candidates := []struct {
		label    string
		in       openapi.Components
		expected []string
	}{
		{"empty", openapi.Components{}, []string{}},
	}
	for _, c := range candidates {
		keys := openapi.ReduceComponentKeys(c.in)
		if !reflect.DeepEqual(keys, c.expected) {
			t.Log(c.label)
			t.Errorf("%+v != %+v", keys, c.expected)
		}
	}
}

func TestReduceComponentObjects(t *testing.T) {
	candidates := []struct {
		label    string
		in       openapi.Components
		expected []openapi.Validater
	}{
		{"empty", openapi.Components{}, []openapi.Validater{}},
	}
	for _, c := range candidates {
		objects := openapi.ReduceComponentObjects(c.in)
		if !reflect.DeepEqual(objects, c.expected) {
			t.Log(c.label)
			t.Errorf("%+v != %+v", objects, c.expected)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Stratumn SAS. All rights reserved.
// Use of this source code is governed by the license
// that can be found in the LICENSE file.

// Package treetestcases contains test cases to test Merkle tree implementation.
package treetestcases

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"io/ioutil"
	"reflect"
	"testing"

	"github.com/stratumn/goprivate/merkle"
	"github.com/stratumn/goprivate/merkle/merkletesting"
)

var (
	pathA0     merkle.Path
	pathAB0    merkle.Path
	pathAB1    merkle.Path
	pathABC0   merkle.Path
	pathABC1   merkle.Path
	pathABC2   merkle.Path
	pathABCD0  merkle.Path
	pathABCD1  merkle.Path
	pathABCD2  merkle.Path
	pathABCD3  merkle.Path
	pathABCDE0 merkle.Path
	pathABCDE1 merkle.Path
	pathABCDE2 merkle.Path
	pathABCDE3 merkle.Path
	pathABCDE4 merkle.Path
)

func loadPath(filename string, path *merkle.Path) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	if err = json.Unmarshal(data, path); err != nil {
		panic(err)
	}
}

// LoadFixtures loads test fixtures and should be called before running the tests.
func LoadFixtures(testdatapath string) {
	// Load fixtures.
	loadPath(testdatapath+"/path-a-0.json", &pathA0)
	loadPath(testdatapath+"/path-ab-0.json", &pathAB0)
	loadPath(testdatapath+"/path-ab-1.json", &pathAB1)
	loadPath(testdatapath+"/path-abc-0.json", &pathABC0)
	loadPath(testdatapath+"/path-abc-1.json", &pathABC1)
	loadPath(testdatapath+"/path-abc-2.json", &pathABC2)
	loadPath(testdatapath+"/path-abcd-0.json", &pathABCD0)
	loadPath(testdatapath+"/path-abcd-1.json", &pathABCD1)
	loadPath(testdatapath+"/path-abcd-2.json", &pathABCD2)
	loadPath(testdatapath+"/path-abcd-3.json", &pathABCD3)
	loadPath(testdatapath+"/path-abcde-0.json", &pathABCDE0)
	loadPath(testdatapath+"/path-abcde-1.json", &pathABCDE1)
	loadPath(testdatapath+"/path-abcde-2.json", &pathABCDE2)
	loadPath(testdatapath+"/path-abcde-3.json", &pathABCDE3)
	loadPath(testdatapath+"/path-abcde-4.json", &pathABCDE4)
}

// Factory contains function to allocate and free a Merkle tree.
type Factory struct {
	// New create a Merkle tree from leaves.
	New func(leaves []merkle.Hash) (merkle.Tree, error)

	// Free is an optional function to free a Merkle tree.
	Free func(tree merkle.Tree)
}

// RunTests runs all the tests.
func (f Factory) RunTests(t *testing.T) {
	t.Run("NumLeaves", f.TestNumLeaves)
	t.Run("Root", f.TestRoot)
	t.Run("Leaf", f.TestLeaf)
	t.Run("Path", f.TestPath)
}

// RunBenchmarks runs all the benchmarks.
func (f Factory) RunBenchmarks(b *testing.B) {
	b.Run("Create", f.BenchmarkCreate)
	b.Run("Path", f.BenchmarkPath)
}

func (f Factory) free(tree merkle.Tree) {
	if f.Free != nil {
		f.Free(tree)
	}
}

// TestNumLeaves tests that the implementation returns the correct number of leaves.
func (f Factory) TestNumLeaves(t *testing.T) {
	tree, err := f.New([]merkle.Hash{merkletesting.RandomHash(), merkletesting.RandomHash(), merkletesting.RandomHash()})
	if err != nil {
		t.Fatal(err)
	}
	if tree == nil {
		t.Fatal("expected tree not to be nil")
	}
	defer f.free(tree)

	var (
		a = tree.NumLeaves()
		e = 3
	)
	if a != e {
		t.Logf("actual: %d; expected: %d\n", a, e)
		t.Error("unexpected number of leaves")
	}
}

// TestRoot tests that the implementation computes the root correctly.
func (f Factory) TestRoot(t *testing.T) {
	grid := [...]struct {
		leaves   []string
		expected string
	}{
		{[]string{"a"}, "ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb"},
		{[]string{"a", "b"}, "e5a01fee14e0ed5c48714f22180f25ad8365b53f9779f79dc4a3d7e93963f94a"},
		{[]string{"a", "b", "c"}, "7075152d03a5cd92104887b476862778ec0c87be5c2fa1c0a90f87c49fad6eff"},
		{[]string{"a", "b", "c", "d"}, "14ede5e8e97ad9372327728f5099b95604a39593cac3bd38a343ad76205213e7"},
		{[]string{"a", "b", "c", "d", "e"}, "d71f8983ad4ee170f8129f1ebcdd7440be7798d8e1c80420bf11f1eced610dba"},
		{[]string{"a", "b", "c", "d", "e", "f"}, "1f7379539707bcaea00564168d1d4d626b09b73f8a2a365234c62d763f854da2"},
		{[]string{"a", "b", "c", "d", "e", "f", "g"}, "e2a80e0e872a6c6eaed37b4c1f220e1935004805585b5f99617e48e9c8fe4034"},
		{[]string{"a", "b", "c", "d", "e", "f", "g", "h"}, "bd7c8a900be9b67ba7df5c78a652a8474aedd78adb5083e80e49d9479138a23f"},
		{[]string{"a", "b", "c", "d", "e", "f", "g", "h", "i"}, "386ced54bdc7456fecfc9b43018bbda2fe0a105f4cf7cad6bbb429c18fe852cc"},
	}

	for _, row := range grid {
		var leaves = make([]merkle.Hash, len(row.leaves), len(row.leaves))
		for i, s := range row.leaves {
			leaves[i] = sha256.Sum256([]byte(s))
		}

		tree, err := f.New(leaves)
		if err != nil {
			t.Fatal(err)
		}
		if tree == nil {
			t.Fatal("expected tree not to be nil")
		}
		defer f.free(tree)

		a := hex.EncodeToString(atos(tree.Root()))
		e := row.expected

		if a != e {
			t.Logf("actual: %s; expected: %s\n", a, e)
			t.Error("unexpected root")
		}
	}
}

// TestLeaf tests that the implementation correctly returns leaves.
func (f Factory) TestLeaf(t *testing.T) {
	for i := 1; i < 128; i++ {
		var leaves []merkle.Hash
		for j := 0; j < i; j++ {
			leaves = append(leaves, merkletesting.RandomHash())
		}

		tree, err := f.New(leaves)
		if err != nil {
			t.Fatal(err)
		}
		if tree == nil {
			t.Fatal("expected tree not to be nil")
		}
		defer f.free(tree)

		for j := 0; j < i; j++ {
			a := tree.Leaf(j)
			e := leaves[j]
			if !reflect.DeepEqual(a, e) {
				t.Logf("actual: %s; expected: %s\n", a, e)
				t.Error("unexpected leaf")
			}
		}
	}
}

// TestPath tests that the implementation correctly computes paths.
func (f Factory) TestPath(t *testing.T) {
	grid := [...]struct {
		leaves   []string
		expected []merkle.Path
	}{
		{
			[]string{"a"},
			[]merkle.Path{pathA0},
		},
		{
			[]string{"a", "b"},
			[]merkle.Path{pathAB0, pathAB1},
		},
		{
			[]string{"a", "b", "c"},
			[]merkle.Path{pathABC0, pathABC1, pathABC2},
		},
		{
			[]string{"a", "b", "c", "d"},
			[]merkle.Path{pathABCD0, pathABCD1, pathABCD2, pathABCD3},
		},
		{
			[]string{"a", "b", "c", "d", "e"},
			[]merkle.Path{pathABCDE0, pathABCDE1, pathABCDE2, pathABCDE3, pathABCDE4},
		},
	}

	for _, row := range grid {
		var leaves = make([]merkle.Hash, len(row.leaves), len(row.leaves))
		for i, s := range row.leaves {
			leaves[i] = sha256.Sum256([]byte(s))
		}

		tree, err := f.New(leaves)
		if err != nil {
			t.Fatal(err)
		}
		if tree == nil {
			t.Fatal("expected tree not to be nil")
		}
		defer f.free(tree)

		for i := range row.leaves {
			var (
				a = tree.Path(i)
				e = row.expected[i]
			)

			if !reflect.DeepEqual(e, a) {
				t.Logf("actual: %v; expected: %v\n", a, e)
				t.Error("unexpected root")
			}
		}
	}
}

// BenchmarkCreateWithSize benchmarks creating trees of given size.
func (f Factory) BenchmarkCreateWithSize(b *testing.B, size int) {
	leaves := make([]merkle.Hash, size)
	for i := 0; i < size; i++ {
		leaves[i] = merkletesting.RandomHash()
	}

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree, err := f.New(leaves)
		if err != nil {
			b.Fatal(err)
		}
		if tree == nil {
			b.Fatal("expected tree not to be nil")
		}
		defer f.free(tree)
	}
}

// BenchmarkCreate benchmarks creating trees of different sizes.
func (f Factory) BenchmarkCreate(b *testing.B) {
	b.Run("10-leaves", func(b *testing.B) { f.BenchmarkCreateWithSize(b, 10) })
	b.Run("100-leaves", func(b *testing.B) { f.BenchmarkCreateWithSize(b, 100) })
	b.Run("1000-leaves", func(b *testing.B) { f.BenchmarkCreateWithSize(b, 1000) })
	b.Run("10000-leaves", func(b *testing.B) { f.BenchmarkCreateWithSize(b, 10000) })
}

// BenchmarkPathWithSize benchmarks computing paths for trees of given size.
func (f Factory) BenchmarkPathWithSize(b *testing.B, size int) {
	leaves := make([]merkle.Hash, size)
	for i := 0; i < size; i++ {
		leaves[i] = merkletesting.RandomHash()
	}

	tree, err := f.New(leaves)
	if err != nil {
		b.Fatal(err)
	}
	if tree == nil {
		b.Fatal("expected tree not to be nil")
	}
	defer f.free(tree)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree.Path(i % size)
	}
}

// BenchmarkPath benchmarks computing paths for different sizes.
func (f Factory) BenchmarkPath(b *testing.B) {
	b.Run("10-leaves", func(b *testing.B) { f.BenchmarkPathWithSize(b, 10) })
	b.Run("100-leaves", func(b *testing.B) { f.BenchmarkPathWithSize(b, 100) })
	b.Run("1000-leaves", func(b *testing.B) { f.BenchmarkPathWithSize(b, 1000) })
	b.Run("10000-leaves", func(b *testing.B) { f.BenchmarkPathWithSize(b, 10000) })
}

func atos(a merkle.Hash) []byte {
	return a[:]
}
<commit_msg>merkle: Doc typo<commit_after>// Copyright 2016 Stratumn SAS. All rights reserved.
// Use of this source code is governed by the license
// that can be found in the LICENSE file.

// Package treetestcases contains test cases to test Merkle tree implementation.
package treetestcases

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"io/ioutil"
	"reflect"
	"testing"

	"github.com/stratumn/goprivate/merkle"
	"github.com/stratumn/goprivate/merkle/merkletesting"
)

var (
	pathA0     merkle.Path
	pathAB0    merkle.Path
	pathAB1    merkle.Path
	pathABC0   merkle.Path
	pathABC1   merkle.Path
	pathABC2   merkle.Path
	pathABCD0  merkle.Path
	pathABCD1  merkle.Path
	pathABCD2  merkle.Path
	pathABCD3  merkle.Path
	pathABCDE0 merkle.Path
	pathABCDE1 merkle.Path
	pathABCDE2 merkle.Path
	pathABCDE3 merkle.Path
	pathABCDE4 merkle.Path
)

func loadPath(filename string, path *merkle.Path) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	if err = json.Unmarshal(data, path); err != nil {
		panic(err)
	}
}

// LoadFixtures loads test fixtures and should be called before running the tests.
func LoadFixtures(testdatapath string) {
	// Load fixtures.
	loadPath(testdatapath+"/path-a-0.json", &pathA0)
	loadPath(testdatapath+"/path-ab-0.json", &pathAB0)
	loadPath(testdatapath+"/path-ab-1.json", &pathAB1)
	loadPath(testdatapath+"/path-abc-0.json", &pathABC0)
	loadPath(testdatapath+"/path-abc-1.json", &pathABC1)
	loadPath(testdatapath+"/path-abc-2.json", &pathABC2)
	loadPath(testdatapath+"/path-abcd-0.json", &pathABCD0)
	loadPath(testdatapath+"/path-abcd-1.json", &pathABCD1)
	loadPath(testdatapath+"/path-abcd-2.json", &pathABCD2)
	loadPath(testdatapath+"/path-abcd-3.json", &pathABCD3)
	loadPath(testdatapath+"/path-abcde-0.json", &pathABCDE0)
	loadPath(testdatapath+"/path-abcde-1.json", &pathABCDE1)
	loadPath(testdatapath+"/path-abcde-2.json", &pathABCDE2)
	loadPath(testdatapath+"/path-abcde-3.json", &pathABCDE3)
	loadPath(testdatapath+"/path-abcde-4.json", &pathABCDE4)
}

// Factory contains functions to allocate and free a Merkle tree.
type Factory struct {
	// New create a Merkle tree from leaves.
	New func(leaves []merkle.Hash) (merkle.Tree, error)

	// Free is an optional function to free a Merkle tree.
	Free func(tree merkle.Tree)
}

// RunTests runs all the tests.
func (f Factory) RunTests(t *testing.T) {
	t.Run("NumLeaves", f.TestNumLeaves)
	t.Run("Root", f.TestRoot)
	t.Run("Leaf", f.TestLeaf)
	t.Run("Path", f.TestPath)
}

// RunBenchmarks runs all the benchmarks.
func (f Factory) RunBenchmarks(b *testing.B) {
	b.Run("Create", f.BenchmarkCreate)
	b.Run("Path", f.BenchmarkPath)
}

func (f Factory) free(tree merkle.Tree) {
	if f.Free != nil {
		f.Free(tree)
	}
}

// TestNumLeaves tests that the implementation returns the correct number of leaves.
func (f Factory) TestNumLeaves(t *testing.T) {
	tree, err := f.New([]merkle.Hash{merkletesting.RandomHash(), merkletesting.RandomHash(), merkletesting.RandomHash()})
	if err != nil {
		t.Fatal(err)
	}
	if tree == nil {
		t.Fatal("expected tree not to be nil")
	}
	defer f.free(tree)

	var (
		a = tree.NumLeaves()
		e = 3
	)
	if a != e {
		t.Logf("actual: %d; expected: %d\n", a, e)
		t.Error("unexpected number of leaves")
	}
}

// TestRoot tests that the implementation computes the root correctly.
func (f Factory) TestRoot(t *testing.T) {
	grid := [...]struct {
		leaves   []string
		expected string
	}{
		{[]string{"a"}, "ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb"},
		{[]string{"a", "b"}, "e5a01fee14e0ed5c48714f22180f25ad8365b53f9779f79dc4a3d7e93963f94a"},
		{[]string{"a", "b", "c"}, "7075152d03a5cd92104887b476862778ec0c87be5c2fa1c0a90f87c49fad6eff"},
		{[]string{"a", "b", "c", "d"}, "14ede5e8e97ad9372327728f5099b95604a39593cac3bd38a343ad76205213e7"},
		{[]string{"a", "b", "c", "d", "e"}, "d71f8983ad4ee170f8129f1ebcdd7440be7798d8e1c80420bf11f1eced610dba"},
		{[]string{"a", "b", "c", "d", "e", "f"}, "1f7379539707bcaea00564168d1d4d626b09b73f8a2a365234c62d763f854da2"},
		{[]string{"a", "b", "c", "d", "e", "f", "g"}, "e2a80e0e872a6c6eaed37b4c1f220e1935004805585b5f99617e48e9c8fe4034"},
		{[]string{"a", "b", "c", "d", "e", "f", "g", "h"}, "bd7c8a900be9b67ba7df5c78a652a8474aedd78adb5083e80e49d9479138a23f"},
		{[]string{"a", "b", "c", "d", "e", "f", "g", "h", "i"}, "386ced54bdc7456fecfc9b43018bbda2fe0a105f4cf7cad6bbb429c18fe852cc"},
	}

	for _, row := range grid {
		var leaves = make([]merkle.Hash, len(row.leaves), len(row.leaves))
		for i, s := range row.leaves {
			leaves[i] = sha256.Sum256([]byte(s))
		}

		tree, err := f.New(leaves)
		if err != nil {
			t.Fatal(err)
		}
		if tree == nil {
			t.Fatal("expected tree not to be nil")
		}
		defer f.free(tree)

		a := hex.EncodeToString(atos(tree.Root()))
		e := row.expected

		if a != e {
			t.Logf("actual: %s; expected: %s\n", a, e)
			t.Error("unexpected root")
		}
	}
}

// TestLeaf tests that the implementation correctly returns leaves.
func (f Factory) TestLeaf(t *testing.T) {
	for i := 1; i < 128; i++ {
		var leaves []merkle.Hash
		for j := 0; j < i; j++ {
			leaves = append(leaves, merkletesting.RandomHash())
		}

		tree, err := f.New(leaves)
		if err != nil {
			t.Fatal(err)
		}
		if tree == nil {
			t.Fatal("expected tree not to be nil")
		}
		defer f.free(tree)

		for j := 0; j < i; j++ {
			a := tree.Leaf(j)
			e := leaves[j]
			if !reflect.DeepEqual(a, e) {
				t.Logf("actual: %s; expected: %s\n", a, e)
				t.Error("unexpected leaf")
			}
		}
	}
}

// TestPath tests that the implementation correctly computes paths.
func (f Factory) TestPath(t *testing.T) {
	grid := [...]struct {
		leaves   []string
		expected []merkle.Path
	}{
		{
			[]string{"a"},
			[]merkle.Path{pathA0},
		},
		{
			[]string{"a", "b"},
			[]merkle.Path{pathAB0, pathAB1},
		},
		{
			[]string{"a", "b", "c"},
			[]merkle.Path{pathABC0, pathABC1, pathABC2},
		},
		{
			[]string{"a", "b", "c", "d"},
			[]merkle.Path{pathABCD0, pathABCD1, pathABCD2, pathABCD3},
		},
		{
			[]string{"a", "b", "c", "d", "e"},
			[]merkle.Path{pathABCDE0, pathABCDE1, pathABCDE2, pathABCDE3, pathABCDE4},
		},
	}

	for _, row := range grid {
		var leaves = make([]merkle.Hash, len(row.leaves), len(row.leaves))
		for i, s := range row.leaves {
			leaves[i] = sha256.Sum256([]byte(s))
		}

		tree, err := f.New(leaves)
		if err != nil {
			t.Fatal(err)
		}
		if tree == nil {
			t.Fatal("expected tree not to be nil")
		}
		defer f.free(tree)

		for i := range row.leaves {
			var (
				a = tree.Path(i)
				e = row.expected[i]
			)

			if !reflect.DeepEqual(e, a) {
				t.Logf("actual: %v; expected: %v\n", a, e)
				t.Error("unexpected root")
			}
		}
	}
}

// BenchmarkCreateWithSize benchmarks creating trees of given size.
func (f Factory) BenchmarkCreateWithSize(b *testing.B, size int) {
	leaves := make([]merkle.Hash, size)
	for i := 0; i < size; i++ {
		leaves[i] = merkletesting.RandomHash()
	}

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree, err := f.New(leaves)
		if err != nil {
			b.Fatal(err)
		}
		if tree == nil {
			b.Fatal("expected tree not to be nil")
		}
		defer f.free(tree)
	}
}

// BenchmarkCreate benchmarks creating trees of different sizes.
func (f Factory) BenchmarkCreate(b *testing.B) {
	b.Run("10-leaves", func(b *testing.B) { f.BenchmarkCreateWithSize(b, 10) })
	b.Run("100-leaves", func(b *testing.B) { f.BenchmarkCreateWithSize(b, 100) })
	b.Run("1000-leaves", func(b *testing.B) { f.BenchmarkCreateWithSize(b, 1000) })
	b.Run("10000-leaves", func(b *testing.B) { f.BenchmarkCreateWithSize(b, 10000) })
}

// BenchmarkPathWithSize benchmarks computing paths for trees of given size.
func (f Factory) BenchmarkPathWithSize(b *testing.B, size int) {
	leaves := make([]merkle.Hash, size)
	for i := 0; i < size; i++ {
		leaves[i] = merkletesting.RandomHash()
	}

	tree, err := f.New(leaves)
	if err != nil {
		b.Fatal(err)
	}
	if tree == nil {
		b.Fatal("expected tree not to be nil")
	}
	defer f.free(tree)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree.Path(i % size)
	}
}

// BenchmarkPath benchmarks computing paths for different sizes.
func (f Factory) BenchmarkPath(b *testing.B) {
	b.Run("10-leaves", func(b *testing.B) { f.BenchmarkPathWithSize(b, 10) })
	b.Run("100-leaves", func(b *testing.B) { f.BenchmarkPathWithSize(b, 100) })
	b.Run("1000-leaves", func(b *testing.B) { f.BenchmarkPathWithSize(b, 1000) })
	b.Run("10000-leaves", func(b *testing.B) { f.BenchmarkPathWithSize(b, 10000) })
}

func atos(a merkle.Hash) []byte {
	return a[:]
}
<|endoftext|>


<commit_before>// +build linux

package ether

import (
	"errors"
	"fmt"
	"net"
	"syscall"
	"unsafe"

	"golang.org/x/sys/unix"
)

func htons(h int) (n int) {
	a := uint16(42)
	if *(*byte)(unsafe.Pointer(&a)) == 42 { // little-endian
		a = uint16(h)
		n = int(a>>8 | a<<8)
	} else { // big-endian
		n = h
	}
	return
}

type afpacket struct {
	name        string
	frameFilter FrameFilter

	fd   int
	mtu  int
	addr net.HardwareAddr

	// for outgoing frames
	sockaddr_ll *unix.SockaddrLinklayer

	// for incoming frames
	buf []byte
}

func NewDev(ifName string, frameFilter FrameFilter) (dev Dev, err error) {
	if len([]byte(ifName)) > unix.IFNAMSIZ {
		err = errors.New("invalid ifName")
		return
	}

	d := new(afpacket)
	d.name = ifName
	d.frameFilter = frameFilter

	d.fd, err = unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, htons(unix.ETH_P_ALL))
	if err != nil {
		return
	}

	// get interface index
	ifindex_st := struct {
		ifr_name    [unix.IFNAMSIZ]byte
		ifr_ifindex uint32
	}{}
	copy(ifindex_st.ifr_name[:], []byte(ifName))
	var errno syscall.Errno
	_, _, errno = unix.Syscall(syscall.SYS_IOCTL, uintptr(d.fd), uintptr(syscall.SIOCGIFINDEX), uintptr(unsafe.Pointer(&ifindex_st)))
	if errno != 0 {
		err = errno
		return
	}
	index := int(ifindex_st.ifr_ifindex)

	// get MAC address
	ifhaddr_st := struct {
		ifr_name   [unix.IFNAMSIZ]byte
		ifr_hwaddr unix.RawSockaddr
	}{}
	copy(ifhaddr_st.ifr_name[:], []byte(ifName))
	_, _, errno = unix.Syscall(syscall.SYS_IOCTL, uintptr(d.fd), uintptr(syscall.SIOCGIFHWADDR), uintptr(unsafe.Pointer(&ifhaddr_st)))
	if errno != 0 {
		err = errno
		return
	}
	for i := 0; i < 6; i++ {
		d.addr = append(d.addr, byte(ifhaddr_st.ifr_hwaddr.Data[i]))
	}

	// get MTU
	ifmtu_st := struct {
		ifr_name [unix.IFNAMSIZ]byte
		ifr_mtu  uint32
	}{}
	copy(ifmtu_st.ifr_name[:], []byte(ifName))
	_, _, errno = unix.Syscall(syscall.SYS_IOCTL, uintptr(d.fd), uintptr(syscall.SIOCGIFMTU), uintptr(unsafe.Pointer(&ifmtu_st)))
	if errno != 0 {
		err = errno
		return
	}
	d.mtu = int(ifmtu_st.ifr_mtu)

	d.sockaddr_ll = new(unix.SockaddrLinklayer)
	d.sockaddr_ll.Ifindex = index
	d.sockaddr_ll.Halen = 6

	d.buf = make([]byte, d.mtu+20)

	dev = d
	return
}

func (d *afpacket) Name() string {
	return d.name
}

func (d *afpacket) GetMTU() int {
	return d.mtu
}

func (d *afpacket) GetHardwareAddr() net.HardwareAddr {
	return append(net.HardwareAddr(nil), d.addr...)
}

func (d *afpacket) Close() error {
	return unix.Close(d.fd)
}

func (d *afpacket) Write(from Frame) (err error) {
	if len(from) > d.mtu {
		err = fmt.Errorf("frame too large (%d); MTU: (%d)", len(from), d.mtu)
	}
	copy(d.sockaddr_ll.Addr[:6], []byte(from.Destination()))
	err = unix.Sendto(d.fd, []byte(from), 0, d.sockaddr_ll)
	if err != nil {
		return
	}
	return
}

func (d *afpacket) Read(to Frame) (n int, err error) {
	n, _, err = unix.Recvfrom(d.fd, d.buf, 0)
	if err != nil {
		return
	}
	if n > len(to) {
		err = fmt.Errorf("destination buffer too small (%d); need (%d)\n", len(to), n)
		return
	}
	copy(to, d.buf[:n])
	return
}
<commit_msg>fix a bug where filtering was not done on Linux<commit_after>// +build linux

package ether

import (
	"errors"
	"fmt"
	"net"
	"syscall"
	"unsafe"

	"golang.org/x/sys/unix"
)

func htons(h int) (n int) {
	a := uint16(42)
	if *(*byte)(unsafe.Pointer(&a)) == 42 { // little-endian
		a = uint16(h)
		n = int(a>>8 | a<<8)
	} else { // big-endian
		n = h
	}
	return
}

type afpacket struct {
	name   string
	filter FrameFilter

	fd   int
	mtu  int
	addr net.HardwareAddr

	// for outgoing frames
	sockaddr_ll *unix.SockaddrLinklayer

	// for incoming frames
	buf []byte
}

func NewDev(ifName string, frameFilter FrameFilter) (dev Dev, err error) {
	if len([]byte(ifName)) > unix.IFNAMSIZ {
		err = errors.New("invalid ifName")
		return
	}

	d := new(afpacket)
	d.name = ifName
	d.filter = frameFilter

	d.fd, err = unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, htons(unix.ETH_P_ALL))
	if err != nil {
		return
	}

	// get interface index
	ifindex_st := struct {
		ifr_name    [unix.IFNAMSIZ]byte
		ifr_ifindex uint32
	}{}
	copy(ifindex_st.ifr_name[:], []byte(ifName))
	var errno syscall.Errno
	_, _, errno = unix.Syscall(syscall.SYS_IOCTL, uintptr(d.fd), uintptr(syscall.SIOCGIFINDEX), uintptr(unsafe.Pointer(&ifindex_st)))
	if errno != 0 {
		err = errno
		return
	}
	index := int(ifindex_st.ifr_ifindex)

	// get MAC address
	ifhaddr_st := struct {
		ifr_name   [unix.IFNAMSIZ]byte
		ifr_hwaddr unix.RawSockaddr
	}{}
	copy(ifhaddr_st.ifr_name[:], []byte(ifName))
	_, _, errno = unix.Syscall(syscall.SYS_IOCTL, uintptr(d.fd), uintptr(syscall.SIOCGIFHWADDR), uintptr(unsafe.Pointer(&ifhaddr_st)))
	if errno != 0 {
		err = errno
		return
	}
	for i := 0; i < 6; i++ {
		d.addr = append(d.addr, byte(ifhaddr_st.ifr_hwaddr.Data[i]))
	}

	// get MTU
	ifmtu_st := struct {
		ifr_name [unix.IFNAMSIZ]byte
		ifr_mtu  uint32
	}{}
	copy(ifmtu_st.ifr_name[:], []byte(ifName))
	_, _, errno = unix.Syscall(syscall.SYS_IOCTL, uintptr(d.fd), uintptr(syscall.SIOCGIFMTU), uintptr(unsafe.Pointer(&ifmtu_st)))
	if errno != 0 {
		err = errno
		return
	}
	d.mtu = int(ifmtu_st.ifr_mtu)

	d.sockaddr_ll = new(unix.SockaddrLinklayer)
	d.sockaddr_ll.Ifindex = index
	d.sockaddr_ll.Halen = 6

	d.buf = make([]byte, d.mtu+20)

	dev = d
	return
}

func (d *afpacket) Name() string {
	return d.name
}

func (d *afpacket) GetMTU() int {
	return d.mtu
}

func (d *afpacket) GetHardwareAddr() net.HardwareAddr {
	return append(net.HardwareAddr(nil), d.addr...)
}

func (d *afpacket) Close() error {
	return unix.Close(d.fd)
}

func (d *afpacket) Write(from Frame) (err error) {
	if len(from) > d.mtu {
		err = fmt.Errorf("frame too large (%d); MTU: (%d)", len(from), d.mtu)
	}
	copy(d.sockaddr_ll.Addr[:6], []byte(from.Destination()))
	err = unix.Sendto(d.fd, []byte(from), 0, d.sockaddr_ll)
	if err != nil {
		return
	}
	return
}

func (d *afpacket) Read(to Frame) (n int, err error) {
	for {
		n, _, err = unix.Recvfrom(d.fd, d.buf, 0)
		if err != nil {
			return
		}
		if n > len(to) {
			err = fmt.Errorf("destination buffer too small (%d); need (%d)\n", len(to), n)
			return
		}
		copy(to, d.buf[:n])
		if !equalMAC(to.Source(), d.addr) && (d.filter == nil || d.filter(to)) {
			return
		}
	}
}
<|endoftext|>


<commit_before>package ledis

import (
	"encoding/json"
	"fmt"
	"github.com/siddontang/go-log/log"
	"github.com/siddontang/ledisdb/leveldb"
	"sync"
	"time"
)

type DB struct {
	l *Ledis

	db *leveldb.DB

	index uint8

	kvTx   *tx
	listTx *tx
	hashTx *tx
	zsetTx *tx
	binTx  *tx
}

type Ledis struct {
	sync.Mutex

	cfg *Config

	ldb *leveldb.DB
	dbs [MaxDBNumber]*DB

	binlog *BinLog

	quit chan struct{}
	jobs *sync.WaitGroup
}

func OpenWithJsonConfig(configJson json.RawMessage) (*Ledis, error) {
	var cfg Config

	if err := json.Unmarshal(configJson, &cfg); err != nil {
		return nil, err
	}

	return Open(&cfg)
}

func Open(cfg *Config) (*Ledis, error) {
	if len(cfg.DataDir) == 0 {
		return nil, fmt.Errorf("must set correct data_dir")
	}

	ldb, err := leveldb.Open(cfg.NewDBConfig())
	if err != nil {
		return nil, err
	}

	l := new(Ledis)

	l.quit = make(chan struct{})
	l.jobs = new(sync.WaitGroup)

	l.ldb = ldb

	if cfg.BinLog.Use {
		l.binlog, err = NewBinLog(cfg.NewBinLogConfig())
		if err != nil {
			return nil, err
		}
	} else {
		l.binlog = nil
	}

	for i := uint8(0); i < MaxDBNumber; i++ {
		l.dbs[i] = newDB(l, i)
	}

	l.activeExpireCycle()

	return l, nil
}

func newDB(l *Ledis, index uint8) *DB {
	d := new(DB)

	d.l = l

	d.db = l.ldb

	d.index = index

	d.kvTx = newTx(l)
	d.listTx = newTx(l)
	d.hashTx = newTx(l)
	d.zsetTx = newTx(l)
	d.binTx = newTx(l)

	return d
}

func (l *Ledis) Close() {
	close(l.quit)
	l.jobs.Wait()

	l.ldb.Close()

	if l.binlog != nil {
		l.binlog.Close()
		l.binlog = nil
	}
}

func (l *Ledis) Select(index int) (*DB, error) {
	if index < 0 || index >= int(MaxDBNumber) {
		return nil, fmt.Errorf("invalid db index %d", index)
	}

	return l.dbs[index], nil
}

func (l *Ledis) FlushAll() error {
	for index, db := range l.dbs {
		if _, err := db.FlushAll(); err != nil {
			log.Error("flush db %d error %s", index, err.Error())
		}
	}

	return nil
}

// very dangerous to use
func (l *Ledis) DataDB() *leveldb.DB {
	return l.ldb
}

func (l *Ledis) activeExpireCycle() {
	var executors []*elimination = make([]*elimination, len(l.dbs))
	for i, db := range l.dbs {
		executors[i] = db.newEliminator()
	}

	l.jobs.Add(1)
	go func() {
		tick := time.NewTicker(1 * time.Second)
		end := false
		done := make(chan struct{})
		for !end {
			select {
			case <-tick.C:
				go func() {
					for _, eli := range executors {
						eli.active()
					}
					done <- struct{}{}
				}()
				<-done
			case <-l.quit:
				end = true
				break
			}
		}

		tick.Stop()
		l.jobs.Done()
	}()
}
<commit_msg>add bin log comment<commit_after>package ledis

import (
	"encoding/json"
	"fmt"
	"github.com/siddontang/go-log/log"
	"github.com/siddontang/ledisdb/leveldb"
	"sync"
	"time"
)

type DB struct {
	l *Ledis

	db *leveldb.DB

	index uint8

	kvTx   *tx
	listTx *tx
	hashTx *tx
	zsetTx *tx
	binTx  *tx
}

type Ledis struct {
	sync.Mutex

	cfg *Config

	ldb *leveldb.DB
	dbs [MaxDBNumber]*DB

	binlog *BinLog

	quit chan struct{}
	jobs *sync.WaitGroup
}

func OpenWithJsonConfig(configJson json.RawMessage) (*Ledis, error) {
	var cfg Config

	if err := json.Unmarshal(configJson, &cfg); err != nil {
		return nil, err
	}

	return Open(&cfg)
}

func Open(cfg *Config) (*Ledis, error) {
	if len(cfg.DataDir) == 0 {
		return nil, fmt.Errorf("must set correct data_dir")
	}

	ldb, err := leveldb.Open(cfg.NewDBConfig())
	if err != nil {
		return nil, err
	}

	l := new(Ledis)

	l.quit = make(chan struct{})
	l.jobs = new(sync.WaitGroup)

	l.ldb = ldb

	if cfg.BinLog.Use {
		println("binlog will be refactored later, use your own risk!!!")
		l.binlog, err = NewBinLog(cfg.NewBinLogConfig())
		if err != nil {
			return nil, err
		}
	} else {
		l.binlog = nil
	}

	for i := uint8(0); i < MaxDBNumber; i++ {
		l.dbs[i] = newDB(l, i)
	}

	l.activeExpireCycle()

	return l, nil
}

func newDB(l *Ledis, index uint8) *DB {
	d := new(DB)

	d.l = l

	d.db = l.ldb

	d.index = index

	d.kvTx = newTx(l)
	d.listTx = newTx(l)
	d.hashTx = newTx(l)
	d.zsetTx = newTx(l)
	d.binTx = newTx(l)

	return d
}

func (l *Ledis) Close() {
	close(l.quit)
	l.jobs.Wait()

	l.ldb.Close()

	if l.binlog != nil {
		l.binlog.Close()
		l.binlog = nil
	}
}

func (l *Ledis) Select(index int) (*DB, error) {
	if index < 0 || index >= int(MaxDBNumber) {
		return nil, fmt.Errorf("invalid db index %d", index)
	}

	return l.dbs[index], nil
}

func (l *Ledis) FlushAll() error {
	for index, db := range l.dbs {
		if _, err := db.FlushAll(); err != nil {
			log.Error("flush db %d error %s", index, err.Error())
		}
	}

	return nil
}

// very dangerous to use
func (l *Ledis) DataDB() *leveldb.DB {
	return l.ldb
}

func (l *Ledis) activeExpireCycle() {
	var executors []*elimination = make([]*elimination, len(l.dbs))
	for i, db := range l.dbs {
		executors[i] = db.newEliminator()
	}

	l.jobs.Add(1)
	go func() {
		tick := time.NewTicker(1 * time.Second)
		end := false
		done := make(chan struct{})
		for !end {
			select {
			case <-tick.C:
				go func() {
					for _, eli := range executors {
						eli.active()
					}
					done <- struct{}{}
				}()
				<-done
			case <-l.quit:
				end = true
				break
			}
		}

		tick.Stop()
		l.jobs.Done()
	}()
}
<|endoftext|>


<commit_before>package instances

import (
	"encoding/json"
	"net/http"
	"strconv"

	"github.com/cozy/cozy-stack/model/instance/lifecycle"
	"github.com/cozy/cozy-stack/model/sharing"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/labstack/echo/v4"
)

func fsckHandler(c echo.Context) (err error) {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	indexIntegrityCheck, _ := strconv.ParseBool(c.QueryParam("IndexIntegrity"))
	filesConsistencyCheck, _ := strconv.ParseBool(c.QueryParam("FilesConsistency"))
	failFast, _ := strconv.ParseBool(c.QueryParam("FailFast"))

	logCh := make(chan *vfs.FsckLog)
	go func() {
		fs := i.VFS()
		if indexIntegrityCheck {
			err = fs.CheckIndexIntegrity(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		} else if filesConsistencyCheck {
			err = fs.CheckFilesConsistency(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		} else {
			err = fs.Fsck(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		}
		close(logCh)
	}()

	w := c.Response().Writer
	w.WriteHeader(200)
	encoder := json.NewEncoder(w)
	for log := range logCh {
		// XXX do not serialize to JSON the children and the cozyMetadata, as
		// it can take more than 64ko and scanner will ignore such lines.
		if log.FileDoc != nil {
			log.FileDoc.DirsChildren = nil  // It can be filled on type mismatch
			log.FileDoc.FilesChildren = nil // Idem
			log.FileDoc.FilesChildrenSize = 0
			log.FileDoc.Metadata = nil
		}
		if log.DirDoc != nil {
			log.DirDoc.DirsChildren = nil
			log.DirDoc.FilesChildren = nil
			log.DirDoc.FilesChildrenSize = 0
			log.DirDoc.Metadata = nil
		}
		if log.VersionDoc != nil {
			log.VersionDoc.Metadata = nil
		}
		if errenc := encoder.Encode(log); errenc != nil {
			i.Logger().WithField("nspace", "fsck").
				Warnf("Cannot encode to JSON: %s (%v)", errenc, log)
		}
		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	if err != nil {
		log := map[string]string{"error": err.Error()}
		if errenc := encoder.Encode(log); errenc != nil {
			i.Logger().WithField("nspace", "fsck").
				Warnf("Cannot encode to JSON: %s (%v)", errenc, log)
		}
		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	return nil
}

func checkShared(c echo.Context) error {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	results, err := sharing.CheckShared(i)
	if err != nil {
		return wrapError(err)
	}
	return c.JSON(http.StatusOK, results)
}

func checkSharings(c echo.Context) error {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	results, err := sharing.CheckSharings(i)
	if err != nil {
		return wrapError(err)
	}
	return c.JSON(http.StatusOK, results)
}
<commit_msg>Fix the response code when checking an instance<commit_after>package instances

import (
	"encoding/json"
	"net/http"
	"strconv"

	"github.com/cozy/cozy-stack/model/instance/lifecycle"
	"github.com/cozy/cozy-stack/model/sharing"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/labstack/echo/v4"
)

func fsckHandler(c echo.Context) (err error) {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	indexIntegrityCheck, _ := strconv.ParseBool(c.QueryParam("IndexIntegrity"))
	filesConsistencyCheck, _ := strconv.ParseBool(c.QueryParam("FilesConsistency"))
	failFast, _ := strconv.ParseBool(c.QueryParam("FailFast"))

	logCh := make(chan *vfs.FsckLog)
	go func() {
		fs := i.VFS()
		if indexIntegrityCheck {
			err = fs.CheckIndexIntegrity(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		} else if filesConsistencyCheck {
			err = fs.CheckFilesConsistency(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		} else {
			err = fs.Fsck(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		}
		close(logCh)
	}()

	w := c.Response().Writer
	w.WriteHeader(200)
	encoder := json.NewEncoder(w)
	for log := range logCh {
		// XXX do not serialize to JSON the children and the cozyMetadata, as
		// it can take more than 64ko and scanner will ignore such lines.
		if log.FileDoc != nil {
			log.FileDoc.DirsChildren = nil  // It can be filled on type mismatch
			log.FileDoc.FilesChildren = nil // Idem
			log.FileDoc.FilesChildrenSize = 0
			log.FileDoc.Metadata = nil
		}
		if log.DirDoc != nil {
			log.DirDoc.DirsChildren = nil
			log.DirDoc.FilesChildren = nil
			log.DirDoc.FilesChildrenSize = 0
			log.DirDoc.Metadata = nil
		}
		if log.VersionDoc != nil {
			log.VersionDoc.Metadata = nil
		}
		if errenc := encoder.Encode(log); errenc != nil {
			i.Logger().WithField("nspace", "fsck").
				Warnf("Cannot encode to JSON: %s (%v)", errenc, log)
		}
		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	if err != nil {
		log := map[string]string{"error": err.Error()}
		if errenc := encoder.Encode(log); errenc != nil {
			i.Logger().WithField("nspace", "fsck").
				Warnf("Cannot encode to JSON: %s (%v)", errenc, log)
		}
		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	return nil
}

func checkShared(c echo.Context) error {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	results, err := sharing.CheckShared(i)
	if err != nil {
		if couchdb.IsNotFoundError(err) {
			return c.JSON(http.StatusOK, echo.Map{"err": err.Error()})
		}
		return wrapError(err)
	}
	return c.JSON(http.StatusOK, results)
}

func checkSharings(c echo.Context) error {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	results, err := sharing.CheckSharings(i)
	if err != nil {
		if couchdb.IsNotFoundError(err) {
			return c.JSON(http.StatusOK, echo.Map{"err": err.Error()})
		}
		return wrapError(err)
	}
	return c.JSON(http.StatusOK, results)
}
<|endoftext|>