<commit_before>package labelservice

/*
Copyright 2017-2018 Crunchy Data Solutions, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import (
	"encoding/json"
	"errors"
	log "github.com/Sirupsen/logrus"
	crv1 "github.com/crunchydata/postgres-operator/apis/cr/v1"
	"github.com/crunchydata/postgres-operator/apiserver"
	msgs "github.com/crunchydata/postgres-operator/apiservermsgs"
	"github.com/crunchydata/postgres-operator/kubeapi"
	"github.com/crunchydata/postgres-operator/util"
	jsonpatch "github.com/evanphx/json-patch"
	"k8s.io/api/extensions/v1beta1"
	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/validation"
	"strings"
)

// Label ... 2 forms ...
// pgo label  myucluser yourcluster --label=env=prod
// pgo label  myucluser yourcluster --label=env=prod --delete-label
// pgo label  --label=env=prod --selector=name=mycluster
func Label(request *msgs.LabelRequest) msgs.LabelResponse {
	var err error
	var labelsMap map[string]string
	resp := msgs.LabelResponse{}
	resp.Status.Code = msgs.Ok
	resp.Status.Msg = ""
	resp.Results = make([]string, 0)

	if len(request.Args) == 0 && request.Selector == "" {
		resp.Status.Code = msgs.Error
		resp.Status.Msg = "no clusters specified"
		return resp
	}

	labelsMap, err = validateLabel(request.LabelCmdLabel)
	if err != nil {
		resp.Status.Code = msgs.Error
		resp.Status.Msg = "lables not formatted correctly"
		return resp
	}

	clusterList := crv1.PgclusterList{}
	if request.Selector != "" {
		log.Debug("selector is " + request.Selector)

		err = kubeapi.GetpgclustersBySelector(apiserver.RESTClient,
			&clusterList, request.Selector, apiserver.Namespace)
		if err != nil {
			log.Error("error getting list of clusters" + err.Error())
			resp.Status.Code = msgs.Error
			resp.Status.Msg = "error getting list of clusters" + err.Error()
			return resp
		}
		if len(clusterList.Items) == 0 {
			resp.Status.Code = msgs.Error
			resp.Status.Msg = "no clusters found"
			return resp
		}
	} else {
		//each arg represents a cluster name or the special 'all' value
		items := make([]crv1.Pgcluster, 0)
		for _, cluster := range request.Args {
			result := crv1.Pgcluster{}
			_, err := kubeapi.Getpgcluster(apiserver.RESTClient,
				&result, cluster, apiserver.Namespace)
			if err != nil {
				resp.Status.Code = msgs.Error
				resp.Status.Msg = "error getting list of clusters" + err.Error()
				return resp
			}

			items = append(items, result)
		}
		clusterList.Items = items
	}

	for _, c := range clusterList.Items {
		resp.Results = append(resp.Results, "adding label to "+c.Spec.Name)
	}

	addLabels(clusterList.Items, request.DryRun, request.LabelCmdLabel, labelsMap)

	return resp

}

func addLabels(items []crv1.Pgcluster, DryRun bool, LabelCmdLabel string, newLabels map[string]string) {
	for i := 0; i < len(items); i++ {
		log.Debug("adding label to " + items[i].Spec.Name)
		if DryRun {
			log.Debug("dry run only")
		} else {
			err := PatchPgcluster(LabelCmdLabel, items[i])
			if err != nil {
				log.Error(err.Error())
			}
		}
	}

	for i := 0; i < len(items); i++ {
		//get deployments for this CRD
		selector := util.LABEL_PG_CLUSTER + "=" + items[i].Spec.Name
		deployments, err := kubeapi.GetDeployments(apiserver.Clientset, selector, apiserver.Namespace)
		if err != nil {
			return
		}

		for _, d := range deployments.Items {
			//update Deployment with the label
			if !DryRun {
				err := updateLabels(&d, items[i].Spec.Name, newLabels)
				if err != nil {
					log.Error(err.Error())
				}
			}
		}

	}
}

func updateLabels(deployment *v1beta1.Deployment, clusterName string, newLabels map[string]string) error {

	var err error

	log.Debugf("%v is the labels to apply\n", newLabels)

	var patchBytes, newData, origData []byte
	origData, err = json.Marshal(deployment)
	if err != nil {
		return err
	}

	accessor, err2 := meta.Accessor(deployment)
	if err2 != nil {
		return err2
	}

	objLabels := accessor.GetLabels()
	if objLabels == nil {
		objLabels = make(map[string]string)
	}

	//update the deployment labels
	for key, value := range newLabels {
		objLabels[key] = value
	}
	log.Debugf("updated labels are %v\n", objLabels)

	accessor.SetLabels(objLabels)
	newData, err = json.Marshal(deployment)
	if err != nil {
		return err
	}

	patchBytes, err = jsonpatch.CreateMergePatch(origData, newData)
	if err != nil {
		return err
	}

	_, err = apiserver.Clientset.ExtensionsV1beta1().Deployments(apiserver.Namespace).Patch(clusterName, types.MergePatchType, patchBytes, "")
	if err != nil {
		log.Debug("error patching deployment " + err.Error())
	}
	return err

}

func PatchPgcluster(newLabel string, oldCRD crv1.Pgcluster) error {

	fields := strings.Split(newLabel, "=")
	labelKey := fields[0]
	labelValue := fields[1]
	oldData, err := json.Marshal(oldCRD)
	if err != nil {
		return err
	}
	if oldCRD.ObjectMeta.Labels == nil {
		oldCRD.ObjectMeta.Labels = make(map[string]string)
	}
	oldCRD.ObjectMeta.Labels[labelKey] = labelValue
	var newData, patchBytes []byte
	newData, err = json.Marshal(oldCRD)
	if err != nil {
		return err
	}
	patchBytes, err = jsonpatch.CreateMergePatch(oldData, newData)
	if err != nil {
		return err
	}

	log.Debug(string(patchBytes))
	_, err6 := apiserver.RESTClient.Patch(types.MergePatchType).
		Namespace(apiserver.Namespace).
		Resource(crv1.PgclusterResourcePlural).
		Name(oldCRD.Spec.Name).
		Body(patchBytes).
		Do().
		Get()

	return err6

}

func validateLabel(LabelCmdLabel string) (map[string]string, error) {
	var err error
	labelMap := make(map[string]string)
	userValues := strings.Split(LabelCmdLabel, ",")
	for _, v := range userValues {
		pair := strings.Split(v, "=")
		if len(pair) != 2 {
			log.Error("label format incorrect, requires name=value")
			return labelMap, errors.New("label format incorrect, requires name=value")
		}

		errs := validation.IsDNS1035Label(pair[0])
		if len(errs) > 0 {
			return labelMap, errors.New("label format incorrect, requires name=value " + errs[0])
		}
		errs = validation.IsDNS1035Label(pair[1])
		if len(errs) > 0 {
			return labelMap, errors.New("label format incorrect, requires name=value " + errs[0])
		}

		labelMap[pair[0]] = pair[1]
	}
	return labelMap, err
}
<commit_msg>small typo fix<commit_after>package labelservice

/*
Copyright 2017-2018 Crunchy Data Solutions, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import (
	"encoding/json"
	"errors"
	log "github.com/Sirupsen/logrus"
	crv1 "github.com/crunchydata/postgres-operator/apis/cr/v1"
	"github.com/crunchydata/postgres-operator/apiserver"
	msgs "github.com/crunchydata/postgres-operator/apiservermsgs"
	"github.com/crunchydata/postgres-operator/kubeapi"
	"github.com/crunchydata/postgres-operator/util"
	jsonpatch "github.com/evanphx/json-patch"
	"k8s.io/api/extensions/v1beta1"
	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/validation"
	"strings"
)

// Label ... 2 forms ...
// pgo label  myucluser yourcluster --label=env=prod
// pgo label  myucluser yourcluster --label=env=prod --delete-label
// pgo label  --label=env=prod --selector=name=mycluster
func Label(request *msgs.LabelRequest) msgs.LabelResponse {
	var err error
	var labelsMap map[string]string
	resp := msgs.LabelResponse{}
	resp.Status.Code = msgs.Ok
	resp.Status.Msg = ""
	resp.Results = make([]string, 0)

	if len(request.Args) == 0 && request.Selector == "" {
		resp.Status.Code = msgs.Error
		resp.Status.Msg = "no clusters specified"
		return resp
	}

	labelsMap, err = validateLabel(request.LabelCmdLabel)
	if err != nil {
		resp.Status.Code = msgs.Error
		resp.Status.Msg = "labels not formatted correctly"
		return resp
	}

	clusterList := crv1.PgclusterList{}
	if request.Selector != "" {
		log.Debug("selector is " + request.Selector)

		err = kubeapi.GetpgclustersBySelector(apiserver.RESTClient,
			&clusterList, request.Selector, apiserver.Namespace)
		if err != nil {
			log.Error("error getting list of clusters" + err.Error())
			resp.Status.Code = msgs.Error
			resp.Status.Msg = "error getting list of clusters" + err.Error()
			return resp
		}
		if len(clusterList.Items) == 0 {
			resp.Status.Code = msgs.Error
			resp.Status.Msg = "no clusters found"
			return resp
		}
	} else {
		//each arg represents a cluster name or the special 'all' value
		items := make([]crv1.Pgcluster, 0)
		for _, cluster := range request.Args {
			result := crv1.Pgcluster{}
			_, err := kubeapi.Getpgcluster(apiserver.RESTClient,
				&result, cluster, apiserver.Namespace)
			if err != nil {
				resp.Status.Code = msgs.Error
				resp.Status.Msg = "error getting list of clusters" + err.Error()
				return resp
			}

			items = append(items, result)
		}
		clusterList.Items = items
	}

	for _, c := range clusterList.Items {
		resp.Results = append(resp.Results, "adding label to "+c.Spec.Name)
	}

	addLabels(clusterList.Items, request.DryRun, request.LabelCmdLabel, labelsMap)

	return resp

}

func addLabels(items []crv1.Pgcluster, DryRun bool, LabelCmdLabel string, newLabels map[string]string) {
	for i := 0; i < len(items); i++ {
		log.Debug("adding label to " + items[i].Spec.Name)
		if DryRun {
			log.Debug("dry run only")
		} else {
			err := PatchPgcluster(LabelCmdLabel, items[i])
			if err != nil {
				log.Error(err.Error())
			}
		}
	}

	for i := 0; i < len(items); i++ {
		//get deployments for this CRD
		selector := util.LABEL_PG_CLUSTER + "=" + items[i].Spec.Name
		deployments, err := kubeapi.GetDeployments(apiserver.Clientset, selector, apiserver.Namespace)
		if err != nil {
			return
		}

		for _, d := range deployments.Items {
			//update Deployment with the label
			if !DryRun {
				err := updateLabels(&d, items[i].Spec.Name, newLabels)
				if err != nil {
					log.Error(err.Error())
				}
			}
		}

	}
}

func updateLabels(deployment *v1beta1.Deployment, clusterName string, newLabels map[string]string) error {

	var err error

	log.Debugf("%v are the labels to apply\n", newLabels)

	var patchBytes, newData, origData []byte
	origData, err = json.Marshal(deployment)
	if err != nil {
		return err
	}

	accessor, err2 := meta.Accessor(deployment)
	if err2 != nil {
		return err2
	}

	objLabels := accessor.GetLabels()
	if objLabels == nil {
		objLabels = make(map[string]string)
	}

	//update the deployment labels
	for key, value := range newLabels {
		objLabels[key] = value
	}
	log.Debugf("updated labels are %v\n", objLabels)

	accessor.SetLabels(objLabels)
	newData, err = json.Marshal(deployment)
	if err != nil {
		return err
	}

	patchBytes, err = jsonpatch.CreateMergePatch(origData, newData)
	if err != nil {
		return err
	}

	_, err = apiserver.Clientset.ExtensionsV1beta1().Deployments(apiserver.Namespace).Patch(clusterName, types.MergePatchType, patchBytes, "")
	if err != nil {
		log.Debug("error patching deployment " + err.Error())
	}
	return err

}

func PatchPgcluster(newLabel string, oldCRD crv1.Pgcluster) error {

	fields := strings.Split(newLabel, "=")
	labelKey := fields[0]
	labelValue := fields[1]
	oldData, err := json.Marshal(oldCRD)
	if err != nil {
		return err
	}
	if oldCRD.ObjectMeta.Labels == nil {
		oldCRD.ObjectMeta.Labels = make(map[string]string)
	}
	oldCRD.ObjectMeta.Labels[labelKey] = labelValue
	var newData, patchBytes []byte
	newData, err = json.Marshal(oldCRD)
	if err != nil {
		return err
	}
	patchBytes, err = jsonpatch.CreateMergePatch(oldData, newData)
	if err != nil {
		return err
	}

	log.Debug(string(patchBytes))
	_, err6 := apiserver.RESTClient.Patch(types.MergePatchType).
		Namespace(apiserver.Namespace).
		Resource(crv1.PgclusterResourcePlural).
		Name(oldCRD.Spec.Name).
		Body(patchBytes).
		Do().
		Get()

	return err6

}

func validateLabel(LabelCmdLabel string) (map[string]string, error) {
	var err error
	labelMap := make(map[string]string)
	userValues := strings.Split(LabelCmdLabel, ",")
	for _, v := range userValues {
		pair := strings.Split(v, "=")
		if len(pair) != 2 {
			log.Error("label format incorrect, requires name=value")
			return labelMap, errors.New("label format incorrect, requires name=value")
		}

		errs := validation.IsDNS1035Label(pair[0])
		if len(errs) > 0 {
			return labelMap, errors.New("label format incorrect, requires name=value " + errs[0])
		}
		errs = validation.IsDNS1035Label(pair[1])
		if len(errs) > 0 {
			return labelMap, errors.New("label format incorrect, requires name=value " + errs[0])
		}

		labelMap[pair[0]] = pair[1]
	}
	return labelMap, err
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package memcache provides a client for the memcached cache server.
package memcache

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"strings"
	"testing"
	"time"
)

const testServer = "localhost:11211"

func setup(t *testing.T) bool {
	c, err := net.Dial("tcp", testServer)
	if err != nil {
		t.Logf("skipping test; no server running at %s", testServer)
		return false
	}
	c.Write([]byte("flush_all\r\n"))
	c.Close()
	return true
}

func TestLocalhost(t *testing.T) {
	if !setup(t) {
		return
	}
	testWithClient(t, New(testServer))
}

// Run the memcached binary as a child process and connect to its unix socket.
func TestUnixSocket(t *testing.T) {
	sock := fmt.Sprintf("/tmp/test-gomemcache-%d.sock", os.Getpid())
	cmd := exec.Command("memcached", "-s", sock)
	if err := cmd.Start(); err != nil {
		t.Logf("skipping test; couldn't find memcached")
		return
	}
	defer cmd.Wait()
	defer cmd.Process.Kill()

	// Wait a bit for the socket to appear.
	for i := 0; i < 10; i++ {
		if _, err := os.Stat(sock); err == nil {
			break
		}
		time.Sleep(time.Duration(25*i) * time.Millisecond)
	}

	testWithClient(t, New(sock))
}

func testWithClient(t *testing.T, c *Client) {
	checkErr := func(err error, format string, args ...interface{}) {
		if err != nil {
			t.Fatalf(format, args...)
		}
	}

	mustSet := func(it *Item) {
		if err := c.Set(it); err != nil {
			t.Fatalf("failed to Set %#v: %v", *it, err)
		}
	}

	// Set
	foo := &Item{Key: "foo", Value: []byte("fooval"), Flags: 123}
	err := c.Set(foo)
	checkErr(err, "first set(foo): %v", err)
	err = c.Set(foo)
	checkErr(err, "second set(foo): %v", err)

	// Get
	it, err := c.Get("foo")
	checkErr(err, "get(foo): %v", err)
	if it.Key != "foo" {
		t.Errorf("get(foo) Key = %q, want foo", it.Key)
	}
	if string(it.Value) != "fooval" {
		t.Errorf("get(foo) Value = %q, want fooval", string(it.Value))
	}
	if it.Flags != 123 {
		t.Errorf("get(foo) Flags = %v, want 123", it.Flags)
	}

	// Add
	bar := &Item{Key: "bar", Value: []byte("barval")}
	err = c.Add(bar)
	checkErr(err, "first add(bar): %v", err)
	if err := c.Add(bar); err != ErrNotStored {
		t.Fatalf("second add(bar) want ErrNotStored, got %v", err)
	}

	// GetMulti
	m, err := c.GetMulti([]string{"foo", "bar"})
	checkErr(err, "GetMulti: %v", err)
	if g, e := len(m), 2; g != e {
		t.Errorf("GetMulti: got len(map) = %d, want = %d", g, e)
	}
	if _, ok := m["foo"]; !ok {
		t.Fatalf("GetMulti: didn't get key 'foo'")
	}
	if _, ok := m["bar"]; !ok {
		t.Fatalf("GetMulti: didn't get key 'bar'")
	}
	if g, e := string(m["foo"].Value), "fooval"; g != e {
		t.Errorf("GetMulti: foo: got %q, want %q", g, e)
	}
	if g, e := string(m["bar"].Value), "barval"; g != e {
		t.Errorf("GetMulti: bar: got %q, want %q", g, e)
	}

	// Delete
	err = c.Delete("foo")
	checkErr(err, "Delete: %v", err)
	it, err = c.Get("foo")
	if err != ErrCacheMiss {
		t.Errorf("post-Delete want ErrCacheMiss, got %v", err)
	}

	// Incr/Decr
	mustSet(&Item{Key: "num", Value: []byte("42")})
	n, err := c.Increment("num", 8)
	checkErr(err, "Increment num + 8: %v", err)
	if n != 50 {
		t.Fatalf("Increment num + 8: want=50, got=%d", n)
	}
	n, err = c.Decrement("num", 49)
	checkErr(err, "Decrement: %v", err)
	if n != 1 {
		t.Fatalf("Decrement 49: want=1, got=%d", n)
	}
	err = c.Delete("num")
	checkErr(err, "delete num: %v", err)
	n, err = c.Increment("num", 1)
	if err != ErrCacheMiss {
		t.Fatalf("increment post-delete: want ErrCacheMiss, got %v", err)
	}
	mustSet(&Item{Key: "num", Value: []byte("not-numeric")})
	n, err = c.Increment("num", 1)
	if err == nil || !strings.Contains(err.Error(), "client error") {
		t.Fatalf("increment non-number: want client error, got %v", err)
	}

}
<commit_msg>Assign unused value to _<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package memcache provides a client for the memcached cache server.
package memcache

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"strings"
	"testing"
	"time"
)

const testServer = "localhost:11211"

func setup(t *testing.T) bool {
	c, err := net.Dial("tcp", testServer)
	if err != nil {
		t.Logf("skipping test; no server running at %s", testServer)
		return false
	}
	c.Write([]byte("flush_all\r\n"))
	c.Close()
	return true
}

func TestLocalhost(t *testing.T) {
	if !setup(t) {
		return
	}
	testWithClient(t, New(testServer))
}

// Run the memcached binary as a child process and connect to its unix socket.
func TestUnixSocket(t *testing.T) {
	sock := fmt.Sprintf("/tmp/test-gomemcache-%d.sock", os.Getpid())
	cmd := exec.Command("memcached", "-s", sock)
	if err := cmd.Start(); err != nil {
		t.Logf("skipping test; couldn't find memcached")
		return
	}
	defer cmd.Wait()
	defer cmd.Process.Kill()

	// Wait a bit for the socket to appear.
	for i := 0; i < 10; i++ {
		if _, err := os.Stat(sock); err == nil {
			break
		}
		time.Sleep(time.Duration(25*i) * time.Millisecond)
	}

	testWithClient(t, New(sock))
}

func testWithClient(t *testing.T, c *Client) {
	checkErr := func(err error, format string, args ...interface{}) {
		if err != nil {
			t.Fatalf(format, args...)
		}
	}

	mustSet := func(it *Item) {
		if err := c.Set(it); err != nil {
			t.Fatalf("failed to Set %#v: %v", *it, err)
		}
	}

	// Set
	foo := &Item{Key: "foo", Value: []byte("fooval"), Flags: 123}
	err := c.Set(foo)
	checkErr(err, "first set(foo): %v", err)
	err = c.Set(foo)
	checkErr(err, "second set(foo): %v", err)

	// Get
	it, err := c.Get("foo")
	checkErr(err, "get(foo): %v", err)
	if it.Key != "foo" {
		t.Errorf("get(foo) Key = %q, want foo", it.Key)
	}
	if string(it.Value) != "fooval" {
		t.Errorf("get(foo) Value = %q, want fooval", string(it.Value))
	}
	if it.Flags != 123 {
		t.Errorf("get(foo) Flags = %v, want 123", it.Flags)
	}

	// Add
	bar := &Item{Key: "bar", Value: []byte("barval")}
	err = c.Add(bar)
	checkErr(err, "first add(bar): %v", err)
	if err := c.Add(bar); err != ErrNotStored {
		t.Fatalf("second add(bar) want ErrNotStored, got %v", err)
	}

	// GetMulti
	m, err := c.GetMulti([]string{"foo", "bar"})
	checkErr(err, "GetMulti: %v", err)
	if g, e := len(m), 2; g != e {
		t.Errorf("GetMulti: got len(map) = %d, want = %d", g, e)
	}
	if _, ok := m["foo"]; !ok {
		t.Fatalf("GetMulti: didn't get key 'foo'")
	}
	if _, ok := m["bar"]; !ok {
		t.Fatalf("GetMulti: didn't get key 'bar'")
	}
	if g, e := string(m["foo"].Value), "fooval"; g != e {
		t.Errorf("GetMulti: foo: got %q, want %q", g, e)
	}
	if g, e := string(m["bar"].Value), "barval"; g != e {
		t.Errorf("GetMulti: bar: got %q, want %q", g, e)
	}

	// Delete
	err = c.Delete("foo")
	checkErr(err, "Delete: %v", err)
	_, err = c.Get("foo")
	if err != ErrCacheMiss {
		t.Errorf("post-Delete want ErrCacheMiss, got %v", err)
	}

	// Incr/Decr
	mustSet(&Item{Key: "num", Value: []byte("42")})
	n, err := c.Increment("num", 8)
	checkErr(err, "Increment num + 8: %v", err)
	if n != 50 {
		t.Fatalf("Increment num + 8: want=50, got=%d", n)
	}
	n, err = c.Decrement("num", 49)
	checkErr(err, "Decrement: %v", err)
	if n != 1 {
		t.Fatalf("Decrement 49: want=1, got=%d", n)
	}
	err = c.Delete("num")
	checkErr(err, "delete num: %v", err)
	n, err = c.Increment("num", 1)
	if err != ErrCacheMiss {
		t.Fatalf("increment post-delete: want ErrCacheMiss, got %v", err)
	}
	mustSet(&Item{Key: "num", Value: []byte("not-numeric")})
	n, err = c.Increment("num", 1)
	if err == nil || !strings.Contains(err.Error(), "client error") {
		t.Fatalf("increment non-number: want client error, got %v", err)
	}

}
<|endoftext|>


<commit_before>package create

import (
	"fmt"
	"io/ioutil"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
	microerror "github.com/giantswarm/microkit/error"

	awsclient "github.com/giantswarm/aws-operator/client/aws"
)

type keyPairProvider interface {
	pubKeyContent() ([]byte, error)
}

type fsKeyPairProvider struct {
	pubKeyFile string
}

func newFsKeyPairProvider(pubKeyFile string) *fsKeyPairProvider {
	return &fsKeyPairProvider{
		pubKeyFile: pubKeyFile,
	}
}

func (f *fsKeyPairProvider) pubKeyContent() ([]byte, error) {
	return ioutil.ReadFile(f.pubKeyFile)
}

type keyPairInput struct {
	ec2Client   *ec2.EC2
	clusterName string
	provider    keyPairProvider
}

func (s *Service) keyPair(input keyPairInput) (string, error) {
	pkc, err := input.provider.pubKeyContent()
	if err != nil {
		return "", microerror.MaskAny(err)
	}

	keyPair, err := input.ec2Client.ImportKeyPair(&ec2.ImportKeyPairInput{
		KeyName:           aws.String(input.clusterName),
		PublicKeyMaterial: pkc,
	})
	if err != nil {
		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == awsclient.KeyPairDuplicate {
			s.logger.Log("info", fmt.Sprintf("keypair '%s' exists, reusing", input.clusterName))
			return input.clusterName, nil
		} else {
			return "", microerror.MaskAny(err)
		}
	}

	if keyPair == nil || keyPair.KeyName == nil {
		return "", fmt.Errorf("Couln't create and find the keypair '%s'", input.clusterName)
	}

	return *keyPair.KeyName, nil
}
<commit_msg>service: remove unused keypair.go<commit_after><|endoftext|>


<commit_before>package main

import (
	"image"
	"image/color"
	"io"
	"os"
	"time"

	"github.com/lucasb-eyer/go-colorful"
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/config"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
	ledmodel "github.com/ninjasphere/sphere-go-led-controller/model"
	"github.com/ninjasphere/sphere-go-led-controller/ui"
	"github.com/ninjasphere/sphere-go-led-controller/util"
)

var log = logger.GetLogger("sphere-go-led-controller")

var fps Tick = Tick{
	name: "Pane FPS",
}

type LedController struct {
	controlEnabled   bool
	controlRequested bool
	controlRendering bool
	commandReceived  bool

	controlLayout *ui.PaneLayout
	pairingLayout *ui.PairingLayout
	conn          *ninja.Connection
	serial        io.ReadWriteCloser
	waiting       chan bool
}

func NewLedController(conn *ninja.Connection) (*LedController, error) {

	s, err := util.GetLEDConnection()

	if err != nil {
		log.Fatalf("Failed to get connection to LED matrix: %s", err)
	}

	// Send a blank image to the led matrix
	util.WriteLEDMatrix(image.NewRGBA(image.Rect(0, 0, 16, 16)), s)

	controller := &LedController{
		conn:          conn,
		pairingLayout: ui.NewPairingLayout(),
		serial:        s,
		waiting:       make(chan bool),
	}

	conn.MustExportService(controller, "$node/"+config.Serial()+"/led-controller", &model.ServiceAnnouncement{
		Schema: "/service/led-controller",
	})

	conn.MustExportService(controller, "$home/led-controller", &model.ServiceAnnouncement{
		Schema: "/service/led-controller",
	})

	if config.HasString("siteId") {
		log.Infof("Have a siteId, checking if homecloud is running")
		// If we have just started, and we have a site, and homecloud is running... enable control!
		go func() {
			siteModel := conn.GetServiceClient("$home/services/SiteModel")
			for {

				if controller.commandReceived {
					log.Infof("Command has been received, stopping search for homecloud.")
					break
				}

				err := siteModel.Call("fetch", config.MustString("siteId"), nil, time.Second*5)

				if err != nil {
					log.Infof("Fetched site to enableControl. Got err: %s", err)
				} else if err == nil && !controller.commandReceived {
					controller.EnableControl()
					break
				}
				time.Sleep(time.Second * 5)
			}
		}()
	}

	return controller, nil
}

func (c *LedController) start(enableControl bool) {
	c.controlRequested = enableControl

	frameWritten := make(chan bool)

	go func() {
		fps.start()

		for {
			fps.tick()

			if c.controlEnabled {
				// Good to go

				image, wake, err := c.controlLayout.Render()
				if err != nil {
					log.Fatalf("Unable to render()", err)
				}

				go func() {
					util.WriteLEDMatrix(image, c.serial)
					frameWritten <- true
				}()

				select {
				case <-frameWritten:
					// All good.
				case <-time.After(10 * time.Second):
					log.Infof("Timeout writing to LED matrix. Quitting.")
					os.Exit(1)
				}

				if wake != nil {
					log.Infof("Waiting as the UI is asleep")
					select {
					case <-wake:
						log.Infof("UI woke up!")
					case <-c.waiting:
						log.Infof("Got a command from rpc...")
					}
				}

			} else if c.controlRequested && !c.controlRendering {

				// We want to display controls, so lets render the pane

				c.controlRendering = true
				go func() {

					log.Infof("Starting control layout")
					c.controlLayout = getPaneLayout(c.conn)
					c.controlRendering = false
					c.controlEnabled = true
					log.Infof("Finished control layout")

				}()
			}

			if c.controlRendering || !c.controlEnabled {
				// We're either already controlling, or waiting for the pane to render

				image, err := c.pairingLayout.Render()
				if err != nil {
					log.Fatalf("Unable to render()", err)
				}
				util.WriteLEDMatrix(image, c.serial)

			}
		}

	}()
}

func (c *LedController) EnableControl() error {
	log.Infof("Enabling control. Already enabled? %t", c.controlEnabled)
	if !c.controlEnabled {
		if c.controlLayout != nil {
			// Pane layout has already been rendered. Just re-enable control.
			c.controlEnabled = true
		} else {
			c.controlRequested = true
		}
		c.gotCommand()
	}
	return nil
}

func (c *LedController) DisableControl() error {
	log.Infof("Disabling control. Currently enabled? %t", c.controlEnabled)

	c.DisplayIcon(&ledmodel.IconRequest{
		Icon: "spinner-red.gif",
	})

	c.controlEnabled = false
	c.controlRequested = false
	c.gotCommand()
	return nil
}

type PairingCodeRequest struct {
	Code        string `json:"code"`
	DisplayTime int    `json:"displayTime"`
}

func (c *LedController) DisplayPairingCode(req *PairingCodeRequest) error {
	c.pairingLayout.ShowCode(req.Code)
	c.gotCommand()
	return nil
}

type ColorRequest struct {
	Color       string `json:"color"`
	DisplayTime int    `json:"displayTime"`
}

func (c *LedController) DisplayColor(req *ColorRequest) error {
	col, err := colorful.Hex(req.Color)

	if err != nil {
		return err
	}

	c.pairingLayout.ShowColor(col)
	c.gotCommand()
	return nil
}

func (c *LedController) DisplayIcon(req *ledmodel.IconRequest) error {
	log.Infof("Displaying icon: %v", req)
	c.pairingLayout.ShowIcon(req.Icon)
	c.gotCommand()
	return nil
}

func (c *LedController) DisplayDrawing() error {
	c.pairingLayout.ShowDrawing()
	return nil
}

func (c *LedController) Draw(updates *[][]uint8) error {
	c.pairingLayout.Draw(updates)
	return nil
}

func (c *LedController) DisplayResetMode(m *ledmodel.ResetMode) error {
	c.DisableControl()
	fade := m.Duration > 0 && !m.Hold
	loading := false
	var col color.Color
	switch m.Mode {
	case "abort":
		col, _ = colorful.Hex("#FFFFFF")
	case "halt":
		col, _ = colorful.Hex("#CDC9C9")
	case "reboot":
		col, _ = colorful.Hex("#00FF00")
	case "reset-userdata":
		col, _ = colorful.Hex("#FFFF00")
	case "reset-root":
		col, _ = colorful.Hex("#FF0000")
	default:
		loading = true
	}

	if loading {
		c.pairingLayout.ShowIcon("spinner-pink.gif")
	} else if fade {
		c.pairingLayout.ShowFadingShrinkingColor(col, m.Duration)
	} else {
		c.pairingLayout.ShowColor(col)
	}

	c.gotCommand()
	return nil
}

func (c *LedController) DisplayUpdateProgress(p *ledmodel.DisplayUpdateProgress) error {
	c.pairingLayout.ShowUpdateProgress(p.Progress)

	return nil
}

func (c *LedController) gotCommand() {
	select {
	case c.waiting <- true:
	default:
	}
	c.commandReceived = true
}

// Load from a config file instead...
func getPaneLayout(conn *ninja.Connection) *ui.PaneLayout {
	layout, wake := ui.NewPaneLayout(false, conn)

	layout.AddPane(ui.NewClockPane())
	layout.AddPane(ui.NewWeatherPane(conn))
	layout.AddPane(ui.NewGesturePane())
	layout.AddPane(ui.NewGameOfLifePane())
	layout.AddPane(ui.NewMediaPane(conn))
	layout.AddPane(ui.NewCertPane(conn.GetMqttClient()))

	//layout.AddPane(ui.NewTextScrollPane("Exit Music (For A Film)"))
	lampPane := ui.NewOnOffPane(util.ResolveImagePath("lamp2-off.gif"), util.ResolveImagePath("lamp2-on.gif"), func(state bool) {
		log.Debugf("Lamp state: %t", state)
	}, conn, "lamp")
	layout.AddPane(lampPane)

	heaterPane := ui.NewOnOffPane(util.ResolveImagePath("heater-off.png"), util.ResolveImagePath("heater-on.gif"), func(state bool) {
		log.Debugf("Heater state: %t", state)
	}, conn, "heater")
	layout.AddPane(heaterPane)

	brightnessPane := ui.NewLightPane(false, util.ResolveImagePath("light-off.png"), util.ResolveImagePath("light-on.png"), conn)
	layout.AddPane(brightnessPane)

	colorPane := ui.NewLightPane(true, util.ResolveImagePath("light-off.png"), util.ResolveImagePath("light-on.png"), conn)
	layout.AddPane(colorPane)

	fanPane := ui.NewOnOffPane(util.ResolveImagePath("fan-off.png"), util.ResolveImagePath("fan-on.gif"), func(state bool) {
		log.Debugf("Fan state: %t", state)
	}, conn, "fan")

	layout.AddPane(fanPane)

	go func() {
		<-wake
	}()

	go layout.Wake()

	return layout
}

type Tick struct {
	count int
	name  string
}

func (t *Tick) tick() {
	t.count++
}

func (t *Tick) start() {
	go func() {
		for {
			time.Sleep(time.Second)
			//log.Debugf("%s - %d", t.name, t.count)
			t.count = 0
		}
	}()
}
<commit_msg>If the mode is "none", then re-enable control.<commit_after>package main

import (
	"image"
	"image/color"
	"io"
	"os"
	"time"

	"github.com/lucasb-eyer/go-colorful"
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/config"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
	ledmodel "github.com/ninjasphere/sphere-go-led-controller/model"
	"github.com/ninjasphere/sphere-go-led-controller/ui"
	"github.com/ninjasphere/sphere-go-led-controller/util"
)

var log = logger.GetLogger("sphere-go-led-controller")

var fps Tick = Tick{
	name: "Pane FPS",
}

type LedController struct {
	controlEnabled   bool
	controlRequested bool
	controlRendering bool
	commandReceived  bool

	controlLayout *ui.PaneLayout
	pairingLayout *ui.PairingLayout
	conn          *ninja.Connection
	serial        io.ReadWriteCloser
	waiting       chan bool
}

func NewLedController(conn *ninja.Connection) (*LedController, error) {

	s, err := util.GetLEDConnection()

	if err != nil {
		log.Fatalf("Failed to get connection to LED matrix: %s", err)
	}

	// Send a blank image to the led matrix
	util.WriteLEDMatrix(image.NewRGBA(image.Rect(0, 0, 16, 16)), s)

	controller := &LedController{
		conn:          conn,
		pairingLayout: ui.NewPairingLayout(),
		serial:        s,
		waiting:       make(chan bool),
	}

	conn.MustExportService(controller, "$node/"+config.Serial()+"/led-controller", &model.ServiceAnnouncement{
		Schema: "/service/led-controller",
	})

	conn.MustExportService(controller, "$home/led-controller", &model.ServiceAnnouncement{
		Schema: "/service/led-controller",
	})

	if config.HasString("siteId") {
		log.Infof("Have a siteId, checking if homecloud is running")
		// If we have just started, and we have a site, and homecloud is running... enable control!
		go func() {
			siteModel := conn.GetServiceClient("$home/services/SiteModel")
			for {

				if controller.commandReceived {
					log.Infof("Command has been received, stopping search for homecloud.")
					break
				}

				err := siteModel.Call("fetch", config.MustString("siteId"), nil, time.Second*5)

				if err != nil {
					log.Infof("Fetched site to enableControl. Got err: %s", err)
				} else if err == nil && !controller.commandReceived {
					controller.EnableControl()
					break
				}
				time.Sleep(time.Second * 5)
			}
		}()
	}

	return controller, nil
}

func (c *LedController) start(enableControl bool) {
	c.controlRequested = enableControl

	frameWritten := make(chan bool)

	go func() {
		fps.start()

		for {
			fps.tick()

			if c.controlEnabled {
				// Good to go

				image, wake, err := c.controlLayout.Render()
				if err != nil {
					log.Fatalf("Unable to render()", err)
				}

				go func() {
					util.WriteLEDMatrix(image, c.serial)
					frameWritten <- true
				}()

				select {
				case <-frameWritten:
					// All good.
				case <-time.After(10 * time.Second):
					log.Infof("Timeout writing to LED matrix. Quitting.")
					os.Exit(1)
				}

				if wake != nil {
					log.Infof("Waiting as the UI is asleep")
					select {
					case <-wake:
						log.Infof("UI woke up!")
					case <-c.waiting:
						log.Infof("Got a command from rpc...")
					}
				}

			} else if c.controlRequested && !c.controlRendering {

				// We want to display controls, so lets render the pane

				c.controlRendering = true
				go func() {

					log.Infof("Starting control layout")
					c.controlLayout = getPaneLayout(c.conn)
					c.controlRendering = false
					c.controlEnabled = true
					log.Infof("Finished control layout")

				}()
			}

			if c.controlRendering || !c.controlEnabled {
				// We're either already controlling, or waiting for the pane to render

				image, err := c.pairingLayout.Render()
				if err != nil {
					log.Fatalf("Unable to render()", err)
				}
				util.WriteLEDMatrix(image, c.serial)

			}
		}

	}()
}

func (c *LedController) EnableControl() error {
	log.Infof("Enabling control. Already enabled? %t", c.controlEnabled)
	if !c.controlEnabled {
		if c.controlLayout != nil {
			// Pane layout has already been rendered. Just re-enable control.
			c.controlEnabled = true
		} else {
			c.controlRequested = true
		}
		c.gotCommand()
	}
	return nil
}

func (c *LedController) DisableControl() error {
	log.Infof("Disabling control. Currently enabled? %t", c.controlEnabled)

	c.DisplayIcon(&ledmodel.IconRequest{
		Icon: "spinner-red.gif",
	})

	c.controlEnabled = false
	c.controlRequested = false
	c.gotCommand()
	return nil
}

type PairingCodeRequest struct {
	Code        string `json:"code"`
	DisplayTime int    `json:"displayTime"`
}

func (c *LedController) DisplayPairingCode(req *PairingCodeRequest) error {
	c.pairingLayout.ShowCode(req.Code)
	c.gotCommand()
	return nil
}

type ColorRequest struct {
	Color       string `json:"color"`
	DisplayTime int    `json:"displayTime"`
}

func (c *LedController) DisplayColor(req *ColorRequest) error {
	col, err := colorful.Hex(req.Color)

	if err != nil {
		return err
	}

	c.pairingLayout.ShowColor(col)
	c.gotCommand()
	return nil
}

func (c *LedController) DisplayIcon(req *ledmodel.IconRequest) error {
	log.Infof("Displaying icon: %v", req)
	c.pairingLayout.ShowIcon(req.Icon)
	c.gotCommand()
	return nil
}

func (c *LedController) DisplayDrawing() error {
	c.pairingLayout.ShowDrawing()
	return nil
}

func (c *LedController) Draw(updates *[][]uint8) error {
	c.pairingLayout.Draw(updates)
	return nil
}

func (c *LedController) DisplayResetMode(m *ledmodel.ResetMode) error {
	c.DisableControl()
	fade := m.Duration > 0 && !m.Hold
	loading := false
	var col color.Color
	switch m.Mode {
	case "abort":
		col, _ = colorful.Hex("#FFFFFF")
	case "halt":
		col, _ = colorful.Hex("#CDC9C9")
	case "reboot":
		col, _ = colorful.Hex("#00FF00")
	case "reset-userdata":
		col, _ = colorful.Hex("#FFFF00")
	case "reset-root":
		col, _ = colorful.Hex("#FF0000")
	case "none":
		c.EnableControl()
		return nil
	default:
		loading = true
	}

	if loading {
		c.pairingLayout.ShowIcon("spinner-pink.gif")
	} else if fade {
		c.pairingLayout.ShowFadingShrinkingColor(col, m.Duration)
	} else {
		c.pairingLayout.ShowColor(col)
	}

	c.gotCommand()
	return nil
}

func (c *LedController) DisplayUpdateProgress(p *ledmodel.DisplayUpdateProgress) error {
	c.pairingLayout.ShowUpdateProgress(p.Progress)

	return nil
}

func (c *LedController) gotCommand() {
	select {
	case c.waiting <- true:
	default:
	}
	c.commandReceived = true
}

// Load from a config file instead...
func getPaneLayout(conn *ninja.Connection) *ui.PaneLayout {
	layout, wake := ui.NewPaneLayout(false, conn)

	layout.AddPane(ui.NewClockPane())
	layout.AddPane(ui.NewWeatherPane(conn))
	layout.AddPane(ui.NewGesturePane())
	layout.AddPane(ui.NewGameOfLifePane())
	layout.AddPane(ui.NewMediaPane(conn))
	layout.AddPane(ui.NewCertPane(conn.GetMqttClient()))

	//layout.AddPane(ui.NewTextScrollPane("Exit Music (For A Film)"))
	lampPane := ui.NewOnOffPane(util.ResolveImagePath("lamp2-off.gif"), util.ResolveImagePath("lamp2-on.gif"), func(state bool) {
		log.Debugf("Lamp state: %t", state)
	}, conn, "lamp")
	layout.AddPane(lampPane)

	heaterPane := ui.NewOnOffPane(util.ResolveImagePath("heater-off.png"), util.ResolveImagePath("heater-on.gif"), func(state bool) {
		log.Debugf("Heater state: %t", state)
	}, conn, "heater")
	layout.AddPane(heaterPane)

	brightnessPane := ui.NewLightPane(false, util.ResolveImagePath("light-off.png"), util.ResolveImagePath("light-on.png"), conn)
	layout.AddPane(brightnessPane)

	colorPane := ui.NewLightPane(true, util.ResolveImagePath("light-off.png"), util.ResolveImagePath("light-on.png"), conn)
	layout.AddPane(colorPane)

	fanPane := ui.NewOnOffPane(util.ResolveImagePath("fan-off.png"), util.ResolveImagePath("fan-on.gif"), func(state bool) {
		log.Debugf("Fan state: %t", state)
	}, conn, "fan")

	layout.AddPane(fanPane)

	go func() {
		<-wake
	}()

	go layout.Wake()

	return layout
}

type Tick struct {
	count int
	name  string
}

func (t *Tick) tick() {
	t.count++
}

func (t *Tick) start() {
	go func() {
		for {
			time.Sleep(time.Second)
			//log.Debugf("%s - %d", t.name, t.count)
			t.count = 0
		}
	}()
}
<|endoftext|>


<commit_before>// Consumer service accpets logs and other events from queue, validates,
// and pushes to index and persistance storage.
package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"regexp"
	"strings"
	"sync"
	"time"

	"bitbucket.org/firstrow/logvoyage/models"
	"github.com/streadway/amqp"
	"gopkg.in/olivere/elastic.v5"
)

const (
	// Default tag name name in case tag does not specified in source message.
	defaultTag = "default"
	// How often in seconds send messages to storge.
	persistTimeout = 2
)

var (
	storage = newInMemStorage()
	// See https://regex101.com/r/sQskdz/1
	msgRegExp = regexp.MustCompile(`^([a-z0-9]{8}-[a-z0-9]{4}-[1-5][a-z0-9]{3}-[a-z0-9]{4}-[a-z0-9]{12})(@([a-zA-Z0-9\_\-\.]{1,20}))?\s*(.*)`)
)

type message struct {
	ProjectUUID string
	Tag         string
	Source      string
	Datetime    int64
}

type doc struct {
	Source   string
	Datetime int64
}

// inMemStorage stores all received valid message in memory.
// Each N seconds messages will be sent to ES via bulk request.
type inMemStorage struct {
	sync.Mutex
	messages []message
}

func newInMemStorage() *inMemStorage {
	s := &inMemStorage{}
	go s.startTimer()
	return s
}

func (s *inMemStorage) Add(msg message) {
	s.Lock()
	s.messages = append(s.messages, msg)
	s.Unlock()
}

func (s *inMemStorage) Persist() {

	// TODO: Handle error
	client, err := elasticClient()
	if err != nil {
		log.Println("Error connecting to elastic:", err)
		return
	}

	s.Lock()
	defer s.Unlock()
	if len(s.messages) == 0 {
		return
	}

	bulkRequest := client.Bulk()
	for _, msg := range s.messages {
		req := elastic.NewBulkIndexRequest().Index("logs-" + msg.ProjectUUID).Type(msg.Tag)
		var userJson map[string]interface{}
		err := json.Unmarshal([]byte(msg.Source), &userJson)

		if err != nil {
			doc := doc{
				Source:   msg.Source,
				Datetime: msg.Datetime,
			}
			req.Doc(doc)
		} else {
			// Save user json
			fmt.Println("Save user json:", userJson)
			userJson["Datetime"] = msg.Datetime
			req.Doc(userJson)
		}
		bulkRequest.Add(req)
	}

	bulkResponse, err := bulkRequest.Do(context.TODO())
	if err != nil {
		log.Println("Bulk request err:", err)
	}
	if bulkResponse == nil {
		log.Println("Bulk response == nil")
	}
	s.messages = []message{}
}

func (s *inMemStorage) startTimer() {
	ticker := time.NewTicker(persistTimeout * time.Second)
	defer ticker.Stop()

	for range ticker.C {
		s.Persist()
	}
}

func elasticClient() (*elastic.Client, error) {
	return elastic.NewClient(elastic.SetURL("http://ubuntu:9200"))
}

// processMessage extracts apiKey, tag(optional) and source message from log line.
// In case tag not found, "default" tag will be returned.
// Error will be returned if uuid or tag has wrong format.
//
// Examples:
// uuid@tag msg -> uuid, tag, msg, nil
// uuid msg     -> uuid, default, msg, nil
func processMessage(msg string) (string, string, string, error) {
	result := msgRegExp.FindAllStringSubmatch(msg, -1)

	if result == nil {
		return "", "", "", errors.New("Error parsing source message")
	}

	var tag string
	if result[0][3] == "" {
		tag = defaultTag
	} else {
		tag = result[0][3]
	}

	return result[0][1], tag, strings.TrimSpace(result[0][4]), nil
}

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
	}
}

func handle(deliveries <-chan amqp.Delivery) {
	for d := range deliveries {
		log.Printf(
			"got %dB delivery: [%v] %q",
			len(d.Body),
			d.DeliveryTag,
			d.Body,
		)
		processDelivery(d)
	}
	log.Fatalln("Deliveries channel closed")
}

func processDelivery(d amqp.Delivery) {
	defer d.Ack(false)

	projectUUID, tag, msgSource, err := processMessage(string(d.Body))

	if err != nil {
		log.Println("Error processing message:", err)
		return
	}

	// TODO: Cache project in mem
	project, err := models.FindProjectByUUID(projectUUID)

	if err != nil {
		log.Println("Project not found")
		return
	}

	storage.Add(message{
		ProjectUUID: project.UUID,
		Tag:         tag,
		Source:      msgSource,
		Datetime:    time.Now().UTC().Unix(),
	})
}

func main() {
	amqpConn, err := amqp.Dial("amqp://guest:guest@ubuntu:5672")
	failOnError(err, "Failed to connect to RabbitMQ")
	defer amqpConn.Close()

	channel, err := amqpConn.Channel()
	failOnError(err, "Failed to open a channel")
	defer channel.Close()

	err = channel.ExchangeDeclare(
		"logs",   // name
		"direct", // type
		true,     // durable
		false,    // auto-deleted
		false,    // internal
		false,    // noWait
		nil,      // arguments
	)
	failOnError(err, "Failed to declare a exchange")

	queue, err := channel.QueueDeclare(
		"all", // name of the queue
		true,  // durable
		false, // delete when unused
		false, // exclusive
		false, // noWait
		nil,   // arguments
	)
	failOnError(err, "Failed to declare a queue")

	err = channel.QueueBind(
		queue.Name, // name of the queue
		"all",      // bindingKey
		"logs",     // sourceExchange
		false,      // noWait
		nil,        // arguments
	)

	deliveries, err := channel.Consume(
		queue.Name, // name
		"tag",      // consumerTag,
		false,      // noAck
		false,      // exclusive
		false,      // noLocal
		false,      // noWait
		nil,        // arguments
	)
	failOnError(err, "Failed to consume")

	// TODO: Handle close method. See example consumer.
	go handle(deliveries)

	select {}
}
<commit_msg>Added json struct annotation<commit_after>// Consumer service accpets logs and other events from queue, validates,
// and pushes to index and persistance storage.
package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"regexp"
	"strings"
	"sync"
	"time"

	"bitbucket.org/firstrow/logvoyage/models"
	"github.com/streadway/amqp"
	"gopkg.in/olivere/elastic.v5"
)

const (
	// Default tag name name in case tag does not specified in source message.
	defaultTag = "default"
	// How often in seconds send messages to storge.
	persistTimeout = 2
)

var (
	storage = newInMemStorage()
	// See https://regex101.com/r/sQskdz/1
	msgRegExp = regexp.MustCompile(`^([a-z0-9]{8}-[a-z0-9]{4}-[1-5][a-z0-9]{3}-[a-z0-9]{4}-[a-z0-9]{12})(@([a-zA-Z0-9\_\-\.]{1,20}))?\s*(.*)`)
)

type message struct {
	ProjectUUID string
	Tag         string
	Source      string
	Datetime    int64
}

type doc struct {
	Source   string `json:"source"`
	Datetime int64  `json:"datetime"`
}

// inMemStorage stores all received valid message in memory.
// Each N seconds messages will be sent to ES via bulk request.
type inMemStorage struct {
	sync.Mutex
	messages []message
}

func newInMemStorage() *inMemStorage {
	s := &inMemStorage{}
	go s.startTimer()
	return s
}

func (s *inMemStorage) Add(msg message) {
	s.Lock()
	s.messages = append(s.messages, msg)
	s.Unlock()
}

func (s *inMemStorage) Persist() {
	client, err := elasticClient()
	if err != nil {
		log.Println("Error connecting to elastic:", err)
		return
	}

	s.Lock()
	defer s.Unlock()
	if len(s.messages) == 0 {
		return
	}

	bulkRequest := client.Bulk()
	for _, msg := range s.messages {
		req := elastic.NewBulkIndexRequest().Index("logs-" + msg.ProjectUUID).Type(msg.Tag)
		var userJSON map[string]interface{}
		err := json.Unmarshal([]byte(msg.Source), &userJSON)

		if err != nil {
			doc := doc{
				Source:   msg.Source,
				Datetime: msg.Datetime,
			}
			req.Doc(doc)
		} else {
			// Save user json
			fmt.Println("Save user json:", userJSON)
			userJSON["Datetime"] = msg.Datetime
			req.Doc(userJSON)
		}
		bulkRequest.Add(req)
	}

	bulkResponse, err := bulkRequest.Do(context.TODO())
	if err != nil {
		log.Println("Bulk request err:", err)
	}
	if bulkResponse == nil {
		log.Println("Bulk response == nil")
	}
	s.messages = []message{}
}

func (s *inMemStorage) startTimer() {
	ticker := time.NewTicker(persistTimeout * time.Second)
	defer ticker.Stop()

	for range ticker.C {
		s.Persist()
	}
}

func elasticClient() (*elastic.Client, error) {
	return elastic.NewClient(elastic.SetURL("http://ubuntu:9200"))
}

// processMessage extracts apiKey, tag(optional) and source message from log line.
// In case tag not found, "default" tag will be returned.
// Error will be returned if uuid or tag has wrong format.
//
// Examples:
// uuid@tag msg -> uuid, tag, msg, nil
// uuid msg     -> uuid, default, msg, nil
func processMessage(msg string) (string, string, string, error) {
	result := msgRegExp.FindAllStringSubmatch(msg, -1)

	if result == nil {
		return "", "", "", errors.New("Error parsing source message")
	}

	var tag string
	if result[0][3] == "" {
		tag = defaultTag
	} else {
		tag = result[0][3]
	}

	return result[0][1], tag, strings.TrimSpace(result[0][4]), nil
}

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
	}
}

func handle(deliveries <-chan amqp.Delivery) {
	for d := range deliveries {
		log.Printf(
			"got %dB delivery: [%v] %q",
			len(d.Body),
			d.DeliveryTag,
			d.Body,
		)
		processDelivery(d)
	}
	log.Fatalln("Deliveries channel closed")
}

func processDelivery(d amqp.Delivery) {
	defer d.Ack(false)

	projectUUID, tag, msgSource, err := processMessage(string(d.Body))

	if err != nil {
		log.Println("Error processing message:", err)
		return
	}

	// TODO: Cache project in mem
	project, err := models.FindProjectByUUID(projectUUID)

	if err != nil {
		log.Println("Project not found")
		return
	}

	storage.Add(message{
		ProjectUUID: project.UUID,
		Tag:         tag,
		Source:      msgSource,
		Datetime:    time.Now().UTC().Unix(),
	})
}

func main() {
	amqpConn, err := amqp.Dial("amqp://guest:guest@ubuntu:5672")
	failOnError(err, "Failed to connect to RabbitMQ")
	defer amqpConn.Close()

	channel, err := amqpConn.Channel()
	failOnError(err, "Failed to open a channel")
	defer channel.Close()

	err = channel.ExchangeDeclare(
		"logs",   // name
		"direct", // type
		true,     // durable
		false,    // auto-deleted
		false,    // internal
		false,    // noWait
		nil,      // arguments
	)
	failOnError(err, "Failed to declare a exchange")

	queue, err := channel.QueueDeclare(
		"all", // name of the queue
		true,  // durable
		false, // delete when unused
		false, // exclusive
		false, // noWait
		nil,   // arguments
	)
	failOnError(err, "Failed to declare a queue")

	err = channel.QueueBind(
		queue.Name, // name of the queue
		"all",      // bindingKey
		"logs",     // sourceExchange
		false,      // noWait
		nil,        // arguments
	)

	deliveries, err := channel.Consume(
		queue.Name, // name
		"tag",      // consumerTag,
		false,      // noAck
		false,      // exclusive
		false,      // noLocal
		false,      // noWait
		nil,        // arguments
	)
	failOnError(err, "Failed to consume")

	// TODO: Handle close method. See example consumer.
	go handle(deliveries)

	select {}
}
<|endoftext|>


<commit_before>package evaluation

import "fmt"

type Object interface {
	fmt.Stringer
	Equals(Object) bool
	Type() Type
}

type Collection interface {
	Object
	Elements() []Object
	GetIndex(int) Object
	SetIndex(int, Object)
}

type Container interface {
	Object
	Get(Object) Object
	Set(Object, Object)
}

type Hasher interface {
	Object
	Hash() string
}
<commit_msg>Add comments to the object interfaces<commit_after>package evaluation

import "fmt"

// Object is the base object interface, which
// every object implements
type Object interface {
	fmt.Stringer
	Equals(Object) bool
	Type() Type
}

// Collection is a child interface of Object,
// which represents an object which can be
// thought of as a list of items
type Collection interface {
	Object
	Elements() []Object
	GetIndex(int) Object
	SetIndex(int, Object)
}

// Container is a child interface of Object,
// which can be accessed by keys - like a map
type Container interface {
	Object
	Get(Object) Object
	Set(Object, Object)
}

// Hasher is any object which can be a key
// in a map
type Hasher interface {
	Object
	Hash() string
}
<|endoftext|>


<commit_before>package spirit

import (
	"fmt"
	"reflect"

	"github.com/gogap/errors"
)

type MessageSenderFactory interface {
	RegisterMessageSenders(senders ...MessageSender)
	IsExist(senderType string) bool
	NewSender(senderType string) (sender MessageSender, err error)
}

type DefaultMessageSenderFactory struct {
	senderDrivers map[string]reflect.Type
}

func NewDefaultMessageSenderFactory() MessageSenderFactory {
	fact := new(DefaultMessageSenderFactory)
	fact.senderDrivers = make(map[string]reflect.Type)
	return fact
}

func (p *DefaultMessageSenderFactory) RegisterMessageSenders(senders ...MessageSender) {
	if senders == nil {
		panic("senders is nil")
	}

	if len(senders) == 0 {
		return
	}

	for _, sender := range senders {
		if _, exist := p.senderDrivers[sender.Type()]; exist {
			panic(fmt.Errorf("sender driver of [%s] already exist", sender.Type()))
		}

		vof := reflect.ValueOf(sender)
		vType := vof.Type()
		if vof.Kind() == reflect.Ptr {
			vType = vof.Elem().Type()
		}

		p.senderDrivers[sender.Type()] = vType
	}
	return
}

func (p *DefaultMessageSenderFactory) IsExist(senderType string) bool {
	if _, exist := p.senderDrivers[senderType]; exist {
		return true
	}
	return false
}

func (p *DefaultMessageSenderFactory) NewSender(senderType string) (sender MessageSender, err error) {
	if senderType, exist := p.senderDrivers[senderType]; !exist {
		err = ERR_RECEIVER_DRIVER_NOT_EXIST.New(errors.Params{"type": senderType})
		return
	} else {
		if vOfMessageSender := reflect.New(senderType); vOfMessageSender.CanInterface() {
			iMessageSender := vOfMessageSender.Interface()
			if r, ok := iMessageSender.(MessageSender); ok {
				sender = r
				return
			} else {
				err = ERR_SENDER_CREATE_FAILED.New(errors.Params{"type": senderType})
				return
			}
		}
		err = ERR_SENDER_BAD_DRIVER.New(errors.Params{"type": senderType})
		return
	}
}
<commit_msg>add instance cache for message sender factory<commit_after>package spirit

import (
	"fmt"
	"reflect"

	"github.com/gogap/errors"
)

type MessageSenderFactory interface {
	RegisterMessageSenders(senders ...MessageSender)
	IsExist(senderType string) bool
	NewSender(senderType string) (sender MessageSender, err error)
}

type DefaultMessageSenderFactory struct {
	senderDrivers map[string]reflect.Type

	instanceCache map[string]MessageSender
}

func NewDefaultMessageSenderFactory() MessageSenderFactory {
	fact := new(DefaultMessageSenderFactory)
	fact.senderDrivers = make(map[string]reflect.Type)
	fact.instanceCache = make(map[string]MessageSender)
	return fact
}

func (p *DefaultMessageSenderFactory) RegisterMessageSenders(senders ...MessageSender) {
	if senders == nil {
		panic("senders is nil")
	}

	if len(senders) == 0 {
		return
	}

	for _, sender := range senders {
		if _, exist := p.senderDrivers[sender.Type()]; exist {
			panic(fmt.Errorf("sender driver of [%s] already exist", sender.Type()))
		}

		vof := reflect.ValueOf(sender)
		vType := vof.Type()
		if vof.Kind() == reflect.Ptr {
			vType = vof.Elem().Type()
		}

		p.senderDrivers[sender.Type()] = vType
	}
	return
}

func (p *DefaultMessageSenderFactory) IsExist(senderType string) bool {
	if _, exist := p.senderDrivers[senderType]; exist {
		return true
	}
	return false
}

func (p *DefaultMessageSenderFactory) NewSender(senderType string) (sender MessageSender, err error) {
	if sender, exist := p.instanceCache[senderType]; exist && sender != nil {
		return sender, nil
	}

	if senderType, exist := p.senderDrivers[senderType]; !exist {
		err = ERR_RECEIVER_DRIVER_NOT_EXIST.New(errors.Params{"type": senderType})
		return
	} else {
		if vOfMessageSender := reflect.New(senderType); vOfMessageSender.CanInterface() {
			iMessageSender := vOfMessageSender.Interface()
			if r, ok := iMessageSender.(MessageSender); ok {
				sender = r
				return
			} else {
				err = ERR_SENDER_CREATE_FAILED.New(errors.Params{"type": senderType})
				return
			}
		}
		err = ERR_SENDER_BAD_DRIVER.New(errors.Params{"type": senderType})
		return
	}
}
<|endoftext|>


<commit_before>package manage

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/graffic/wanon/bot"
	"github.com/graffic/wanon/telegram"
	"github.com/op/go-logging"
)

var log = logging.MustGetLogger("wanon.messages.manage")

type handler struct {
	allowed map[int]bool
}

type configuration struct {
	Admins []int
}

type manager struct {
	storage managerStorage
	answer  telegram.AnswerBack
}

func (handler *handler) Check(message *telegram.Message, context *bot.Context) int {
	isManage := strings.Index(message.Text, "/manage") == 0
	if handler.allowed[message.Chat.ID] && isManage {
		log.Debug("Accepted")
		return bot.RouteAccept
	}
	log.Debug("Ignored")
	return bot.RouteStop
}

func (handler *handler) Handle(message *telegram.Message, context *bot.Context) {
	actions := manager{
		managerStorage{context.Storage},
		telegram.AnswerBack{API: context.API, Message: message},
	}

	switch {
	case strings.Index(message.Text, "/manage list") == 0:
		actions.list(message.Text)
	case strings.Index(message.Text, "/manage delete") == 0:
		actions.delete(message.Text)
	case strings.Index(message.Text, "/manage chats") == 0:
		actions.chats()
	default:
		help()
	}
}

func (actions *manager) list(text string) {
	items := strings.Split(text, " ")
	amount := len(items)
	skip := 0

	if amount < 3 {
		return
	}
	if amount == 4 {
		skip, _ = strconv.Atoi(items[3])
	}

	quotes, _ := actions.storage.List(items[2], skip)
	var result string
	for _, quote := range *quotes {
		quoteStr := fmt.Sprintf("%s: <%s> %s\n", quote.ID.Hex(), quote.SaidBy, quote.What)
		result += quoteStr
	}
	actions.answer.Reply(result)
}

func (actions *manager) chats() {
	log.Debug("Chat database status")
	chats, err := actions.storage.Chats()
	if err != nil {
		log.Error(fmt.Sprint(err))
		return
	}

	var result string
	for _, chat := range *chats {
		result += fmt.Sprintf("%s: %d\n", chat.Name, chat.Records)
	}

	actions.answer.Reply(result)
}

func (actions *manager) delete(text string) {
	items := strings.Split(text, " ")

	if len(items) != 4 {
		return
	}

	err := actions.storage.Delete(items[2], items[3])
	if err != nil {
		actions.answer.Reply(fmt.Sprint(err))
	} else {
		actions.answer.Reply("Deleted")
	}
}

func help() {

}

// Create creates the ignore handler
func Create(conf *bot.ConfService) bot.Handler {
	myConf := new(configuration)
	conf.Get(myConf)
	log.Notice("Manage only from: %d", myConf.Admins)

	allowed := make(map[int]bool)
	for _, chatID := range myConf.Admins {
		allowed[chatID] = true
	}

	return &handler{allowed}
}
<commit_msg>Better ignore message for manage<commit_after>package manage

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/graffic/wanon/bot"
	"github.com/graffic/wanon/telegram"
	"github.com/op/go-logging"
)

var log = logging.MustGetLogger("wanon.messages.manage")

type handler struct {
	allowed map[int]bool
}

type configuration struct {
	Admins []int
}

type manager struct {
	storage managerStorage
	answer  telegram.AnswerBack
}

func (handler *handler) Check(message *telegram.Message, context *bot.Context) int {
	isManage := strings.Index(message.Text, "/manage") == 0
	if handler.allowed[message.Chat.ID] && isManage {
		log.Debug("Accepted")
		return bot.RouteAccept
	}
	if isManage {
		log.Debug(fmt.Sprintf("Not allowed: %d", message.Chat.ID))
	}
	return bot.RouteStop
}

func (handler *handler) Handle(message *telegram.Message, context *bot.Context) {
	actions := manager{
		managerStorage{context.Storage},
		telegram.AnswerBack{API: context.API, Message: message},
	}

	switch {
	case strings.Index(message.Text, "/manage list") == 0:
		actions.list(message.Text)
	case strings.Index(message.Text, "/manage delete") == 0:
		actions.delete(message.Text)
	case strings.Index(message.Text, "/manage chats") == 0:
		actions.chats()
	default:
		help()
	}
}

func (actions *manager) list(text string) {
	items := strings.Split(text, " ")
	amount := len(items)
	skip := 0

	if amount < 3 {
		return
	}
	if amount == 4 {
		skip, _ = strconv.Atoi(items[3])
	}

	quotes, _ := actions.storage.List(items[2], skip)
	var result string
	for _, quote := range *quotes {
		quoteStr := fmt.Sprintf("%s: <%s> %s\n", quote.ID.Hex(), quote.SaidBy, quote.What)
		result += quoteStr
	}
	actions.answer.Reply(result)
}

func (actions *manager) chats() {
	log.Debug("Chat database status")
	chats, err := actions.storage.Chats()
	if err != nil {
		log.Error(fmt.Sprint(err))
		return
	}

	var result string
	for _, chat := range *chats {
		result += fmt.Sprintf("%s: %d\n", chat.Name, chat.Records)
	}

	actions.answer.Reply(result)
}

func (actions *manager) delete(text string) {
	items := strings.Split(text, " ")

	if len(items) != 4 {
		return
	}

	err := actions.storage.Delete(items[2], items[3])
	if err != nil {
		actions.answer.Reply(fmt.Sprint(err))
	} else {
		actions.answer.Reply("Deleted")
	}
}

func help() {

}

// Create creates the ignore handler
func Create(conf *bot.ConfService) bot.Handler {
	myConf := new(configuration)
	conf.Get(myConf)
	log.Notice("Manage only from: %d", myConf.Admins)

	allowed := make(map[int]bool)
	for _, chatID := range myConf.Admins {
		allowed[chatID] = true
	}

	return &handler{allowed}
}
<|endoftext|>


<commit_before>// Copyright 2016 Palantir Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package defaulttasks

import (
	"sort"
	"strings"

	"github.com/pkg/errors"

	"github.com/palantir/godel/framework/godel/config"
)

const defaultResolver = "https://palantir.bintray.com/releases/{{GroupPath}}/{{Product}}/{{Version}}/{{Product}}-{{Version}}-{{OS}}-{{Arch}}.tgz"

var defaultPluginsConfig = config.PluginsConfig{
	DefaultResolvers: []string{
		defaultResolver,
	},
	Plugins: config.ToSinglePluginConfigs([]config.SinglePluginConfig{
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.distgo:dist-plugin:1.0.0-rc6",
				}),
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.godel-format-plugin:format-plugin:1.0.0-rc5",
				}),
			}),
			Assets: config.ToLocatorWithResolverConfigs([]config.LocatorWithResolverConfig{
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-format-asset-ptimports:ptimports-asset:1.0.0-rc4",
					}),
				},
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.godel-goland-plugin:goland-plugin:1.0.0-rc2",
				}),
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.okgo:check-plugin:1.0.0-rc4",
				}),
			}),
			Assets: config.ToLocatorWithResolverConfigs([]config.LocatorWithResolverConfig{
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-compiles:compiles-asset:1.0.0-rc3",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-deadcode:deadcode-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-errcheck:errcheck-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-extimport:extimport-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-golint:golint-asset:1.0.0-rc3",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-govet:govet-asset:1.0.0-rc3",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-importalias:importalias-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-ineffassign:ineffassign-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-novendor:novendor-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-outparamcheck:outparamcheck-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-unconvert:unconvert-asset:1.0.0-rc3",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-varcheck:varcheck-asset:1.0.0-rc2",
					}),
				},
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.go-license:license-plugin:1.0.0-rc3",
				}),
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.godel-test-plugin:test-plugin:1.0.0-rc4",
				}),
			}),
		},
	}),
}

func BuiltinPluginsConfig() config.PluginsConfig {
	return defaultPluginsConfig
}

func PluginsConfig(cfg config.DefaultTasksConfig) (config.PluginsConfig, error) {
	// start with configuration that uses default resolver
	pluginsCfg := config.PluginsConfig{
		DefaultResolvers: defaultPluginsConfig.DefaultResolvers,
	}
	// append default resolvers provided by the configuration
	pluginsCfg.DefaultResolvers = append(pluginsCfg.DefaultResolvers, cfg.DefaultResolvers...)

	defaultPluginKeys := make(map[string]struct{})
	for _, currPlugin := range defaultPluginsConfig.Plugins {
		currKey := locatorIDWithoutVersion(currPlugin.Locator.ID)
		defaultPluginKeys[currKey] = struct{}{}

		var assets []config.LocatorWithResolverConfig
		for _, asset := range currPlugin.Assets {
			assets = append(assets, config.LocatorWithResolverConfig(asset))
		}
		taskCfgV0, ok := cfg.Tasks[currKey]
		if !ok {
			// if custom configuration is not specified, use default and continue
			pluginsCfg.Plugins = append(pluginsCfg.Plugins, currPlugin)
			continue
		}
		taskCfg := config.SingleDefaultTaskConfig(taskCfgV0)

		// custom configuration was non-empty: start it with default LocatorWithResolver configuration
		currCfg := config.SinglePluginConfig{
			LocatorWithResolverConfig: currPlugin.LocatorWithResolverConfig,
		}
		if taskCfg.Locator.ID != "" {
			currCfg.Locator = taskCfg.Locator
		}
		if taskCfg.Resolver != "" {
			currCfg.Resolver = taskCfg.Resolver
		}

		currCfg.Assets = append(currCfg.Assets, config.ToLocatorWithResolverConfigs(assetConfigFromDefault(assets, taskCfg))...)
		currCfg.Assets = append(currCfg.Assets, taskCfg.Assets...)
		pluginsCfg.Plugins = append(pluginsCfg.Plugins, config.ToSinglePluginConfig(currCfg))
	}

	var invalidKeys []string
	for providedDefaultCfgKey := range cfg.Tasks {
		if _, ok := defaultPluginKeys[providedDefaultCfgKey]; ok {
			continue
		}
		invalidKeys = append(invalidKeys, providedDefaultCfgKey)
	}
	sort.Strings(invalidKeys)

	if len(invalidKeys) > 0 {
		var validKeys []string
		for k := range defaultPluginKeys {
			validKeys = append(validKeys, k)
		}
		sort.Strings(validKeys)
		return config.PluginsConfig{}, errors.Errorf("default-task key(s) specified but are not valid: %v. Valid values: %v", invalidKeys, validKeys)
	}

	return pluginsCfg, nil
}

func assetConfigFromDefault(baseCfg []config.LocatorWithResolverConfig, cfg config.SingleDefaultTaskConfig) []config.LocatorWithResolverConfig {
	if cfg.ExcludeAllDefaultAssets {
		return nil
	}
	exclude := make(map[string]struct{})
	for _, currExclude := range cfg.DefaultAssetsToExclude {
		exclude[currExclude] = struct{}{}
	}
	var out []config.LocatorWithResolverConfig
	for _, asset := range baseCfg {
		if _, ok := exclude[locatorIDWithoutVersion(asset.Locator.ID)]; ok {
			continue
		}
		out = append(out, asset)
	}
	return out
}

func locatorIDWithoutVersion(locatorID string) string {
	parts := strings.Split(locatorID, ":")
	return strings.Join(parts[:2], ":")
}
<commit_msg>Update distgo to use v1.0.0-rc7 (#332)<commit_after>// Copyright 2016 Palantir Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package defaulttasks

import (
	"sort"
	"strings"

	"github.com/pkg/errors"

	"github.com/palantir/godel/framework/godel/config"
)

const defaultResolver = "https://palantir.bintray.com/releases/{{GroupPath}}/{{Product}}/{{Version}}/{{Product}}-{{Version}}-{{OS}}-{{Arch}}.tgz"

var defaultPluginsConfig = config.PluginsConfig{
	DefaultResolvers: []string{
		defaultResolver,
	},
	Plugins: config.ToSinglePluginConfigs([]config.SinglePluginConfig{
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.distgo:dist-plugin:1.0.0-rc7",
				}),
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.godel-format-plugin:format-plugin:1.0.0-rc5",
				}),
			}),
			Assets: config.ToLocatorWithResolverConfigs([]config.LocatorWithResolverConfig{
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-format-asset-ptimports:ptimports-asset:1.0.0-rc4",
					}),
				},
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.godel-goland-plugin:goland-plugin:1.0.0-rc2",
				}),
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.okgo:check-plugin:1.0.0-rc4",
				}),
			}),
			Assets: config.ToLocatorWithResolverConfigs([]config.LocatorWithResolverConfig{
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-compiles:compiles-asset:1.0.0-rc3",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-deadcode:deadcode-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-errcheck:errcheck-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-extimport:extimport-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-golint:golint-asset:1.0.0-rc3",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-govet:govet-asset:1.0.0-rc3",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-importalias:importalias-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-ineffassign:ineffassign-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-novendor:novendor-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-outparamcheck:outparamcheck-asset:1.0.0-rc2",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-unconvert:unconvert-asset:1.0.0-rc3",
					}),
				},
				{
					Locator: config.ToLocatorConfig(config.LocatorConfig{
						ID: "com.palantir.godel-okgo-asset-varcheck:varcheck-asset:1.0.0-rc2",
					}),
				},
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.go-license:license-plugin:1.0.0-rc3",
				}),
			}),
		},
		{
			LocatorWithResolverConfig: config.ToLocatorWithResolverConfig(config.LocatorWithResolverConfig{
				Locator: config.ToLocatorConfig(config.LocatorConfig{
					ID: "com.palantir.godel-test-plugin:test-plugin:1.0.0-rc4",
				}),
			}),
		},
	}),
}

func BuiltinPluginsConfig() config.PluginsConfig {
	return defaultPluginsConfig
}

func PluginsConfig(cfg config.DefaultTasksConfig) (config.PluginsConfig, error) {
	// start with configuration that uses default resolver
	pluginsCfg := config.PluginsConfig{
		DefaultResolvers: defaultPluginsConfig.DefaultResolvers,
	}
	// append default resolvers provided by the configuration
	pluginsCfg.DefaultResolvers = append(pluginsCfg.DefaultResolvers, cfg.DefaultResolvers...)

	defaultPluginKeys := make(map[string]struct{})
	for _, currPlugin := range defaultPluginsConfig.Plugins {
		currKey := locatorIDWithoutVersion(currPlugin.Locator.ID)
		defaultPluginKeys[currKey] = struct{}{}

		var assets []config.LocatorWithResolverConfig
		for _, asset := range currPlugin.Assets {
			assets = append(assets, config.LocatorWithResolverConfig(asset))
		}
		taskCfgV0, ok := cfg.Tasks[currKey]
		if !ok {
			// if custom configuration is not specified, use default and continue
			pluginsCfg.Plugins = append(pluginsCfg.Plugins, currPlugin)
			continue
		}
		taskCfg := config.SingleDefaultTaskConfig(taskCfgV0)

		// custom configuration was non-empty: start it with default LocatorWithResolver configuration
		currCfg := config.SinglePluginConfig{
			LocatorWithResolverConfig: currPlugin.LocatorWithResolverConfig,
		}
		if taskCfg.Locator.ID != "" {
			currCfg.Locator = taskCfg.Locator
		}
		if taskCfg.Resolver != "" {
			currCfg.Resolver = taskCfg.Resolver
		}

		currCfg.Assets = append(currCfg.Assets, config.ToLocatorWithResolverConfigs(assetConfigFromDefault(assets, taskCfg))...)
		currCfg.Assets = append(currCfg.Assets, taskCfg.Assets...)
		pluginsCfg.Plugins = append(pluginsCfg.Plugins, config.ToSinglePluginConfig(currCfg))
	}

	var invalidKeys []string
	for providedDefaultCfgKey := range cfg.Tasks {
		if _, ok := defaultPluginKeys[providedDefaultCfgKey]; ok {
			continue
		}
		invalidKeys = append(invalidKeys, providedDefaultCfgKey)
	}
	sort.Strings(invalidKeys)

	if len(invalidKeys) > 0 {
		var validKeys []string
		for k := range defaultPluginKeys {
			validKeys = append(validKeys, k)
		}
		sort.Strings(validKeys)
		return config.PluginsConfig{}, errors.Errorf("default-task key(s) specified but are not valid: %v. Valid values: %v", invalidKeys, validKeys)
	}

	return pluginsCfg, nil
}

func assetConfigFromDefault(baseCfg []config.LocatorWithResolverConfig, cfg config.SingleDefaultTaskConfig) []config.LocatorWithResolverConfig {
	if cfg.ExcludeAllDefaultAssets {
		return nil
	}
	exclude := make(map[string]struct{})
	for _, currExclude := range cfg.DefaultAssetsToExclude {
		exclude[currExclude] = struct{}{}
	}
	var out []config.LocatorWithResolverConfig
	for _, asset := range baseCfg {
		if _, ok := exclude[locatorIDWithoutVersion(asset.Locator.ID)]; ok {
			continue
		}
		out = append(out, asset)
	}
	return out
}

func locatorIDWithoutVersion(locatorID string) string {
	parts := strings.Split(locatorID, ":")
	return strings.Join(parts[:2], ":")
}
<|endoftext|>


<commit_before>package decoder

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"time"

	"github.com/garyburd/redigo/redis"
	log "github.com/glendc/go-mini-log"
	"github.com/golang/snappy"
	"github.com/templexxx/reedsolomon"
	"zombiezen.com/go/capnproto2"

	"github.com/g8os/blockstor/tlog"
	"github.com/g8os/blockstor/tlog/schema"
)

// Decoder defines tlog data decoder
type Decoder struct {
	objstorAddr []string
	vdiskID     string

	// erasure encoder variable
	k int
	m int

	// encryption variable
	decrypter tlog.AESDecrypter
	privKey   []byte

	// prefix of the last hash key
	lastHashPrefix string

	// pools of redis connection
	redisPools []*redis.Pool
}

// DecodedAggregation defines a decoded tlog aggregation
// from decoder.
type DecodedAggregation struct {
	Agg *schema.TlogAggregation
	Err error
}

// New creates a tlog decoder
func New(objstorAddr []string, k, m int, vdiskID, privKey, nonce string) (*Decoder, error) {
	if len(objstorAddr) != k+m+1 {
		return nil, fmt.Errorf("invalid number of objstor")
	}

	// create decrypter
	decrypter, err := tlog.NewAESDecrypter(privKey, nonce)
	if err != nil {
		return nil, err
	}

	// create redis pools
	pools := make([]*redis.Pool, len(objstorAddr))

	for i := 0; i < len(objstorAddr); i++ {
		addr := objstorAddr[i]
		pools[i] = &redis.Pool{
			MaxIdle:     3,
			IdleTimeout: 240 * time.Second,
			Dial:        func() (redis.Conn, error) { return redis.Dial("tcp", addr) },
		}
	}

	return &Decoder{
		objstorAddr:    objstorAddr,
		vdiskID:        vdiskID,
		k:              k,
		m:              m,
		decrypter:      decrypter,
		privKey:        []byte(privKey),
		lastHashPrefix: "last_hash_",
		redisPools:     pools,
	}, nil
}

// Decode decodes the tlog with timestamp >= startTs.
func (d *Decoder) Decode(startTs uint64) <-chan *DecodedAggregation {
	daChan := make(chan *DecodedAggregation, 1)

	go func() {
		defer close(daChan)

		// get all keys after the specified timestamp
		keys, err := d.getKeysAfter(startTs)
		if err != nil {
			da := &DecodedAggregation{
				Agg: nil,
				Err: err,
			}
			daChan <- da
			return
		}

		// get the aggregations
		for i := len(keys) - 1; i >= 0; i-- {
			key := keys[i]
			agg, err := d.get(key)
			da := &DecodedAggregation{
				Agg: agg,
				Err: err,
			}
			daChan <- da
		}
	}()
	return daChan
}

// getAllKeys of this vdisk ID after specified timestamp
func (d *Decoder) getKeysAfter(startTs uint64) ([][]byte, error) {
	key, err := d.getLastHash()
	if err != nil {
		return nil, err
	}

	keys := [][]byte{}

	for {
		agg, err := d.get(key)
		if err != nil {
			return keys, err
		}

		// we'have passed the startTs
		if agg.Timestamp() < startTs {
			return keys, nil
		}

		keys = append(keys, key)

		// check if we already in very first key
		prev, err := agg.Prev()
		if err != nil {
			return keys, err
		}

		// this is very first key
		if bytes.Compare(prev, d.privKey) == 0 {
			return keys, nil
		}

		key = prev
	}
}

// get tlog aggregation by it's key
func (d *Decoder) get(key []byte) (*schema.TlogAggregation, error) {
	merged, err := d.getAllPieces(key)
	if err != nil {
		return nil, err
	}

	// get the real len
	var realLen uint64
	if err := binary.Read(bytes.NewBuffer(merged), binary.LittleEndian, &realLen); err != nil {
		return nil, err
	}

	merged = merged[uint64Size : uint64Size+realLen]

	// decrypt
	decrypted, err := d.decrypt(merged)
	if err != nil {
		return nil, err
	}

	//uncompressed, err := d.uncompress(decrypted)
	uncompressed, err := snappy.Decode(nil, decrypted)
	if err != nil {
		return nil, err
	}

	return d.decodeCapnp(uncompressed)
}

func (d *Decoder) decrypt(encrypted []byte) ([]byte, error) {
	return d.decrypter.Decrypt(encrypted)
}

// get and merge all pieces of the data
func (d *Decoder) getAllPieces(key []byte) ([]byte, error) {
	lost := []int{}
	pieces := make([][]byte, d.k+d.m)

	var pieceSize int

	// get pieces from storage
	for i := 0; i < d.k+d.m; i++ {
		rc := d.redisPools[i+1].Get()

		b, err := redis.Bytes(rc.Do("GET", key))
		if err != nil {
			log.Infof("tlog decoder : failed to get piece num:%v, key:%v,err:%v", i, key, err)
			lost = append(lost, i)
			continue
		}

		pieces[i] = b
		pieceSize = len(b)
	}

	if len(lost) > d.m {
		return nil, fmt.Errorf("tlog decoder : too much lost pieces:%v, max:%v", len(lost), d.m)
	}

	// erasure encoded it
	if len(lost) > 0 {
		// fill up the pieces size
		for _, v := range lost {
			pieces[v] = make([]byte, pieceSize)
		}

		rs, err := reedsolomon.New(d.k, d.m)
		if err != nil {
			return nil, err
		}
		if err := rs.Reconst(pieces, lost, true); err != nil {
			return nil, err
		}
	}

	// merge the pieces
	all := make([]byte, 0, len(pieces[0])*d.k)
	for i := 0; i < d.k; i++ {
		all = append(all, pieces[i]...)
	}
	return all, nil
}

func (d *Decoder) getLastHash() ([]byte, error) {
	rc := d.redisPools[0].Get()

	key := d.lastHashPrefix + d.vdiskID
	b, err := redis.Bytes(rc.Do("get", key))
	if err != nil {
		return nil, fmt.Errorf("failed to get key:%v, err:%v", key, err)
	}
	return b, nil
}

func (d *Decoder) decodeCapnp(data []byte) (*schema.TlogAggregation, error) {
	buf := bytes.NewBuffer(data)

	msg, err := capnp.NewDecoder(buf).Decode()
	if err != nil {
		return nil, err
	}

	agg, err := schema.ReadRootTlogAggregation(msg)
	return &agg, err
}

const (
	uint64Size = 8
)
<commit_msg>tlog decoder : close redis after usage.<commit_after>package decoder

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"time"

	"github.com/garyburd/redigo/redis"
	log "github.com/glendc/go-mini-log"
	"github.com/golang/snappy"
	"github.com/templexxx/reedsolomon"
	"zombiezen.com/go/capnproto2"

	"github.com/g8os/blockstor/tlog"
	"github.com/g8os/blockstor/tlog/schema"
)

// Decoder defines tlog data decoder
type Decoder struct {
	objstorAddr []string
	vdiskID     string

	// erasure encoder variable
	k int
	m int

	// encryption variable
	decrypter tlog.AESDecrypter
	privKey   []byte

	// prefix of the last hash key
	lastHashPrefix string

	// pools of redis connection
	redisPools []*redis.Pool
}

// DecodedAggregation defines a decoded tlog aggregation
// from decoder.
type DecodedAggregation struct {
	Agg *schema.TlogAggregation
	Err error
}

// New creates a tlog decoder
func New(objstorAddr []string, k, m int, vdiskID, privKey, nonce string) (*Decoder, error) {
	if len(objstorAddr) != k+m+1 {
		return nil, fmt.Errorf("invalid number of objstor")
	}

	// create decrypter
	decrypter, err := tlog.NewAESDecrypter(privKey, nonce)
	if err != nil {
		return nil, err
	}

	// create redis pools
	pools := make([]*redis.Pool, len(objstorAddr))

	for i := 0; i < len(objstorAddr); i++ {
		addr := objstorAddr[i]
		pools[i] = &redis.Pool{
			MaxIdle:     3,
			IdleTimeout: 240 * time.Second,
			Dial:        func() (redis.Conn, error) { return redis.Dial("tcp", addr) },
		}
	}

	return &Decoder{
		objstorAddr:    objstorAddr,
		vdiskID:        vdiskID,
		k:              k,
		m:              m,
		decrypter:      decrypter,
		privKey:        []byte(privKey),
		lastHashPrefix: "last_hash_",
		redisPools:     pools,
	}, nil
}

// Decode decodes the tlog with timestamp >= startTs.
func (d *Decoder) Decode(startTs uint64) <-chan *DecodedAggregation {
	daChan := make(chan *DecodedAggregation, 1)

	go func() {
		defer close(daChan)

		// get all keys after the specified timestamp
		keys, err := d.getKeysAfter(startTs)
		if err != nil {
			da := &DecodedAggregation{
				Agg: nil,
				Err: err,
			}
			daChan <- da
			return
		}

		// get the aggregations
		for i := len(keys) - 1; i >= 0; i-- {
			key := keys[i]
			agg, err := d.get(key)
			da := &DecodedAggregation{
				Agg: agg,
				Err: err,
			}
			daChan <- da
		}
	}()
	return daChan
}

// getAllKeys of this vdisk ID after specified timestamp
func (d *Decoder) getKeysAfter(startTs uint64) ([][]byte, error) {
	key, err := d.getLastHash()
	if err != nil {
		return nil, err
	}

	keys := [][]byte{}

	for {
		agg, err := d.get(key)
		if err != nil {
			return keys, err
		}

		// we'have passed the startTs
		if agg.Timestamp() < startTs {
			return keys, nil
		}

		keys = append(keys, key)

		// check if we already in very first key
		prev, err := agg.Prev()
		if err != nil {
			return keys, err
		}

		// this is very first key
		if bytes.Compare(prev, d.privKey) == 0 {
			return keys, nil
		}

		key = prev
	}
}

// get tlog aggregation by it's key
func (d *Decoder) get(key []byte) (*schema.TlogAggregation, error) {
	merged, err := d.getAllPieces(key)
	if err != nil {
		return nil, err
	}

	// get the real len
	var realLen uint64
	if err := binary.Read(bytes.NewBuffer(merged), binary.LittleEndian, &realLen); err != nil {
		return nil, err
	}

	merged = merged[uint64Size : uint64Size+realLen]

	// decrypt
	decrypted, err := d.decrypt(merged)
	if err != nil {
		return nil, err
	}

	//uncompressed, err := d.uncompress(decrypted)
	uncompressed, err := snappy.Decode(nil, decrypted)
	if err != nil {
		return nil, err
	}

	return d.decodeCapnp(uncompressed)
}

func (d *Decoder) decrypt(encrypted []byte) ([]byte, error) {
	return d.decrypter.Decrypt(encrypted)
}

// get and merge all pieces of the data
func (d *Decoder) getAllPieces(key []byte) ([]byte, error) {
	lost := []int{}
	pieces := make([][]byte, d.k+d.m)

	var pieceSize int

	// get pieces from storage
	for i := 0; i < d.k+d.m; i++ {
		rc := d.redisPools[i+1].Get()
		defer rc.Close()

		b, err := redis.Bytes(rc.Do("GET", key))
		if err != nil {
			log.Infof("tlog decoder : failed to get piece num:%v, key:%v,err:%v", i, key, err)
			lost = append(lost, i)
			continue
		}

		pieces[i] = b
		pieceSize = len(b)
	}

	if len(lost) > d.m {
		return nil, fmt.Errorf("tlog decoder : too much lost pieces:%v, max:%v", len(lost), d.m)
	}

	// erasure encoded it
	if len(lost) > 0 {
		// fill up the pieces size
		for _, v := range lost {
			pieces[v] = make([]byte, pieceSize)
		}

		rs, err := reedsolomon.New(d.k, d.m)
		if err != nil {
			return nil, err
		}
		if err := rs.Reconst(pieces, lost, true); err != nil {
			return nil, err
		}
	}

	// merge the pieces
	all := make([]byte, 0, len(pieces[0])*d.k)
	for i := 0; i < d.k; i++ {
		all = append(all, pieces[i]...)
	}
	return all, nil
}

func (d *Decoder) getLastHash() ([]byte, error) {
	rc := d.redisPools[0].Get()
	defer rc.Close()

	key := d.lastHashPrefix + d.vdiskID
	b, err := redis.Bytes(rc.Do("get", key))
	if err != nil {
		return nil, fmt.Errorf("failed to get key:%v, err:%v", key, err)
	}
	return b, nil
}

func (d *Decoder) decodeCapnp(data []byte) (*schema.TlogAggregation, error) {
	buf := bytes.NewBuffer(data)

	msg, err := capnp.NewDecoder(buf).Decode()
	if err != nil {
		return nil, err
	}

	agg, err := schema.ReadRootTlogAggregation(msg)
	return &agg, err
}

const (
	uint64Size = 8
)
<|endoftext|>