<commit_before>package hive

import (
	"bufio"
	"errors"
	// "fmt"
	"github.com/eaciit/errorlib"
	// "github.com/eaciit/toolkit"
	"io"
	"log"
	"os/exec"
	"reflect"
	"strings"
)

const (
	BEE_CLI_STR  = "0: jdbc:hive2:"
	CLOSE_SCRIPT = "!quit"
)

type DuplexTerm struct {
	Writer     *bufio.Writer
	Reader     *bufio.Reader
	Cmd        *exec.Cmd
	CmdStr     string
	Stdin      io.WriteCloser
	Stdout     io.ReadCloser
	FnReceive  FnHiveReceive
	Fn         interface{}
	OutputType string
	DateFormat string
}

/*func (d *DuplexTerm) Open() (e error) {
	if d.Stdin, e = d.Cmd.StdinPipe(); e != nil {
		return
	}

	if d.Stdout, e = d.Cmd.StdoutPipe(); e != nil {
		return
	}

	d.Writer = bufio.NewWriter(d.Stdin)
	d.Reader = bufio.NewReader(d.Stdout)

	e = d.Cmd.Start()
	return
}*/

var hr HiveResult

func (d *DuplexTerm) Open() (e error) {
	if d.CmdStr != "" {
		arg := append([]string{"-c"}, d.CmdStr)
		d.Cmd = exec.Command("sh", arg...)

		if d.Stdin, e = d.Cmd.StdinPipe(); e != nil {
			return
		}

		if d.Stdout, e = d.Cmd.StdoutPipe(); e != nil {
			return
		}

		d.Writer = bufio.NewWriter(d.Stdin)
		d.Reader = bufio.NewReader(d.Stdout)

		if d.FnReceive != nil {
			go func() {
				_, e = d.Wait()
			}()
		}
		e = d.Cmd.Start()
	} else {
		errorlib.Error("", "", "Open", "The Connection Config not Set")
	}

	return
}

func (d *DuplexTerm) Close() {
	result, e := d.SendInput(CLOSE_SCRIPT)

	_ = result
	_ = e

	d.Cmd.Wait()
	d.Stdin.Close()
	d.Stdout.Close()
}

func (d *DuplexTerm) SendInput(input string) (result []string, e error) {
	iwrite, e := d.Writer.WriteString(input + "\n")
	if iwrite == 0 {
		e = errors.New("Writing only 0 byte")
	} else {
		e = d.Writer.Flush()
	}

	if e != nil {
		return
	}

	if d.FnReceive == nil {
		result, e = d.Wait()
	}

	return
}

func (d *DuplexTerm) SetFn(f interface{}) {
	fn := reflect.ValueOf(f)
	fnType := fn.Type()
	if fnType.Kind() != reflect.Func || fnType.NumIn() != 1 || fnType.NumOut() != 1 {
		panic("Expected a unary function returning a single value")
	}

	d.Fn = f
}

func (d *DuplexTerm) Wait() (result []string, e error) {
	isHeader := false
	for {
		peekBefore, _ := d.Reader.Peek(14)
		peekBeforeStr := string(peekBefore)

		bread, e := d.Reader.ReadString('\n')
		bread = strings.TrimRight(bread, "\n")

		peek, _ := d.Reader.Peek(14)
		peekStr := string(peek)

		delimiter := "\t"

		if d.OutputType == CSV {
			delimiter = ","
		}

		if isHeader {
			hr.constructHeader(bread, delimiter)
			isHeader = false
		}

		if BEE_CLI_STR == peekBeforeStr {
			isHeader = true
		}

		if !strings.Contains(bread, BEE_CLI_STR) {
			if d.FnReceive != nil {
				fn := reflect.ValueOf(d.Fn)
				tp := fn.Type().In(0)
				tmp := reflect.New(tp)

				Parse(hr.Header, bread, &tmp, d.OutputType, d.DateFormat)
				log.Printf("fn: %v\n", tmp)
				res := fn.Call([]reflect.Value{reflect.ValueOf(hr.ResultObj)})
				log.Printf("test: %v\n", res)
				d.FnReceive(res)
			} else {
				result = append(result, bread)
			}
		}

		if d.FnReceive != nil {
			if (e != nil && e.Error() == "EOF") || (strings.Contains(peekStr, CLOSE_SCRIPT)) {
				break
			}
		} else {
			if (e != nil && e.Error() == "EOF") || (BEE_CLI_STR == peekStr) {
				break
			}
		}

	}

	return
}
<commit_msg>bug fixing<commit_after>package hive

import (
	"bufio"
	"errors"
	// "fmt"
	"github.com/eaciit/errorlib"
	"github.com/eaciit/toolkit"
	"io"
	"log"
	"os/exec"
	"reflect"
	"strings"
)

const (
	BEE_CLI_STR  = "0: jdbc:hive2:"
	CLOSE_SCRIPT = "!quit"
)

type DuplexTerm struct {
	Writer     *bufio.Writer
	Reader     *bufio.Reader
	Cmd        *exec.Cmd
	CmdStr     string
	Stdin      io.WriteCloser
	Stdout     io.ReadCloser
	FnReceive  FnHiveReceive
	Fn         interface{}
	OutputType string
	DateFormat string
}

/*func (d *DuplexTerm) Open() (e error) {
	if d.Stdin, e = d.Cmd.StdinPipe(); e != nil {
		return
	}

	if d.Stdout, e = d.Cmd.StdoutPipe(); e != nil {
		return
	}

	d.Writer = bufio.NewWriter(d.Stdin)
	d.Reader = bufio.NewReader(d.Stdout)

	e = d.Cmd.Start()
	return
}*/

var hr HiveResult

func (d *DuplexTerm) Open() (e error) {
	if d.CmdStr != "" {
		arg := append([]string{"-c"}, d.CmdStr)
		d.Cmd = exec.Command("sh", arg...)

		if d.Stdin, e = d.Cmd.StdinPipe(); e != nil {
			return
		}

		if d.Stdout, e = d.Cmd.StdoutPipe(); e != nil {
			return
		}

		d.Writer = bufio.NewWriter(d.Stdin)
		d.Reader = bufio.NewReader(d.Stdout)

		if d.FnReceive != nil {
			go func() {
				_, e = d.Wait()
			}()
		}
		e = d.Cmd.Start()
	} else {
		errorlib.Error("", "", "Open", "The Connection Config not Set")
	}

	return
}

func (d *DuplexTerm) Close() {
	result, e := d.SendInput(CLOSE_SCRIPT)

	_ = result
	_ = e

	d.Cmd.Wait()
	d.Stdin.Close()
	d.Stdout.Close()
}

func (d *DuplexTerm) SendInput(input string) (result []string, e error) {
	iwrite, e := d.Writer.WriteString(input + "\n")
	if iwrite == 0 {
		e = errors.New("Writing only 0 byte")
	} else {
		e = d.Writer.Flush()
	}

	if e != nil {
		return
	}

	if d.FnReceive == nil {
		result, e = d.Wait()
	}

	return
}

func (d *DuplexTerm) SetFn(f interface{}) {
	fn := reflect.ValueOf(f)
	fnType := fn.Type()
	if fnType.Kind() != reflect.Func || fnType.NumIn() != 1 || fnType.NumOut() != 1 {
		panic("Expected a unary function returning a single value")
	}

	d.Fn = f
}

func (d *DuplexTerm) Wait() (result []string, e error) {
	isHeader := false
	for {
		peekBefore, _ := d.Reader.Peek(14)
		peekBeforeStr := string(peekBefore)

		bread, e := d.Reader.ReadString('\n')
		bread = strings.TrimRight(bread, "\n")

		peek, _ := d.Reader.Peek(14)
		peekStr := string(peek)

		delimiter := "\t"

		if d.OutputType == CSV {
			delimiter = ","
		}

		if isHeader {
			hr.constructHeader(bread, delimiter)
			isHeader = false
		}

		if BEE_CLI_STR == peekBeforeStr {
			isHeader = true
		}

		if !strings.Contains(bread, BEE_CLI_STR) {
			if d.Fn != nil {
				fn := reflect.ValueOf(d.Fn)
				tp := fn.Type().In(0)
				tmp := reflect.New(tp)

				xTmp := toolkit.M{}

				Parse(hr.Header, bread, &xTmp, d.OutputType, d.DateFormat)
				log.Printf("tmp: %v\n", xTmp)
				// log.Printf("tmp: %v\n", tmp)

				/*res := fn.Call([]reflect.Value{reflect.ValueOf(hr.ResultObj)})
				log.Printf("test: %v\n", res)
				d.FnReceive(res)*/
			} else {
				result = append(result, bread)
			}
		}

		if d.FnReceive != nil {
			if (e != nil && e.Error() == "EOF") || (strings.Contains(peekStr, CLOSE_SCRIPT)) {
				break
			}
		} else {
			if (e != nil && e.Error() == "EOF") || (BEE_CLI_STR == peekStr) {
				break
			}
		}

	}

	return
}
<|endoftext|>


<commit_before>// +build linux darwin freebsd netbsd openbsd

package hostsfile

var HostsPath = `/etc/hosts`
<commit_msg>Use UNIX path on all OSs other than Windows and Plan9 (#3)<commit_after>// +build !plan9 !windows

package hostsfile

var HostsPath = `/etc/hosts`
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api_test

import (
	"encoding/hex"
	"math/rand"
	"testing"

	"github.com/gogo/protobuf/proto"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/testapi"
	apitesting "k8s.io/kubernetes/pkg/api/testing"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/api/v1"
	_ "k8s.io/kubernetes/pkg/apis/extensions"
	_ "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/runtime/serializer/protobuf"
	"k8s.io/kubernetes/pkg/util/diff"
)

func init() {
	codecsToTest = append(codecsToTest, func(version unversioned.GroupVersion, item runtime.Object) (runtime.Codec, error) {
		s := protobuf.NewSerializer(api.Scheme, runtime.ObjectTyperToTyper(api.Scheme), "application/arbitrary.content.type")
		return api.Codecs.CodecForVersions(s, testapi.ExternalGroupVersions(), nil), nil
	})
}

func TestProtobufRoundTrip(t *testing.T) {
	obj := &v1.Pod{}
	apitesting.FuzzerFor(t, v1.SchemeGroupVersion, rand.NewSource(benchmarkSeed)).Fuzz(obj)
	data, err := obj.Marshal()
	if err != nil {
		t.Fatal(err)
	}
	out := &v1.Pod{}
	if err := out.Unmarshal(data); err != nil {
		t.Fatal(err)
	}
	if !api.Semantic.Equalities.DeepEqual(out, obj) {
		t.Logf("marshal\n%s", hex.Dump(data))
		t.Fatalf("Unmarshal is unequal\n%s", diff.ObjectGoPrintSideBySide(out, obj))
	}
}

// BenchmarkEncodeCodec measures the cost of performing a codec encode, which includes
// reflection (to clear APIVersion and Kind)
func BenchmarkEncodeCodecProtobuf(b *testing.B) {
	items := benchmarkItems()
	width := len(items)
	s := protobuf.NewSerializer(nil, nil, "application/arbitrary.content.type")
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if _, err := runtime.Encode(s, &items[i%width]); err != nil {
			b.Fatal(err)
		}
	}
	b.StopTimer()
}

// BenchmarkEncodeCodecFromInternalProtobuf measures the cost of performing a codec encode,
// including conversions and any type setting. This is a "full" encode.
func BenchmarkEncodeCodecFromInternalProtobuf(b *testing.B) {
	items := benchmarkItems()
	width := len(items)
	encodable := make([]api.Pod, width)
	for i := range items {
		if err := api.Scheme.Convert(&items[i], &encodable[i]); err != nil {
			b.Fatal(err)
		}
	}
	s := protobuf.NewSerializer(nil, nil, "application/arbitrary.content.type")
	codec := api.Codecs.EncoderForVersion(s, v1.SchemeGroupVersion)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if _, err := runtime.Encode(codec, &encodable[i%width]); err != nil {
			b.Fatal(err)
		}
	}
	b.StopTimer()
}

func BenchmarkEncodeProtobufGeneratedMarshal(b *testing.B) {
	items := benchmarkItems()
	width := len(items)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if _, err := items[i%width].Marshal(); err != nil {
			b.Fatal(err)
		}
	}
	b.StopTimer()
}

// BenchmarkDecodeJSON provides a baseline for regular JSON decode performance
func BenchmarkDecodeIntoProtobuf(b *testing.B) {
	items := benchmarkItems()
	width := len(items)
	encoded := make([][]byte, width)
	for i := range items {
		data, err := (&items[i]).Marshal()
		if err != nil {
			b.Fatal(err)
		}
		encoded[i] = data
		validate := &v1.Pod{}
		if err := proto.Unmarshal(data, validate); err != nil {
			b.Fatalf("Failed to unmarshal %d: %v\n%#v", i, err, items[i])
		}
	}

	for i := 0; i < b.N; i++ {
		obj := v1.Pod{}
		if err := proto.Unmarshal(encoded[i%width], &obj); err != nil {
			b.Fatal(err)
		}
	}
}
<commit_msg>RecognizingDecoder didn't handle ambiguous input<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api_test

import (
	"bytes"
	"encoding/hex"
	"math/rand"
	"testing"

	"github.com/gogo/protobuf/proto"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/testapi"
	apitesting "k8s.io/kubernetes/pkg/api/testing"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/api/v1"
	_ "k8s.io/kubernetes/pkg/apis/extensions"
	_ "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/runtime/serializer/protobuf"
	"k8s.io/kubernetes/pkg/util/diff"
)

func init() {
	codecsToTest = append(codecsToTest, func(version unversioned.GroupVersion, item runtime.Object) (runtime.Codec, error) {
		s := protobuf.NewSerializer(api.Scheme, runtime.ObjectTyperToTyper(api.Scheme), "application/arbitrary.content.type")
		return api.Codecs.CodecForVersions(s, testapi.ExternalGroupVersions(), nil), nil
	})
}

func TestUniversalDeserializer(t *testing.T) {
	expected := &v1.Pod{ObjectMeta: v1.ObjectMeta{Name: "test"}}
	d := api.Codecs.UniversalDeserializer()
	for _, mediaType := range []string{"application/json", "application/yaml", "application/vnd.kubernetes.protobuf"} {
		e, ok := api.Codecs.SerializerForMediaType(mediaType, nil)
		if !ok {
			t.Fatal(mediaType)
		}
		buf := &bytes.Buffer{}
		if err := e.EncodeToStream(expected, buf); err != nil {
			t.Fatalf("%s: %v", mediaType, err)
		}
		obj, _, err := d.Decode(buf.Bytes(), &unversioned.GroupVersionKind{Kind: "Pod", Version: "v1"}, nil)
		if err != nil {
			t.Fatalf("%s: %v", mediaType, err)
		}
		if !api.Semantic.DeepEqual(expected, obj) {
			t.Fatalf("%s: %#v", mediaType, obj)
		}
	}
}

func TestProtobufRoundTrip(t *testing.T) {
	obj := &v1.Pod{}
	apitesting.FuzzerFor(t, v1.SchemeGroupVersion, rand.NewSource(benchmarkSeed)).Fuzz(obj)
	data, err := obj.Marshal()
	if err != nil {
		t.Fatal(err)
	}
	out := &v1.Pod{}
	if err := out.Unmarshal(data); err != nil {
		t.Fatal(err)
	}
	if !api.Semantic.Equalities.DeepEqual(out, obj) {
		t.Logf("marshal\n%s", hex.Dump(data))
		t.Fatalf("Unmarshal is unequal\n%s", diff.ObjectGoPrintSideBySide(out, obj))
	}
}

// BenchmarkEncodeCodec measures the cost of performing a codec encode, which includes
// reflection (to clear APIVersion and Kind)
func BenchmarkEncodeCodecProtobuf(b *testing.B) {
	items := benchmarkItems()
	width := len(items)
	s := protobuf.NewSerializer(nil, nil, "application/arbitrary.content.type")
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if _, err := runtime.Encode(s, &items[i%width]); err != nil {
			b.Fatal(err)
		}
	}
	b.StopTimer()
}

// BenchmarkEncodeCodecFromInternalProtobuf measures the cost of performing a codec encode,
// including conversions and any type setting. This is a "full" encode.
func BenchmarkEncodeCodecFromInternalProtobuf(b *testing.B) {
	items := benchmarkItems()
	width := len(items)
	encodable := make([]api.Pod, width)
	for i := range items {
		if err := api.Scheme.Convert(&items[i], &encodable[i]); err != nil {
			b.Fatal(err)
		}
	}
	s := protobuf.NewSerializer(nil, nil, "application/arbitrary.content.type")
	codec := api.Codecs.EncoderForVersion(s, v1.SchemeGroupVersion)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if _, err := runtime.Encode(codec, &encodable[i%width]); err != nil {
			b.Fatal(err)
		}
	}
	b.StopTimer()
}

func BenchmarkEncodeProtobufGeneratedMarshal(b *testing.B) {
	items := benchmarkItems()
	width := len(items)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if _, err := items[i%width].Marshal(); err != nil {
			b.Fatal(err)
		}
	}
	b.StopTimer()
}

// BenchmarkDecodeJSON provides a baseline for regular JSON decode performance
func BenchmarkDecodeIntoProtobuf(b *testing.B) {
	items := benchmarkItems()
	width := len(items)
	encoded := make([][]byte, width)
	for i := range items {
		data, err := (&items[i]).Marshal()
		if err != nil {
			b.Fatal(err)
		}
		encoded[i] = data
		validate := &v1.Pod{}
		if err := proto.Unmarshal(data, validate); err != nil {
			b.Fatalf("Failed to unmarshal %d: %v\n%#v", i, err, items[i])
		}
	}

	for i := 0; i < b.N; i++ {
		obj := v1.Pod{}
		if err := proto.Unmarshal(encoded[i%width], &obj); err != nil {
			b.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>package launch

import (
	"github.com/sapcc/kubernikus/pkg/api/models"
	"github.com/sapcc/kubernikus/pkg/apis/kubernikus/v1"
	"github.com/sapcc/kubernikus/pkg/controller/base"
	"github.com/sapcc/kubernikus/pkg/controller/config"
	"github.com/sapcc/kubernikus/pkg/controller/metrics"

	"github.com/go-kit/kit/log"
	"k8s.io/client-go/tools/record"
)

type LaunchReconciler struct {
	config.Clients

	Recorder record.EventRecorder
	Logger   log.Logger
}

func NewController(factories config.Factories, clients config.Clients, recorder record.EventRecorder, logger log.Logger) base.Controller {
	logger = log.With(logger,
		"controller", "launch")

	var reconciler base.Reconciler
	reconciler = &LaunchReconciler{clients, recorder, logger}
	reconciler = &base.LoggingReconciler{reconciler, logger}
	reconciler = &base.EventingReconciler{reconciler}
	reconciler = &base.InstrumentingReconciler{
		reconciler,
		metrics.LaunchOperationsLatency,
		metrics.LaunchOperationsTotal,
		metrics.LaunchSuccessfulOperationsTotal,
		metrics.LaunchFailedOperationsTotal,
	}

	return base.NewController(factories, clients, reconciler, logger)
}

func (lr *LaunchReconciler) Reconcile(kluster *v1.Kluster) (requeueRequested bool, err error) {
	if !(kluster.Status.Phase == models.KlusterPhaseRunning || kluster.Status.Phase == models.KlusterPhaseTerminating) {
		return false, nil
	}

	for _, pool := range kluster.Spec.NodePools {
		_, requeue, err := lr.reconcilePool(kluster, &pool)
		if err != nil {
			return false, err
		}

		if requeue {
			requeueRequested = true
		}
	}

	return requeueRequested, nil
}

func (lr *LaunchReconciler) reconcilePool(kluster *v1.Kluster, pool *models.NodePool) (status *PoolStatus, requeue bool, err error) {

	pm := lr.newPoolManager(kluster, pool)
	status, err = pm.GetStatus()
	if err != nil {
		return
	}

	switch {
	case kluster.Status.Phase == models.KlusterPhaseTerminating:
		for _, node := range status.Nodes {
			requeue = true
			if err = pm.DeleteNode(node); err != nil {
				break
			}
		}
		break
	case status.Needed > 0:
		for i := 0; i < int(status.Needed); i++ {
			requeue = true
			if _, err = pm.CreateNode(); err != nil {
				break
			}
		}
		break
	case status.UnNeeded > 0:
		for i := 0; i < int(status.UnNeeded); i++ {
			requeue = true
			if err = pm.DeleteNode(status.Nodes[i]); err != nil {
				break
			}
		}
		break
	case status.Starting > 0:
		requeue = true
	case status.Stopping > 0:
		requeue = true
	default:
		return
	}

	err = pm.SetStatus(status)
	return
}
<commit_msg>reverts optimization<commit_after>package launch

import (
	"github.com/sapcc/kubernikus/pkg/api/models"
	"github.com/sapcc/kubernikus/pkg/apis/kubernikus/v1"
	"github.com/sapcc/kubernikus/pkg/controller/base"
	"github.com/sapcc/kubernikus/pkg/controller/config"
	"github.com/sapcc/kubernikus/pkg/controller/metrics"

	"github.com/go-kit/kit/log"
	"k8s.io/client-go/tools/record"
)

type LaunchReconciler struct {
	config.Clients

	Recorder record.EventRecorder
	Logger   log.Logger
}

func NewController(factories config.Factories, clients config.Clients, recorder record.EventRecorder, logger log.Logger) base.Controller {
	logger = log.With(logger,
		"controller", "launch")

	var reconciler base.Reconciler
	reconciler = &LaunchReconciler{clients, recorder, logger}
	reconciler = &base.LoggingReconciler{reconciler, logger}
	reconciler = &base.EventingReconciler{reconciler}
	reconciler = &base.InstrumentingReconciler{
		reconciler,
		metrics.LaunchOperationsLatency,
		metrics.LaunchOperationsTotal,
		metrics.LaunchSuccessfulOperationsTotal,
		metrics.LaunchFailedOperationsTotal,
	}

	return base.NewController(factories, clients, reconciler, logger)
}

func (lr *LaunchReconciler) Reconcile(kluster *v1.Kluster) (requeueRequested bool, err error) {
	if !(kluster.Status.Phase == models.KlusterPhaseRunning || kluster.Status.Phase == models.KlusterPhaseTerminating) {
		return false, nil
	}

	for _, pool := range kluster.Spec.NodePools {
		_, requeue, err := lr.reconcilePool(kluster, &pool)
		if err != nil {
			return false, err
		}

		if requeue {
			requeueRequested = true
		}
	}

	return requeueRequested, nil
}

func (lr *LaunchReconciler) reconcilePool(kluster *v1.Kluster, pool *models.NodePool) (status *PoolStatus, requeue bool, err error) {

	pm := lr.newPoolManager(kluster, pool)
	status, err = pm.GetStatus()
	if err != nil {
		return
	}

	switch {
	case kluster.Status.Phase == models.KlusterPhaseTerminating:
		for _, node := range status.Nodes {
			requeue = true
			if err = pm.DeleteNode(node); err != nil {
				return
			}
		}
		return
	case status.Needed > 0:
		for i := 0; i < int(status.Needed); i++ {
			requeue = true
			if _, err = pm.CreateNode(); err != nil {
				return
			}
		}
		return
	case status.UnNeeded > 0:
		for i := 0; i < int(status.UnNeeded); i++ {
			requeue = true
			if err = pm.DeleteNode(status.Nodes[i]); err != nil {
				return
			}
		}
		return
	case status.Starting > 0:
		requeue = true
	case status.Stopping > 0:
		requeue = true
	default:
		return
	}

	err = pm.SetStatus(status)
	return
}
<|endoftext|>


<commit_before>// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:build arm64
// +build arm64

package cpuid

func archSkipFeature(feature Feature, version hostos.Version) bool {
	return false
}
<commit_msg>Import hostos in ARM cpuid test.<commit_after>// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:build arm64
// +build arm64

package cpuid

import "gvisor.dev/gvisor/pkg/hostos"

func archSkipFeature(feature Feature, version hostos.Version) bool {
	return false
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package generator

import (
	"fmt"
	"time"

	"knative.dev/net-kourier/pkg/envoy"
	"knative.dev/net-kourier/pkg/knative"

	endpoint "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"

	"go.uber.org/zap"
	kubev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	kubeclient "k8s.io/client-go/kubernetes"
	corev1listers "k8s.io/client-go/listers/core/v1"
	"knative.dev/pkg/tracker"
	"knative.dev/serving/pkg/apis/networking/v1alpha1"

	v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2"
	route "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
)

type IngressTranslator struct {
	kubeclient      kubeclient.Interface
	endpointsLister corev1listers.EndpointsLister
	localDomainName string
	tracker         tracker.Interface
	logger          *zap.SugaredLogger
}

type translatedIngress struct {
	ingressName          string
	ingressNamespace     string
	sniMatches           []*envoy.SNIMatch
	routes               []*route.Route
	clusters             []*v2.Cluster
	externalVirtualHosts []*route.VirtualHost
	internalVirtualHosts []*route.VirtualHost
}

func NewIngressTranslator(kubeclient kubeclient.Interface, endpointsLister corev1listers.EndpointsLister, localDomainName string, tracker tracker.Interface,
	logger *zap.SugaredLogger) IngressTranslator {
	return IngressTranslator{
		kubeclient:      kubeclient,
		endpointsLister: endpointsLister,
		localDomainName: localDomainName,
		tracker:         tracker,
		logger:          logger,
	}
}

func (translator *IngressTranslator) translateIngress(ingress *v1alpha1.Ingress, extAuthzEnabled bool) (*translatedIngress, error) {
	res := &translatedIngress{
		ingressName:      ingress.Name,
		ingressNamespace: ingress.Namespace,
	}

	for _, ingressTLS := range ingress.Spec.TLS {
		sniMatch, err := sniMatchFromIngressTLS(ingressTLS, translator.kubeclient)

		if err != nil {
			translator.logger.Errorf("%s", err)

			// We need to propagate this error to the reconciler so the current
			// event can be retried. This error might be caused because the
			// secrets referenced in the TLS section of the spec do not exist
			// yet. That's expected when auto TLS is configured.
			// See the "TestPerKsvcCert_localCA" test in Knative Serving. It's a
			// test that fails if this error is not propagated:
			// https://github.com/knative/serving/blob/571e4db2392839082c559870ea8d4b72ef61e59d/test/e2e/autotls/auto_tls_test.go#L68
			return nil, err
		}
		res.sniMatches = append(res.sniMatches, sniMatch)
	}

	for _, rule := range ingress.Spec.Rules {

		var ruleRoute []*route.Route

		for _, httpPath := range rule.HTTP.Paths {

			path := "/"
			if httpPath.Path != "" {
				path = httpPath.Path
			}

			var wrs []*route.WeightedCluster_ClusterWeight

			for _, split := range httpPath.Splits {
				if err := trackService(translator.tracker, split.ServiceName, ingress); err != nil {
					return nil, err
				}

				endpoints, err := translator.endpointsLister.Endpoints(split.ServiceNamespace).Get(split.ServiceName)
				if apierrors.IsNotFound(err) {
					translator.logger.Warnf("Endpoints '%s/%s' not yet created", split.ServiceNamespace, split.ServiceName)
					break
				} else if err != nil {
					return nil, fmt.Errorf("failed to fetch endpoints '%s/%s': %w", split.ServiceNamespace, split.ServiceName, err)
				}

				service, err := translator.kubeclient.CoreV1().Services(split.ServiceNamespace).Get(split.ServiceName, metav1.GetOptions{})
				if apierrors.IsNotFound(err) {
					translator.logger.Warnf("Service '%s/%s' not yet created", split.ServiceNamespace, split.ServiceName)
					break
				} else if err != nil {
					return nil, fmt.Errorf("failed to fetch service '%s/%s': %w", split.ServiceNamespace, split.ServiceName, err)
				}

				var targetPort int32
				http2 := false
				for _, port := range service.Spec.Ports {
					if port.Port == split.ServicePort.IntVal || port.Name == split.ServicePort.StrVal {
						targetPort = port.TargetPort.IntVal
						http2 = port.Name == "http2" || port.Name == "h2c"
					}
				}

				publicLbEndpoints := lbEndpointsForKubeEndpoints(endpoints, targetPort)

				connectTimeout := 5 * time.Second
				cluster := envoy.NewCluster(split.ServiceName+path, connectTimeout, publicLbEndpoints, http2, v2.Cluster_STATIC)

				res.clusters = append(res.clusters, cluster)

				// Workaround: in some environments, split.Percent can be 0
				// instead of 100 when there's only one split. We need to set it
				// to 100 to avoid errors.
				clusterPercent := split.Percent
				if clusterPercent == 0 && len(httpPath.Splits) == 1 {
					clusterPercent = 100
				}

				weightedCluster := envoy.NewWeightedCluster(split.ServiceName+path, uint32(clusterPercent), split.AppendHeaders)

				wrs = append(wrs, weightedCluster)
			}

			if len(wrs) != 0 {
				r := createRouteForRevision(ingress.Name, ingress.Namespace, httpPath, wrs)
				ruleRoute = append(ruleRoute, r)
				res.routes = append(res.routes, r)
			}

		}

		if len(ruleRoute) == 0 {
			// Return nothing if there are not routes to generate.
			return nil, nil
		}

		externalDomains := knative.ExternalDomains(rule, translator.localDomainName)

		// External should also be accessible internally
		internalDomains := append(knative.InternalDomains(rule, translator.localDomainName), externalDomains...)

		var virtualHost, internalVirtualHost route.VirtualHost
		if extAuthzEnabled {

			visibility := ingress.Spec.Visibility
			if visibility == "" { // Needed because visibility is optional
				visibility = v1alpha1.IngressVisibilityClusterLocal
			}

			ContextExtensions := map[string]string{
				"client":     "kourier",
				"visibility": string(visibility),
			}

			ContextExtensions = mergeMapString(ContextExtensions, ingress.GetLabels())

			virtualHost = envoy.NewVirtualHostWithExtAuthz(ingress.Name, ContextExtensions, externalDomains, ruleRoute)
			internalVirtualHost = envoy.NewVirtualHostWithExtAuthz(ingress.Name, ContextExtensions, internalDomains,
				ruleRoute)
		} else {
			virtualHost = envoy.NewVirtualHost(ingress.GetName(), externalDomains, ruleRoute)
			internalVirtualHost = envoy.NewVirtualHost(ingress.GetName(), internalDomains, ruleRoute)
		}

		if knative.RuleIsExternal(rule, ingress.Spec.Visibility) {
			res.externalVirtualHosts = append(res.externalVirtualHosts, &virtualHost)
		}

		res.internalVirtualHosts = append(res.internalVirtualHosts, &internalVirtualHost)
	}

	return res, nil
}

func trackService(t tracker.Interface, svcName string, ingress *v1alpha1.Ingress) error {
	if err := t.TrackReference(tracker.Reference{
		Kind:       "Service",
		APIVersion: "v1",
		Namespace:  ingress.Namespace,
		Name:       svcName,
	}, ingress); err != nil {
		return fmt.Errorf("could not track service reference: %w", err)
	}

	if err := t.TrackReference(tracker.Reference{
		Kind:       "Endpoints",
		APIVersion: "v1",
		Namespace:  ingress.Namespace,
		Name:       svcName,
	}, ingress); err != nil {
		return fmt.Errorf("could not track endpoints reference: %w", err)
	}
	return nil
}

func lbEndpointsForKubeEndpoints(kubeEndpoints *kubev1.Endpoints, targetPort int32) (publicLbEndpoints []*endpoint.LbEndpoint) {
	for _, subset := range kubeEndpoints.Subsets {
		for _, address := range subset.Addresses {
			lbEndpoint := envoy.NewLBEndpoint(address.IP, uint32(targetPort))
			publicLbEndpoints = append(publicLbEndpoints, lbEndpoint)
		}
	}

	return publicLbEndpoints
}

func createRouteForRevision(ingressName string, ingressNamespace string, httpPath v1alpha1.HTTPIngressPath, wrs []*route.WeightedCluster_ClusterWeight) *route.Route {
	routeName := ingressName + "_" + ingressNamespace + "_" + httpPath.Path

	path := "/"
	if httpPath.Path != "" {
		path = httpPath.Path
	}

	var routeTimeout time.Duration
	if httpPath.Timeout != nil {
		routeTimeout = httpPath.Timeout.Duration
	}

	attempts := 0
	var perTryTimeout time.Duration
	if httpPath.Retries != nil {
		attempts = httpPath.Retries.Attempts

		if httpPath.Retries.PerTryTimeout != nil {
			perTryTimeout = httpPath.Retries.PerTryTimeout.Duration
		}
	}

	return envoy.NewRoute(
		routeName, path, wrs, routeTimeout, uint32(attempts), perTryTimeout, httpPath.AppendHeaders,
	)
}

func sniMatchFromIngressTLS(ingressTLS v1alpha1.IngressTLS, kubeClient kubeclient.Interface) (*envoy.SNIMatch, error) {
	certChain, privateKey, err := sslCreds(
		kubeClient, ingressTLS.SecretNamespace, ingressTLS.SecretName,
	)

	if err != nil {
		return nil, err
	}

	sniMatch := envoy.NewSNIMatch(ingressTLS.Hosts, certChain, privateKey)
	return &sniMatch, nil
}

func mergeMapString(a, b map[string]string) map[string]string {
	merged := make(map[string]string)
	for k, v := range a {
		merged[k] = v
	}
	for k, v := range b {
		merged[k] = v
	}
	return merged
}
<commit_msg>generator/ingress_translator: remove unnecessary workaround for split perc (#9)<commit_after>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package generator

import (
	"fmt"
	"time"

	"knative.dev/net-kourier/pkg/envoy"
	"knative.dev/net-kourier/pkg/knative"

	endpoint "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint"

	"go.uber.org/zap"
	kubev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	kubeclient "k8s.io/client-go/kubernetes"
	corev1listers "k8s.io/client-go/listers/core/v1"
	"knative.dev/pkg/tracker"
	"knative.dev/serving/pkg/apis/networking/v1alpha1"

	v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2"
	route "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
)

type IngressTranslator struct {
	kubeclient      kubeclient.Interface
	endpointsLister corev1listers.EndpointsLister
	localDomainName string
	tracker         tracker.Interface
	logger          *zap.SugaredLogger
}

type translatedIngress struct {
	ingressName          string
	ingressNamespace     string
	sniMatches           []*envoy.SNIMatch
	routes               []*route.Route
	clusters             []*v2.Cluster
	externalVirtualHosts []*route.VirtualHost
	internalVirtualHosts []*route.VirtualHost
}

func NewIngressTranslator(kubeclient kubeclient.Interface, endpointsLister corev1listers.EndpointsLister, localDomainName string, tracker tracker.Interface,
	logger *zap.SugaredLogger) IngressTranslator {
	return IngressTranslator{
		kubeclient:      kubeclient,
		endpointsLister: endpointsLister,
		localDomainName: localDomainName,
		tracker:         tracker,
		logger:          logger,
	}
}

func (translator *IngressTranslator) translateIngress(ingress *v1alpha1.Ingress, extAuthzEnabled bool) (*translatedIngress, error) {
	res := &translatedIngress{
		ingressName:      ingress.Name,
		ingressNamespace: ingress.Namespace,
	}

	for _, ingressTLS := range ingress.Spec.TLS {
		sniMatch, err := sniMatchFromIngressTLS(ingressTLS, translator.kubeclient)

		if err != nil {
			translator.logger.Errorf("%s", err)

			// We need to propagate this error to the reconciler so the current
			// event can be retried. This error might be caused because the
			// secrets referenced in the TLS section of the spec do not exist
			// yet. That's expected when auto TLS is configured.
			// See the "TestPerKsvcCert_localCA" test in Knative Serving. It's a
			// test that fails if this error is not propagated:
			// https://github.com/knative/serving/blob/571e4db2392839082c559870ea8d4b72ef61e59d/test/e2e/autotls/auto_tls_test.go#L68
			return nil, err
		}
		res.sniMatches = append(res.sniMatches, sniMatch)
	}

	for _, rule := range ingress.Spec.Rules {

		var ruleRoute []*route.Route

		for _, httpPath := range rule.HTTP.Paths {

			path := "/"
			if httpPath.Path != "" {
				path = httpPath.Path
			}

			var wrs []*route.WeightedCluster_ClusterWeight

			for _, split := range httpPath.Splits {
				if err := trackService(translator.tracker, split.ServiceName, ingress); err != nil {
					return nil, err
				}

				endpoints, err := translator.endpointsLister.Endpoints(split.ServiceNamespace).Get(split.ServiceName)
				if apierrors.IsNotFound(err) {
					translator.logger.Warnf("Endpoints '%s/%s' not yet created", split.ServiceNamespace, split.ServiceName)
					break
				} else if err != nil {
					return nil, fmt.Errorf("failed to fetch endpoints '%s/%s': %w", split.ServiceNamespace, split.ServiceName, err)
				}

				service, err := translator.kubeclient.CoreV1().Services(split.ServiceNamespace).Get(split.ServiceName, metav1.GetOptions{})
				if apierrors.IsNotFound(err) {
					translator.logger.Warnf("Service '%s/%s' not yet created", split.ServiceNamespace, split.ServiceName)
					break
				} else if err != nil {
					return nil, fmt.Errorf("failed to fetch service '%s/%s': %w", split.ServiceNamespace, split.ServiceName, err)
				}

				var targetPort int32
				http2 := false
				for _, port := range service.Spec.Ports {
					if port.Port == split.ServicePort.IntVal || port.Name == split.ServicePort.StrVal {
						targetPort = port.TargetPort.IntVal
						http2 = port.Name == "http2" || port.Name == "h2c"
					}
				}

				publicLbEndpoints := lbEndpointsForKubeEndpoints(endpoints, targetPort)

				connectTimeout := 5 * time.Second
				cluster := envoy.NewCluster(split.ServiceName+path, connectTimeout, publicLbEndpoints, http2, v2.Cluster_STATIC)

				res.clusters = append(res.clusters, cluster)

				weightedCluster := envoy.NewWeightedCluster(split.ServiceName+path, uint32(split.Percent), split.AppendHeaders)

				wrs = append(wrs, weightedCluster)
			}

			if len(wrs) != 0 {
				r := createRouteForRevision(ingress.Name, ingress.Namespace, httpPath, wrs)
				ruleRoute = append(ruleRoute, r)
				res.routes = append(res.routes, r)
			}

		}

		if len(ruleRoute) == 0 {
			// Return nothing if there are not routes to generate.
			return nil, nil
		}

		externalDomains := knative.ExternalDomains(rule, translator.localDomainName)

		// External should also be accessible internally
		internalDomains := append(knative.InternalDomains(rule, translator.localDomainName), externalDomains...)

		var virtualHost, internalVirtualHost route.VirtualHost
		if extAuthzEnabled {

			visibility := ingress.Spec.Visibility
			if visibility == "" { // Needed because visibility is optional
				visibility = v1alpha1.IngressVisibilityClusterLocal
			}

			ContextExtensions := map[string]string{
				"client":     "kourier",
				"visibility": string(visibility),
			}

			ContextExtensions = mergeMapString(ContextExtensions, ingress.GetLabels())

			virtualHost = envoy.NewVirtualHostWithExtAuthz(ingress.Name, ContextExtensions, externalDomains, ruleRoute)
			internalVirtualHost = envoy.NewVirtualHostWithExtAuthz(ingress.Name, ContextExtensions, internalDomains,
				ruleRoute)
		} else {
			virtualHost = envoy.NewVirtualHost(ingress.GetName(), externalDomains, ruleRoute)
			internalVirtualHost = envoy.NewVirtualHost(ingress.GetName(), internalDomains, ruleRoute)
		}

		if knative.RuleIsExternal(rule, ingress.Spec.Visibility) {
			res.externalVirtualHosts = append(res.externalVirtualHosts, &virtualHost)
		}

		res.internalVirtualHosts = append(res.internalVirtualHosts, &internalVirtualHost)
	}

	return res, nil
}

func trackService(t tracker.Interface, svcName string, ingress *v1alpha1.Ingress) error {
	if err := t.TrackReference(tracker.Reference{
		Kind:       "Service",
		APIVersion: "v1",
		Namespace:  ingress.Namespace,
		Name:       svcName,
	}, ingress); err != nil {
		return fmt.Errorf("could not track service reference: %w", err)
	}

	if err := t.TrackReference(tracker.Reference{
		Kind:       "Endpoints",
		APIVersion: "v1",
		Namespace:  ingress.Namespace,
		Name:       svcName,
	}, ingress); err != nil {
		return fmt.Errorf("could not track endpoints reference: %w", err)
	}
	return nil
}

func lbEndpointsForKubeEndpoints(kubeEndpoints *kubev1.Endpoints, targetPort int32) (publicLbEndpoints []*endpoint.LbEndpoint) {
	for _, subset := range kubeEndpoints.Subsets {
		for _, address := range subset.Addresses {
			lbEndpoint := envoy.NewLBEndpoint(address.IP, uint32(targetPort))
			publicLbEndpoints = append(publicLbEndpoints, lbEndpoint)
		}
	}

	return publicLbEndpoints
}

func createRouteForRevision(ingressName string, ingressNamespace string, httpPath v1alpha1.HTTPIngressPath, wrs []*route.WeightedCluster_ClusterWeight) *route.Route {
	routeName := ingressName + "_" + ingressNamespace + "_" + httpPath.Path

	path := "/"
	if httpPath.Path != "" {
		path = httpPath.Path
	}

	var routeTimeout time.Duration
	if httpPath.Timeout != nil {
		routeTimeout = httpPath.Timeout.Duration
	}

	attempts := 0
	var perTryTimeout time.Duration
	if httpPath.Retries != nil {
		attempts = httpPath.Retries.Attempts

		if httpPath.Retries.PerTryTimeout != nil {
			perTryTimeout = httpPath.Retries.PerTryTimeout.Duration
		}
	}

	return envoy.NewRoute(
		routeName, path, wrs, routeTimeout, uint32(attempts), perTryTimeout, httpPath.AppendHeaders,
	)
}

func sniMatchFromIngressTLS(ingressTLS v1alpha1.IngressTLS, kubeClient kubeclient.Interface) (*envoy.SNIMatch, error) {
	certChain, privateKey, err := sslCreds(
		kubeClient, ingressTLS.SecretNamespace, ingressTLS.SecretName,
	)

	if err != nil {
		return nil, err
	}

	sniMatch := envoy.NewSNIMatch(ingressTLS.Hosts, certChain, privateKey)
	return &sniMatch, nil
}

func mergeMapString(a, b map[string]string) map[string]string {
	merged := make(map[string]string)
	for k, v := range a {
		merged[k] = v
	}
	for k, v := range b {
		merged[k] = v
	}
	return merged
}
<|endoftext|>


<commit_before>package resolver

import (
	"sync"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/patrickmn/go-cache"

	"nimona.io/internal/rand"
	"nimona.io/pkg/context"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/hyperspace"
	"nimona.io/pkg/hyperspace/peerstore"
	"nimona.io/pkg/localpeer"
	"nimona.io/pkg/log"
	"nimona.io/pkg/network"
	"nimona.io/pkg/object"
	"nimona.io/pkg/peer"
)

var (
	hyperspaceAnnouncementType   = new(hyperspace.Announcement).Type()
	hyperspaceLookupResponseType = new(hyperspace.LookupResponse).Type()

	peerCacheTTL = 1 * time.Minute
)

const (
	ErrNoPeersToAsk = errors.Error("no peers to ask")
)

//go:generate mockgen -destination=../resolvermock/resolvermock_generated.go -package=resolvermock -source=resolver.go

type (
	Resolver interface {
		Lookup(
			ctx context.Context,
			opts ...LookupOption,
		) ([]*peer.ConnectionInfo, error)
	}
	resolver struct {
		context                        context.Context
		network                        network.Network
		localpeer                      localpeer.LocalPeer
		peerCache                      *peerstore.PeerCache
		localPeerAnnouncementCache     *hyperspace.Announcement
		localPeerAnnouncementCacheLock sync.RWMutex
		bootstrapPeers                 []*peer.ConnectionInfo
		blocklist                      *cache.Cache
	}
	// Option for customizing a new resolver
	Option func(*resolver)
)

// New returns a new resolver
func New(
	ctx context.Context,
	netw network.Network,
	opts ...Option,
) Resolver {
	r := &resolver{
		context: ctx,
		network: netw,
		peerCache: peerstore.NewPeerCache(
			time.Minute,
			"nimona_hyperspace_resolver",
		),
		localPeerAnnouncementCacheLock: sync.RWMutex{},
		bootstrapPeers:                 []*peer.ConnectionInfo{},
		blocklist:                      cache.New(time.Second*5, time.Second*60),
	}

	for _, opt := range opts {
		opt(r)
	}

	r.localpeer = r.network.LocalPeer()

	// we are listening for all incoming object types in order to learn about
	// new peers that are talking to us so we can announce ourselves to them
	go network.HandleEnvelopeSubscription(
		r.network.Subscribe(),
		r.handleObject,
	)

	for _, p := range r.bootstrapPeers {
		r.peerCache.Put(&hyperspace.Announcement{
			ConnectionInfo: p,
		}, 0)
	}

	go func() {
		r.announceSelf()
		announceTimer := time.NewTicker(30 * time.Second)
		for range announceTimer.C {
			r.announceSelf()
		}
	}()

	return r
}

// Lookup finds and returns peer infos from a fingerprint
// TODO consider returning peers synchronously
func (r *resolver) Lookup(
	ctx context.Context,
	opts ...LookupOption,
) ([]*peer.ConnectionInfo, error) {
	if len(r.bootstrapPeers) == 0 {
		return nil, errors.New("no peers to ask")
	}

	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.Lookup"),
	)
	logger.Debug("looking up")

	opt := ParseLookupOptions(opts...)
	bl := hyperspace.New(opt.Lookups...)

	// send content requests to recipients
	req := &hyperspace.LookupRequest{
		Metadata: object.Metadata{
			Owner: r.localpeer.GetPrimaryPeerKey().PublicKey(),
		},
		Nonce:       rand.String(12),
		QueryVector: bl,
	}
	reqObject := req.ToObject()

	// listen for lookup responses
	resSub := r.network.Subscribe(
		network.FilterByObjectType(hyperspaceLookupResponseType),
		func(e *network.Envelope) bool {
			v := e.Payload.Data["nonce:s"]
			rn, ok := v.(string)
			return ok && rn == req.Nonce
		},
	)

	go func() {
		for _, bp := range r.bootstrapPeers {
			err := r.network.Send(
				ctx,
				reqObject,
				bp,
			)
			if err != nil {
				logger.Debug("could send request to peer", log.Error(err))
				continue
			}
			logger.Debug(
				"asked peer",
				log.String("peer", bp.PublicKey.String()),
			)
		}
	}()

	// create channel to keep peers we find
	peers := []*peer.ConnectionInfo{}
	done := make(chan struct{})

	go func() {
		for {
			e, err := resSub.Next()
			if err != nil {
				break
			}
			r := &hyperspace.LookupResponse{}
			if err := r.FromObject(e.Payload); err != nil {
				continue
			}
			// TODO verify peer?
			for _, ann := range r.Announcements {
				peers = append(peers, ann.ConnectionInfo)
			}
			close(done)
			break
		}
	}()

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case <-done:
		return peers, nil
	}
}

func (r *resolver) handleObject(
	e *network.Envelope,
) error {
	// attempt to recover correlation id from request id
	ctx := r.context

	// handle payload
	o := e.Payload
	if o.Type == hyperspaceAnnouncementType {
		v := &hyperspace.Announcement{}
		if err := v.FromObject(o); err != nil {
			return err
		}
		r.handleAnnouncement(ctx, v)
	}
	return nil
}

func (r *resolver) handleAnnouncement(
	ctx context.Context,
	p *hyperspace.Announcement,
) {
	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.handleAnnouncement"),
		log.String("peer.publicKey", p.ConnectionInfo.PublicKey.String()),
		log.Strings("peer.addresses", p.ConnectionInfo.Addresses),
	)
	logger.Debug("adding peer to cache")
	r.peerCache.Put(p, peerCacheTTL)
}

func (r *resolver) announceSelf() {
	ctx := context.New(
		context.WithParent(r.context),
	)
	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.announceSelf"),
	)
	n := 0
	for _, p := range r.bootstrapPeers {
		if err := r.network.Send(
			context.New(
				context.WithParent(ctx),
				context.WithTimeout(time.Second*3),
			),
			r.getLocalPeerAnnouncement().ToObject(),
			p,
		); err != nil {
			logger.Error(
				"error announcing self to bootstrap",
				log.String("peer", p.PublicKey.String()),
				log.Error(err),
			)
			continue
		}
		n++
	}
	logger.Info("announced self to bootstrap peers", log.Int("n", n))
}

func (r *resolver) getLocalPeerAnnouncement() *hyperspace.Announcement {
	r.localPeerAnnouncementCacheLock.RLock()
	lastAnnouncement := r.localPeerAnnouncementCache
	r.localPeerAnnouncementCacheLock.RUnlock()

	peerKey := r.localpeer.GetPrimaryPeerKey().PublicKey()
	certificates := r.localpeer.GetCertificates()
	contentHashes := r.localpeer.GetContentHashes()
	contentTypes := r.localpeer.GetContentTypes()
	addresses := r.localpeer.GetAddresses()
	relays := r.localpeer.GetRelays()

	// gather up peer key, certificates, content ids and types
	hs := contentTypes
	hs = append(hs, peerKey.String())
	for _, c := range contentHashes {
		hs = append(hs, c.String())
	}
	for _, c := range certificates {
		if !c.Metadata.Signature.IsEmpty() {
			hs = append(hs, c.Metadata.Signature.Signer.String())
		}
	}
	vec := hyperspace.New(hs...)

	if lastAnnouncement != nil &&
		cmp.Equal(lastAnnouncement.ConnectionInfo.Addresses, addresses) &&
		cmp.Equal(lastAnnouncement.PeerVector, vec) {
		return lastAnnouncement
	}

	localPeerAnnouncementCache := &hyperspace.Announcement{
		Metadata: object.Metadata{
			Owner: peerKey,
		},
		Version: time.Now().Unix(),
		ConnectionInfo: &peer.ConnectionInfo{
			PublicKey: peerKey,
			Addresses: addresses,
			Relays:    relays,
		},
		PeerVector:       vec,
		PeerCapabilities: contentTypes,
	}

	r.localPeerAnnouncementCacheLock.Lock()
	r.localPeerAnnouncementCache = localPeerAnnouncementCache
	r.localPeerAnnouncementCacheLock.Unlock()

	return localPeerAnnouncementCache
}
<commit_msg>feat(resolver): listen for local peer updates and announce self<commit_after>package resolver

import (
	"sync"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/patrickmn/go-cache"

	"nimona.io/internal/rand"
	"nimona.io/pkg/context"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/hyperspace"
	"nimona.io/pkg/hyperspace/peerstore"
	"nimona.io/pkg/localpeer"
	"nimona.io/pkg/log"
	"nimona.io/pkg/network"
	"nimona.io/pkg/object"
	"nimona.io/pkg/peer"
)

var (
	hyperspaceAnnouncementType   = new(hyperspace.Announcement).Type()
	hyperspaceLookupResponseType = new(hyperspace.LookupResponse).Type()

	peerCacheTTL = 1 * time.Minute
)

const (
	ErrNoPeersToAsk = errors.Error("no peers to ask")
)

//go:generate mockgen -destination=../resolvermock/resolvermock_generated.go -package=resolvermock -source=resolver.go

type (
	Resolver interface {
		Lookup(
			ctx context.Context,
			opts ...LookupOption,
		) ([]*peer.ConnectionInfo, error)
	}
	resolver struct {
		context                        context.Context
		network                        network.Network
		localpeer                      localpeer.LocalPeer
		peerCache                      *peerstore.PeerCache
		localPeerAnnouncementCache     *hyperspace.Announcement
		localPeerAnnouncementCacheLock sync.RWMutex
		bootstrapPeers                 []*peer.ConnectionInfo
		blocklist                      *cache.Cache
	}
	// Option for customizing a new resolver
	Option func(*resolver)
)

// New returns a new resolver
func New(
	ctx context.Context,
	netw network.Network,
	opts ...Option,
) Resolver {
	r := &resolver{
		context: ctx,
		network: netw,
		peerCache: peerstore.NewPeerCache(
			time.Minute,
			"nimona_hyperspace_resolver",
		),
		localPeerAnnouncementCacheLock: sync.RWMutex{},
		bootstrapPeers:                 []*peer.ConnectionInfo{},
		blocklist:                      cache.New(time.Second*5, time.Second*60),
	}

	for _, opt := range opts {
		opt(r)
	}

	r.localpeer = r.network.LocalPeer()

	// we are listening for all incoming object types in order to learn about
	// new peers that are talking to us so we can announce ourselves to them
	go network.HandleEnvelopeSubscription(
		r.network.Subscribe(),
		r.handleObject,
	)

	for _, p := range r.bootstrapPeers {
		r.peerCache.Put(&hyperspace.Announcement{
			ConnectionInfo: p,
		}, 0)
	}

	go func() {
		r.announceSelf()
		announceOnUpdate, cf := r.localpeer.ListenForUpdates()
		defer cf()
		announceTimer := time.NewTicker(30 * time.Second)
		select {
		case <-announceTimer.C:
			r.announceSelf()
		case <-announceOnUpdate:
			r.announceSelf()
		}
	}()

	return r
}

// Lookup finds and returns peer infos from a fingerprint
// TODO consider returning peers synchronously
func (r *resolver) Lookup(
	ctx context.Context,
	opts ...LookupOption,
) ([]*peer.ConnectionInfo, error) {
	if len(r.bootstrapPeers) == 0 {
		return nil, errors.New("no peers to ask")
	}

	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.Lookup"),
	)
	logger.Debug("looking up")

	opt := ParseLookupOptions(opts...)
	bl := hyperspace.New(opt.Lookups...)

	// send content requests to recipients
	req := &hyperspace.LookupRequest{
		Metadata: object.Metadata{
			Owner: r.localpeer.GetPrimaryPeerKey().PublicKey(),
		},
		Nonce:       rand.String(12),
		QueryVector: bl,
	}
	reqObject := req.ToObject()

	// listen for lookup responses
	resSub := r.network.Subscribe(
		network.FilterByObjectType(hyperspaceLookupResponseType),
		func(e *network.Envelope) bool {
			v := e.Payload.Data["nonce:s"]
			rn, ok := v.(string)
			return ok && rn == req.Nonce
		},
	)

	go func() {
		for _, bp := range r.bootstrapPeers {
			err := r.network.Send(
				ctx,
				reqObject,
				bp,
			)
			if err != nil {
				logger.Debug("could send request to peer", log.Error(err))
				continue
			}
			logger.Debug(
				"asked peer",
				log.String("peer", bp.PublicKey.String()),
			)
		}
	}()

	// create channel to keep peers we find
	peers := []*peer.ConnectionInfo{}
	done := make(chan struct{})

	go func() {
		for {
			e, err := resSub.Next()
			if err != nil {
				break
			}
			r := &hyperspace.LookupResponse{}
			if err := r.FromObject(e.Payload); err != nil {
				continue
			}
			// TODO verify peer?
			for _, ann := range r.Announcements {
				peers = append(peers, ann.ConnectionInfo)
			}
			close(done)
			break
		}
	}()

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case <-done:
		return peers, nil
	}
}

func (r *resolver) handleObject(
	e *network.Envelope,
) error {
	// attempt to recover correlation id from request id
	ctx := r.context

	// handle payload
	o := e.Payload
	if o.Type == hyperspaceAnnouncementType {
		v := &hyperspace.Announcement{}
		if err := v.FromObject(o); err != nil {
			return err
		}
		r.handleAnnouncement(ctx, v)
	}
	return nil
}

func (r *resolver) handleAnnouncement(
	ctx context.Context,
	p *hyperspace.Announcement,
) {
	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.handleAnnouncement"),
		log.String("peer.publicKey", p.ConnectionInfo.PublicKey.String()),
		log.Strings("peer.addresses", p.ConnectionInfo.Addresses),
	)
	logger.Debug("adding peer to cache")
	r.peerCache.Put(p, peerCacheTTL)
}

func (r *resolver) announceSelf() {
	ctx := context.New(
		context.WithParent(r.context),
	)
	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.announceSelf"),
	)
	n := 0
	for _, p := range r.bootstrapPeers {
		if err := r.network.Send(
			context.New(
				context.WithParent(ctx),
				context.WithTimeout(time.Second*3),
			),
			r.getLocalPeerAnnouncement().ToObject(),
			p,
		); err != nil {
			logger.Error(
				"error announcing self to bootstrap",
				log.String("peer", p.PublicKey.String()),
				log.Error(err),
			)
			continue
		}
		n++
	}
	logger.Info(
		"announced self to bootstrap peers",
		log.Int("bootstrapPeers", n),
	)
}

func (r *resolver) getLocalPeerAnnouncement() *hyperspace.Announcement {
	r.localPeerAnnouncementCacheLock.RLock()
	lastAnnouncement := r.localPeerAnnouncementCache
	r.localPeerAnnouncementCacheLock.RUnlock()

	peerKey := r.localpeer.GetPrimaryPeerKey().PublicKey()
	certificates := r.localpeer.GetCertificates()
	contentHashes := r.localpeer.GetContentHashes()
	contentTypes := r.localpeer.GetContentTypes()
	addresses := r.localpeer.GetAddresses()
	relays := r.localpeer.GetRelays()

	// gather up peer key, certificates, content ids and types
	hs := contentTypes
	hs = append(hs, peerKey.String())
	for _, c := range contentHashes {
		hs = append(hs, c.String())
	}
	for _, c := range certificates {
		if !c.Metadata.Signature.IsEmpty() {
			hs = append(hs, c.Metadata.Signature.Signer.String())
		}
	}
	vec := hyperspace.New(hs...)

	if lastAnnouncement != nil &&
		cmp.Equal(lastAnnouncement.ConnectionInfo.Addresses, addresses) &&
		cmp.Equal(lastAnnouncement.PeerVector, vec) {
		return lastAnnouncement
	}

	localPeerAnnouncementCache := &hyperspace.Announcement{
		Metadata: object.Metadata{
			Owner: peerKey,
		},
		Version: time.Now().Unix(),
		ConnectionInfo: &peer.ConnectionInfo{
			PublicKey: peerKey,
			Addresses: addresses,
			Relays:    relays,
		},
		PeerVector:       vec,
		PeerCapabilities: contentTypes,
	}

	r.localPeerAnnouncementCacheLock.Lock()
	r.localPeerAnnouncementCache = localPeerAnnouncementCache
	r.localPeerAnnouncementCacheLock.Unlock()

	return localPeerAnnouncementCache
}
<|endoftext|>


<commit_before>// package hook is used to ensure execution of a set of pre-start, post-start,
// pre-stop and post-stop hooks. The transitioning between states is gated by
// the consumer of the package, through use of the Hooks.Transition function.
package hook

import (
	"fmt"
	"sync"

	"github.com/golang/glog"
	"github.com/jetstack/navigator/pkg/apis/navigator/v1alpha1"
)

type Interface interface {
	// The name of the Hook
	Name() string
	// Execute the action with the given Pilot resource
	Execute(pilot *v1alpha1.Pilot) error
}

func New(name string, fn func(*v1alpha1.Pilot) error) Interface {
	return &hookAdapter{
		name: name,
		fn:   fn,
	}
}

type hookAdapter struct {
	name string
	fn   func(pilot *v1alpha1.Pilot) error
}

var _ Interface = &hookAdapter{}

func (h *hookAdapter) Name() string {
	return h.name
}

func (h *hookAdapter) Execute(pilot *v1alpha1.Pilot) error {
	return h.fn(pilot)
}

type Hooks struct {
	// PreStart are hooks to be run before the application starts
	PreStart         []Interface
	executedPreStart map[string]struct{}
	// PostStart are hooks to be run after the application starts
	PostStart         []Interface
	executedPostStart map[string]struct{}
	// PreStop are hooks to be run before the application stops
	PreStop         []Interface
	executedPreStop map[string]struct{}
	// PostStop are hooks to be run after the application stops
	PostStop         []Interface
	executedPostStop map[string]struct{}

	lock sync.Mutex
}

// CurrentPhase returns the current phase
func (h *Hooks) CurrentPhase() v1alpha1.PilotPhase {
	switch {
	case len(h.executedPostStop) == len(h.PostStop):
		return v1alpha1.PilotPhasePostStop
	case len(h.executedPreStop) == len(h.PreStop):
		return v1alpha1.PilotPhasePreStop
	case len(h.executedPostStart) == len(h.PostStart):
		return v1alpha1.PilotPhasePostStart
	case len(h.executedPreStart) == len(h.PreStart):
		return v1alpha1.PilotPhasePreStart
	default:
		return ""
	}
}

func (h *Hooks) Transition(p v1alpha1.PilotPhase, pilot *v1alpha1.Pilot) error {
	h.lock.Lock()
	defer h.lock.Unlock()
	if pilot == nil {
		return fmt.Errorf("pilot resource must not be nil")
	}
	var hooks []Interface
	executed := map[string]struct{}{}
	switch p {
	case v1alpha1.PilotPhasePreStart:
		hooks = h.PreStart
		if h.executedPreStart == nil {
			h.executedPreStart = executed
		}
		executed = h.executedPreStart
	case v1alpha1.PilotPhasePostStart:
		hooks = h.PostStart
		if h.executedPostStart == nil {
			h.executedPostStart = executed
		}
		executed = h.executedPostStart
	case v1alpha1.PilotPhasePreStop:
		hooks = h.PreStop
		if h.executedPreStop == nil {
			h.executedPreStop = executed
		}
		executed = h.executedPreStop
	case v1alpha1.PilotPhasePostStop:
		hooks = h.PostStop
		if h.executedPostStop == nil {
			h.executedPostStop = executed
		}
		executed = h.executedPostStop
	default:
		return fmt.Errorf("invalid phase: %q", p)
	}
	for _, hook := range hooks {
		if _, ok := executed[hook.Name()]; ok {
			glog.V(4).Infof("Skipping already executed hook for %q in phase %q", hook.Name, p)
			continue
		}
		glog.V(4).Infof("Executing %s hook '%s'", p, hook.Name())
		if err := hook.Execute(pilot); err != nil {
			glog.V(4).Infof("Error executing %s hook %q: %s", p, hook.Name(), err.Error())
			return fmt.Errorf("error executing %s hook %q: %s", p, hook.Name(), err.Error())
		}
		glog.V(4).Infof("Executed %s hook %q", p, hook.Name())
		executed[hook.Name()] = struct{}{}
	}
	return nil
}
<commit_msg>Fix typo in hook debug message<commit_after>// package hook is used to ensure execution of a set of pre-start, post-start,
// pre-stop and post-stop hooks. The transitioning between states is gated by
// the consumer of the package, through use of the Hooks.Transition function.
package hook

import (
	"fmt"
	"sync"

	"github.com/golang/glog"
	"github.com/jetstack/navigator/pkg/apis/navigator/v1alpha1"
)

type Interface interface {
	// The name of the Hook
	Name() string
	// Execute the action with the given Pilot resource
	Execute(pilot *v1alpha1.Pilot) error
}

func New(name string, fn func(*v1alpha1.Pilot) error) Interface {
	return &hookAdapter{
		name: name,
		fn:   fn,
	}
}

type hookAdapter struct {
	name string
	fn   func(pilot *v1alpha1.Pilot) error
}

var _ Interface = &hookAdapter{}

func (h *hookAdapter) Name() string {
	return h.name
}

func (h *hookAdapter) Execute(pilot *v1alpha1.Pilot) error {
	return h.fn(pilot)
}

type Hooks struct {
	// PreStart are hooks to be run before the application starts
	PreStart         []Interface
	executedPreStart map[string]struct{}
	// PostStart are hooks to be run after the application starts
	PostStart         []Interface
	executedPostStart map[string]struct{}
	// PreStop are hooks to be run before the application stops
	PreStop         []Interface
	executedPreStop map[string]struct{}
	// PostStop are hooks to be run after the application stops
	PostStop         []Interface
	executedPostStop map[string]struct{}

	lock sync.Mutex
}

// CurrentPhase returns the current phase
func (h *Hooks) CurrentPhase() v1alpha1.PilotPhase {
	switch {
	case len(h.executedPostStop) == len(h.PostStop):
		return v1alpha1.PilotPhasePostStop
	case len(h.executedPreStop) == len(h.PreStop):
		return v1alpha1.PilotPhasePreStop
	case len(h.executedPostStart) == len(h.PostStart):
		return v1alpha1.PilotPhasePostStart
	case len(h.executedPreStart) == len(h.PreStart):
		return v1alpha1.PilotPhasePreStart
	default:
		return ""
	}
}

func (h *Hooks) Transition(p v1alpha1.PilotPhase, pilot *v1alpha1.Pilot) error {
	h.lock.Lock()
	defer h.lock.Unlock()
	if pilot == nil {
		return fmt.Errorf("pilot resource must not be nil")
	}
	var hooks []Interface
	executed := map[string]struct{}{}
	switch p {
	case v1alpha1.PilotPhasePreStart:
		hooks = h.PreStart
		if h.executedPreStart == nil {
			h.executedPreStart = executed
		}
		executed = h.executedPreStart
	case v1alpha1.PilotPhasePostStart:
		hooks = h.PostStart
		if h.executedPostStart == nil {
			h.executedPostStart = executed
		}
		executed = h.executedPostStart
	case v1alpha1.PilotPhasePreStop:
		hooks = h.PreStop
		if h.executedPreStop == nil {
			h.executedPreStop = executed
		}
		executed = h.executedPreStop
	case v1alpha1.PilotPhasePostStop:
		hooks = h.PostStop
		if h.executedPostStop == nil {
			h.executedPostStop = executed
		}
		executed = h.executedPostStop
	default:
		return fmt.Errorf("invalid phase: %q", p)
	}
	for _, hook := range hooks {
		if _, ok := executed[hook.Name()]; ok {
			glog.V(4).Infof("Skipping already executed hook for %q in phase %q", hook.Name(), p)
			continue
		}
		glog.V(4).Infof("Executing %s hook '%s'", p, hook.Name())
		if err := hook.Execute(pilot); err != nil {
			glog.V(4).Infof("Error executing %s hook %q: %s", p, hook.Name(), err.Error())
			return fmt.Errorf("error executing %s hook %q: %s", p, hook.Name(), err.Error())
		}
		glog.V(4).Infof("Executed %s hook %q", p, hook.Name())
		executed[hook.Name()] = struct{}{}
	}
	return nil
}
<|endoftext|>


<commit_before>package scheduler

import (
	"context"
	"sync"
	"testing"
	"time"

	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/stretchr/testify/assert"
)

func TestTriggersBadArguments(t *testing.T) {
	var err error
	_, err = NewTrigger(&TriggerInfos{
		Domain:    "cozy.local",
		Type:      "@at",
		Arguments: "garbage",
	})
	assert.Error(t, err)

	_, err = NewTrigger(&TriggerInfos{
		Type:      "@in",
		Arguments: "garbage",
	})
	assert.Error(t, err)

	_, err = NewTrigger(&TriggerInfos{
		Domain:    "cozy.local",
		Type:      "@unknown",
		Arguments: "",
	})
	if assert.Error(t, err) {
		assert.Equal(t, ErrUnknownTrigger, err)
	}
}

func TestMemSchedulerWithTimeTriggers(t *testing.T) {
	var wAt sync.WaitGroup
	var wIn sync.WaitGroup
	bro := jobs.NewMemBroker(1)
	bro.Start(jobs.WorkersList{
		"worker": {
			Concurrency:  1,
			MaxExecCount: 1,
			Timeout:      1 * time.Millisecond,
			WorkerFunc: func(ctx context.Context, m *jobs.Message) error {
				var msg string
				if err := m.Unmarshal(&msg); err != nil {
					return err
				}
				switch msg {
				case "@at":
					wAt.Done()
				case "@in":
					wIn.Done()
				}
				return nil
			},
		},
	})

	msg1, _ := jobs.NewMessage("json", "@at")
	msg2, _ := jobs.NewMessage("json", "@in")

	wAt.Add(1) // 1 time in @at
	wIn.Add(1) // 1 time in @in

	at := &TriggerInfos{
		Type:       "@at",
		Domain:     "cozy.local.memsched_timetriggers",
		Arguments:  time.Now().Add(2 * time.Second).Format(time.RFC3339),
		WorkerType: "worker",
		Message:    msg1,
	}
	in := &TriggerInfos{
		Domain:     "cozy.local.memsched_timetriggers",
		Type:       "@in",
		Arguments:  "1s",
		WorkerType: "worker",
		Message:    msg2,
	}

	triggers := []*TriggerInfos{at, in}
	sch := newMemScheduler()
	sch.Start(bro)

	for _, infos := range triggers {
		trigger, err := NewTrigger(infos)
		if !assert.NoError(t, err) {
			return
		}
		err = sch.Add(trigger)
		if !assert.NoError(t, err) {
			return
		}
	}

	ts, err := sch.GetAll("cozy.local.memsched_timetriggers")
	assert.NoError(t, err)
	assert.Len(t, ts, len(triggers))

	for _, trigger := range ts {
		switch trigger.Infos().Type {
		case "@at":
			assert.EqualValues(t, at, trigger.Infos())
		case "@in":
			assert.EqualValues(t, in, trigger.Infos())
		default:
			t.Fatalf("unknown trigger ID %s", trigger.Infos().TID)
		}
	}

	done := make(chan bool)
	go func() {
		wAt.Wait()
		done <- true
	}()

	go func() {
		wIn.Wait()
		done <- true
	}()

	for i := 0; i < len(ts); i++ {
		<-done
	}

	for _, trigger := range triggers {
		err := sch.Delete(trigger.Domain, trigger.TID)
		assert.NoError(t, err)
	}

	err = sch.Shutdown(context.Background())
	assert.NoError(t, err)
}

func TestMemSchedulerWithDebounce(t *testing.T) {
	called := 0
	bro := jobs.NewMemBroker(1)
	bro.Start(jobs.WorkersList{
		"worker": {
			Concurrency:  1,
			MaxExecCount: 1,
			Timeout:      1 * time.Millisecond,
			WorkerFunc: func(ctx context.Context, m *jobs.Message) error {
				called++
				return nil
			},
		},
	})

	msg, _ := jobs.NewMessage("json", "@event")
	ti := &TriggerInfos{
		Type:       "@event",
		Domain:     "cozy.local.withdebounce",
		Arguments:  "io.cozy.testdebounce",
		Debounce:   "100ms",
		WorkerType: "worker",
		Message:    msg,
	}

	triggers := []*TriggerInfos{ti}
	sch := newMemScheduler()
	sch.Start(bro)

	for _, infos := range triggers {
		trigger, err := NewTrigger(infos)
		if !assert.NoError(t, err) {
			return
		}
		err = sch.Add(trigger)
		if !assert.NoError(t, err) {
			return
		}
	}

	ts, err := sch.GetAll("cozy.local.withdebounce")
	assert.NoError(t, err)
	assert.Len(t, ts, len(triggers))

	doc := couchdb.JSONDoc{
		Type: "io.cozy.testdebounce",
		M: map[string]interface{}{
			"_id":  "test-id",
			"_rev": "1-xxabxx",
			"test": "value",
		},
	}
	event := &realtime.Event{
		Verb:   realtime.EventCreate,
		Doc:    &doc,
		Domain: "cozy.local.withdebounce",
	}

	for i := 0; i < 24; i++ {
		time.Sleep(10 * time.Millisecond)
		realtime.GetHub().Publish(event)
	}

	time.Sleep(150 * time.Millisecond)
	assert.Equal(t, 3, called)

	for _, trigger := range triggers {
		err := sch.Delete(trigger.Domain, trigger.TID)
		assert.NoError(t, err)
	}

	err = sch.Shutdown(context.Background())
	assert.NoError(t, err)
}
<commit_msg>Fix linter<commit_after>package scheduler

import (
	"context"
	"sync"
	"testing"
	"time"

	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/stretchr/testify/assert"
)

func TestTriggersBadArguments(t *testing.T) {
	var err error
	_, err = NewTrigger(&TriggerInfos{
		Domain:    "cozy.local",
		Type:      "@at",
		Arguments: "garbage",
	})
	assert.Error(t, err)

	_, err = NewTrigger(&TriggerInfos{
		Type:      "@in",
		Arguments: "garbage",
	})
	assert.Error(t, err)

	_, err = NewTrigger(&TriggerInfos{
		Domain:    "cozy.local",
		Type:      "@unknown",
		Arguments: "",
	})
	if assert.Error(t, err) {
		assert.Equal(t, ErrUnknownTrigger, err)
	}
}

func TestMemSchedulerWithTimeTriggers(t *testing.T) {
	var wAt sync.WaitGroup
	var wIn sync.WaitGroup
	bro := jobs.NewMemBroker(1)
	bro.Start(jobs.WorkersList{
		"worker": {
			Concurrency:  1,
			MaxExecCount: 1,
			Timeout:      1 * time.Millisecond,
			WorkerFunc: func(ctx context.Context, m *jobs.Message) error {
				var msg string
				if err := m.Unmarshal(&msg); err != nil {
					return err
				}
				switch msg {
				case "@at":
					wAt.Done()
				case "@in":
					wIn.Done()
				}
				return nil
			},
		},
	})

	msg1, _ := jobs.NewMessage("json", "@at")
	msg2, _ := jobs.NewMessage("json", "@in")

	wAt.Add(1) // 1 time in @at
	wIn.Add(1) // 1 time in @in

	at := &TriggerInfos{
		Type:       "@at",
		Domain:     "cozy.local.memsched_timetriggers",
		Arguments:  time.Now().Add(2 * time.Second).Format(time.RFC3339),
		WorkerType: "worker",
		Message:    msg1,
	}
	in := &TriggerInfos{
		Domain:     "cozy.local.memsched_timetriggers",
		Type:       "@in",
		Arguments:  "1s",
		WorkerType: "worker",
		Message:    msg2,
	}

	triggers := []*TriggerInfos{at, in}
	sch := newMemScheduler()
	sch.Start(bro)

	for _, infos := range triggers {
		trigger, err := NewTrigger(infos)
		if !assert.NoError(t, err) {
			return
		}
		err = sch.Add(trigger)
		if !assert.NoError(t, err) {
			return
		}
	}

	ts, err := sch.GetAll("cozy.local.memsched_timetriggers")
	assert.NoError(t, err)
	assert.Len(t, ts, len(triggers))

	for _, trigger := range ts {
		switch trigger.Infos().Type {
		case "@at":
			assert.EqualValues(t, at, trigger.Infos())
		case "@in":
			assert.EqualValues(t, in, trigger.Infos())
		default:
			t.Fatalf("unknown trigger ID %s", trigger.Infos().TID)
		}
	}

	done := make(chan bool)
	go func() {
		wAt.Wait()
		done <- true
	}()

	go func() {
		wIn.Wait()
		done <- true
	}()

	for i := 0; i < len(ts); i++ {
		<-done
	}

	for _, trigger := range triggers {
		err = sch.Delete(trigger.Domain, trigger.TID)
		assert.NoError(t, err)
	}

	err = sch.Shutdown(context.Background())
	assert.NoError(t, err)
}

func TestMemSchedulerWithDebounce(t *testing.T) {
	called := 0
	bro := jobs.NewMemBroker(1)
	bro.Start(jobs.WorkersList{
		"worker": {
			Concurrency:  1,
			MaxExecCount: 1,
			Timeout:      1 * time.Millisecond,
			WorkerFunc: func(ctx context.Context, m *jobs.Message) error {
				called++
				return nil
			},
		},
	})

	msg, _ := jobs.NewMessage("json", "@event")
	ti := &TriggerInfos{
		Type:       "@event",
		Domain:     "cozy.local.withdebounce",
		Arguments:  "io.cozy.testdebounce",
		Debounce:   "100ms",
		WorkerType: "worker",
		Message:    msg,
	}

	triggers := []*TriggerInfos{ti}
	sch := newMemScheduler()
	sch.Start(bro)

	for _, infos := range triggers {
		trigger, err := NewTrigger(infos)
		if !assert.NoError(t, err) {
			return
		}
		err = sch.Add(trigger)
		if !assert.NoError(t, err) {
			return
		}
	}

	ts, err := sch.GetAll("cozy.local.withdebounce")
	assert.NoError(t, err)
	assert.Len(t, ts, len(triggers))

	doc := couchdb.JSONDoc{
		Type: "io.cozy.testdebounce",
		M: map[string]interface{}{
			"_id":  "test-id",
			"_rev": "1-xxabxx",
			"test": "value",
		},
	}
	event := &realtime.Event{
		Verb:   realtime.EventCreate,
		Doc:    &doc,
		Domain: "cozy.local.withdebounce",
	}

	for i := 0; i < 24; i++ {
		time.Sleep(10 * time.Millisecond)
		realtime.GetHub().Publish(event)
	}

	time.Sleep(150 * time.Millisecond)
	assert.Equal(t, 3, called)

	for _, trigger := range triggers {
		err = sch.Delete(trigger.Domain, trigger.TID)
		assert.NoError(t, err)
	}

	err = sch.Shutdown(context.Background())
	assert.NoError(t, err)
}
<|endoftext|>


<commit_before>/*
 * Minimalist Object Storage, (C) 2014 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package httpserver

import (
	"fmt"
	"net"
	"net/http"
	"strings"
)

// Config - http server config
type Config struct {
	Address   string
	TLS       bool
	CertFile  string
	KeyFile   string
	RateLimit int
}

// Server - http server related
type Server struct{}

// Start http server
func Start(handler http.Handler, config Config) (chan<- string, <-chan error, *Server) {
	ctrlChannel := make(chan string)
	errorChannel := make(chan error)
	server := Server{}
	go start(ctrlChannel, errorChannel, handler, config, &server)
	return ctrlChannel, errorChannel, &server
}

func start(ctrlChannel <-chan string, errorChannel chan<- error, router http.Handler, config Config, server *Server) {
	defer close(errorChannel)

	var err error
	// Minio server config
	httpServer := &http.Server{
		Addr:           config.Address,
		Handler:        router,
		MaxHeaderBytes: 1 << 20,
	}

	host, port, err := net.SplitHostPort(config.Address)
	if err != nil {
		errorChannel <- err
		return
	}

	var hosts []string
	switch {
	case host != "":
		hosts = append(hosts, host)
	default:
		addrs, err := net.InterfaceAddrs()
		errorChannel <- err
		for _, addr := range addrs {
			if addr.Network() == "ip+net" {
				host := strings.Split(addr.String(), "/")[0]
				if ip := net.ParseIP(host); ip.To4() != nil {
					hosts = append(hosts, host)
				}
			}
		}
	}
	switch {
	default:
		for _, host := range hosts {
			fmt.Printf("Starting minio server on: http://%s:%s\n", host, port)
		}
		err = httpServer.ListenAndServe()
	case config.TLS == true:
		for _, host := range hosts {
			fmt.Printf("Starting minio server on: https://%s:%s\n", host, port)
		}
		err = httpServer.ListenAndServeTLS(config.CertFile, config.KeyFile)
	}
	if err != nil {
		errorChannel <- err
	}

}
<commit_msg>Expand http server struct to store more values<commit_after>/*
 * Minimalist Object Storage, (C) 2014 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package httpserver

import (
	"fmt"
	"net"
	"net/http"
	"strings"
)

// Config - http server config
type Config struct {
	Address   string
	TLS       bool
	CertFile  string
	KeyFile   string
	RateLimit int
}

// Server - http server related
type Server struct {
	config  Config
	handler http.Handler
}

// Start http server
func Start(handler http.Handler, config Config) (chan<- string, <-chan error, Server) {
	ctrlChannel := make(chan string)
	errorChannel := make(chan error)
	server := Server{
		config:  config,
		handler: handler,
	}
	go start(ctrlChannel, errorChannel, server)
	return ctrlChannel, errorChannel, server
}

func start(ctrlChannel <-chan string, errorChannel chan<- error, server Server) {
	defer close(errorChannel)

	var err error
	// Minio server config
	httpServer := &http.Server{
		Addr:           server.config.Address,
		Handler:        server.handler,
		MaxHeaderBytes: 1 << 20,
	}

	host, port, err := net.SplitHostPort(server.config.Address)
	if err != nil {
		errorChannel <- err
		return
	}

	var hosts []string
	switch {
	case host != "":
		hosts = append(hosts, host)
	default:
		addrs, err := net.InterfaceAddrs()
		if err != nil {
			errorChannel <- err
			return
		}
		for _, addr := range addrs {
			if addr.Network() == "ip+net" {
				host := strings.Split(addr.String(), "/")[0]
				if ip := net.ParseIP(host); ip.To4() != nil {
					hosts = append(hosts, host)
				}
			}
		}
	}
	switch {
	default:
		for _, host := range hosts {
			fmt.Printf("Starting minio server on: http://%s:%s\n", host, port)
		}
		err = httpServer.ListenAndServe()
	case server.config.TLS == true:
		for _, host := range hosts {
			fmt.Printf("Starting minio server on: https://%s:%s\n", host, port)
		}
		err = httpServer.ListenAndServeTLS(server.config.CertFile, server.config.KeyFile)
	}
	if err != nil {
		errorChannel <- err
	}

}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package acr

import (
	"bufio"
	"context"
	"io"
	"net/http"
	"regexp"
	"time"

	cr "github.com/Azure/azure-sdk-for-go/services/containerregistry/mgmt/2018-09-01/containerregistry"
	"github.com/Azure/go-autorest/autorest/azure/auth"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/tag"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/pkg/errors"
)

const BuildStatusHeader = "x-ms-meta-Complete"

func (b *Builder) Build(ctx context.Context, out io.Writer, tagger tag.Tagger, artifacts []*latest.Artifact) ([]build.Artifact, error) {
	return build.InParallel(ctx, out, tagger, artifacts, b.buildArtifact)
}

func (b *Builder) buildArtifact(ctx context.Context, out io.Writer, tagger tag.Tagger, artifact *latest.Artifact) (string, error) {
	client := cr.NewRegistriesClient(b.Credentials.SubscriptionID)
	authorizer, err := auth.NewClientCredentialsConfig(b.Credentials.ClientID, b.Credentials.ClientSecret, b.Credentials.TenantID).Authorizer()
	if err != nil {
		return "", errors.Wrap(err, "authorizing client")
	}
	client.Authorizer = authorizer

	result, err := client.GetBuildSourceUploadURL(ctx, b.ResourceGroup, b.ContainerRegistry)
	if err != nil {
		return "", errors.Wrap(err, "build source upload url")
	}
	blob := NewBlobStorage(*result.UploadURL)

	err = docker.CreateDockerTarGzContext(blob.Buffer, artifact.Workspace, artifact.DockerArtifact)
	if err != nil {
		return "", errors.Wrap(err, "create context tar.gz")
	}

	err = blob.UploadFileToBlob()
	if err != nil {
		return "", errors.Wrap(err, "upload file to blob")
	}

	imageTag, err := tagger.GenerateFullyQualifiedImageName(artifact.Workspace, &tag.Options{
		Digest:    util.RandomID(),
		ImageName: artifact.ImageName,
	})
	if err != nil {
		return "", errors.Wrap(err, "create fully qualified image name")
	}

	imageTag, err = getImageTagWithoutFQDN(imageTag)
	if err != nil {
		return "", errors.Wrap(err, "get azure image tag")
	}

	buildRequest := cr.DockerBuildRequest{
		ImageNames:     &[]string{imageTag},
		IsPushEnabled:  util.BoolPtr(true),
		SourceLocation: result.RelativePath,
		Platform: &cr.PlatformProperties{
			Variant:      cr.V8,
			Os:           cr.Linux,
			Architecture: cr.Amd64,
		},
		DockerFilePath: &artifact.DockerArtifact.DockerfilePath,
		Type:           cr.TypeDockerBuildRequest,
	}
	future, err := client.ScheduleRun(ctx, b.ResourceGroup, b.ContainerRegistry, buildRequest)
	if err != nil {
		return "", errors.Wrap(err, "schedule build request")
	}

	run, err := future.Result(client)
	if err != nil {
		return "", errors.Wrap(err, "get run id")
	}
	runID := *run.RunID

	runsClient := cr.NewRunsClient(b.Credentials.SubscriptionID)
	runsClient.Authorizer = client.Authorizer
	logURL, err := runsClient.GetLogSasURL(ctx, b.ResourceGroup, b.ContainerRegistry, runID)
	if err != nil {
		return "", errors.Wrap(err, "get log url")
	}

	err = streamBuildLogs(*logURL.LogLink, out)
	if err != nil {
		return "", errors.Wrap(err, "polling build status")
	}

	return imageTag, nil
}

func streamBuildLogs(logURL string, out io.Writer) error {
	offset := int32(0)
	for {
		resp, err := http.Get(logURL)
		if err != nil {
			return err
		}

		if resp.StatusCode == http.StatusNotFound {
			//if blob is not available yet, try again
			time.Sleep(2 * time.Second)
			continue
		}

		scanner := bufio.NewScanner(resp.Body)
		line := int32(0)
		for scanner.Scan() {
			if line >= offset {
				out.Write(scanner.Bytes())
				out.Write([]byte("\n"))
				offset++
			}
			line++
		}
		resp.Body.Close()

		if offset > 0 {
			switch resp.Header.Get(BuildStatusHeader) {
			case "":
				continue
			case "internalerror":
			case "failed":
				return errors.New("run failed")
			case "timedout":
				return errors.New("run timed out")
			case "canceled":
				return errors.New("run was canceled")
			default:
				return nil
			}
		}

		time.Sleep(2 * time.Second)
	}
}

// ACR needs the image tag in the following format
// <repository>:<tag>
func getImageTagWithoutFQDN(imageTag string) (string, error) {
	r, err := regexp.Compile(`.*\..*\..*/(.*)`)
	if err != nil {
		return "", errors.Wrap(err, "create regexp")
	}

	matches := r.FindStringSubmatch(imageTag)
	if len(matches) < 2 {
		return "", errors.New("invalid image tag")
	}

	return matches[1], nil
}
<commit_msg>Simplified removing FQDN from image tag<commit_after>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package acr

import (
	"bufio"
	"context"
	"io"
	"net/http"
	"strings"
	"time"

	cr "github.com/Azure/azure-sdk-for-go/services/containerregistry/mgmt/2018-09-01/containerregistry"
	"github.com/Azure/go-autorest/autorest/azure/auth"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/tag"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/pkg/errors"
)

const BuildStatusHeader = "x-ms-meta-Complete"

func (b *Builder) Build(ctx context.Context, out io.Writer, tagger tag.Tagger, artifacts []*latest.Artifact) ([]build.Artifact, error) {
	return build.InParallel(ctx, out, tagger, artifacts, b.buildArtifact)
}

func (b *Builder) buildArtifact(ctx context.Context, out io.Writer, tagger tag.Tagger, artifact *latest.Artifact) (string, error) {
	client := cr.NewRegistriesClient(b.Credentials.SubscriptionID)
	authorizer, err := auth.NewClientCredentialsConfig(b.Credentials.ClientID, b.Credentials.ClientSecret, b.Credentials.TenantID).Authorizer()
	if err != nil {
		return "", errors.Wrap(err, "authorizing client")
	}
	client.Authorizer = authorizer

	result, err := client.GetBuildSourceUploadURL(ctx, b.ResourceGroup, b.ContainerRegistry)
	if err != nil {
		return "", errors.Wrap(err, "build source upload url")
	}
	blob := NewBlobStorage(*result.UploadURL)

	err = docker.CreateDockerTarGzContext(blob.Buffer, artifact.Workspace, artifact.DockerArtifact)
	if err != nil {
		return "", errors.Wrap(err, "create context tar.gz")
	}

	err = blob.UploadFileToBlob()
	if err != nil {
		return "", errors.Wrap(err, "upload file to blob")
	}

	imageTag, err := tagger.GenerateFullyQualifiedImageName(artifact.Workspace, &tag.Options{
		Digest:    util.RandomID(),
		ImageName: artifact.ImageName,
	})
	if err != nil {
		return "", errors.Wrap(err, "create fully qualified image name")
	}

	//remove fqdn from image tag since we only need <repository>:<tag>
	imageTag = imageTag[strings.Index(imageTag, "/")+1:]

	buildRequest := cr.DockerBuildRequest{
		ImageNames:     &[]string{imageTag},
		IsPushEnabled:  util.BoolPtr(true),
		SourceLocation: result.RelativePath,
		Platform: &cr.PlatformProperties{
			Variant:      cr.V8,
			Os:           cr.Linux,
			Architecture: cr.Amd64,
		},
		DockerFilePath: &artifact.DockerArtifact.DockerfilePath,
		Type:           cr.TypeDockerBuildRequest,
	}
	future, err := client.ScheduleRun(ctx, b.ResourceGroup, b.ContainerRegistry, buildRequest)
	if err != nil {
		return "", errors.Wrap(err, "schedule build request")
	}

	run, err := future.Result(client)
	if err != nil {
		return "", errors.Wrap(err, "get run id")
	}
	runID := *run.RunID

	runsClient := cr.NewRunsClient(b.Credentials.SubscriptionID)
	runsClient.Authorizer = client.Authorizer
	logURL, err := runsClient.GetLogSasURL(ctx, b.ResourceGroup, b.ContainerRegistry, runID)
	if err != nil {
		return "", errors.Wrap(err, "get log url")
	}

	err = streamBuildLogs(*logURL.LogLink, out)
	if err != nil {
		return "", errors.Wrap(err, "polling build status")
	}

	return imageTag, nil
}

func streamBuildLogs(logURL string, out io.Writer) error {
	offset := int32(0)
	for {
		resp, err := http.Get(logURL)
		if err != nil {
			return err
		}

		if resp.StatusCode == http.StatusNotFound {
			//if blob is not available yet, try again
			time.Sleep(2 * time.Second)
			continue
		}

		scanner := bufio.NewScanner(resp.Body)
		line := int32(0)
		for scanner.Scan() {
			if line >= offset {
				out.Write(scanner.Bytes())
				out.Write([]byte("\n"))
				offset++
			}
			line++
		}
		resp.Body.Close()

		if offset > 0 {
			switch resp.Header.Get(BuildStatusHeader) {
			case "":
				continue
			case "internalerror":
			case "failed":
				return errors.New("run failed")
			case "timedout":
				return errors.New("run timed out")
			case "canceled":
				return errors.New("run was canceled")
			default:
				return nil
			}
		}

		time.Sleep(2 * time.Second)
	}
}
<|endoftext|>


<commit_before>package ble

import (
	"fmt"
	"github.com/currantlabs/gatt"
	"github.com/hybridgroup/gobot"
	"log"
	"strings"
)

var _ gobot.Adaptor = (*BLEClientAdaptor)(nil)

// Represents a Client Connection to a BLE Peripheral
type BLEClientAdaptor struct {
	name       string
	uuid       string
	device     gatt.Device
	peripheral gatt.Peripheral
	services   map[string]*BLEService
	connected  bool
	ready      chan struct{}
}

// NewBLEClientAdaptor returns a new BLEClientAdaptor given a name and uuid
func NewBLEClientAdaptor(name string, uuid string) *BLEClientAdaptor {
	return &BLEClientAdaptor{
		name:      name,
		uuid:      uuid,
		connected: false,
		ready:     make(chan struct{}),
		services:  make(map[string]*BLEService),
	}
}

func (b *BLEClientAdaptor) Name() string                { return b.name }
func (b *BLEClientAdaptor) UUID() string                { return b.uuid }
func (b *BLEClientAdaptor) Peripheral() gatt.Peripheral { return b.peripheral }

// Connect initiates a connection to the BLE peripheral. Returns true on successful connection.
func (b *BLEClientAdaptor) Connect() (errs []error) {
	device, err := gatt.NewDevice(DefaultClientOptions...)
	if err != nil {
		log.Fatalf("Failed to open BLE device, err: %s\n", err)
		return
	}

	b.device = device

	// Register handlers.
	device.Handle(
		gatt.PeripheralDiscovered(b.DiscoveryHandler),
		gatt.PeripheralConnected(b.ConnectHandler),
		gatt.PeripheralDisconnected(b.DisconnectHandler),
	)

	device.Init(b.StateChangeHandler)
	<-b.ready
	// TODO: make sure peripheral currently exists for this UUID before returning
	return nil
}

// Reconnect attempts to reconnect to the BLE peripheral. If it has an active connection
// it will first close that connection and then establish a new connection.
// Returns true on Successful reconnection
func (b *BLEClientAdaptor) Reconnect() (errs []error) {
	if b.connected {
		b.Disconnect()
	}
	return b.Connect()
}

// Disconnect terminates the connection to the BLE peripheral. Returns true on successful disconnect.
func (b *BLEClientAdaptor) Disconnect() (errs []error) {
	b.peripheral.Device().CancelConnection(b.peripheral)

	return
}

// Finalize finalizes the BLEAdaptor
func (b *BLEClientAdaptor) Finalize() (errs []error) {
	return b.Disconnect()
}

// ReadCharacteristic returns bytes from the BLE device for the
// requested service and characteristic
func (b *BLEClientAdaptor) ReadCharacteristic(sUUID string, cUUID string) (data []byte, err error) {
	if !b.connected {
		log.Fatalf("Cannot read from BLE device until connected")
		return
	}

	characteristic := b.lookupCharacteristic(sUUID, cUUID)
	if characteristic == nil {
		log.Println("Cannot read from unknown characteristic")
		return
	}

	val, err := b.peripheral.ReadCharacteristic(characteristic)
	if err != nil {
		fmt.Printf("Failed to read characteristic, err: %s\n", err)
		return nil, err
	}

	return val, nil
}

// WriteCharacteristic writes bytes to the BLE device for the
// requested service and characteristic
func (b *BLEClientAdaptor) WriteCharacteristic(sUUID string, cUUID string, data []byte) (err error) {
	if !b.connected {
		log.Fatalf("Cannot write to BLE device until connected")
		return
	}

	characteristic := b.lookupCharacteristic(sUUID, cUUID)
	if characteristic == nil {
		log.Println("Cannot write to unknown characteristic")
		return
	}

	err = b.peripheral.WriteCharacteristic(characteristic, data, true)
	if err != nil {
		fmt.Printf("Failed to write characteristic, err: %s\n", err)
		return err
	}

	return
}

// Subscribe subscribes to notifications from the BLE device for the
// requested service and characteristic
func (b *BLEClientAdaptor) Subscribe(sUUID string, cUUID string, f func([]byte, error)) (err error) {
	if !b.connected {
		log.Fatalf("Cannot subscribe to BLE device until connected")
		return
	}

	characteristic := b.lookupCharacteristic(sUUID, cUUID)
	if characteristic == nil {
		log.Println("Cannot subscribe to unknown characteristic")
		return
	}

	fn := func(c *gatt.Characteristic, b []byte, err error) {
		f(b, err)
	}

	err = b.peripheral.SetNotifyValue(characteristic, fn)
	if err != nil {
		fmt.Printf("Failed to subscribe to characteristic, err: %s\n", err)
		return err
	}

	return
}

func (b *BLEClientAdaptor) StateChangeHandler(d gatt.Device, s gatt.State) {
	fmt.Println("State:", s)
	switch s {
	case gatt.StatePoweredOn:
		fmt.Println("scanning...")
		d.Scan([]gatt.UUID{}, false)
		return
	default:
		d.StopScanning()
	}
}

func (b *BLEClientAdaptor) DiscoveryHandler(p gatt.Peripheral, a *gatt.Advertisement, rssi int) {
	// try looking by local name
	if a.LocalName == b.UUID() {
		b.uuid = p.ID()
	} else {
		// try looking by ID
		id := strings.ToUpper(b.UUID())
		if strings.ToUpper(p.ID()) != id {
			return
		}
	}

	// Stop scanning once we've got the peripheral we're looking for.
	p.Device().StopScanning()

	// and connect to it
	p.Device().Connect(p)
}

func (b *BLEClientAdaptor) ConnectHandler(p gatt.Peripheral, err error) {
	fmt.Printf("\nConnected Peripheral ID:%s, NAME:(%s)\n", p.ID(), p.Name())

	b.peripheral = p

	if err := p.SetMTU(250); err != nil {
		fmt.Printf("Failed to set MTU, err: %s\n", err)
	}

	ss, err := p.DiscoverServices(nil)
	if err != nil {
		fmt.Printf("Failed to discover services, err: %s\n", err)
		return
	}

	for _, s := range ss {
		b.services[s.UUID().String()] = NewBLEService(s.UUID().String(), s)

		cs, err := p.DiscoverCharacteristics(nil, s)
		if err != nil {
			fmt.Printf("Failed to discover characteristics, err: %s\n", err)
			continue
		}

		for _, c := range cs {
			b.services[s.UUID().String()].characteristics[c.UUID().String()] = c
		}
	}

	b.connected = true
	close(b.ready)
}

func (b *BLEClientAdaptor) DisconnectHandler(p gatt.Peripheral, err error) {
	fmt.Println("Disconnected")
}

// Finalize finalizes the BLEAdaptor
func (b *BLEClientAdaptor) lookupCharacteristic(sUUID string, cUUID string) *gatt.Characteristic {
	service := b.services[sUUID]
	if service == nil {
		log.Printf("Unknown service ID: %s\n", sUUID)
		return nil
	}

	characteristic := service.characteristics[cUUID]
	if characteristic == nil {
		log.Printf("Unknown characteristic ID: %s\n", cUUID)
		return nil
	}

	return characteristic
}

// Represents a BLE Peripheral's Service
type BLEService struct {
	uuid            string
	service         *gatt.Service
	characteristics map[string]*gatt.Characteristic
}

// NewBLEAdaptor returns a new BLEService given a uuid
func NewBLEService(sUuid string, service *gatt.Service) *BLEService {
	return &BLEService{
		uuid:            sUuid,
		service:         service,
		characteristics: make(map[string]*gatt.Characteristic),
	}
}
<commit_msg>ble: populate descriptors after descovering characterisitcs<commit_after>package ble

import (
	"fmt"
	"log"
	"strings"

	"github.com/currantlabs/gatt"
	"github.com/hybridgroup/gobot"
)

var _ gobot.Adaptor = (*BLEClientAdaptor)(nil)

// Represents a Client Connection to a BLE Peripheral
type BLEClientAdaptor struct {
	name       string
	uuid       string
	device     gatt.Device
	peripheral gatt.Peripheral
	services   map[string]*BLEService
	connected  bool
	ready      chan struct{}
}

// NewBLEClientAdaptor returns a new BLEClientAdaptor given a name and uuid
func NewBLEClientAdaptor(name string, uuid string) *BLEClientAdaptor {
	return &BLEClientAdaptor{
		name:      name,
		uuid:      uuid,
		connected: false,
		ready:     make(chan struct{}),
		services:  make(map[string]*BLEService),
	}
}

func (b *BLEClientAdaptor) Name() string                { return b.name }
func (b *BLEClientAdaptor) UUID() string                { return b.uuid }
func (b *BLEClientAdaptor) Peripheral() gatt.Peripheral { return b.peripheral }

// Connect initiates a connection to the BLE peripheral. Returns true on successful connection.
func (b *BLEClientAdaptor) Connect() (errs []error) {
	device, err := gatt.NewDevice(DefaultClientOptions...)
	if err != nil {
		log.Fatalf("Failed to open BLE device, err: %s\n", err)
		return
	}

	b.device = device

	// Register handlers.
	device.Handle(
		gatt.PeripheralDiscovered(b.DiscoveryHandler),
		gatt.PeripheralConnected(b.ConnectHandler),
		gatt.PeripheralDisconnected(b.DisconnectHandler),
	)

	device.Init(b.StateChangeHandler)
	<-b.ready
	// TODO: make sure peripheral currently exists for this UUID before returning
	return nil
}

// Reconnect attempts to reconnect to the BLE peripheral. If it has an active connection
// it will first close that connection and then establish a new connection.
// Returns true on Successful reconnection
func (b *BLEClientAdaptor) Reconnect() (errs []error) {
	if b.connected {
		b.Disconnect()
	}
	return b.Connect()
}

// Disconnect terminates the connection to the BLE peripheral. Returns true on successful disconnect.
func (b *BLEClientAdaptor) Disconnect() (errs []error) {
	b.peripheral.Device().CancelConnection(b.peripheral)

	return
}

// Finalize finalizes the BLEAdaptor
func (b *BLEClientAdaptor) Finalize() (errs []error) {
	return b.Disconnect()
}

// ReadCharacteristic returns bytes from the BLE device for the
// requested service and characteristic
func (b *BLEClientAdaptor) ReadCharacteristic(sUUID string, cUUID string) (data []byte, err error) {
	if !b.connected {
		log.Fatalf("Cannot read from BLE device until connected")
		return
	}

	characteristic := b.lookupCharacteristic(sUUID, cUUID)
	if characteristic == nil {
		log.Println("Cannot read from unknown characteristic")
		return
	}

	val, err := b.peripheral.ReadCharacteristic(characteristic)
	if err != nil {
		fmt.Printf("Failed to read characteristic, err: %s\n", err)
		return nil, err
	}

	return val, nil
}

// WriteCharacteristic writes bytes to the BLE device for the
// requested service and characteristic
func (b *BLEClientAdaptor) WriteCharacteristic(sUUID string, cUUID string, data []byte) (err error) {
	if !b.connected {
		log.Fatalf("Cannot write to BLE device until connected")
		return
	}

	characteristic := b.lookupCharacteristic(sUUID, cUUID)
	if characteristic == nil {
		log.Println("Cannot write to unknown characteristic")
		return
	}

	err = b.peripheral.WriteCharacteristic(characteristic, data, true)
	if err != nil {
		fmt.Printf("Failed to write characteristic, err: %s\n", err)
		return err
	}

	return
}

// Subscribe subscribes to notifications from the BLE device for the
// requested service and characteristic
func (b *BLEClientAdaptor) Subscribe(sUUID string, cUUID string, f func([]byte, error)) (err error) {
	if !b.connected {
		log.Fatalf("Cannot subscribe to BLE device until connected")
		return
	}

	characteristic := b.lookupCharacteristic(sUUID, cUUID)
	if characteristic == nil {
		log.Println("Cannot subscribe to unknown characteristic")
		return
	}

	fn := func(c *gatt.Characteristic, b []byte, err error) {
		f(b, err)
	}

	err = b.peripheral.SetNotifyValue(characteristic, fn)
	if err != nil {
		fmt.Printf("Failed to subscribe to characteristic, err: %s\n", err)
		return err
	}

	return
}

func (b *BLEClientAdaptor) StateChangeHandler(d gatt.Device, s gatt.State) {
	fmt.Println("State:", s)
	switch s {
	case gatt.StatePoweredOn:
		fmt.Println("scanning...")
		d.Scan([]gatt.UUID{}, false)
		return
	default:
		d.StopScanning()
	}
}

func (b *BLEClientAdaptor) DiscoveryHandler(p gatt.Peripheral, a *gatt.Advertisement, rssi int) {
	// try looking by local name
	if a.LocalName == b.UUID() {
		b.uuid = p.ID()
	} else {
		// try looking by ID
		id := strings.ToUpper(b.UUID())
		if strings.ToUpper(p.ID()) != id {
			return
		}
	}

	// Stop scanning once we've got the peripheral we're looking for.
	p.Device().StopScanning()

	// and connect to it
	p.Device().Connect(p)
}

func (b *BLEClientAdaptor) ConnectHandler(p gatt.Peripheral, err error) {
	fmt.Printf("\nConnected Peripheral ID:%s, NAME:(%s)\n", p.ID(), p.Name())

	b.peripheral = p

	if err := p.SetMTU(250); err != nil {
		fmt.Printf("Failed to set MTU, err: %s\n", err)
	}

	ss, err := p.DiscoverServices(nil)
	if err != nil {
		fmt.Printf("Failed to discover services, err: %s\n", err)
		return
	}

outer:
	for _, s := range ss {
		b.services[s.UUID().String()] = NewBLEService(s.UUID().String(), s)

		cs, err := p.DiscoverCharacteristics(nil, s)
		if err != nil {
			fmt.Printf("Failed to discover characteristics, err: %s\n", err)
			continue
		}

		for _, c := range cs {
			desc, err := p.DiscoverDescriptors(nil, c)
			if err != nil {
				fmt.Printf("Failed to discover descriptors: %v\n", err)
				continue outer
			}
			b.services[s.UUID().String()].characteristics[c.UUID().String()] = c
		}
	}

	b.connected = true
	close(b.ready)
}

func (b *BLEClientAdaptor) DisconnectHandler(p gatt.Peripheral, err error) {
	fmt.Println("Disconnected")
}

// Finalize finalizes the BLEAdaptor
func (b *BLEClientAdaptor) lookupCharacteristic(sUUID string, cUUID string) *gatt.Characteristic {
	service := b.services[sUUID]
	if service == nil {
		log.Printf("Unknown service ID: %s\n", sUUID)
		return nil
	}

	characteristic := service.characteristics[cUUID]
	if characteristic == nil {
		log.Printf("Unknown characteristic ID: %s\n", cUUID)
		return nil
	}

	return characteristic
}

// Represents a BLE Peripheral's Service
type BLEService struct {
	uuid            string
	service         *gatt.Service
	characteristics map[string]*gatt.Characteristic
}

// NewBLEAdaptor returns a new BLEService given a uuid
func NewBLEService(sUuid string, service *gatt.Service) *BLEService {
	return &BLEService{
		uuid:            sUuid,
		service:         service,
		characteristics: make(map[string]*gatt.Characteristic),
	}
}
<|endoftext|>