<commit_before>/*--------------------------------------------------------*\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: https://hprose.com                     |
|                                                          |
| encoding/int_decoder_test.go                             |
|                                                          |
| LastModified: Jun 2, 2020                                |
| Author: Ma Bingyao <andot@hprose.com>                    |
|                                                          |
\*________________________________________________________*/

package encoding

import (
	"math"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestDecodeFloat32(t *testing.T) {
	sb := new(strings.Builder)
	enc := NewEncoder(sb, true)
	enc.Encode(-1)
	enc.Encode(0)
	enc.Encode(1)
	enc.Encode(123)
	enc.Encode(math.MinInt64)
	enc.Encode(-math.MaxInt64)
	enc.Encode(math.MaxInt64)
	enc.Encode(true)
	enc.Encode(false)
	enc.Encode(nil)
	enc.Encode(3.14)
	enc.Encode(math.NaN())
	enc.Encode(math.Inf(1))
	enc.Encode(math.Inf(-1))
	enc.Encode("")
	enc.Encode("1")
	enc.Encode("123")
	enc.Encode("N")
	enc.Encode("NaN")
	enc.Encode([]byte{1})
	dec := NewDecoder(([]byte)(sb.String()))
	var f float32
	dec.Decode(&f)
	assert.Equal(t, float32(-1), f)
	dec.Decode(&f)
	assert.Equal(t, float32(0), f)
	dec.Decode(&f)
	assert.Equal(t, float32(1), f)
	dec.Decode(&f)
	assert.Equal(t, float32(123), f)
	dec.Decode(&f)
	assert.Equal(t, float32(math.MinInt64), f)
	dec.Decode(&f)
	assert.Equal(t, float32(-math.MaxInt64), f)
	dec.Decode(&f)
	assert.Equal(t, float32(math.MaxInt64), f)
	dec.Decode(&f)
	assert.Equal(t, float32(1), f)
	dec.Decode(&f)
	assert.Equal(t, float32(0), f)
	dec.Decode(&f)
	assert.Equal(t, float32(0), f)
	dec.Decode(&f)
	assert.Equal(t, float32(3.14), f)
	dec.Decode(&f)
	assert.True(t, math.IsNaN(float64(f)))
	dec.Decode(&f)
	assert.True(t, math.IsInf(float64(f), 1))
	dec.Decode(&f)
	assert.True(t, math.IsInf(float64(f), -1))
	dec.Decode(&f)
	assert.Equal(t, float32(0), f)
	dec.Decode(&f)
	assert.Equal(t, float32(1), f)
	dec.Decode(&f)
	assert.Equal(t, float32(123), f)
	assert.NoError(t, dec.Error)
	dec.Decode(&f)
	assert.EqualError(t, dec.Error, `strconv.ParseFloat: parsing "N": invalid syntax`)
	dec.Error = nil
	dec.Decode(&f)
	assert.True(t, math.IsNaN(float64(f)))
	dec.Decode(&f)
	assert.True(t, math.IsNaN(float64(f)))
}

func TestDecodeFloat64(t *testing.T) {
	sb := new(strings.Builder)
	enc := NewEncoder(sb, true)
	enc.Encode(-1)
	enc.Encode(0)
	enc.Encode(1)
	enc.Encode(123)
	enc.Encode(math.MinInt64)
	enc.Encode(-math.MaxInt64)
	enc.Encode(math.MaxInt64)
	enc.Encode(true)
	enc.Encode(false)
	enc.Encode(nil)
	enc.Encode(3.14)
	enc.Encode(math.NaN())
	enc.Encode(math.Inf(1))
	enc.Encode(math.Inf(-1))
	enc.Encode("")
	enc.Encode("1")
	enc.Encode("123")
	enc.Encode("N")
	enc.Encode("NaN")
	enc.Encode([]byte{1})
	dec := NewDecoder(([]byte)(sb.String()))
	var f float64
	dec.Decode(&f)
	assert.Equal(t, float64(-1), f)
	dec.Decode(&f)
	assert.Equal(t, float64(0), f)
	dec.Decode(&f)
	assert.Equal(t, float64(1), f)
	dec.Decode(&f)
	assert.Equal(t, float64(123), f)
	dec.Decode(&f)
	assert.Equal(t, float64(math.MinInt64), f)
	dec.Decode(&f)
	assert.Equal(t, float64(-math.MaxInt64), f)
	dec.Decode(&f)
	assert.Equal(t, float64(math.MaxInt64), f)
	dec.Decode(&f)
	assert.Equal(t, float64(1), f)
	dec.Decode(&f)
	assert.Equal(t, float64(0), f)
	dec.Decode(&f)
	assert.Equal(t, float64(0), f)
	dec.Decode(&f)
	assert.Equal(t, float64(3.14), f)
	dec.Decode(&f)
	assert.True(t, math.IsNaN(f))
	dec.Decode(&f)
	assert.True(t, math.IsInf(f, 1))
	dec.Decode(&f)
	assert.True(t, math.IsInf(f, -1))
	dec.Decode(&f)
	assert.Equal(t, float64(0), f)
	dec.Decode(&f)
	assert.Equal(t, float64(1), f)
	dec.Decode(&f)
	assert.Equal(t, float64(123), f)
	assert.NoError(t, dec.Error)
	dec.Decode(&f)
	assert.EqualError(t, dec.Error, `strconv.ParseFloat: parsing "N": invalid syntax`)
	dec.Error = nil
	dec.Decode(&f)
	assert.True(t, math.IsNaN(f))
	dec.Decode(&f)
	assert.True(t, math.IsNaN(f))
}

func TestDecodeFloat32Ptr(t *testing.T) {
	sb := new(strings.Builder)
	enc := NewEncoder(sb, true)
	enc.Encode(-1)
	enc.Encode(0)
	enc.Encode(1)
	enc.Encode(123)
	enc.Encode(math.MinInt64)
	enc.Encode(-math.MaxInt64)
	enc.Encode(math.MaxInt64)
	enc.Encode(true)
	enc.Encode(false)
	enc.Encode(nil)
	enc.Encode(3.14)
	enc.Encode(math.NaN())
	enc.Encode(math.Inf(1))
	enc.Encode(math.Inf(-1))
	enc.Encode("")
	enc.Encode("1")
	enc.Encode("123")
	enc.Encode("N")
	enc.Encode("NaN")
	enc.Encode([]byte{1})
	dec := NewDecoder(([]byte)(sb.String()))
	var f *float32
	dec.Decode(&f)
	assert.Equal(t, float32(-1), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(0), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(1), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(123), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(math.MinInt64), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(-math.MaxInt64), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(math.MaxInt64), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(1), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(0), *f)
	dec.Decode(&f)
	assert.Nil(t, f)
	dec.Decode(&f)
	assert.Equal(t, float32(3.14), *f)
	dec.Decode(&f)
	assert.True(t, math.IsNaN(float64(*f)))
	dec.Decode(&f)
	assert.True(t, math.IsInf(float64(*f), 1))
	dec.Decode(&f)
	assert.True(t, math.IsInf(float64(*f), -1))
	dec.Decode(&f)
	assert.Equal(t, float32(0), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(1), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(123), *f)
	assert.NoError(t, dec.Error)
	dec.Decode(&f)
	assert.EqualError(t, dec.Error, `strconv.ParseFloat: parsing "N": invalid syntax`)
	dec.Error = nil
	dec.Decode(&f)
	assert.True(t, math.IsNaN(float64(*f)))
	dec.Decode(&f)
	assert.True(t, math.IsNaN(float64(*f)))
}

func TestDecodeFloat64Ptr(t *testing.T) {
	sb := new(strings.Builder)
	enc := NewEncoder(sb, true)
	enc.Encode(-1)
	enc.Encode(0)
	enc.Encode(1)
	enc.Encode(123)
	enc.Encode(math.MinInt64)
	enc.Encode(-math.MaxInt64)
	enc.Encode(math.MaxInt64)
	enc.Encode(true)
	enc.Encode(false)
	enc.Encode(nil)
	enc.Encode(3.14)
	enc.Encode(math.NaN())
	enc.Encode(math.Inf(1))
	enc.Encode(math.Inf(-1))
	enc.Encode("")
	enc.Encode("1")
	enc.Encode("123")
	enc.Encode("N")
	enc.Encode("NaN")
	enc.Encode([]byte{1})
	dec := NewDecoder(([]byte)(sb.String()))
	var f *float64
	dec.Decode(&f)
	assert.Equal(t, float64(-1), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(0), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(1), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(123), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(math.MinInt64), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(-math.MaxInt64), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(math.MaxInt64), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(1), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(0), *f)
	dec.Decode(&f)
	assert.Nil(t, f)
	dec.Decode(&f)
	assert.Equal(t, float64(3.14), *f)
	dec.Decode(&f)
	assert.True(t, math.IsNaN(*f))
	dec.Decode(&f)
	assert.True(t, math.IsInf(*f, 1))
	dec.Decode(&f)
	assert.True(t, math.IsInf(*f, -1))
	dec.Decode(&f)
	assert.Equal(t, float64(0), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(1), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(123), *f)
	assert.NoError(t, dec.Error)
	dec.Decode(&f)
	assert.EqualError(t, dec.Error, `strconv.ParseFloat: parsing "N": invalid syntax`)
	dec.Error = nil
	dec.Decode(&f)
	assert.True(t, math.IsNaN(*f))
	dec.Decode(&f)
	assert.True(t, math.IsNaN(*f))
}
<commit_msg>Update float_decoder_test.go<commit_after>/*--------------------------------------------------------*\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: https://hprose.com                     |
|                                                          |
| encoding/float_decoder_test.go                           |
|                                                          |
| LastModified: Jun 6, 2020                                |
| Author: Ma Bingyao <andot@hprose.com>                    |
|                                                          |
\*________________________________________________________*/

package encoding

import (
	"math"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestDecodeFloat32(t *testing.T) {
	sb := new(strings.Builder)
	enc := NewEncoder(sb, true)
	enc.Encode(-1)
	enc.Encode(0)
	enc.Encode(1)
	enc.Encode(123)
	enc.Encode(math.MinInt64)
	enc.Encode(-math.MaxInt64)
	enc.Encode(math.MaxInt64)
	enc.Encode(true)
	enc.Encode(false)
	enc.Encode(nil)
	enc.Encode(3.14)
	enc.Encode(math.NaN())
	enc.Encode(math.Inf(1))
	enc.Encode(math.Inf(-1))
	enc.Encode("")
	enc.Encode("1")
	enc.Encode("123")
	enc.Encode("N")
	enc.Encode("NaN")
	enc.Encode([]byte{1})
	dec := NewDecoder(([]byte)(sb.String()))
	var f float32
	dec.Decode(&f)
	assert.Equal(t, float32(-1), f)
	dec.Decode(&f)
	assert.Equal(t, float32(0), f)
	dec.Decode(&f)
	assert.Equal(t, float32(1), f)
	dec.Decode(&f)
	assert.Equal(t, float32(123), f)
	dec.Decode(&f)
	assert.Equal(t, float32(math.MinInt64), f)
	dec.Decode(&f)
	assert.Equal(t, float32(-math.MaxInt64), f)
	dec.Decode(&f)
	assert.Equal(t, float32(math.MaxInt64), f)
	dec.Decode(&f)
	assert.Equal(t, float32(1), f)
	dec.Decode(&f)
	assert.Equal(t, float32(0), f)
	dec.Decode(&f)
	assert.Equal(t, float32(0), f)
	dec.Decode(&f)
	assert.Equal(t, float32(3.14), f)
	dec.Decode(&f)
	assert.True(t, math.IsNaN(float64(f)))
	dec.Decode(&f)
	assert.True(t, math.IsInf(float64(f), 1))
	dec.Decode(&f)
	assert.True(t, math.IsInf(float64(f), -1))
	dec.Decode(&f)
	assert.Equal(t, float32(0), f)
	dec.Decode(&f)
	assert.Equal(t, float32(1), f)
	dec.Decode(&f)
	assert.Equal(t, float32(123), f)
	assert.NoError(t, dec.Error)
	dec.Decode(&f)
	assert.EqualError(t, dec.Error, `strconv.ParseFloat: parsing "N": invalid syntax`)
	dec.Error = nil
	dec.Decode(&f)
	assert.True(t, math.IsNaN(float64(f)))
	dec.Decode(&f)
	assert.True(t, math.IsNaN(float64(f)))
}

func TestDecodeFloat64(t *testing.T) {
	sb := new(strings.Builder)
	enc := NewEncoder(sb, true)
	enc.Encode(-1)
	enc.Encode(0)
	enc.Encode(1)
	enc.Encode(123)
	enc.Encode(math.MinInt64)
	enc.Encode(-math.MaxInt64)
	enc.Encode(math.MaxInt64)
	enc.Encode(true)
	enc.Encode(false)
	enc.Encode(nil)
	enc.Encode(3.14)
	enc.Encode(math.NaN())
	enc.Encode(math.Inf(1))
	enc.Encode(math.Inf(-1))
	enc.Encode("")
	enc.Encode("1")
	enc.Encode("123")
	enc.Encode("N")
	enc.Encode("NaN")
	enc.Encode([]byte{1})
	dec := NewDecoder(([]byte)(sb.String()))
	var f float64
	dec.Decode(&f)
	assert.Equal(t, float64(-1), f)
	dec.Decode(&f)
	assert.Equal(t, float64(0), f)
	dec.Decode(&f)
	assert.Equal(t, float64(1), f)
	dec.Decode(&f)
	assert.Equal(t, float64(123), f)
	dec.Decode(&f)
	assert.Equal(t, float64(math.MinInt64), f)
	dec.Decode(&f)
	assert.Equal(t, float64(-math.MaxInt64), f)
	dec.Decode(&f)
	assert.Equal(t, float64(math.MaxInt64), f)
	dec.Decode(&f)
	assert.Equal(t, float64(1), f)
	dec.Decode(&f)
	assert.Equal(t, float64(0), f)
	dec.Decode(&f)
	assert.Equal(t, float64(0), f)
	dec.Decode(&f)
	assert.Equal(t, float64(3.14), f)
	dec.Decode(&f)
	assert.True(t, math.IsNaN(f))
	dec.Decode(&f)
	assert.True(t, math.IsInf(f, 1))
	dec.Decode(&f)
	assert.True(t, math.IsInf(f, -1))
	dec.Decode(&f)
	assert.Equal(t, float64(0), f)
	dec.Decode(&f)
	assert.Equal(t, float64(1), f)
	dec.Decode(&f)
	assert.Equal(t, float64(123), f)
	assert.NoError(t, dec.Error)
	dec.Decode(&f)
	assert.EqualError(t, dec.Error, `strconv.ParseFloat: parsing "N": invalid syntax`)
	dec.Error = nil
	dec.Decode(&f)
	assert.True(t, math.IsNaN(f))
	dec.Decode(&f)
	assert.True(t, math.IsNaN(f))
}

func TestDecodeFloat32Ptr(t *testing.T) {
	sb := new(strings.Builder)
	enc := NewEncoder(sb, true)
	enc.Encode(-1)
	enc.Encode(0)
	enc.Encode(1)
	enc.Encode(123)
	enc.Encode(math.MinInt64)
	enc.Encode(-math.MaxInt64)
	enc.Encode(math.MaxInt64)
	enc.Encode(true)
	enc.Encode(false)
	enc.Encode(nil)
	enc.Encode(3.14)
	enc.Encode(math.NaN())
	enc.Encode(math.Inf(1))
	enc.Encode(math.Inf(-1))
	enc.Encode("")
	enc.Encode("1")
	enc.Encode("123")
	enc.Encode("N")
	enc.Encode("NaN")
	enc.Encode([]byte{1})
	dec := NewDecoder(([]byte)(sb.String()))
	var f *float32
	dec.Decode(&f)
	assert.Equal(t, float32(-1), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(0), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(1), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(123), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(math.MinInt64), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(-math.MaxInt64), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(math.MaxInt64), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(1), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(0), *f)
	dec.Decode(&f)
	assert.Nil(t, f)
	dec.Decode(&f)
	assert.Equal(t, float32(3.14), *f)
	dec.Decode(&f)
	assert.True(t, math.IsNaN(float64(*f)))
	dec.Decode(&f)
	assert.True(t, math.IsInf(float64(*f), 1))
	dec.Decode(&f)
	assert.True(t, math.IsInf(float64(*f), -1))
	dec.Decode(&f)
	assert.Equal(t, float32(0), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(1), *f)
	dec.Decode(&f)
	assert.Equal(t, float32(123), *f)
	assert.NoError(t, dec.Error)
	dec.Decode(&f)
	assert.EqualError(t, dec.Error, `strconv.ParseFloat: parsing "N": invalid syntax`)
	dec.Error = nil
	dec.Decode(&f)
	assert.True(t, math.IsNaN(float64(*f)))
	dec.Decode(&f)
	assert.True(t, math.IsNaN(float64(*f)))
}

func TestDecodeFloat64Ptr(t *testing.T) {
	sb := new(strings.Builder)
	enc := NewEncoder(sb, true)
	enc.Encode(-1)
	enc.Encode(0)
	enc.Encode(1)
	enc.Encode(123)
	enc.Encode(math.MinInt64)
	enc.Encode(-math.MaxInt64)
	enc.Encode(math.MaxInt64)
	enc.Encode(true)
	enc.Encode(false)
	enc.Encode(nil)
	enc.Encode(3.14)
	enc.Encode(math.NaN())
	enc.Encode(math.Inf(1))
	enc.Encode(math.Inf(-1))
	enc.Encode("")
	enc.Encode("1")
	enc.Encode("123")
	enc.Encode("N")
	enc.Encode("NaN")
	enc.Encode([]byte{1})
	dec := NewDecoder(([]byte)(sb.String()))
	var f *float64
	dec.Decode(&f)
	assert.Equal(t, float64(-1), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(0), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(1), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(123), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(math.MinInt64), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(-math.MaxInt64), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(math.MaxInt64), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(1), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(0), *f)
	dec.Decode(&f)
	assert.Nil(t, f)
	dec.Decode(&f)
	assert.Equal(t, float64(3.14), *f)
	dec.Decode(&f)
	assert.True(t, math.IsNaN(*f))
	dec.Decode(&f)
	assert.True(t, math.IsInf(*f, 1))
	dec.Decode(&f)
	assert.True(t, math.IsInf(*f, -1))
	dec.Decode(&f)
	assert.Equal(t, float64(0), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(1), *f)
	dec.Decode(&f)
	assert.Equal(t, float64(123), *f)
	assert.NoError(t, dec.Error)
	dec.Decode(&f)
	assert.EqualError(t, dec.Error, `strconv.ParseFloat: parsing "N": invalid syntax`)
	dec.Error = nil
	dec.Decode(&f)
	assert.True(t, math.IsNaN(*f))
	dec.Decode(&f)
	assert.True(t, math.IsNaN(*f))
}
<|endoftext|>


<commit_before>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"fmt"
	"strings"

	"github.com/coreos/etcd/clientv3"
	"github.com/spf13/cobra"
)

var (
	getConsistency string
	getLimit       int64
	getSortOrder   string
	getSortTarget  string
	getPrefix      bool
	getFromKey     bool
	getRev         int64
	getKeysOnly    bool
	printValueOnly bool
)

// NewGetCommand returns the cobra command for "get".
func NewGetCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "get [options] <key> [range_end]",
		Short: "Gets the key or a range of keys",
		Run:   getCommandFunc,
	}

	cmd.Flags().StringVar(&getConsistency, "consistency", "l", "Linearizable(l) or Serializable(s)")
	cmd.Flags().StringVar(&getSortOrder, "order", "", "Order of results; ASCEND or DESCEND (ASCEND by default)")
	cmd.Flags().StringVar(&getSortTarget, "sort-by", "", "Sort target; CREATE, KEY, MODIFY, VALUE, or VERSION")
	cmd.Flags().Int64Var(&getLimit, "limit", 0, "Maximum number of results")
	cmd.Flags().BoolVar(&getPrefix, "prefix", false, "Get keys with matching prefix")
	cmd.Flags().BoolVar(&getFromKey, "from-key", false, "Get keys that are greater than or equal to the given key using byte compare")
	cmd.Flags().Int64Var(&getRev, "rev", 0, "Specify the kv revision")
	cmd.Flags().BoolVar(&getKeysOnly, "keys-only", false, "Get only the keys")
	cmd.Flags().BoolVar(&printValueOnly, "print-value-only", false, `Only write values when using the "simple" output format`)
	return cmd
}

// getCommandFunc executes the "get" command.
func getCommandFunc(cmd *cobra.Command, args []string) {
	key, opts := getGetOp(args)
	ctx, cancel := commandCtx(cmd)
	resp, err := mustClientFromCmd(cmd).Get(ctx, key, opts...)
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}

	if printValueOnly {
		dp, simple := (display).(*simplePrinter)
		if !simple {
			ExitWithError(ExitBadArgs, fmt.Errorf("print-value-only is only for `--write-out=simple`."))
		}
		dp.valueOnly = true
	}
	display.Get(*resp)
}

func getGetOp(args []string) (string, []clientv3.OpOption) {
	if len(args) == 0 {
		ExitWithError(ExitBadArgs, fmt.Errorf("range command needs arguments."))
	}

	if getPrefix && getFromKey {
		ExitWithError(ExitBadArgs, fmt.Errorf("`--prefix` and `--from-key` cannot be set at the same time, choose one."))
	}

	opts := []clientv3.OpOption{}
	switch getConsistency {
	case "s":
		opts = append(opts, clientv3.WithSerializable())
	case "l":
	default:
		ExitWithError(ExitBadFeature, fmt.Errorf("unknown consistency flag %q", getConsistency))
	}

	key := args[0]
	if len(args) > 1 {
		if getPrefix || getFromKey {
			ExitWithError(ExitBadArgs, fmt.Errorf("too many arguments, only accept one argument when `--prefix` or `--from-key` is set."))
		}
		opts = append(opts, clientv3.WithRange(args[1]))
	}

	opts = append(opts, clientv3.WithLimit(getLimit))
	if getRev > 0 {
		opts = append(opts, clientv3.WithRev(getRev))
	}

	sortByOrder := clientv3.SortNone
	sortOrder := strings.ToUpper(getSortOrder)
	switch {
	case sortOrder == "ASCEND":
		sortByOrder = clientv3.SortAscend
	case sortOrder == "DESCEND":
		sortByOrder = clientv3.SortDescend
	case sortOrder == "":
		// nothing
	default:
		ExitWithError(ExitBadFeature, fmt.Errorf("bad sort order %v", getSortOrder))
	}

	sortByTarget := clientv3.SortByKey
	sortTarget := strings.ToUpper(getSortTarget)
	switch {
	case sortTarget == "CREATE":
		sortByTarget = clientv3.SortByCreateRevision
	case sortTarget == "KEY":
		sortByTarget = clientv3.SortByKey
	case sortTarget == "MODIFY":
		sortByTarget = clientv3.SortByModRevision
	case sortTarget == "VALUE":
		sortByTarget = clientv3.SortByValue
	case sortTarget == "VERSION":
		sortByTarget = clientv3.SortByVersion
	case sortTarget == "":
		// nothing
	default:
		ExitWithError(ExitBadFeature, fmt.Errorf("bad sort target %v", getSortTarget))
	}

	opts = append(opts, clientv3.WithSort(sortByTarget, sortByOrder))

	if getPrefix {
		if len(key) == 0 {
			key = "\x00"
			opts = append(opts, clientv3.WithFromKey())
		} else {
			opts = append(opts, clientv3.WithPrefix())
		}
	}

	if getFromKey {
		if len(key) == 0 {
			key = "\x00"
		}
		opts = append(opts, clientv3.WithFromKey())
	}

	if getKeysOnly {
		opts = append(opts, clientv3.WithKeysOnly())
	}

	return key, opts
}
<commit_msg>etcdctl: fix get command error when no arg provided<commit_after>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"fmt"
	"strings"

	"github.com/coreos/etcd/clientv3"
	"github.com/spf13/cobra"
)

var (
	getConsistency string
	getLimit       int64
	getSortOrder   string
	getSortTarget  string
	getPrefix      bool
	getFromKey     bool
	getRev         int64
	getKeysOnly    bool
	printValueOnly bool
)

// NewGetCommand returns the cobra command for "get".
func NewGetCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "get [options] <key> [range_end]",
		Short: "Gets the key or a range of keys",
		Run:   getCommandFunc,
	}

	cmd.Flags().StringVar(&getConsistency, "consistency", "l", "Linearizable(l) or Serializable(s)")
	cmd.Flags().StringVar(&getSortOrder, "order", "", "Order of results; ASCEND or DESCEND (ASCEND by default)")
	cmd.Flags().StringVar(&getSortTarget, "sort-by", "", "Sort target; CREATE, KEY, MODIFY, VALUE, or VERSION")
	cmd.Flags().Int64Var(&getLimit, "limit", 0, "Maximum number of results")
	cmd.Flags().BoolVar(&getPrefix, "prefix", false, "Get keys with matching prefix")
	cmd.Flags().BoolVar(&getFromKey, "from-key", false, "Get keys that are greater than or equal to the given key using byte compare")
	cmd.Flags().Int64Var(&getRev, "rev", 0, "Specify the kv revision")
	cmd.Flags().BoolVar(&getKeysOnly, "keys-only", false, "Get only the keys")
	cmd.Flags().BoolVar(&printValueOnly, "print-value-only", false, `Only write values when using the "simple" output format`)
	return cmd
}

// getCommandFunc executes the "get" command.
func getCommandFunc(cmd *cobra.Command, args []string) {
	key, opts := getGetOp(args)
	ctx, cancel := commandCtx(cmd)
	resp, err := mustClientFromCmd(cmd).Get(ctx, key, opts...)
	cancel()
	if err != nil {
		ExitWithError(ExitError, err)
	}

	if printValueOnly {
		dp, simple := (display).(*simplePrinter)
		if !simple {
			ExitWithError(ExitBadArgs, fmt.Errorf("print-value-only is only for `--write-out=simple`."))
		}
		dp.valueOnly = true
	}
	display.Get(*resp)
}

func getGetOp(args []string) (string, []clientv3.OpOption) {
	if len(args) == 0 {
		ExitWithError(ExitBadArgs, fmt.Errorf("get command needs one argument as key and an optional argument as range_end."))
	}

	if getPrefix && getFromKey {
		ExitWithError(ExitBadArgs, fmt.Errorf("`--prefix` and `--from-key` cannot be set at the same time, choose one."))
	}

	opts := []clientv3.OpOption{}
	switch getConsistency {
	case "s":
		opts = append(opts, clientv3.WithSerializable())
	case "l":
	default:
		ExitWithError(ExitBadFeature, fmt.Errorf("unknown consistency flag %q", getConsistency))
	}

	key := args[0]
	if len(args) > 1 {
		if getPrefix || getFromKey {
			ExitWithError(ExitBadArgs, fmt.Errorf("too many arguments, only accept one argument when `--prefix` or `--from-key` is set."))
		}
		opts = append(opts, clientv3.WithRange(args[1]))
	}

	opts = append(opts, clientv3.WithLimit(getLimit))
	if getRev > 0 {
		opts = append(opts, clientv3.WithRev(getRev))
	}

	sortByOrder := clientv3.SortNone
	sortOrder := strings.ToUpper(getSortOrder)
	switch {
	case sortOrder == "ASCEND":
		sortByOrder = clientv3.SortAscend
	case sortOrder == "DESCEND":
		sortByOrder = clientv3.SortDescend
	case sortOrder == "":
		// nothing
	default:
		ExitWithError(ExitBadFeature, fmt.Errorf("bad sort order %v", getSortOrder))
	}

	sortByTarget := clientv3.SortByKey
	sortTarget := strings.ToUpper(getSortTarget)
	switch {
	case sortTarget == "CREATE":
		sortByTarget = clientv3.SortByCreateRevision
	case sortTarget == "KEY":
		sortByTarget = clientv3.SortByKey
	case sortTarget == "MODIFY":
		sortByTarget = clientv3.SortByModRevision
	case sortTarget == "VALUE":
		sortByTarget = clientv3.SortByValue
	case sortTarget == "VERSION":
		sortByTarget = clientv3.SortByVersion
	case sortTarget == "":
		// nothing
	default:
		ExitWithError(ExitBadFeature, fmt.Errorf("bad sort target %v", getSortTarget))
	}

	opts = append(opts, clientv3.WithSort(sortByTarget, sortByOrder))

	if getPrefix {
		if len(key) == 0 {
			key = "\x00"
			opts = append(opts, clientv3.WithFromKey())
		} else {
			opts = append(opts, clientv3.WithPrefix())
		}
	}

	if getFromKey {
		if len(key) == 0 {
			key = "\x00"
		}
		opts = append(opts, clientv3.WithFromKey())
	}

	if getKeysOnly {
		opts = append(opts, clientv3.WithKeysOnly())
	}

	return key, opts
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"io/ioutil"
	"log"
	"path/filepath"
	"time"

	"github.com/mably/btcrpcclient"
	"github.com/mably/btcutil"
	"github.com/davecgh/go-spew/spew"
)

func main() {
	// Only override the handlers for notifications you care about.
	// Also note most of the handlers will only be called if you register
	// for notifications.  See the documentation of the btcrpcclient
	// NotificationHandlers type for more details about each handler.
	ntfnHandlers := btcrpcclient.NotificationHandlers{
		OnAccountBalance: func(account string, balance btcutil.Amount, confirmed bool) {
			log.Printf("New balance for account %s: %v", account,
				balance)
		},
	}

	// Connect to local btcwallet RPC server using websockets.
	certHomeDir := btcutil.AppDataDir("btcwallet", false)
	certs, err := ioutil.ReadFile(filepath.Join(certHomeDir, "rpc.cert"))
	if err != nil {
		log.Fatal(err)
	}
	connCfg := &btcrpcclient.ConnConfig{
		Host:         "localhost:18332",
		Endpoint:     "ws",
		User:         "yourrpcuser",
		Pass:         "yourrpcpass",
		Certificates: certs,
	}
	client, err := btcrpcclient.New(connCfg, &ntfnHandlers)
	if err != nil {
		log.Fatal(err)
	}

	// Get the list of unspent transaction outputs (utxos) that the
	// connected wallet has at least one private key for.
	unspent, err := client.ListUnspent()
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("Num unspent outputs (utxos): %d", len(unspent))
	if len(unspent) > 0 {
		log.Printf("First utxo:\n%v", spew.Sdump(unspent[0]))
	}

	// For this example gracefully shutdown the client after 10 seconds.
	// Ordinarily when to shutdown the client is highly application
	// specific.
	log.Println("Client shutdown in 10 seconds...")
	time.AfterFunc(time.Second*10, func() {
		log.Println("Client shutting down...")
		client.Shutdown()
		log.Println("Client shutdown complete.")
	})

	// Wait until the client either shuts down gracefully (or the user
	// terminates the process with Ctrl+C).
	client.WaitForShutdown()
}
<commit_msg>Few changes for use with ppcwallet<commit_after>// Copyright (c) 2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"io/ioutil"
	"log"
	"path/filepath"
	"time"

	"github.com/mably/btcrpcclient"
	"github.com/mably/btcutil"
	"github.com/davecgh/go-spew/spew"
)

func main() {
	// Only override the handlers for notifications you care about.
	// Also note most of the handlers will only be called if you register
	// for notifications.  See the documentation of the btcrpcclient
	// NotificationHandlers type for more details about each handler.
	ntfnHandlers := btcrpcclient.NotificationHandlers{
		OnAccountBalance: func(account string, balance btcutil.Amount, confirmed bool) {
			log.Printf("New balance for account %s: %v", account,
				balance)
		},
	}

	// Connect to local btcwallet RPC server using websockets.
	certHomeDir := btcutil.AppDataDir("ppcwallet", false)
	certs, err := ioutil.ReadFile(filepath.Join(certHomeDir, "rpc.cert"))
	if err != nil {
		log.Fatal(err)
	}
	connCfg := &btcrpcclient.ConnConfig{
		Host:         "localhost:8332",
		Endpoint:     "ws",
		User:         "rpcuser",
		Pass:         "rpcpass",
		Certificates: certs,
	}
	client, err := btcrpcclient.New(connCfg, &ntfnHandlers)
	if err != nil {
		log.Fatal(err)
	}

	// Get the list of unspent transaction outputs (utxos) that the
	// connected wallet has at least one private key for.
	unspent, err := client.ListUnspent()
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("Num unspent outputs (utxos): %d", len(unspent))
	if len(unspent) > 0 {
		log.Printf("First utxo:\n%v", spew.Sdump(unspent[0]))
	}

	// For this example gracefully shutdown the client after 10 seconds.
	// Ordinarily when to shutdown the client is highly application
	// specific.
	log.Println("Client shutdown in 10 seconds...")
	time.AfterFunc(time.Second*10, func() {
		log.Println("Client shutting down...")
		client.Shutdown()
		log.Println("Client shutdown complete.")
	})

	// Wait until the client either shuts down gracefully (or the user
	// terminates the process with Ctrl+C).
	client.WaitForShutdown()
}
<|endoftext|>


<commit_before>// Stub provider for OpenStack, using goose will be implemented here

package openstack

import (
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"sync"
)

type environProvider struct{}

var _ environs.EnvironProvider = (*environProvider)(nil)

var providerInstance environProvider

func init() {
	environs.RegisterProvider("openstack", environProvider{})
}

func (p environProvider) Open(cfg *config.Config) (environs.Environ, error) {
	log.Printf("environs/openstack: opening environment %q", cfg.Name())
	e := new(environ)
	err := e.SetConfig(cfg)
	if err != nil {
		return nil, err
	}
	return e, nil
}

func (p environProvider) SecretAttrs(cfg *config.Config) (map[string]interface{}, error) {
	m := make(map[string]interface{})
	ecfg, err := providerInstance.newConfig(cfg)
	if err != nil {
		return nil, err
	}
	m["username"] = ecfg.username()
	m["password"] = ecfg.password()
	m["tenant-name"] = ecfg.tenantName()
	return m, nil
}

func (p environProvider) PublicAddress() (string, error) {
	panic("not implemented")
}

func (p environProvider) PrivateAddress() (string, error) {
	panic("not implemented")
}

type environ struct {
	name string

	ecfgMutex    sync.Mutex
	ecfgUnlocked *environConfig
}

var _ environs.Environ = (*environ)(nil)

func (e *environ) ecfg() *environConfig {
	e.ecfgMutex.Lock()
	ecfg := e.ecfgUnlocked
	e.ecfgMutex.Unlock()
	return ecfg
}

func (e *environ) Name() string {
	return e.name
}

func (e *environ) Bootstrap(uploadTools bool, stateServerPEM []byte) error {
	panic("not implemented")
}

func (e *environ) StateInfo() (*state.Info, error) {
	panic("not implemented")
}

func (e *environ) Config() *config.Config {
	panic("not implemented")
}

func (e *environ) SetConfig(cfg *config.Config) error {
	ecfg, err := providerInstance.newConfig(cfg)
	if err != nil {
		return err
	}
	e.ecfgMutex.Lock()
	defer e.ecfgMutex.Unlock()
	e.name = ecfg.Name()
	e.ecfgUnlocked = ecfg

	// TODO(dimitern): setup the goose client auth/compute, etc. here
	return nil
}

func (e *environ) StartInstance(machineId int, info *state.Info, tools *state.Tools) (environs.Instance, error) {
	panic("not implemented")
}

func (e *environ) StopInstances([]environs.Instance) error {
	panic("not implemented")
}

func (e *environ) Instances(ids []string) ([]environs.Instance, error) {
	panic("not implemented")
}

func (e *environ) AllInstances() ([]environs.Instance, error) {
	panic("not implemented")
}

func (e *environ) Storage() environs.Storage {
	panic("not implemented")
}

func (e *environ) PublicStorage() environs.StorageReader {
	panic("not implemented")
}

func (e *environ) Destroy(insts []environs.Instance) error {
	panic("not implemented")
}

func (e *environ) AssignmentPolicy() state.AssignmentPolicy {
	panic("not implemented")
}

func (e *environ) OpenPorts(ports []state.Port) error {
	panic("not implemented")
}

func (e *environ) ClosePorts(ports []state.Port) error {
	panic("not implemented")
}

func (e *environ) Ports() ([]state.Port, error) {
	panic("not implemented")
}

func (e *environ) Provider() environs.EnvironProvider {
	return &providerInstance
}
<commit_msg>Implemeted getting Private/PublicAddress (tests will follow)<commit_after>// Stub provider for OpenStack, using goose will be implemented here

package openstack

import (
	"fmt"
	"io/ioutil"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/trivial"
	"net/http"
	"strings"
	"sync"
	"time"
)

type environProvider struct{}

var _ environs.EnvironProvider = (*environProvider)(nil)

var providerInstance environProvider

// A request may fail to due "eventual consistency" semantics, which
// should resolve fairly quickly.  A request may also fail due to a slow
// state transition (for instance an instance taking a while to release
// a security group after termination).  The former failure mode is
// dealt with by shortAttempt, the latter by longAttempt.
var shortAttempt = trivial.AttemptStrategy{
	Total: 5 * time.Second,
	Delay: 200 * time.Millisecond,
}

var longAttempt = trivial.AttemptStrategy{
	Total: 3 * time.Minute,
	Delay: 1 * time.Second,
}

func init() {
	environs.RegisterProvider("openstack", environProvider{})
}

func (p environProvider) Open(cfg *config.Config) (environs.Environ, error) {
	log.Printf("environs/openstack: opening environment %q", cfg.Name())
	e := new(environ)
	err := e.SetConfig(cfg)
	if err != nil {
		return nil, err
	}
	return e, nil
}

func (p environProvider) SecretAttrs(cfg *config.Config) (map[string]interface{}, error) {
	m := make(map[string]interface{})
	ecfg, err := providerInstance.newConfig(cfg)
	if err != nil {
		return nil, err
	}
	m["username"] = ecfg.username()
	m["password"] = ecfg.password()
	m["tenant-name"] = ecfg.tenantName()
	return m, nil
}

func (p environProvider) PublicAddress() (string, error) {
	return fetchMetadata("public-hostname")
}

func (p environProvider) PrivateAddress() (string, error) {
	return fetchMetadata("local-hostname")
}

type environ struct {
	name string

	ecfgMutex    sync.Mutex
	ecfgUnlocked *environConfig
}

var _ environs.Environ = (*environ)(nil)

func (e *environ) ecfg() *environConfig {
	e.ecfgMutex.Lock()
	ecfg := e.ecfgUnlocked
	e.ecfgMutex.Unlock()
	return ecfg
}

func (e *environ) Name() string {
	return e.name
}

func (e *environ) Bootstrap(uploadTools bool, stateServerPEM []byte) error {
	panic("not implemented")
}

func (e *environ) StateInfo() (*state.Info, error) {
	panic("not implemented")
}

func (e *environ) Config() *config.Config {
	panic("not implemented")
}

func (e *environ) SetConfig(cfg *config.Config) error {
	ecfg, err := providerInstance.newConfig(cfg)
	if err != nil {
		return err
	}
	e.ecfgMutex.Lock()
	defer e.ecfgMutex.Unlock()
	e.name = ecfg.Name()
	e.ecfgUnlocked = ecfg

	// TODO(dimitern): setup the goose client auth/compute, etc. here
	return nil
}

func (e *environ) StartInstance(machineId int, info *state.Info, tools *state.Tools) (environs.Instance, error) {
	panic("not implemented")
}

func (e *environ) StopInstances([]environs.Instance) error {
	panic("not implemented")
}

func (e *environ) Instances(ids []string) ([]environs.Instance, error) {
	panic("not implemented")
}

func (e *environ) AllInstances() ([]environs.Instance, error) {
	panic("not implemented")
}

func (e *environ) Storage() environs.Storage {
	panic("not implemented")
}

func (e *environ) PublicStorage() environs.StorageReader {
	panic("not implemented")
}

func (e *environ) Destroy(insts []environs.Instance) error {
	panic("not implemented")
}

func (e *environ) AssignmentPolicy() state.AssignmentPolicy {
	panic("not implemented")
}

func (e *environ) OpenPorts(ports []state.Port) error {
	panic("not implemented")
}

func (e *environ) ClosePorts(ports []state.Port) error {
	panic("not implemented")
}

func (e *environ) Ports() ([]state.Port, error) {
	panic("not implemented")
}

func (e *environ) Provider() environs.EnvironProvider {
	return &providerInstance
}

// metadataHost holds the address of the instance metadata service.
// It is a variable so that tests can change it to refer to a local
// server when needed.
var metadataHost = "http://169.254.169.254"

// fetchMetadata fetches a single atom of data from the openstack instance metadata service.
// http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/AESDG-chapter-instancedata.html
func fetchMetadata(name string) (value string, err error) {
	uri := fmt.Sprintf("%s/2011-01-01/meta-data/%s", metadataHost, name)
	defer trivial.ErrorContextf(&err, "cannot get %q", uri)
	for a := shortAttempt.Start(); a.Next(); {
		var resp *http.Response
		resp, err = http.Get(uri)
		if err != nil {
			continue
		}
		defer resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			err = fmt.Errorf("bad http response %v", resp.Status)
			continue
		}
		var data []byte
		data, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			continue
		}
		return strings.TrimSpace(string(data)), nil
	}
	return
}
<|endoftext|>


<commit_before>package image

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"sort"
	"strings"
	"time"

	"github.com/cenk/backoff"
	"github.com/pkg/errors"
	workererrors "github.com/travis-ci/worker/errors"
)

const (
	imageAPIRequestContentType = "application/x-www-form-urlencoded; boundary=NL"
)

type APISelector struct {
	baseURL *url.URL

	maxInterval    time.Duration
	maxElapsedTime time.Duration
}

func NewAPISelector(u *url.URL) *APISelector {
	return &APISelector{
		baseURL: u,

		maxInterval:    10 * time.Second,
		maxElapsedTime: time.Minute,
	}
}

func (as *APISelector) Select(params *Params) (string, error) {
	tagSets, err := as.buildCandidateTags(params)
	if err != nil {
		return "default", err
	}

	imageName, err := as.queryWithTags(params.Infra, tagSets)
	if err != nil {
		return "default", err
	}

	if imageName != "" {
		return imageName, nil
	}

	return "default", nil
}

func (as *APISelector) queryWithTags(infra string, tags []*tagSet) (string, error) {
	bodyLines := []string{}
	lastJobID := uint64(0)
	lastRepo := ""

	for _, ts := range tags {
		qs := url.Values{}
		qs.Set("infra", infra)
		qs.Set("fields[images]", "name")
		qs.Set("limit", "1")
		qs.Set("job_id", fmt.Sprintf("%v", ts.JobID))
		qs.Set("repo", ts.Repo)
		qs.Set("is_default", fmt.Sprintf("%v", ts.IsDefault))
		if len(ts.Tags) > 0 {
			qs.Set("tags", strings.Join(ts.Tags, ","))
		}

		bodyLines = append(bodyLines, qs.Encode())
		lastJobID = ts.JobID
		lastRepo = ts.Repo
	}

	qs := url.Values{}
	qs.Set("infra", infra)
	qs.Set("is_default", "true")
	qs.Set("fields[images]", "name")
	qs.Set("limit", "1")
	qs.Set("job_id", fmt.Sprintf("%v", lastJobID))
	qs.Set("repo", lastRepo)

	bodyLines = append(bodyLines, qs.Encode())

	u, err := url.Parse(as.baseURL.String())
	if err != nil {
		return "", err
	}

	imageResp, err := as.makeImageRequest(u.String(), bodyLines)
	if err != nil {
		return "", err
	}

	if len(imageResp.Data) == 0 {
		return "", nil
	}

	return imageResp.Data[0].Name, nil
}

func (as *APISelector) makeImageRequest(urlString string, bodyLines []string) (*apiSelectorImageResponse, error) {
	var responseBody []byte

	b := backoff.NewExponentialBackOff()
	b.MaxInterval = 10 * time.Second
	b.MaxElapsedTime = time.Minute

	err := backoff.Retry(func() error {
		resp, err := http.Post(urlString, imageAPIRequestContentType,
			strings.NewReader(strings.Join(bodyLines, "\n")+"\n"))

		if err != nil {
			return err
		}
		defer resp.Body.Close()

		responseBody, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}

		if resp.StatusCode != 200 {
			return errors.Errorf("expected 200 status code from job-board, received status=%d body=%q",
				resp.StatusCode,
				responseBody)
		}

		return nil
	}, b)

	if err != nil {
		return nil, err
	}

	imageResp := &apiSelectorImageResponse{
		Data: []*apiSelectorImageRef{},
	}

	err = json.Unmarshal(responseBody, imageResp)
	if err != nil {
		return nil, err
	}

	return imageResp, nil
}

type tagSet struct {
	Tags      []string
	IsDefault bool

	JobID uint64
	Repo  string
}

func (ts *tagSet) GoString() string {
	return fmt.Sprintf("&image.tagSet{IsDefault: %v, Tags: %#v}", ts.IsDefault, ts.Tags)
}

func (as *APISelector) buildCandidateTags(params *Params) ([]*tagSet, error) {
	fullTagSet := &tagSet{
		Tags:  []string{},
		JobID: params.JobID,
		Repo:  params.Repo,
	}
	candidateTags := []*tagSet{}

	addDefaultTag := func(tag string) {
		fullTagSet.Tags = append(fullTagSet.Tags, tag)
		candidateTags = append(candidateTags,
			&tagSet{
				IsDefault: true,
				Tags:      []string{tag},
				JobID:     params.JobID,
				Repo:      params.Repo,
			})
	}

	addTags := func(tags ...string) {
		candidateTags = append(candidateTags,
			&tagSet{
				IsDefault: false,
				Tags:      tags,
				JobID:     params.JobID,
				Repo:      params.Repo,
			})
	}

	hasLang := params.Language != ""

	if params.OS == "osx" && params.OsxImage != "" {
		addTags("osx_image:"+params.OsxImage, "os:osx")
	}

	if params.Dist != "" && params.Group != "" && hasLang {
		addTags("dist:"+params.Dist, "group_"+params.Group+":true", "language_"+params.Language+":true")
	}

	if params.Dist != "" && hasLang {
		addTags("dist:"+params.Dist, "language_"+params.Language+":true")
	}

	if params.Group != "" && hasLang {
		addTags("group_"+params.Group+":true", "language_"+params.Language+":true")
	}

	if params.OS != "" && hasLang {
		addTags("os:"+params.OS, "language_"+params.Language+":true")
	}

	if hasLang {
		addDefaultTag("language_" + params.Language + ":true")
	}

	if params.OS == "osx" && params.OsxImage != "" {
		addDefaultTag("osx_image:" + params.OsxImage)
	}

	if params.Dist != "" {
		addDefaultTag("dist:" + params.Dist)
	}

	if params.Group != "" {
		addDefaultTag("group_" + params.Group + ":true")
	}

	if params.OS != "" {
		addDefaultTag("os:" + params.OS)
	}

	result := append([]*tagSet{fullTagSet}, candidateTags...)
	for _, ts := range result {
		sort.Strings(ts.Tags)
	}

	for _, ts := range result {
		for _, tag := range ts.Tags {
			if strings.Contains(tag, ",") {
				return result, workererrors.NewWrappedJobAbortError(errors.Errorf("tag %v contained \",\", which is not supported by job-board -- check .travis.yml for trailing comma", tag))
			}
		}
	}

	return result, nil
}

type apiSelectorImageResponse struct {
	Data []*apiSelectorImageRef `json:"data"`
}

type apiSelectorImageRef struct {
	ID        int               `json:"id"`
	Infra     string            `json:"infra"`
	Name      string            `json:"name"`
	Tags      map[string]string `json:"tags"`
	IsDefault bool              `json:"is_default"`
	CreatedAt string            `json:"created_at"`
	UpdatedAt string            `json:"updated_at"`
}
<commit_msg>update error message in api_selector to not mention job-board<commit_after>package image

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"sort"
	"strings"
	"time"

	"github.com/cenk/backoff"
	"github.com/pkg/errors"
	workererrors "github.com/travis-ci/worker/errors"
)

const (
	imageAPIRequestContentType = "application/x-www-form-urlencoded; boundary=NL"
)

type APISelector struct {
	baseURL *url.URL

	maxInterval    time.Duration
	maxElapsedTime time.Duration
}

func NewAPISelector(u *url.URL) *APISelector {
	return &APISelector{
		baseURL: u,

		maxInterval:    10 * time.Second,
		maxElapsedTime: time.Minute,
	}
}

func (as *APISelector) Select(params *Params) (string, error) {
	tagSets, err := as.buildCandidateTags(params)
	if err != nil {
		return "default", err
	}

	imageName, err := as.queryWithTags(params.Infra, tagSets)
	if err != nil {
		return "default", err
	}

	if imageName != "" {
		return imageName, nil
	}

	return "default", nil
}

func (as *APISelector) queryWithTags(infra string, tags []*tagSet) (string, error) {
	bodyLines := []string{}
	lastJobID := uint64(0)
	lastRepo := ""

	for _, ts := range tags {
		qs := url.Values{}
		qs.Set("infra", infra)
		qs.Set("fields[images]", "name")
		qs.Set("limit", "1")
		qs.Set("job_id", fmt.Sprintf("%v", ts.JobID))
		qs.Set("repo", ts.Repo)
		qs.Set("is_default", fmt.Sprintf("%v", ts.IsDefault))
		if len(ts.Tags) > 0 {
			qs.Set("tags", strings.Join(ts.Tags, ","))
		}

		bodyLines = append(bodyLines, qs.Encode())
		lastJobID = ts.JobID
		lastRepo = ts.Repo
	}

	qs := url.Values{}
	qs.Set("infra", infra)
	qs.Set("is_default", "true")
	qs.Set("fields[images]", "name")
	qs.Set("limit", "1")
	qs.Set("job_id", fmt.Sprintf("%v", lastJobID))
	qs.Set("repo", lastRepo)

	bodyLines = append(bodyLines, qs.Encode())

	u, err := url.Parse(as.baseURL.String())
	if err != nil {
		return "", err
	}

	imageResp, err := as.makeImageRequest(u.String(), bodyLines)
	if err != nil {
		return "", err
	}

	if len(imageResp.Data) == 0 {
		return "", nil
	}

	return imageResp.Data[0].Name, nil
}

func (as *APISelector) makeImageRequest(urlString string, bodyLines []string) (*apiSelectorImageResponse, error) {
	var responseBody []byte

	b := backoff.NewExponentialBackOff()
	b.MaxInterval = 10 * time.Second
	b.MaxElapsedTime = time.Minute

	err := backoff.Retry(func() error {
		resp, err := http.Post(urlString, imageAPIRequestContentType,
			strings.NewReader(strings.Join(bodyLines, "\n")+"\n"))

		if err != nil {
			return err
		}
		defer resp.Body.Close()

		responseBody, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}

		if resp.StatusCode != 200 {
			return errors.Errorf("expected 200 status code from job-board, received status=%d body=%q",
				resp.StatusCode,
				responseBody)
		}

		return nil
	}, b)

	if err != nil {
		return nil, err
	}

	imageResp := &apiSelectorImageResponse{
		Data: []*apiSelectorImageRef{},
	}

	err = json.Unmarshal(responseBody, imageResp)
	if err != nil {
		return nil, err
	}

	return imageResp, nil
}

type tagSet struct {
	Tags      []string
	IsDefault bool

	JobID uint64
	Repo  string
}

func (ts *tagSet) GoString() string {
	return fmt.Sprintf("&image.tagSet{IsDefault: %v, Tags: %#v}", ts.IsDefault, ts.Tags)
}

func (as *APISelector) buildCandidateTags(params *Params) ([]*tagSet, error) {
	fullTagSet := &tagSet{
		Tags:  []string{},
		JobID: params.JobID,
		Repo:  params.Repo,
	}
	candidateTags := []*tagSet{}

	addDefaultTag := func(tag string) {
		fullTagSet.Tags = append(fullTagSet.Tags, tag)
		candidateTags = append(candidateTags,
			&tagSet{
				IsDefault: true,
				Tags:      []string{tag},
				JobID:     params.JobID,
				Repo:      params.Repo,
			})
	}

	addTags := func(tags ...string) {
		candidateTags = append(candidateTags,
			&tagSet{
				IsDefault: false,
				Tags:      tags,
				JobID:     params.JobID,
				Repo:      params.Repo,
			})
	}

	hasLang := params.Language != ""

	if params.OS == "osx" && params.OsxImage != "" {
		addTags("osx_image:"+params.OsxImage, "os:osx")
	}

	if params.Dist != "" && params.Group != "" && hasLang {
		addTags("dist:"+params.Dist, "group_"+params.Group+":true", "language_"+params.Language+":true")
	}

	if params.Dist != "" && hasLang {
		addTags("dist:"+params.Dist, "language_"+params.Language+":true")
	}

	if params.Group != "" && hasLang {
		addTags("group_"+params.Group+":true", "language_"+params.Language+":true")
	}

	if params.OS != "" && hasLang {
		addTags("os:"+params.OS, "language_"+params.Language+":true")
	}

	if hasLang {
		addDefaultTag("language_" + params.Language + ":true")
	}

	if params.OS == "osx" && params.OsxImage != "" {
		addDefaultTag("osx_image:" + params.OsxImage)
	}

	if params.Dist != "" {
		addDefaultTag("dist:" + params.Dist)
	}

	if params.Group != "" {
		addDefaultTag("group_" + params.Group + ":true")
	}

	if params.OS != "" {
		addDefaultTag("os:" + params.OS)
	}

	result := append([]*tagSet{fullTagSet}, candidateTags...)
	for _, ts := range result {
		sort.Strings(ts.Tags)
	}

	for _, ts := range result {
		for _, tag := range ts.Tags {
			if strings.Contains(tag, ",") {
				return result, workererrors.NewWrappedJobAbortError(errors.Errorf("job was aborted because tag %v contained \",\", this can happen when .travis.yml has a trailing comma", tag))
			}
		}
	}

	return result, nil
}

type apiSelectorImageResponse struct {
	Data []*apiSelectorImageRef `json:"data"`
}

type apiSelectorImageRef struct {
	ID        int               `json:"id"`
	Infra     string            `json:"infra"`
	Name      string            `json:"name"`
	Tags      map[string]string `json:"tags"`
	IsDefault bool              `json:"is_default"`
	CreatedAt string            `json:"created_at"`
	UpdatedAt string            `json:"updated_at"`
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package membership

import (
	"encoding/json"
	"fmt"
	"path"

	"github.com/coreos/etcd/mvcc/backend"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/store"

	"github.com/coreos/go-semver/semver"
)

const (
	attributesSuffix     = "attributes"
	raftAttributesSuffix = "raftAttributes"

	// the prefix for stroing membership related information in store provided by store pkg.
	storePrefix = "/0"
)

var (
	membersBucketName        = []byte("members")
	membersRemovedBuckedName = []byte("members_removed")
	clusterBucketName        = []byte("cluster")

	StoreMembersPrefix        = path.Join(storePrefix, "members")
	storeRemovedMembersPrefix = path.Join(storePrefix, "removed_members")
)

func mustSaveMemberToBackend(be backend.Backend, m *Member) {
	mkey := backendMemberKey(m.ID)
	mvalue, err := json.Marshal(m)
	if err != nil {
		plog.Panicf("marshal raftAttributes should never fail: %v", err)
	}

	tx := be.BatchTx()
	tx.Lock()
	tx.UnsafePut(membersBucketName, mkey, mvalue)
	tx.Unlock()
}

func mustDeleteMemberFromBackend(be backend.Backend, id types.ID) {
	mkey := backendMemberKey(id)

	tx := be.BatchTx()
	tx.Lock()
	tx.UnsafeDelete(membersBucketName, mkey)
	tx.UnsafePut(membersRemovedBuckedName, mkey, []byte("removed"))
	tx.Unlock()
}

func mustSaveClusterVersionToBackend(be backend.Backend, ver *semver.Version) {
	ckey := backendClusterVersionKey()

	tx := be.BatchTx()
	tx.Lock()
	defer tx.Unlock()
	tx.UnsafePut(clusterBucketName, ckey, []byte(ver.String()))
}

func mustSaveMemberToStore(s store.Store, m *Member) {
	b, err := json.Marshal(m.RaftAttributes)
	if err != nil {
		plog.Panicf("marshal raftAttributes should never fail: %v", err)
	}
	p := path.Join(MemberStoreKey(m.ID), raftAttributesSuffix)
	if _, err := s.Create(p, false, string(b), false, store.TTLOptionSet{ExpireTime: store.Permanent}); err != nil {
		plog.Panicf("create raftAttributes should never fail: %v", err)
	}
}

func mustDeleteMemberFromStore(s store.Store, id types.ID) {
	if _, err := s.Delete(MemberStoreKey(id), true, true); err != nil {
		plog.Panicf("delete member should never fail: %v", err)
	}
	if _, err := s.Create(RemovedMemberStoreKey(id), false, "", false, store.TTLOptionSet{ExpireTime: store.Permanent}); err != nil {
		plog.Panicf("create removedMember should never fail: %v", err)
	}
}

func mustUpdateMemberInStore(s store.Store, m *Member) {
	b, err := json.Marshal(m.RaftAttributes)
	if err != nil {
		plog.Panicf("marshal raftAttributes should never fail: %v", err)
	}
	p := path.Join(MemberStoreKey(m.ID), raftAttributesSuffix)
	if _, err := s.Update(p, string(b), store.TTLOptionSet{ExpireTime: store.Permanent}); err != nil {
		plog.Panicf("update raftAttributes should never fail: %v", err)
	}
}

func mustUpdateMemberAttrInStore(s store.Store, m *Member) {
	b, err := json.Marshal(m.Attributes)
	if err != nil {
		plog.Panicf("marshal raftAttributes should never fail: %v", err)
	}
	p := path.Join(MemberStoreKey(m.ID), attributesSuffix)
	if _, err := s.Set(p, false, string(b), store.TTLOptionSet{ExpireTime: store.Permanent}); err != nil {
		plog.Panicf("update raftAttributes should never fail: %v", err)
	}
}

func mustSaveClusterVersionToStore(s store.Store, ver *semver.Version) {
	if _, err := s.Set(StoreClusterVersionKey(), false, ver.String(), store.TTLOptionSet{ExpireTime: store.Permanent}); err != nil {
		plog.Panicf("save cluster version should never fail: %v", err)
	}
}

// nodeToMember builds member from a key value node.
// the child nodes of the given node MUST be sorted by key.
func nodeToMember(n *store.NodeExtern) (*Member, error) {
	m := &Member{ID: MustParseMemberIDFromKey(n.Key)}
	attrs := make(map[string][]byte)
	raftAttrKey := path.Join(n.Key, raftAttributesSuffix)
	attrKey := path.Join(n.Key, attributesSuffix)
	for _, nn := range n.Nodes {
		if nn.Key != raftAttrKey && nn.Key != attrKey {
			return nil, fmt.Errorf("unknown key %q", nn.Key)
		}
		attrs[nn.Key] = []byte(*nn.Value)
	}
	if data := attrs[raftAttrKey]; data != nil {
		if err := json.Unmarshal(data, &m.RaftAttributes); err != nil {
			return nil, fmt.Errorf("unmarshal raftAttributes error: %v", err)
		}
	} else {
		return nil, fmt.Errorf("raftAttributes key doesn't exist")
	}
	if data := attrs[attrKey]; data != nil {
		if err := json.Unmarshal(data, &m.Attributes); err != nil {
			return m, fmt.Errorf("unmarshal attributes error: %v", err)
		}
	}
	return m, nil
}

func backendMemberKey(id types.ID) []byte {
	return []byte(id.String())
}

func backendClusterVersionKey() []byte {
	return []byte("clusterVersion")
}

func mustCreateBackendBuckets(be backend.Backend) {
	tx := be.BatchTx()
	tx.Lock()
	defer tx.Unlock()
	tx.UnsafeCreateBucket(membersBucketName)
	tx.UnsafeCreateBucket(membersRemovedBuckedName)
	tx.UnsafeCreateBucket(clusterBucketName)
}

func MemberStoreKey(id types.ID) string {
	return path.Join(StoreMembersPrefix, id.String())
}

func StoreClusterVersionKey() string {
	return path.Join(storePrefix, "version")
}

func MemberAttributesStorePath(id types.ID) string {
	return path.Join(MemberStoreKey(id), attributesSuffix)
}

func MustParseMemberIDFromKey(key string) types.ID {
	id, err := types.IDFromString(path.Base(key))
	if err != nil {
		plog.Panicf("unexpected parse member id error: %v", err)
	}
	return id
}

func RemovedMemberStoreKey(id types.ID) string {
	return path.Join(storeRemovedMembersPrefix, id.String())
}
<commit_msg>etcdserver: fix a typo in bucket name var<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package membership

import (
	"encoding/json"
	"fmt"
	"path"

	"github.com/coreos/etcd/mvcc/backend"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/store"

	"github.com/coreos/go-semver/semver"
)

const (
	attributesSuffix     = "attributes"
	raftAttributesSuffix = "raftAttributes"

	// the prefix for stroing membership related information in store provided by store pkg.
	storePrefix = "/0"
)

var (
	membersBucketName        = []byte("members")
	membersRemovedBucketName = []byte("members_removed")
	clusterBucketName        = []byte("cluster")

	StoreMembersPrefix        = path.Join(storePrefix, "members")
	storeRemovedMembersPrefix = path.Join(storePrefix, "removed_members")
)

func mustSaveMemberToBackend(be backend.Backend, m *Member) {
	mkey := backendMemberKey(m.ID)
	mvalue, err := json.Marshal(m)
	if err != nil {
		plog.Panicf("marshal raftAttributes should never fail: %v", err)
	}

	tx := be.BatchTx()
	tx.Lock()
	tx.UnsafePut(membersBucketName, mkey, mvalue)
	tx.Unlock()
}

func mustDeleteMemberFromBackend(be backend.Backend, id types.ID) {
	mkey := backendMemberKey(id)

	tx := be.BatchTx()
	tx.Lock()
	tx.UnsafeDelete(membersBucketName, mkey)
	tx.UnsafePut(membersRemovedBucketName, mkey, []byte("removed"))
	tx.Unlock()
}

func mustSaveClusterVersionToBackend(be backend.Backend, ver *semver.Version) {
	ckey := backendClusterVersionKey()

	tx := be.BatchTx()
	tx.Lock()
	defer tx.Unlock()
	tx.UnsafePut(clusterBucketName, ckey, []byte(ver.String()))
}

func mustSaveMemberToStore(s store.Store, m *Member) {
	b, err := json.Marshal(m.RaftAttributes)
	if err != nil {
		plog.Panicf("marshal raftAttributes should never fail: %v", err)
	}
	p := path.Join(MemberStoreKey(m.ID), raftAttributesSuffix)
	if _, err := s.Create(p, false, string(b), false, store.TTLOptionSet{ExpireTime: store.Permanent}); err != nil {
		plog.Panicf("create raftAttributes should never fail: %v", err)
	}
}

func mustDeleteMemberFromStore(s store.Store, id types.ID) {
	if _, err := s.Delete(MemberStoreKey(id), true, true); err != nil {
		plog.Panicf("delete member should never fail: %v", err)
	}
	if _, err := s.Create(RemovedMemberStoreKey(id), false, "", false, store.TTLOptionSet{ExpireTime: store.Permanent}); err != nil {
		plog.Panicf("create removedMember should never fail: %v", err)
	}
}

func mustUpdateMemberInStore(s store.Store, m *Member) {
	b, err := json.Marshal(m.RaftAttributes)
	if err != nil {
		plog.Panicf("marshal raftAttributes should never fail: %v", err)
	}
	p := path.Join(MemberStoreKey(m.ID), raftAttributesSuffix)
	if _, err := s.Update(p, string(b), store.TTLOptionSet{ExpireTime: store.Permanent}); err != nil {
		plog.Panicf("update raftAttributes should never fail: %v", err)
	}
}

func mustUpdateMemberAttrInStore(s store.Store, m *Member) {
	b, err := json.Marshal(m.Attributes)
	if err != nil {
		plog.Panicf("marshal raftAttributes should never fail: %v", err)
	}
	p := path.Join(MemberStoreKey(m.ID), attributesSuffix)
	if _, err := s.Set(p, false, string(b), store.TTLOptionSet{ExpireTime: store.Permanent}); err != nil {
		plog.Panicf("update raftAttributes should never fail: %v", err)
	}
}

func mustSaveClusterVersionToStore(s store.Store, ver *semver.Version) {
	if _, err := s.Set(StoreClusterVersionKey(), false, ver.String(), store.TTLOptionSet{ExpireTime: store.Permanent}); err != nil {
		plog.Panicf("save cluster version should never fail: %v", err)
	}
}

// nodeToMember builds member from a key value node.
// the child nodes of the given node MUST be sorted by key.
func nodeToMember(n *store.NodeExtern) (*Member, error) {
	m := &Member{ID: MustParseMemberIDFromKey(n.Key)}
	attrs := make(map[string][]byte)
	raftAttrKey := path.Join(n.Key, raftAttributesSuffix)
	attrKey := path.Join(n.Key, attributesSuffix)
	for _, nn := range n.Nodes {
		if nn.Key != raftAttrKey && nn.Key != attrKey {
			return nil, fmt.Errorf("unknown key %q", nn.Key)
		}
		attrs[nn.Key] = []byte(*nn.Value)
	}
	if data := attrs[raftAttrKey]; data != nil {
		if err := json.Unmarshal(data, &m.RaftAttributes); err != nil {
			return nil, fmt.Errorf("unmarshal raftAttributes error: %v", err)
		}
	} else {
		return nil, fmt.Errorf("raftAttributes key doesn't exist")
	}
	if data := attrs[attrKey]; data != nil {
		if err := json.Unmarshal(data, &m.Attributes); err != nil {
			return m, fmt.Errorf("unmarshal attributes error: %v", err)
		}
	}
	return m, nil
}

func backendMemberKey(id types.ID) []byte {
	return []byte(id.String())
}

func backendClusterVersionKey() []byte {
	return []byte("clusterVersion")
}

func mustCreateBackendBuckets(be backend.Backend) {
	tx := be.BatchTx()
	tx.Lock()
	defer tx.Unlock()
	tx.UnsafeCreateBucket(membersBucketName)
	tx.UnsafeCreateBucket(membersRemovedBucketName)
	tx.UnsafeCreateBucket(clusterBucketName)
}

func MemberStoreKey(id types.ID) string {
	return path.Join(StoreMembersPrefix, id.String())
}

func StoreClusterVersionKey() string {
	return path.Join(storePrefix, "version")
}

func MemberAttributesStorePath(id types.ID) string {
	return path.Join(MemberStoreKey(id), attributesSuffix)
}

func MustParseMemberIDFromKey(key string) types.ID {
	id, err := types.IDFromString(path.Base(key))
	if err != nil {
		plog.Panicf("unexpected parse member id error: %v", err)
	}
	return id
}

func RemovedMemberStoreKey(id types.ID) string {
	return path.Join(storeRemovedMembersPrefix, id.String())
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os"
	"os/signal"
	"runtime"
	"sort"
	"strings"
	"time"

	"github.com/google/gops/agent"
	defaults "github.com/mcuadros/go-defaults"
	"github.com/spf13/cobra"
	_ "github.com/spf13/viper/remote"
	"github.com/yesnault/go-toml"

	"github.com/ovh/cds/engine/api"
	"github.com/ovh/cds/engine/api/database"
	"github.com/ovh/cds/engine/hatchery/kubernetes"
	"github.com/ovh/cds/engine/hatchery/local"
	"github.com/ovh/cds/engine/hatchery/marathon"
	"github.com/ovh/cds/engine/hatchery/openstack"
	"github.com/ovh/cds/engine/hatchery/swarm"
	"github.com/ovh/cds/engine/hatchery/vsphere"
	"github.com/ovh/cds/engine/hooks"
	"github.com/ovh/cds/engine/repositories"
	"github.com/ovh/cds/engine/vcs"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/doc"
	"github.com/ovh/cds/sdk/log"
)

var (
	cfgFile      string
	remoteCfg    string
	remoteCfgKey string
	vaultAddr    string
	vaultToken   string
	vaultConfKey = "/secret/cds/conf"
	conf         = &Configuration{}
)

func init() {
	startCmd.Flags().StringVar(&cfgFile, "config", "", "config file")
	startCmd.Flags().StringVar(&remoteCfg, "remote-config", "", "(optional) consul configuration store")
	startCmd.Flags().StringVar(&remoteCfgKey, "remote-config-key", "cds/config.api.toml", "(optional) consul configuration store key")
	startCmd.Flags().StringVar(&vaultAddr, "vault-addr", "", "(optional) Vault address to fetch secrets from vault (example: https://vault.mydomain.net:8200)")
	startCmd.Flags().StringVar(&vaultToken, "vault-token", "", "(optional) Vault token to fetch secrets from vault")
	//Version  command
	mainCmd.AddCommand(versionCmd)
	//Update  command
	mainCmd.AddCommand(updateCmd)
	updateCmd.Flags().BoolVar(&updateFromGithub, "from-github", false, "Update binary from latest github release")
	updateCmd.Flags().StringVar(&updateURLAPI, "api", "", "Update binary from a CDS Engine API")

	//Database command
	mainCmd.AddCommand(database.DBCmd)
	//Start command
	mainCmd.AddCommand(startCmd)
	//Config command
	mainCmd.AddCommand(configCmd)
	configNewCmd.Flags().BoolVar(&configNewAsEnvFlag, "env", false, "Print configuration as environment variable")

	configCmd.AddCommand(configNewCmd)
	configCmd.AddCommand(configCheckCmd)

	// doc command (hidden command)
	mainCmd.AddCommand(docCmd)
}

func main() {
	mainCmd.Execute()
}

var mainCmd = &cobra.Command{
	Use:   "engine",
	Short: "CDS Engine",
	Long: `
CDS

Continuous Delivery Service

Enterprise-Grade Continuous Delivery & DevOps Automation Open Source Platform

https://ovh.github.io/cds/

## Download

You'll find last release of CDS ` + "`engine`" + ` on [Github Releases](https://github.com/ovh/cds/releases/latest).
`,
}

var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "Display CDS version",
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Printf("CDS Engine version:%s os:%s architecture:%s\n", sdk.VERSION, runtime.GOOS, runtime.GOARCH)
	},
}

var docCmd = &cobra.Command{
	Use:    "doc <generation-path> <git-directory>",
	Short:  "generate hugo doc for building http://ovh.github.com/cds",
	Hidden: true,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) != 2 {
			cmd.Usage()
			os.Exit(1)
		}
		if err := doc.GenerateDocumentation(mainCmd, args[0], args[1]); err != nil {
			sdk.Exit(err.Error())
		}
	},
}

var configCmd = &cobra.Command{
	Use:   "config",
	Short: "Manage CDS Configuration",
}

var configNewAsEnvFlag bool

var configNewCmd = &cobra.Command{
	Use:   "new",
	Short: "CDS configuration file assistant",
	Long: `
Comming soon...`,
	Run: func(cmd *cobra.Command, args []string) {
		defaults.SetDefaults(conf)

		conf.API.Auth.SharedInfraToken = sdk.RandomString(128)
		conf.API.Secrets.Key = sdk.RandomString(32)
		conf.Hatchery.Local.API.Token = conf.API.Auth.SharedInfraToken
		conf.Hatchery.Openstack.API.Token = conf.API.Auth.SharedInfraToken
		conf.Hatchery.VSphere.API.Token = conf.API.Auth.SharedInfraToken
		conf.Hatchery.Swarm.API.Token = conf.API.Auth.SharedInfraToken
		conf.Hatchery.Marathon.API.Token = conf.API.Auth.SharedInfraToken
		conf.Hooks.API.Token = conf.API.Auth.SharedInfraToken
		conf.Repositories.API.Token = conf.API.Auth.SharedInfraToken
		conf.VCS.API.Token = conf.API.Auth.SharedInfraToken
		conf.VCS.Servers = map[string]vcs.ServerConfiguration{}
		conf.VCS.Servers["Github"] = vcs.ServerConfiguration{
			URL: "https://github.com",
			Github: &vcs.GithubServerConfiguration{
				ClientID:     "xxxx",
				ClientSecret: "xxxx",
			},
		}
		conf.VCS.Servers["Bitbucket"] = vcs.ServerConfiguration{
			URL: "https://mybitbucket.com",
			Bitbucket: &vcs.BitbucketServerConfiguration{
				ConsumerKey: "xxx",
				PrivateKey:  "xxx",
			},
		}
		conf.VCS.Servers["Gitlab"] = vcs.ServerConfiguration{
			URL: "https://gitlab.com",
			Gitlab: &vcs.GitlabServerConfiguration{
				AppID:  "xxxx",
				Secret: "xxxx",
			},
		}

		if !configNewAsEnvFlag {
			btes, err := toml.Marshal(*conf)
			if err != nil {
				sdk.Exit("%v", err)
			}
			fmt.Println(string(btes))
		} else {
			m := AsEnvVariables(conf, "cds", true)
			keys := []string{}

			for k := range m {
				keys = append(keys, k)
			}

			sort.Strings(keys)
			for _, k := range keys {
				fmt.Printf("export %s=\"%s\"\n", k, m[k])
			}
		}
	},
}

var configCheckCmd = &cobra.Command{
	Use:   "check",
	Short: "Check CDS configuration file",
	Long:  `$ engine config check <path>`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) != 1 {
			cmd.Help()
			sdk.Exit("Wrong usage")
		}

		cfgFile = args[0]
		//Initialize config
		config()

		var hasError bool
		if conf.API.URL.API != "" {
			if err := api.New().CheckConfiguration(conf.API); err != nil {
				fmt.Println(err)
				hasError = true
			}
		}

		if conf.Hatchery.Local.API.HTTP.URL != "" {
			if err := local.New().CheckConfiguration(conf.Hatchery.Local); err != nil {
				fmt.Println(err)
				hasError = true
			}
		}

		if conf.Hatchery.Marathon.API.HTTP.URL != "" {
			if err := marathon.New().CheckConfiguration(conf.Hatchery.Marathon); err != nil {
				fmt.Println(err)
				hasError = true
			}
		}

		if conf.Hatchery.Openstack.API.HTTP.URL != "" {
			if err := openstack.New().CheckConfiguration(conf.Hatchery.Openstack); err != nil {
				fmt.Println(err)
				hasError = true
			}
		}

		if conf.Hatchery.Swarm.API.HTTP.URL != "" {
			if err := swarm.New().CheckConfiguration(conf.Hatchery.Swarm); err != nil {
				fmt.Println(err)
				hasError = true
			}
		}

		if !hasError {
			fmt.Println("Configuration file OK")
		}
	},
}

var startCmd = &cobra.Command{
	Use:   "start",
	Short: "Start CDS",
	Long: `
Start CDS Engine Services

#### API

This is the core component of CDS.

#### Hatcheries

They are the components responsible for spawning workers. Supported platforms/orchestrators are:

* Local machine
* Openstack
* Docker Swarm
* Openstack
* Vsphere

#### Hooks
This component operates CDS workflow hooks

#### Repositories
This component operates CDS workflow repositories

#### VCS
This component operates CDS VCS connectivity

Start all of this with a single command:

	$ engine start [api] [hatchery:local] [hatchery:marathon] [hatchery:openstack] [hatchery:swarm] [hatchery:vsphere] [hooks] [vcs] [repositories]

All the services are using the same configuration file format.

You have to specify where the toml configuration is. It can be a local file, provided by consul or vault.

You can also use or override toml file with environment variable.

See $ engine config command for more details.

`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) == 0 {
			cmd.Help()
			return
		}

		//Initialize config
		config()

		// gops debug
		if conf.Debug.Enable {
			if conf.Debug.RemoteDebugURL != "" {
				log.Info("Starting gops agent on %s", conf.Debug.RemoteDebugURL)
				if err := agent.Listen(&agent.Options{Addr: conf.Debug.RemoteDebugURL}); err != nil {
					log.Error("Error on starting gops agent", err)
				}
			} else {
				log.Info("Starting gops agent locally")
				if err := agent.Listen(nil); err != nil {
					log.Error("Error on starting gops agent locally", err)
				}
			}
		}

		//Initialize context
		ctx := context.Background()
		ctx, cancel := context.WithCancel(ctx)
		defer cancel()

		// Gracefully shutdown all
		c := make(chan os.Signal, 1)
		signal.Notify(c)
		go func() {
			<-c
			signal.Stop(c)
			cancel()
		}()

		type serviceConf struct {
			arg     string
			service Service
			cfg     interface{}
		}
		services := []serviceConf{}

		names := []string{}
		for _, a := range args {
			fmt.Printf("Starting service %s\n", a)
			switch a {
			case "api":
				services = append(services, serviceConf{arg: a, service: api.New(), cfg: conf.API})
				names = append(names, conf.API.Name)
			case "hatchery:local":
				services = append(services, serviceConf{arg: a, service: local.New(), cfg: conf.Hatchery.Local})
				names = append(names, conf.Hatchery.Local.Name)
			case "hatchery:kubernetes":
				services = append(services, serviceConf{arg: a, service: kubernetes.New(), cfg: conf.Hatchery.Kubernetes})
				names = append(names, conf.Hatchery.Kubernetes.Name)
			case "hatchery:marathon":
				services = append(services, serviceConf{arg: a, service: marathon.New(), cfg: conf.Hatchery.Marathon})
				names = append(names, conf.Hatchery.Marathon.Name)
			case "hatchery:openstack":
				services = append(services, serviceConf{arg: a, service: openstack.New(), cfg: conf.Hatchery.Openstack})
				names = append(names, conf.Hatchery.Openstack.Name)
			case "hatchery:swarm":
				services = append(services, serviceConf{arg: a, service: swarm.New(), cfg: conf.Hatchery.Swarm})
				names = append(names, conf.Hatchery.Swarm.Name)
			case "hatchery:vsphere":
				services = append(services, serviceConf{arg: a, service: vsphere.New(), cfg: conf.Hatchery.VSphere})
				names = append(names, conf.Hatchery.VSphere.Name)
			case "hooks":
				services = append(services, serviceConf{arg: a, service: hooks.New(), cfg: conf.Hooks})
				names = append(names, conf.Hooks.Name)
			case "vcs":
				services = append(services, serviceConf{arg: a, service: vcs.New(), cfg: conf.VCS})
				names = append(names, conf.VCS.Name)
			case "repositories":
				services = append(services, serviceConf{arg: a, service: repositories.New(), cfg: conf.Repositories})
				names = append(names, conf.Repositories.Name)
			default:
				fmt.Printf("Error: service '%s' unknown\n", a)
				os.Exit(1)
			}
		}

		//Initialize logs
		log.Initialize(&log.Conf{
			Level:                  conf.Log.Level,
			GraylogProtocol:        conf.Log.Graylog.Protocol,
			GraylogHost:            conf.Log.Graylog.Host,
			GraylogPort:            fmt.Sprintf("%d", conf.Log.Graylog.Port),
			GraylogExtraKey:        conf.Log.Graylog.ExtraKey,
			GraylogExtraValue:      conf.Log.Graylog.ExtraValue,
			GraylogFieldCDSVersion: sdk.VERSION,
			GraylogFieldCDSName:    strings.Join(names, "_"),
			Ctx:                    ctx,
		})

		for _, s := range services {
			go start(ctx, s.service, s.cfg)

			//Stupid trick: when API is starting wait a bit before start the other
			if s.arg == "API" || s.arg == "api" {
				time.Sleep(2 * time.Second)
			}
		}

		//Wait for the end
		<-ctx.Done()
		if ctx.Err() != nil {
			fmt.Printf("Exiting (%v)\n", ctx.Err())
		}
	},
}

func start(c context.Context, s Service, cfg interface{}) {
	if err := s.ApplyConfiguration(cfg); err != nil {
		sdk.Exit("Unable to init service: %v", err)
	}
	if err := s.Serve(c); err != nil {
		sdk.Exit("Service has been stopped: %v", err)
	}
}
<commit_msg>fix (engine): OS signals (#2183)<commit_after>package main

import (
	"context"
	"fmt"
	"os"
	"os/signal"
	"runtime"
	"sort"
	"strings"
	"syscall"
	"time"

	"github.com/google/gops/agent"
	defaults "github.com/mcuadros/go-defaults"
	"github.com/spf13/cobra"
	_ "github.com/spf13/viper/remote"
	"github.com/yesnault/go-toml"

	"github.com/ovh/cds/engine/api"
	"github.com/ovh/cds/engine/api/database"
	"github.com/ovh/cds/engine/hatchery/kubernetes"
	"github.com/ovh/cds/engine/hatchery/local"
	"github.com/ovh/cds/engine/hatchery/marathon"
	"github.com/ovh/cds/engine/hatchery/openstack"
	"github.com/ovh/cds/engine/hatchery/swarm"
	"github.com/ovh/cds/engine/hatchery/vsphere"
	"github.com/ovh/cds/engine/hooks"
	"github.com/ovh/cds/engine/repositories"
	"github.com/ovh/cds/engine/vcs"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/doc"
	"github.com/ovh/cds/sdk/log"
)

var (
	cfgFile      string
	remoteCfg    string
	remoteCfgKey string
	vaultAddr    string
	vaultToken   string
	vaultConfKey = "/secret/cds/conf"
	conf         = &Configuration{}
)

func init() {
	startCmd.Flags().StringVar(&cfgFile, "config", "", "config file")
	startCmd.Flags().StringVar(&remoteCfg, "remote-config", "", "(optional) consul configuration store")
	startCmd.Flags().StringVar(&remoteCfgKey, "remote-config-key", "cds/config.api.toml", "(optional) consul configuration store key")
	startCmd.Flags().StringVar(&vaultAddr, "vault-addr", "", "(optional) Vault address to fetch secrets from vault (example: https://vault.mydomain.net:8200)")
	startCmd.Flags().StringVar(&vaultToken, "vault-token", "", "(optional) Vault token to fetch secrets from vault")
	//Version  command
	mainCmd.AddCommand(versionCmd)
	//Update  command
	mainCmd.AddCommand(updateCmd)
	updateCmd.Flags().BoolVar(&updateFromGithub, "from-github", false, "Update binary from latest github release")
	updateCmd.Flags().StringVar(&updateURLAPI, "api", "", "Update binary from a CDS Engine API")

	//Database command
	mainCmd.AddCommand(database.DBCmd)
	//Start command
	mainCmd.AddCommand(startCmd)
	//Config command
	mainCmd.AddCommand(configCmd)
	configNewCmd.Flags().BoolVar(&configNewAsEnvFlag, "env", false, "Print configuration as environment variable")

	configCmd.AddCommand(configNewCmd)
	configCmd.AddCommand(configCheckCmd)

	// doc command (hidden command)
	mainCmd.AddCommand(docCmd)
}

func main() {
	mainCmd.Execute()
}

var mainCmd = &cobra.Command{
	Use:   "engine",
	Short: "CDS Engine",
	Long: `
CDS

Continuous Delivery Service

Enterprise-Grade Continuous Delivery & DevOps Automation Open Source Platform

https://ovh.github.io/cds/

## Download

You'll find last release of CDS ` + "`engine`" + ` on [Github Releases](https://github.com/ovh/cds/releases/latest).
`,
}

var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "Display CDS version",
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Printf("CDS Engine version:%s os:%s architecture:%s\n", sdk.VERSION, runtime.GOOS, runtime.GOARCH)
	},
}

var docCmd = &cobra.Command{
	Use:    "doc <generation-path> <git-directory>",
	Short:  "generate hugo doc for building http://ovh.github.com/cds",
	Hidden: true,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) != 2 {
			cmd.Usage()
			os.Exit(1)
		}
		if err := doc.GenerateDocumentation(mainCmd, args[0], args[1]); err != nil {
			sdk.Exit(err.Error())
		}
	},
}

var configCmd = &cobra.Command{
	Use:   "config",
	Short: "Manage CDS Configuration",
}

var configNewAsEnvFlag bool

var configNewCmd = &cobra.Command{
	Use:   "new",
	Short: "CDS configuration file assistant",
	Long: `
Comming soon...`,
	Run: func(cmd *cobra.Command, args []string) {
		defaults.SetDefaults(conf)

		conf.API.Auth.SharedInfraToken = sdk.RandomString(128)
		conf.API.Secrets.Key = sdk.RandomString(32)
		conf.Hatchery.Local.API.Token = conf.API.Auth.SharedInfraToken
		conf.Hatchery.Openstack.API.Token = conf.API.Auth.SharedInfraToken
		conf.Hatchery.VSphere.API.Token = conf.API.Auth.SharedInfraToken
		conf.Hatchery.Swarm.API.Token = conf.API.Auth.SharedInfraToken
		conf.Hatchery.Marathon.API.Token = conf.API.Auth.SharedInfraToken
		conf.Hooks.API.Token = conf.API.Auth.SharedInfraToken
		conf.Repositories.API.Token = conf.API.Auth.SharedInfraToken
		conf.VCS.API.Token = conf.API.Auth.SharedInfraToken
		conf.VCS.Servers = map[string]vcs.ServerConfiguration{}
		conf.VCS.Servers["Github"] = vcs.ServerConfiguration{
			URL: "https://github.com",
			Github: &vcs.GithubServerConfiguration{
				ClientID:     "xxxx",
				ClientSecret: "xxxx",
			},
		}
		conf.VCS.Servers["Bitbucket"] = vcs.ServerConfiguration{
			URL: "https://mybitbucket.com",
			Bitbucket: &vcs.BitbucketServerConfiguration{
				ConsumerKey: "xxx",
				PrivateKey:  "xxx",
			},
		}
		conf.VCS.Servers["Gitlab"] = vcs.ServerConfiguration{
			URL: "https://gitlab.com",
			Gitlab: &vcs.GitlabServerConfiguration{
				AppID:  "xxxx",
				Secret: "xxxx",
			},
		}

		if !configNewAsEnvFlag {
			btes, err := toml.Marshal(*conf)
			if err != nil {
				sdk.Exit("%v", err)
			}
			fmt.Println(string(btes))
		} else {
			m := AsEnvVariables(conf, "cds", true)
			keys := []string{}

			for k := range m {
				keys = append(keys, k)
			}

			sort.Strings(keys)
			for _, k := range keys {
				fmt.Printf("export %s=\"%s\"\n", k, m[k])
			}
		}
	},
}

var configCheckCmd = &cobra.Command{
	Use:   "check",
	Short: "Check CDS configuration file",
	Long:  `$ engine config check <path>`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) != 1 {
			cmd.Help()
			sdk.Exit("Wrong usage")
		}

		cfgFile = args[0]
		//Initialize config
		config()

		var hasError bool
		if conf.API.URL.API != "" {
			if err := api.New().CheckConfiguration(conf.API); err != nil {
				fmt.Println(err)
				hasError = true
			}
		}

		if conf.Hatchery.Local.API.HTTP.URL != "" {
			if err := local.New().CheckConfiguration(conf.Hatchery.Local); err != nil {
				fmt.Println(err)
				hasError = true
			}
		}

		if conf.Hatchery.Marathon.API.HTTP.URL != "" {
			if err := marathon.New().CheckConfiguration(conf.Hatchery.Marathon); err != nil {
				fmt.Println(err)
				hasError = true
			}
		}

		if conf.Hatchery.Openstack.API.HTTP.URL != "" {
			if err := openstack.New().CheckConfiguration(conf.Hatchery.Openstack); err != nil {
				fmt.Println(err)
				hasError = true
			}
		}

		if conf.Hatchery.Swarm.API.HTTP.URL != "" {
			if err := swarm.New().CheckConfiguration(conf.Hatchery.Swarm); err != nil {
				fmt.Println(err)
				hasError = true
			}
		}

		if !hasError {
			fmt.Println("Configuration file OK")
		}
	},
}

var startCmd = &cobra.Command{
	Use:   "start",
	Short: "Start CDS",
	Long: `
Start CDS Engine Services

#### API

This is the core component of CDS.

#### Hatcheries

They are the components responsible for spawning workers. Supported platforms/orchestrators are:

* Local machine
* Openstack
* Docker Swarm
* Openstack
* Vsphere

#### Hooks
This component operates CDS workflow hooks

#### Repositories
This component operates CDS workflow repositories

#### VCS
This component operates CDS VCS connectivity

Start all of this with a single command:

	$ engine start [api] [hatchery:local] [hatchery:marathon] [hatchery:openstack] [hatchery:swarm] [hatchery:vsphere] [hooks] [vcs] [repositories]

All the services are using the same configuration file format.

You have to specify where the toml configuration is. It can be a local file, provided by consul or vault.

You can also use or override toml file with environment variable.

See $ engine config command for more details.

`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) == 0 {
			cmd.Help()
			return
		}

		//Initialize config
		config()

		// gops debug
		if conf.Debug.Enable {
			if conf.Debug.RemoteDebugURL != "" {
				log.Info("Starting gops agent on %s", conf.Debug.RemoteDebugURL)
				if err := agent.Listen(&agent.Options{Addr: conf.Debug.RemoteDebugURL}); err != nil {
					log.Error("Error on starting gops agent", err)
				}
			} else {
				log.Info("Starting gops agent locally")
				if err := agent.Listen(nil); err != nil {
					log.Error("Error on starting gops agent locally", err)
				}
			}
		}

		//Initialize context
		ctx := context.Background()
		ctx, cancel := context.WithCancel(ctx)
		defer cancel()

		// Gracefully shutdown all
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGKILL)
		go func() {
			<-c
			signal.Stop(c)
			cancel()
		}()

		type serviceConf struct {
			arg     string
			service Service
			cfg     interface{}
		}
		services := []serviceConf{}

		names := []string{}
		for _, a := range args {
			fmt.Printf("Starting service %s\n", a)
			switch a {
			case "api":
				services = append(services, serviceConf{arg: a, service: api.New(), cfg: conf.API})
				names = append(names, conf.API.Name)
			case "hatchery:local":
				services = append(services, serviceConf{arg: a, service: local.New(), cfg: conf.Hatchery.Local})
				names = append(names, conf.Hatchery.Local.Name)
			case "hatchery:kubernetes":
				services = append(services, serviceConf{arg: a, service: kubernetes.New(), cfg: conf.Hatchery.Kubernetes})
				names = append(names, conf.Hatchery.Kubernetes.Name)
			case "hatchery:marathon":
				services = append(services, serviceConf{arg: a, service: marathon.New(), cfg: conf.Hatchery.Marathon})
				names = append(names, conf.Hatchery.Marathon.Name)
			case "hatchery:openstack":
				services = append(services, serviceConf{arg: a, service: openstack.New(), cfg: conf.Hatchery.Openstack})
				names = append(names, conf.Hatchery.Openstack.Name)
			case "hatchery:swarm":
				services = append(services, serviceConf{arg: a, service: swarm.New(), cfg: conf.Hatchery.Swarm})
				names = append(names, conf.Hatchery.Swarm.Name)
			case "hatchery:vsphere":
				services = append(services, serviceConf{arg: a, service: vsphere.New(), cfg: conf.Hatchery.VSphere})
				names = append(names, conf.Hatchery.VSphere.Name)
			case "hooks":
				services = append(services, serviceConf{arg: a, service: hooks.New(), cfg: conf.Hooks})
				names = append(names, conf.Hooks.Name)
			case "vcs":
				services = append(services, serviceConf{arg: a, service: vcs.New(), cfg: conf.VCS})
				names = append(names, conf.VCS.Name)
			case "repositories":
				services = append(services, serviceConf{arg: a, service: repositories.New(), cfg: conf.Repositories})
				names = append(names, conf.Repositories.Name)
			default:
				fmt.Printf("Error: service '%s' unknown\n", a)
				os.Exit(1)
			}
		}

		//Initialize logs
		log.Initialize(&log.Conf{
			Level:                  conf.Log.Level,
			GraylogProtocol:        conf.Log.Graylog.Protocol,
			GraylogHost:            conf.Log.Graylog.Host,
			GraylogPort:            fmt.Sprintf("%d", conf.Log.Graylog.Port),
			GraylogExtraKey:        conf.Log.Graylog.ExtraKey,
			GraylogExtraValue:      conf.Log.Graylog.ExtraValue,
			GraylogFieldCDSVersion: sdk.VERSION,
			GraylogFieldCDSName:    strings.Join(names, "_"),
			Ctx:                    ctx,
		})

		for _, s := range services {
			go start(ctx, s.service, s.cfg)

			//Stupid trick: when API is starting wait a bit before start the other
			if s.arg == "API" || s.arg == "api" {
				time.Sleep(2 * time.Second)
			}
		}

		//Wait for the end
		<-ctx.Done()
		if ctx.Err() != nil {
			fmt.Printf("Exiting (%v)\n", ctx.Err())
		}
	},
}

func start(c context.Context, s Service, cfg interface{}) {
	if err := s.ApplyConfiguration(cfg); err != nil {
		sdk.Exit("Unable to init service: %v", err)
	}
	if err := s.Serve(c); err != nil {
		sdk.Exit("Service has been stopped: %v", err)
	}
}
<|endoftext|>


<commit_before>package exchange_actions

import (
	"encoding/csv"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/qor/admin"
	"github.com/qor/i18n"
	"github.com/qor/media/oss"
	"github.com/qor/worker"
)

type ExportTranslationArgument struct {
	Scope string
}

type ImportTranslationArgument struct {
	TranslationsFile oss.OSS
}

// RegisterExchangeJobs register i18n jobs into worker
func RegisterExchangeJobs(I18n *i18n.I18n, Worker *worker.Worker) {
	Worker.Admin.RegisterViewPath("github.com/qor/i18n/exchange_actions/views")

	// Export Translations
	exportTranslationResource := Worker.Admin.NewResource(&ExportTranslationArgument{})
	exportTranslationResource.Meta(&admin.Meta{Name: "Scope", Type: "select_one", Collection: []string{"All", "Backend", "Frontend"}})

	Worker.RegisterJob(&worker.Job{
		Name:     "Export Translations",
		Group:    "Export/Import Translations From CSV file",
		Resource: exportTranslationResource,
		Handler: func(arg interface{}, qorJob worker.QorJobInterface) (err error) {
			var (
				locales          []string
				translationKeys  []string
				translationsMap  = map[string]bool{}
				filename         = fmt.Sprintf("/downloads/translations.%v.csv", time.Now().UnixNano())
				fullFilename     = path.Join("public", filename)
				i18nTranslations = I18n.LoadTranslations()
				scope            = arg.(*ExportTranslationArgument).Scope
			)
			qorJob.AddLog("Exporting translations...")

			// Sort locales
			for locale := range i18nTranslations {
				locales = append(locales, locale)
			}
			sort.Strings(locales)

			// Create download file
			if _, err = os.Stat(filepath.Dir(fullFilename)); os.IsNotExist(err) {
				err = os.MkdirAll(filepath.Dir(fullFilename), os.ModePerm)
			}
			csvfile, err := os.OpenFile(fullFilename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
			defer csvfile.Close()
			if err != nil {
				return err
			}

			writer := csv.NewWriter(csvfile)

			// Append Headers
			writer.Write(append([]string{"Translation Keys"}, locales...))

			// Sort translation keys
			for _, locale := range locales {
				for key := range i18nTranslations[locale] {
					translationsMap[key] = true
				}
			}

			for key := range translationsMap {
				translationKeys = append(translationKeys, key)
			}
			sort.Strings(translationKeys)

			// Write CSV file
			var (
				recordCount         = len(translationKeys)
				perCount            = recordCount/20 + 1
				processedRecordLogs = []string{}
				index               = 0
				progressCount       = 0
			)
			for _, translationKey := range translationKeys {
				// Filter out translation by scope
				index++
				if scope == "Backend" && !strings.HasPrefix(translationKey, "qor_") {
					continue
				}
				if scope == "Frontend" && strings.HasPrefix(translationKey, "qor_") {
					continue
				}
				var translations = []string{translationKey}
				for _, locale := range locales {
					var value string
					if translation := i18nTranslations[locale][translationKey]; translation != nil {
						value = translation.Value
					}
					translations = append(translations, value)
				}
				writer.Write(translations)
				processedRecordLogs = append(processedRecordLogs, fmt.Sprintf("Exported %v\n", strings.Join(translations, ",")))
				if index == perCount {
					qorJob.AddLog(strings.Join(processedRecordLogs, ""))
					processedRecordLogs = []string{}
					progressCount++
					qorJob.SetProgress(uint(float32(progressCount) / float32(20) * 100))
					index = 0
				}
			}
			writer.Flush()

			qorJob.SetProgressText(fmt.Sprintf("Download exported translations", filename))
			return
		},
	})

	// Import Translations

	Worker.RegisterJob(&worker.Job{
		Name:     "Import Translations",
		Group:    "Export/Import Translations From CSV file",
		Resource: Worker.Admin.NewResource(&ImportTranslationArgument{}),
		Handler: func(arg interface{}, qorJob worker.QorJobInterface) (err error) {
			importTranslationArgument := arg.(*ImportTranslationArgument)
			qorJob.AddLog("Importing translations...")
			if csvfile, err := os.Open(path.Join("public", importTranslationArgument.TranslationsFile.URL())); err == nil {
				reader := csv.NewReader(csvfile)
				reader.TrimLeadingSpace = true
				if records, err := reader.ReadAll(); err == nil {
					if len(records) > 1 && len(records[0]) > 1 {
						var (
							recordCount         = len(records) - 1
							perCount            = recordCount/20 + 1
							processedRecordLogs = []string{}
							locales             = records[0][1:]
							index               = 1
						)
						for _, values := range records[1:] {
							logMsg := ""
							for idx, value := range values[1:] {
								if value == "" {
									if values[0] != "" && locales[idx] != "" {
										I18n.DeleteTranslation(&i18n.Translation{
											Key:    values[0],
											Locale: locales[idx],
										})
										logMsg += fmt.Sprintf("%v/%v Deleted %v,%v\n", index, recordCount, locales[idx], values[0])
									}
								} else {
									I18n.SaveTranslation(&i18n.Translation{
										Key:    values[0],
										Locale: locales[idx],
										Value:  value,
									})
									logMsg += fmt.Sprintf("%v/%v Imported %v,%v,%v\n", index, recordCount, locales[idx], values[0], value)
								}
							}
							processedRecordLogs = append(processedRecordLogs, logMsg)
							if len(processedRecordLogs) == perCount {
								qorJob.AddLog(strings.Join(processedRecordLogs, ""))
								processedRecordLogs = []string{}
								qorJob.SetProgress(uint(float32(index) / float32(recordCount+1) * 100))
							}
							index++
						}
						qorJob.AddLog(strings.Join(processedRecordLogs, ""))
					}
				}
				qorJob.AddLog("Imported translations")
			}
			return
		},
	})
}
<commit_msg>Show warning message if when register jobs before mount i18n into admin<commit_after>package exchange_actions

import (
	"encoding/csv"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"runtime/debug"
	"sort"
	"strings"
	"time"

	"github.com/qor/admin"
	"github.com/qor/i18n"
	"github.com/qor/media/oss"
	"github.com/qor/worker"
)

type ExportTranslationArgument struct {
	Scope string
}

type ImportTranslationArgument struct {
	TranslationsFile oss.OSS
}

// RegisterExchangeJobs register i18n jobs into worker
func RegisterExchangeJobs(I18n *i18n.I18n, Worker *worker.Worker) {
	if I18n.Resource == nil {
		debug.PrintStack()
		fmt.Println("I18n should be registered into `Admin` before register jobs")
		return
	}

	Admin := I18n.Resource.GetAdmin()
	Admin.RegisterViewPath("github.com/qor/i18n/exchange_actions/views")

	// Export Translations
	exportTranslationResource := Admin.NewResource(&ExportTranslationArgument{})
	exportTranslationResource.Meta(&admin.Meta{Name: "Scope", Type: "select_one", Collection: []string{"All", "Backend", "Frontend"}})

	Worker.RegisterJob(&worker.Job{
		Name:     "Export Translations",
		Group:    "Export/Import Translations From CSV file",
		Resource: exportTranslationResource,
		Handler: func(arg interface{}, qorJob worker.QorJobInterface) (err error) {
			var (
				locales          []string
				translationKeys  []string
				translationsMap  = map[string]bool{}
				filename         = fmt.Sprintf("/downloads/translations.%v.csv", time.Now().UnixNano())
				fullFilename     = path.Join("public", filename)
				i18nTranslations = I18n.LoadTranslations()
				scope            = arg.(*ExportTranslationArgument).Scope
			)
			qorJob.AddLog("Exporting translations...")

			// Sort locales
			for locale := range i18nTranslations {
				locales = append(locales, locale)
			}
			sort.Strings(locales)

			// Create download file
			if _, err = os.Stat(filepath.Dir(fullFilename)); os.IsNotExist(err) {
				err = os.MkdirAll(filepath.Dir(fullFilename), os.ModePerm)
			}
			csvfile, err := os.OpenFile(fullFilename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
			defer csvfile.Close()
			if err != nil {
				return err
			}

			writer := csv.NewWriter(csvfile)

			// Append Headers
			writer.Write(append([]string{"Translation Keys"}, locales...))

			// Sort translation keys
			for _, locale := range locales {
				for key := range i18nTranslations[locale] {
					translationsMap[key] = true
				}
			}

			for key := range translationsMap {
				translationKeys = append(translationKeys, key)
			}
			sort.Strings(translationKeys)

			// Write CSV file
			var (
				recordCount         = len(translationKeys)
				perCount            = recordCount/20 + 1
				processedRecordLogs = []string{}
				index               = 0
				progressCount       = 0
			)
			for _, translationKey := range translationKeys {
				// Filter out translation by scope
				index++
				if scope == "Backend" && !strings.HasPrefix(translationKey, "qor_") {
					continue
				}
				if scope == "Frontend" && strings.HasPrefix(translationKey, "qor_") {
					continue
				}
				var translations = []string{translationKey}
				for _, locale := range locales {
					var value string
					if translation := i18nTranslations[locale][translationKey]; translation != nil {
						value = translation.Value
					}
					translations = append(translations, value)
				}
				writer.Write(translations)
				processedRecordLogs = append(processedRecordLogs, fmt.Sprintf("Exported %v\n", strings.Join(translations, ",")))
				if index == perCount {
					qorJob.AddLog(strings.Join(processedRecordLogs, ""))
					processedRecordLogs = []string{}
					progressCount++
					qorJob.SetProgress(uint(float32(progressCount) / float32(20) * 100))
					index = 0
				}
			}
			writer.Flush()

			qorJob.SetProgressText(fmt.Sprintf("Download exported translations", filename))
			return
		},
	})

	// Import Translations

	Worker.RegisterJob(&worker.Job{
		Name:     "Import Translations",
		Group:    "Export/Import Translations From CSV file",
		Resource: Admin.NewResource(&ImportTranslationArgument{}),
		Handler: func(arg interface{}, qorJob worker.QorJobInterface) (err error) {
			importTranslationArgument := arg.(*ImportTranslationArgument)
			qorJob.AddLog("Importing translations...")
			if csvfile, err := os.Open(path.Join("public", importTranslationArgument.TranslationsFile.URL())); err == nil {
				reader := csv.NewReader(csvfile)
				reader.TrimLeadingSpace = true
				if records, err := reader.ReadAll(); err == nil {
					if len(records) > 1 && len(records[0]) > 1 {
						var (
							recordCount         = len(records) - 1
							perCount            = recordCount/20 + 1
							processedRecordLogs = []string{}
							locales             = records[0][1:]
							index               = 1
						)
						for _, values := range records[1:] {
							logMsg := ""
							for idx, value := range values[1:] {
								if value == "" {
									if values[0] != "" && locales[idx] != "" {
										I18n.DeleteTranslation(&i18n.Translation{
											Key:    values[0],
											Locale: locales[idx],
										})
										logMsg += fmt.Sprintf("%v/%v Deleted %v,%v\n", index, recordCount, locales[idx], values[0])
									}
								} else {
									I18n.SaveTranslation(&i18n.Translation{
										Key:    values[0],
										Locale: locales[idx],
										Value:  value,
									})
									logMsg += fmt.Sprintf("%v/%v Imported %v,%v,%v\n", index, recordCount, locales[idx], values[0], value)
								}
							}
							processedRecordLogs = append(processedRecordLogs, logMsg)
							if len(processedRecordLogs) == perCount {
								qorJob.AddLog(strings.Join(processedRecordLogs, ""))
								processedRecordLogs = []string{}
								qorJob.SetProgress(uint(float32(index) / float32(recordCount+1) * 100))
							}
							index++
						}
						qorJob.AddLog(strings.Join(processedRecordLogs, ""))
					}
				}
				qorJob.AddLog("Imported translations")
			}
			return
		},
	})
}
<|endoftext|>


<commit_before>package activitypub

import "time"

// CreateActivity is the type for a create activity message
type CreateActivity struct {
	Activity  *Create
	Published time.Time
	To        ObjectsArr
	CC        ObjectsArr
}

// CreateActivityNew initializes a new CreateActivity message
func CreateActivityNew(id ObjectID, a ObjectOrLink, o ObjectOrLink) CreateActivity {
	act := CreateNew(id, o)

	if a != nil {
		typ := a.GetType()
		switch typ {
		case ApplicationType:
			var app Application
			app, _ = a.(Application)
			if app.Inbox == nil {
				app.Inbox = InboxNew()
			}
			app.Inbox.Append(o)
			act.Actor = app
		case GroupType:
			var grp Group
			grp, _ = a.(Group)
			if grp.Inbox == nil {
				grp.Inbox = InboxNew()
			}
			grp.Inbox.Append(o)
			act.Actor = grp
		case OrganizationType:
			var org Organization
			org, _ = a.(Organization)
			if org.Inbox == nil {
				org.Inbox = InboxNew()
			}
			org.Inbox.Append(o)
			act.Actor = org
		case PersonType:
			var pers Person
			pers, _ = a.(Person)
			if pers.Inbox == nil {
				pers.Inbox = InboxNew()
			}
			pers.Inbox.Append(o)
			act.Actor = pers
		case ServiceType:
			var serv Service
			serv, _ = a.(Service)
			serv.Inbox.Append(o)
			act.Actor = serv
		default:
			actor, _ := a.(Actor)
			if actor.Inbox == nil {
				actor.Inbox = InboxNew()
			}
			actor.Inbox.Append(o)
			act.Actor = actor
		}
	}

	c := CreateActivity{
		Activity:  act,
		Published: time.Now(),
	}

	return c
}
<commit_msg>Add case for CreateActivity that receives an IRI as Actor<commit_after>package activitypub

import "time"

// CreateActivity is the type for a create activity message
type CreateActivity struct {
	Activity  *Create
	Published time.Time
	To        ObjectsArr
	CC        ObjectsArr
}

func loadActorWithInboxObject(a ObjectOrLink, o ObjectOrLink) ObjectOrLink {
	typ := a.GetType()
	switch typ {
	case ApplicationType:
		var app Application
		app, _ = a.(Application)
		if app.Inbox == nil {
			app.Inbox = InboxNew()
		}
		app.Inbox.Append(o)
		return app
	case GroupType:
		var grp Group
		grp, _ = a.(Group)
		if grp.Inbox == nil {
			grp.Inbox = InboxNew()
		}
		grp.Inbox.Append(o)
		return grp
	case OrganizationType:
		var org Organization
		org, _ = a.(Organization)
		if org.Inbox == nil {
			org.Inbox = InboxNew()
		}
		org.Inbox.Append(o)
		return org
	case PersonType:
		var pers Person
		pers, _ = a.(Person)
		if pers.Inbox == nil {
			pers.Inbox = InboxNew()
		}
		pers.Inbox.Append(o)
		return pers
	case ServiceType:
		var serv Service
		serv, _ = a.(Service)
		serv.Inbox.Append(o)
		return serv
	default:
		actor, _ := a.(Actor)
		if actor.Inbox == nil {
			actor.Inbox = InboxNew()
		}
		actor.Inbox.Append(o)
		return actor
	}
}

// CreateActivityNew initializes a new CreateActivity message
func CreateActivityNew(id ObjectID, a ObjectOrLink, o ObjectOrLink) CreateActivity {
	act := CreateNew(id, o)

	if a != nil {
		if a.IsObject() {
			act.Actor = loadActorWithInboxObject(a, o)
		}
		if a.IsLink() {
			act.Actor = a
		}
	}

	act.RecipientsDeduplication()

	c := CreateActivity{
		Activity:  act,
		Published: time.Now(),
	}

	return c
}
<|endoftext|>