<commit_before>package liqui

import (
	"errors"
	"fmt"
	"log"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/thrasher-/gocryptotrader/common"
	"github.com/thrasher-/gocryptotrader/config"
	"github.com/thrasher-/gocryptotrader/exchanges"
	"github.com/thrasher-/gocryptotrader/exchanges/ticker"
)

const (
	LIQUI_API_PUBLIC_URL      = "https://api.Liqui.io/api"
	LIQUI_API_PRIVATE_URL     = "https://api.Liqui.io/tapi"
	LIQUI_API_PUBLIC_VERSION  = "3"
	LIQUI_API_PRIVATE_VERSION = "1"
	LIQUI_INFO                = "info"
	LIQUI_TICKER              = "ticker"
	LIQUI_DEPTH               = "depth"
	LIQUI_TRADES              = "trades"
	LIQUI_ACCOUNT_INFO        = "getInfo"
	LIQUI_TRADE               = "Trade"
	LIQUI_ACTIVE_ORDERS       = "ActiveOrders"
	LIQUI_ORDER_INFO          = "OrderInfo"
	LIQUI_CANCEL_ORDER        = "CancelOrder"
	LIQUI_TRADE_HISTORY       = "TradeHistory"
	LIQUI_WITHDRAW_COIN       = "WithdrawCoin"
)

type Liqui struct {
	exchange.Base
	Ticker map[string]LiquiTicker
	Info   LiquiInfo
}

func (l *Liqui) SetDefaults() {
	l.Name = "Liqui"
	l.Enabled = false
	l.Fee = 0.25
	l.Verbose = false
	l.Websocket = false
	l.RESTPollingDelay = 10
	l.Ticker = make(map[string]LiquiTicker)
	l.RequestCurrencyPairFormat.Delimiter = "_"
	l.RequestCurrencyPairFormat.Uppercase = false
	l.RequestCurrencyPairFormat.Separator = "-"
	l.ConfigCurrencyPairFormat.Delimiter = "_"
	l.ConfigCurrencyPairFormat.Uppercase = true
	l.AssetTypes = []string{ticker.Spot}
}

func (l *Liqui) Setup(exch config.ExchangeConfig) {
	if !exch.Enabled {
		l.SetEnabled(false)
	} else {
		l.Enabled = true
		l.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
		l.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
		l.RESTPollingDelay = exch.RESTPollingDelay
		l.Verbose = exch.Verbose
		l.Websocket = exch.Websocket
		l.BaseCurrencies = common.SplitStrings(exch.BaseCurrencies, ",")
		l.AvailablePairs = common.SplitStrings(exch.AvailablePairs, ",")
		l.EnabledPairs = common.SplitStrings(exch.EnabledPairs, ",")
		err := l.SetCurrencyPairFormat()
		if err != nil {
			log.Fatal(err)
		}
		err = l.SetAssetTypes()
		if err != nil {
			log.Fatal(err)
		}
	}
}

func (l *Liqui) GetFee(currency string) (float64, error) {
	val, ok := l.Info.Pairs[common.StringToLower(currency)]
	if !ok {
		return 0, errors.New("Currency does not exist")
	}

	return val.Fee, nil
}

func (l *Liqui) GetAvailablePairs(nonHidden bool) []string {
	var pairs []string
	for x, y := range l.Info.Pairs {
		if nonHidden && y.Hidden == 1 {
			continue
		}
		pairs = append(pairs, common.StringToUpper(x))
	}
	return pairs
}

func (l *Liqui) GetInfo() (LiquiInfo, error) {
	req := fmt.Sprintf("%s/%s/%s/", LIQUI_API_PUBLIC_URL, LIQUI_API_PUBLIC_VERSION, LIQUI_INFO)
	resp := LiquiInfo{}
	err := common.SendHTTPGetRequest(req, true, &resp)

	if err != nil {
		return resp, err
	}

	return resp, nil
}

func (l *Liqui) GetTicker(symbol string) (map[string]LiquiTicker, error) {
	type Response struct {
		Data map[string]LiquiTicker
	}

	response := Response{}
	req := fmt.Sprintf("%s/%s/%s/%s", LIQUI_API_PUBLIC_URL, LIQUI_API_PUBLIC_VERSION, LIQUI_TICKER, symbol)
	err := common.SendHTTPGetRequest(req, true, &response.Data)

	if err != nil {
		return nil, err
	}
	return response.Data, nil
}

func (l *Liqui) GetDepth(symbol string) (LiquiOrderbook, error) {
	type Response struct {
		Data map[string]LiquiOrderbook
	}

	response := Response{}
	req := fmt.Sprintf("%s/%s/%s/%s", LIQUI_API_PUBLIC_URL, LIQUI_API_PUBLIC_VERSION, LIQUI_DEPTH, symbol)

	err := common.SendHTTPGetRequest(req, true, &response.Data)
	if err != nil {
		return LiquiOrderbook{}, err
	}

	depth := response.Data[symbol]
	return depth, nil
}

func (l *Liqui) GetTrades(symbol string) ([]LiquiTrades, error) {
	type Response struct {
		Data map[string][]LiquiTrades
	}

	response := Response{}
	req := fmt.Sprintf("%s/%s/%s/%s", LIQUI_API_PUBLIC_URL, LIQUI_API_PUBLIC_VERSION, LIQUI_TRADES, symbol)

	err := common.SendHTTPGetRequest(req, true, &response.Data)
	if err != nil {
		return []LiquiTrades{}, err
	}

	trades := response.Data[symbol]
	return trades, nil
}

func (l *Liqui) GetAccountInfo() (LiquiAccountInfo, error) {
	var result LiquiAccountInfo
	err := l.SendAuthenticatedHTTPRequest(LIQUI_ACCOUNT_INFO, url.Values{}, &result)

	if err != nil {
		return result, err
	}

	return result, nil
}

//to-do: convert orderid to int64
func (l *Liqui) Trade(pair, orderType string, amount, price float64) (float64, error) {
	req := url.Values{}
	req.Add("pair", pair)
	req.Add("type", orderType)
	req.Add("amount", strconv.FormatFloat(amount, 'f', -1, 64))
	req.Add("rate", strconv.FormatFloat(price, 'f', -1, 64))

	var result LiquiTrade
	err := l.SendAuthenticatedHTTPRequest(LIQUI_TRADE, req, &result)

	if err != nil {
		return 0, err
	}

	return result.OrderID, nil
}

func (l *Liqui) GetActiveOrders(pair string) (map[string]LiquiActiveOrders, error) {
	req := url.Values{}
	req.Add("pair", pair)

	var result map[string]LiquiActiveOrders
	err := l.SendAuthenticatedHTTPRequest(LIQUI_ACTIVE_ORDERS, req, &result)

	if err != nil {
		return result, err
	}

	return result, nil
}

func (l *Liqui) GetOrderInfo(OrderID int64) (map[string]LiquiOrderInfo, error) {
	req := url.Values{}
	req.Add("order_id", strconv.FormatInt(OrderID, 10))

	var result map[string]LiquiOrderInfo
	err := l.SendAuthenticatedHTTPRequest(LIQUI_ORDER_INFO, req, &result)

	if err != nil {
		return result, err
	}

	return result, nil
}

func (l *Liqui) CancelOrder(OrderID int64) (bool, error) {
	req := url.Values{}
	req.Add("order_id", strconv.FormatInt(OrderID, 10))

	var result LiquiCancelOrder
	err := l.SendAuthenticatedHTTPRequest(LIQUI_CANCEL_ORDER, req, &result)

	if err != nil {
		return false, err
	}

	return true, nil
}

func (l *Liqui) GetTradeHistory(vals url.Values, pair string) (map[string]LiquiTradeHistory, error) {
	if pair != "" {
		vals.Add("pair", pair)
	}

	var result map[string]LiquiTradeHistory
	err := l.SendAuthenticatedHTTPRequest(LIQUI_TRADE_HISTORY, vals, &result)

	if err != nil {
		return result, err
	}

	return result, nil
}

// API mentions that this isn't active now, but will be soon - you must provide the first 8 characters of the key
// in your ticket to support.
func (l *Liqui) WithdrawCoins(coin string, amount float64, address string) (LiquiWithdrawCoins, error) {
	req := url.Values{}
	req.Add("coinName", coin)
	req.Add("amount", strconv.FormatFloat(amount, 'f', -1, 64))
	req.Add("address", address)

	var result LiquiWithdrawCoins
	err := l.SendAuthenticatedHTTPRequest(LIQUI_WITHDRAW_COIN, req, &result)

	if err != nil {
		return result, err
	}
	return result, nil
}

func (l *Liqui) SendAuthenticatedHTTPRequest(method string, values url.Values, result interface{}) (err error) {
	if !l.AuthenticatedAPISupport {
		return fmt.Errorf(exchange.WarningAuthenticatedRequestWithoutCredentialsSet, l.Name)
	}

	if l.Nonce.Get() == 0 {
		l.Nonce.Set(time.Now().Unix())
	} else {
		l.Nonce.Inc()
	}
	values.Set("nonce", l.Nonce.String())
	values.Set("method", method)

	encoded := values.Encode()
	hmac := common.GetHMAC(common.HashSHA512, []byte(encoded), []byte(l.APISecret))

	if l.Verbose {
		log.Printf("Sending POST request to %s calling method %s with params %s\n", LIQUI_API_PRIVATE_URL, method, encoded)
	}

	headers := make(map[string]string)
	headers["Key"] = l.APIKey
	headers["Sign"] = common.HexEncodeToString(hmac)
	headers["Content-Type"] = "application/x-www-form-urlencoded"

	resp, err := common.SendHTTPRequest("POST", LIQUI_API_PRIVATE_URL, headers, strings.NewReader(encoded))

	if err != nil {
		return err
	}

	response := LiquiResponse{}
	err = common.JSONDecode([]byte(resp), &response)

	if err != nil {
		return err
	}

	if response.Success != 1 {
		return errors.New(response.Error)
	}

	jsonEncoded, err := common.JSONEncode(response.Return)

	if err != nil {
		return err
	}

	err = common.JSONDecode(jsonEncoded, &result)

	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Liqui skip empty pairs<commit_after>package liqui

import (
	"errors"
	"fmt"
	"log"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/thrasher-/gocryptotrader/common"
	"github.com/thrasher-/gocryptotrader/config"
	"github.com/thrasher-/gocryptotrader/exchanges"
	"github.com/thrasher-/gocryptotrader/exchanges/ticker"
)

const (
	LIQUI_API_PUBLIC_URL      = "https://api.Liqui.io/api"
	LIQUI_API_PRIVATE_URL     = "https://api.Liqui.io/tapi"
	LIQUI_API_PUBLIC_VERSION  = "3"
	LIQUI_API_PRIVATE_VERSION = "1"
	LIQUI_INFO                = "info"
	LIQUI_TICKER              = "ticker"
	LIQUI_DEPTH               = "depth"
	LIQUI_TRADES              = "trades"
	LIQUI_ACCOUNT_INFO        = "getInfo"
	LIQUI_TRADE               = "Trade"
	LIQUI_ACTIVE_ORDERS       = "ActiveOrders"
	LIQUI_ORDER_INFO          = "OrderInfo"
	LIQUI_CANCEL_ORDER        = "CancelOrder"
	LIQUI_TRADE_HISTORY       = "TradeHistory"
	LIQUI_WITHDRAW_COIN       = "WithdrawCoin"
)

type Liqui struct {
	exchange.Base
	Ticker map[string]LiquiTicker
	Info   LiquiInfo
}

func (l *Liqui) SetDefaults() {
	l.Name = "Liqui"
	l.Enabled = false
	l.Fee = 0.25
	l.Verbose = false
	l.Websocket = false
	l.RESTPollingDelay = 10
	l.Ticker = make(map[string]LiquiTicker)
	l.RequestCurrencyPairFormat.Delimiter = "_"
	l.RequestCurrencyPairFormat.Uppercase = false
	l.RequestCurrencyPairFormat.Separator = "-"
	l.ConfigCurrencyPairFormat.Delimiter = "_"
	l.ConfigCurrencyPairFormat.Uppercase = true
	l.AssetTypes = []string{ticker.Spot}
}

func (l *Liqui) Setup(exch config.ExchangeConfig) {
	if !exch.Enabled {
		l.SetEnabled(false)
	} else {
		l.Enabled = true
		l.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
		l.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
		l.RESTPollingDelay = exch.RESTPollingDelay
		l.Verbose = exch.Verbose
		l.Websocket = exch.Websocket
		l.BaseCurrencies = common.SplitStrings(exch.BaseCurrencies, ",")
		l.AvailablePairs = common.SplitStrings(exch.AvailablePairs, ",")
		l.EnabledPairs = common.SplitStrings(exch.EnabledPairs, ",")
		err := l.SetCurrencyPairFormat()
		if err != nil {
			log.Fatal(err)
		}
		err = l.SetAssetTypes()
		if err != nil {
			log.Fatal(err)
		}
	}
}

func (l *Liqui) GetFee(currency string) (float64, error) {
	val, ok := l.Info.Pairs[common.StringToLower(currency)]
	if !ok {
		return 0, errors.New("Currency does not exist")
	}

	return val.Fee, nil
}

func (l *Liqui) GetAvailablePairs(nonHidden bool) []string {
	var pairs []string
	for x, y := range l.Info.Pairs {
		if nonHidden && y.Hidden == 1 || x == "" {
			continue
		}
		pairs = append(pairs, common.StringToUpper(x))
	}
	return pairs
}

func (l *Liqui) GetInfo() (LiquiInfo, error) {
	req := fmt.Sprintf("%s/%s/%s/", LIQUI_API_PUBLIC_URL, LIQUI_API_PUBLIC_VERSION, LIQUI_INFO)
	resp := LiquiInfo{}
	err := common.SendHTTPGetRequest(req, true, &resp)

	if err != nil {
		return resp, err
	}

	return resp, nil
}

func (l *Liqui) GetTicker(symbol string) (map[string]LiquiTicker, error) {
	type Response struct {
		Data map[string]LiquiTicker
	}

	response := Response{}
	req := fmt.Sprintf("%s/%s/%s/%s", LIQUI_API_PUBLIC_URL, LIQUI_API_PUBLIC_VERSION, LIQUI_TICKER, symbol)
	err := common.SendHTTPGetRequest(req, true, &response.Data)

	if err != nil {
		return nil, err
	}
	return response.Data, nil
}

func (l *Liqui) GetDepth(symbol string) (LiquiOrderbook, error) {
	type Response struct {
		Data map[string]LiquiOrderbook
	}

	response := Response{}
	req := fmt.Sprintf("%s/%s/%s/%s", LIQUI_API_PUBLIC_URL, LIQUI_API_PUBLIC_VERSION, LIQUI_DEPTH, symbol)

	err := common.SendHTTPGetRequest(req, true, &response.Data)
	if err != nil {
		return LiquiOrderbook{}, err
	}

	depth := response.Data[symbol]
	return depth, nil
}

func (l *Liqui) GetTrades(symbol string) ([]LiquiTrades, error) {
	type Response struct {
		Data map[string][]LiquiTrades
	}

	response := Response{}
	req := fmt.Sprintf("%s/%s/%s/%s", LIQUI_API_PUBLIC_URL, LIQUI_API_PUBLIC_VERSION, LIQUI_TRADES, symbol)

	err := common.SendHTTPGetRequest(req, true, &response.Data)
	if err != nil {
		return []LiquiTrades{}, err
	}

	trades := response.Data[symbol]
	return trades, nil
}

func (l *Liqui) GetAccountInfo() (LiquiAccountInfo, error) {
	var result LiquiAccountInfo
	err := l.SendAuthenticatedHTTPRequest(LIQUI_ACCOUNT_INFO, url.Values{}, &result)

	if err != nil {
		return result, err
	}

	return result, nil
}

//to-do: convert orderid to int64
func (l *Liqui) Trade(pair, orderType string, amount, price float64) (float64, error) {
	req := url.Values{}
	req.Add("pair", pair)
	req.Add("type", orderType)
	req.Add("amount", strconv.FormatFloat(amount, 'f', -1, 64))
	req.Add("rate", strconv.FormatFloat(price, 'f', -1, 64))

	var result LiquiTrade
	err := l.SendAuthenticatedHTTPRequest(LIQUI_TRADE, req, &result)

	if err != nil {
		return 0, err
	}

	return result.OrderID, nil
}

func (l *Liqui) GetActiveOrders(pair string) (map[string]LiquiActiveOrders, error) {
	req := url.Values{}
	req.Add("pair", pair)

	var result map[string]LiquiActiveOrders
	err := l.SendAuthenticatedHTTPRequest(LIQUI_ACTIVE_ORDERS, req, &result)

	if err != nil {
		return result, err
	}

	return result, nil
}

func (l *Liqui) GetOrderInfo(OrderID int64) (map[string]LiquiOrderInfo, error) {
	req := url.Values{}
	req.Add("order_id", strconv.FormatInt(OrderID, 10))

	var result map[string]LiquiOrderInfo
	err := l.SendAuthenticatedHTTPRequest(LIQUI_ORDER_INFO, req, &result)

	if err != nil {
		return result, err
	}

	return result, nil
}

func (l *Liqui) CancelOrder(OrderID int64) (bool, error) {
	req := url.Values{}
	req.Add("order_id", strconv.FormatInt(OrderID, 10))

	var result LiquiCancelOrder
	err := l.SendAuthenticatedHTTPRequest(LIQUI_CANCEL_ORDER, req, &result)

	if err != nil {
		return false, err
	}

	return true, nil
}

func (l *Liqui) GetTradeHistory(vals url.Values, pair string) (map[string]LiquiTradeHistory, error) {
	if pair != "" {
		vals.Add("pair", pair)
	}

	var result map[string]LiquiTradeHistory
	err := l.SendAuthenticatedHTTPRequest(LIQUI_TRADE_HISTORY, vals, &result)

	if err != nil {
		return result, err
	}

	return result, nil
}

// API mentions that this isn't active now, but will be soon - you must provide the first 8 characters of the key
// in your ticket to support.
func (l *Liqui) WithdrawCoins(coin string, amount float64, address string) (LiquiWithdrawCoins, error) {
	req := url.Values{}
	req.Add("coinName", coin)
	req.Add("amount", strconv.FormatFloat(amount, 'f', -1, 64))
	req.Add("address", address)

	var result LiquiWithdrawCoins
	err := l.SendAuthenticatedHTTPRequest(LIQUI_WITHDRAW_COIN, req, &result)

	if err != nil {
		return result, err
	}
	return result, nil
}

func (l *Liqui) SendAuthenticatedHTTPRequest(method string, values url.Values, result interface{}) (err error) {
	if !l.AuthenticatedAPISupport {
		return fmt.Errorf(exchange.WarningAuthenticatedRequestWithoutCredentialsSet, l.Name)
	}

	if l.Nonce.Get() == 0 {
		l.Nonce.Set(time.Now().Unix())
	} else {
		l.Nonce.Inc()
	}
	values.Set("nonce", l.Nonce.String())
	values.Set("method", method)

	encoded := values.Encode()
	hmac := common.GetHMAC(common.HashSHA512, []byte(encoded), []byte(l.APISecret))

	if l.Verbose {
		log.Printf("Sending POST request to %s calling method %s with params %s\n", LIQUI_API_PRIVATE_URL, method, encoded)
	}

	headers := make(map[string]string)
	headers["Key"] = l.APIKey
	headers["Sign"] = common.HexEncodeToString(hmac)
	headers["Content-Type"] = "application/x-www-form-urlencoded"

	resp, err := common.SendHTTPRequest("POST", LIQUI_API_PRIVATE_URL, headers, strings.NewReader(encoded))

	if err != nil {
		return err
	}

	response := LiquiResponse{}
	err = common.JSONDecode([]byte(resp), &response)

	if err != nil {
		return err
	}

	if response.Success != 1 {
		return errors.New(response.Error)
	}

	jsonEncoded, err := common.JSONEncode(response.Return)

	if err != nil {
		return err
	}

	err = common.JSONDecode(jsonEncoded, &result)

	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package wrappers

import (
	"bytes"
	"testing"
)

const (
	ByteSentinal  = 0
)

func TestPackerPackByte(t *testing.T) {
	p := Packer{MaxSize: 1}

	p.PackByte(0x01)

	if p.Errored() {
		t.Fatal(p.Err)
	}

	if size := len(p.Bytes); size != 1 {
		t.Fatalf("Packer.PackByte wrote %d byte(s) but expected %d byte(s)", size, 1)
	}

	expected := []byte{0x01}
	if !bytes.Equal(p.Bytes, expected) {
		t.Fatalf("Packer.PackByte wrote:\n%v\nExpected:\n%v", p.Bytes, expected)
	}

	p.PackByte(0x02)
	if !p.Errored() {
		t.Fatal("Packer.PackByte did not fail when attempt was beyond p.MaxSize")
	}
}

func TestPackerUnpackByte(t *testing.T) {
	var (
		p                = Packer{Bytes: []byte{0x01}, Offset: 0}
		actual           = p.UnpackByte()
		expected    byte = 1
		expectedLen      = ByteLen
	)
	if p.Errored() {
		t.Fatalf("Packer.UnpackByte unexpectedly raised %s", p.Err)
	} else if actual != expected {
		t.Fatalf("Packer.UnpackByte returned %d, but expected %d", actual, expected)
	} else if p.Offset != expectedLen {
		t.Fatalf("Packer.UnpackByte left Offset %d, expected %d", p.Offset, expectedLen)
	}

	actual = p.UnpackByte()
	if !p.Errored() {
		t.Fatalf("Packer.UnpackByte should have set error, due to attempted out of bounds read")
	} else if actual != ByteSentinal {
		t.Fatalf("Packer.UnpackByte returned %d, expected sentinal value %d", actual, ByteSentinal)
	}
}

func TestPackerShort(t *testing.T) {
	p := Packer{MaxSize: 2}

	p.PackShort(0x0102)

	if p.Errored() {
		t.Fatal(p.Err)
	}

	if size := len(p.Bytes); size != 2 {
		t.Fatalf("Packer.PackShort wrote %d byte(s) but expected %d byte(s)", size, 2)
	}

	expected := []byte{0x01, 0x02}
	if !bytes.Equal(p.Bytes, expected) {
		t.Fatalf("Packer.PackShort wrote:\n%v\nExpected:\n%v", p.Bytes, expected)
	}
}

func TestPackerInt(t *testing.T) {
	p := Packer{MaxSize: 4}

	p.PackInt(0x01020304)

	if p.Errored() {
		t.Fatal(p.Err)
	}

	if size := len(p.Bytes); size != 4 {
		t.Fatalf("Packer.PackInt wrote %d byte(s) but expected %d byte(s)", size, 4)
	}

	expected := []byte{0x01, 0x02, 0x03, 0x04}
	if !bytes.Equal(p.Bytes, expected) {
		t.Fatalf("Packer.PackInt wrote:\n%v\nExpected:\n%v", p.Bytes, expected)
	}
}

func TestPackerLong(t *testing.T) {
	p := Packer{MaxSize: 8}

	p.PackLong(0x0102030405060708)

	if p.Errored() {
		t.Fatal(p.Err)
	}

	if size := len(p.Bytes); size != 8 {
		t.Fatalf("Packer.PackLong wrote %d byte(s) but expected %d byte(s)", size, 8)
	}

	expected := []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}
	if !bytes.Equal(p.Bytes, expected) {
		t.Fatalf("Packer.PackLong wrote:\n%v\nExpected:\n%v", p.Bytes, expected)
	}
}

func TestPackerString(t *testing.T) {
	p := Packer{MaxSize: 5}

	p.PackStr("Ava")

	if p.Errored() {
		t.Fatal(p.Err)
	}

	if size := len(p.Bytes); size != 5 {
		t.Fatalf("Packer.PackStr wrote %d byte(s) but expected %d byte(s)", size, 5)
	}

	expected := []byte{0x00, 0x03, 0x41, 0x76, 0x61}
	if !bytes.Equal(p.Bytes, expected) {
		t.Fatalf("Packer.PackStr wrote:\n%v\nExpected:\n%v", p.Bytes, expected)
	}
}

func TestPacker(t *testing.T) {
	packer := Packer{
		MaxSize: 3,
	}

	if packer.Errored() {
		t.Fatalf("Packer has error %s", packer.Err)
	}

	packer.PackShort(17)
	if len(packer.Bytes) != 2 {
		t.Fatalf("Wrong byte length")
	}

	packer.PackShort(1)
	if !packer.Errored() {
		t.Fatalf("Packer should have error")
	}

	newPacker := Packer{
		Bytes: packer.Bytes,
	}

	if newPacker.UnpackShort() != 17 {
		t.Fatalf("Unpacked wrong value")
	}
}

func TestPackBool(t *testing.T) {
	p := Packer{MaxSize: 3}
	p.PackBool(false)
	p.PackBool(true)
	p.PackBool(false)
	if p.Errored() {
		t.Fatal("should have been able to pack 3 bools")
	}

	p2 := Packer{Bytes: p.Bytes}
	bool1, bool2, bool3 := p2.UnpackBool(), p2.UnpackBool(), p2.UnpackBool()

	if p.Errored() {
		t.Fatalf("errors while unpacking bools: %v", p.Errs)
	}

	if bool1 || !bool2 || bool3 {
		t.Fatal("got back wrong values")
	}
}
<commit_msg>utils: Add test for Packer.UnpackShort<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package wrappers

import (
	"bytes"
	"testing"
)

const (
	ByteSentinal  = 0
	ShortSentinal = 0
)

func TestPackerPackByte(t *testing.T) {
	p := Packer{MaxSize: 1}

	p.PackByte(0x01)

	if p.Errored() {
		t.Fatal(p.Err)
	}

	if size := len(p.Bytes); size != 1 {
		t.Fatalf("Packer.PackByte wrote %d byte(s) but expected %d byte(s)", size, 1)
	}

	expected := []byte{0x01}
	if !bytes.Equal(p.Bytes, expected) {
		t.Fatalf("Packer.PackByte wrote:\n%v\nExpected:\n%v", p.Bytes, expected)
	}

	p.PackByte(0x02)
	if !p.Errored() {
		t.Fatal("Packer.PackByte did not fail when attempt was beyond p.MaxSize")
	}
}

func TestPackerUnpackByte(t *testing.T) {
	var (
		p                = Packer{Bytes: []byte{0x01}, Offset: 0}
		actual           = p.UnpackByte()
		expected    byte = 1
		expectedLen      = ByteLen
	)
	if p.Errored() {
		t.Fatalf("Packer.UnpackByte unexpectedly raised %s", p.Err)
	} else if actual != expected {
		t.Fatalf("Packer.UnpackByte returned %d, but expected %d", actual, expected)
	} else if p.Offset != expectedLen {
		t.Fatalf("Packer.UnpackByte left Offset %d, expected %d", p.Offset, expectedLen)
	}

	actual = p.UnpackByte()
	if !p.Errored() {
		t.Fatalf("Packer.UnpackByte should have set error, due to attempted out of bounds read")
	} else if actual != ByteSentinal {
		t.Fatalf("Packer.UnpackByte returned %d, expected sentinal value %d", actual, ByteSentinal)
	}
}

func TestPackerPackShort(t *testing.T) {
	p := Packer{MaxSize: 2}

	p.PackShort(0x0102)

	if p.Errored() {
		t.Fatal(p.Err)
	}

	if size := len(p.Bytes); size != 2 {
		t.Fatalf("Packer.PackShort wrote %d byte(s) but expected %d byte(s)", size, 2)
	}

	expected := []byte{0x01, 0x02}
	if !bytes.Equal(p.Bytes, expected) {
		t.Fatalf("Packer.PackShort wrote:\n%v\nExpected:\n%v", p.Bytes, expected)
	}
}

func TestPackerUnpackShort(t *testing.T) {
	var (
		p                  = Packer{Bytes: []byte{0x01, 0x02}, Offset: 0}
		actual             = p.UnpackShort()
		expected    uint16 = 0x0102
		expectedLen        = ShortLen
	)
	if p.Errored() {
		t.Fatalf("Packer.UnpackShort unexpectedly raised %s", p.Err)
	} else if actual != expected {
		t.Fatalf("Packer.UnpackShort returned %d, but expected %d", actual, expected)
	} else if p.Offset != expectedLen {
		t.Fatalf("Packer.UnpackShort left Offset %d, expected %d", p.Offset, expectedLen)
	}

	actual = p.UnpackShort()
	if !p.Errored() {
		t.Fatalf("Packer.UnpackShort should have set error, due to attempted out of bounds read")
	} else if actual != ShortSentinal {
		t.Fatalf("Packer.UnpackShort returned %d, expected sentinal value %d", actual, ShortSentinal)
	}
}

func TestPackerInt(t *testing.T) {
	p := Packer{MaxSize: 4}

	p.PackInt(0x01020304)

	if p.Errored() {
		t.Fatal(p.Err)
	}

	if size := len(p.Bytes); size != 4 {
		t.Fatalf("Packer.PackInt wrote %d byte(s) but expected %d byte(s)", size, 4)
	}

	expected := []byte{0x01, 0x02, 0x03, 0x04}
	if !bytes.Equal(p.Bytes, expected) {
		t.Fatalf("Packer.PackInt wrote:\n%v\nExpected:\n%v", p.Bytes, expected)
	}
}

func TestPackerLong(t *testing.T) {
	p := Packer{MaxSize: 8}

	p.PackLong(0x0102030405060708)

	if p.Errored() {
		t.Fatal(p.Err)
	}

	if size := len(p.Bytes); size != 8 {
		t.Fatalf("Packer.PackLong wrote %d byte(s) but expected %d byte(s)", size, 8)
	}

	expected := []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}
	if !bytes.Equal(p.Bytes, expected) {
		t.Fatalf("Packer.PackLong wrote:\n%v\nExpected:\n%v", p.Bytes, expected)
	}
}

func TestPackerString(t *testing.T) {
	p := Packer{MaxSize: 5}

	p.PackStr("Ava")

	if p.Errored() {
		t.Fatal(p.Err)
	}

	if size := len(p.Bytes); size != 5 {
		t.Fatalf("Packer.PackStr wrote %d byte(s) but expected %d byte(s)", size, 5)
	}

	expected := []byte{0x00, 0x03, 0x41, 0x76, 0x61}
	if !bytes.Equal(p.Bytes, expected) {
		t.Fatalf("Packer.PackStr wrote:\n%v\nExpected:\n%v", p.Bytes, expected)
	}
}

func TestPacker(t *testing.T) {
	packer := Packer{
		MaxSize: 3,
	}

	if packer.Errored() {
		t.Fatalf("Packer has error %s", packer.Err)
	}

	packer.PackShort(17)
	if len(packer.Bytes) != 2 {
		t.Fatalf("Wrong byte length")
	}

	packer.PackShort(1)
	if !packer.Errored() {
		t.Fatalf("Packer should have error")
	}

	newPacker := Packer{
		Bytes: packer.Bytes,
	}

	if newPacker.UnpackShort() != 17 {
		t.Fatalf("Unpacked wrong value")
	}
}

func TestPackBool(t *testing.T) {
	p := Packer{MaxSize: 3}
	p.PackBool(false)
	p.PackBool(true)
	p.PackBool(false)
	if p.Errored() {
		t.Fatal("should have been able to pack 3 bools")
	}

	p2 := Packer{Bytes: p.Bytes}
	bool1, bool2, bool3 := p2.UnpackBool(), p2.UnpackBool(), p2.UnpackBool()

	if p.Errored() {
		t.Fatalf("errors while unpacking bools: %v", p.Errs)
	}

	if bool1 || !bool2 || bool3 {
		t.Fatal("got back wrong values")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/go.exp/go/types"
	. "code.google.com/p/gordon-go/gui"
)

type valueNode struct {
	*nodeBase
	obj types.Object // package var or struct field, or nil if this is an assign (=) or indirect node
	set bool
	x   *port // the target of the operation (struct or pointer)
	y   *port // the result of the read (output) or the argument to write (input)
}

func newValueNode(obj types.Object, set bool) *valueNode {
	n := &valueNode{obj: obj, set: set}
	n.nodeBase = newNodeBase(n)
	text := ""
	switch obj.(type) {
	case field, method, nil:
		n.x = n.newInput(&types.Var{})
		n.x.connsChanged = n.reform
		text = "."
	default:
	}
	if obj != nil {
		text += obj.GetName()
		n.text.SetText(text)
	}
	if set {
		n.y = n.newInput(&types.Var{})
	} else {
		n.y = n.newOutput(&types.Var{})
	}
	switch obj.(type) {
	case *types.Var, field:
		n.addSeqPorts()
	default:
	}
	n.reform()
	return n
}

func (n *valueNode) reform() {
	if n.set {
		if n.y.out {
			n.removePortBase(n.y)
			n.y = n.newInput(&types.Var{})
		}
	} else {
		if !n.y.out {
			n.removePortBase(n.y)
			n.y = n.newOutput(&types.Var{})
		}
	}
	var yt types.Type
	if n.obj != nil {
		yt = n.obj.GetType()
		switch n.obj.(type) {
		case *types.Var, field:
			yt = &types.Pointer{yt}
		default:
		}
	}
	if n.x != nil {
		var xt types.Type
		if n.obj != nil {
			switch obj := n.obj.(type) {
			case field:
				xt = obj.recv
			case method:
				xt = obj.Type.Recv.Type
			}
			// TODO: use indirect result of types.LookupFieldOrMethod, or types.Selection.Indirect()
			if n.set {
				xt = &types.Pointer{xt}
			} else {
				if len(n.x.conns) > 0 {
					xt = n.x.conns[0].src.obj.Type
				}
				if _, ok := xt.(*types.Pointer); !ok {
					yt = n.obj.GetType()
				}
			}
		} else {
			if len(n.x.conns) > 0 {
				xt = n.x.conns[0].src.obj.Type
				yt, _ = indirect(xt)
			}
			if n.set {
				n.text.SetText("=")
			} else {
				n.text.SetText("indirect")
			}
		}
		n.x.setType(xt)
	}
	n.y.setType(yt)
}

func (n *valueNode) KeyPress(event KeyEvent) {
	if _, ok := n.obj.(*types.Const); ok || n.obj == nil {
		n.nodeBase.KeyPress(event)
	} else {
		switch event.Text {
		case "=":
			if n.x != nil {
				// TODO: use indirect result of types.LookupFieldOrMethod, or types.Selection.Indirect()
				if _, ok := n.x.obj.Type.(*types.Pointer); !ok {
					break
				}
			}
			n.set = !n.set
			n.reform()
			SetKeyFocus(n)
		default:
			n.nodeBase.KeyPress(event)
		}
	}
}
<commit_msg>Flux:  Fixed bug in valueNode and cleaned up that code.<commit_after>package main

import (
	"code.google.com/p/go.exp/go/types"
	. "code.google.com/p/gordon-go/gui"
)

type valueNode struct {
	*nodeBase
	obj types.Object // package var or struct field, or nil if this is an assign (=) or indirect node
	set bool
	x   *port // the target of the operation (struct or pointer)
	y   *port // the result of the read (output) or the argument to write (input)
}

func newValueNode(obj types.Object, set bool) *valueNode {
	n := &valueNode{obj: obj, set: set}
	n.nodeBase = newNodeBase(n)
	text := ""
	switch obj.(type) {
	case field, method, nil:
		n.x = n.newInput(&types.Var{})
		n.x.connsChanged = n.reform
		text = "."
	default:
	}
	if obj != nil {
		text += obj.GetName()
		n.text.SetText(text)
	}
	if set {
		n.y = n.newInput(&types.Var{})
	} else {
		n.y = n.newOutput(&types.Var{})
	}
	switch obj.(type) {
	case *types.Var, field:
		n.addSeqPorts()
	default:
	}
	n.reform()
	return n
}

func (n *valueNode) reform() {
	if n.set {
		if n.y.out {
			n.removePortBase(n.y)
			n.y = n.newInput(&types.Var{})
		}
	} else {
		if !n.y.out {
			n.removePortBase(n.y)
			n.y = n.newOutput(&types.Var{})
		}
	}
	var xt, yt types.Type
	if n.obj != nil {
		yt = n.obj.GetType()
	}
	switch obj := n.obj.(type) {
	case *types.Const:
	case *types.Var:
		if !n.set {
			yt = &types.Pointer{yt}
		}
	case *types.Func:
	case field:
		xt = obj.recv
		// TODO: use indirect result of types.LookupFieldOrMethod, or types.Selection.Indirect()
		if n.set {
			xt = &types.Pointer{xt}
		} else {
			if len(n.x.conns) > 0 {
				xt = n.x.conns[0].src.obj.Type
			}
			if _, ok := xt.(*types.Pointer); ok {
				yt = &types.Pointer{yt}
			}
		}
	case method:
		xt = obj.Type.Recv.Type
		// TODO: remove Recv? (from copy)
	case nil:
		if len(n.x.conns) > 0 {
			xt = n.x.conns[0].src.obj.Type
			yt, _ = indirect(xt)
		}
		if n.set {
			n.text.SetText("=")
		} else {
			n.text.SetText("indirect")
		}
	}
	if n.x != nil {
		n.x.setType(xt)
	}
	n.y.setType(yt)
}

func (n *valueNode) KeyPress(event KeyEvent) {
	if _, ok := n.obj.(*types.Const); ok || n.obj == nil {
		n.nodeBase.KeyPress(event)
	} else {
		switch event.Text {
		case "=":
			if n.x != nil {
				// TODO: use indirect result of types.LookupFieldOrMethod, or types.Selection.Indirect()
				if _, ok := n.x.obj.Type.(*types.Pointer); !ok {
					break
				}
			}
			n.set = !n.set
			n.reform()
			SetKeyFocus(n)
		default:
			n.nodeBase.KeyPress(event)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/Unknwon/com"
	"github.com/howeyc/fsnotify"
)

var (
	cmd       *exec.Cmd
	state     sync.Mutex
	eventTime = make(map[string]int64)
)

func NewWatcher(paths []string) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		com.ColorLog("[ERRO] Fail to create new Watcher[ %s ]\n", err)
		os.Exit(2)
	}

	go func() {
		for {
			select {
			case e := <-watcher.Event:
				isbuild := true

				// Skip TMP files for Sublime Text.
				if checkTMPFile(e.Name) {
					continue
				}
				if !checkIsGoFile(e.Name) {
					continue
				}

				mt := getFileModTime(e.Name)
				if t := eventTime[e.Name]; mt == t {
					com.ColorLog("[SKIP] # %s #\n", e.String())
					isbuild = false
				}

				eventTime[e.Name] = mt

				if isbuild {
					com.ColorLog("[EVEN] %s\n", e)
					go Autobuild()
				}
			case err := <-watcher.Error:
				log.Fatal("error:", err)
			}
		}

		time.Sleep(500 * time.Millisecond)
	}()

	com.ColorLog("[INFO] Initializing watcher...\n")
	for _, path := range paths {
		com.ColorLog("[TRAC] Directory( %s )\n", path)
		err = watcher.Watch(path)
		if err != nil {
			com.ColorLog("[ERRO] Fail to watch directory[ %s ]\n", err)
			os.Exit(2)
		}
	}

}

// getFileModTime retuens unix timestamp of `os.File.ModTime` by given path.
func getFileModTime(path string) int64 {
	path = strings.Replace(path, "\\", "/", -1)
	f, err := os.Open(path)
	if err != nil {
		com.ColorLog("[ERRO] Fail to open file[ %s ]\n", err)
		return time.Now().Unix()
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		com.ColorLog("[ERRO] Fail to get file information[ %s ]\n", err)
		return time.Now().Unix()
	}

	return fi.ModTime().Unix()
}

func Autobuild() {
	state.Lock()
	defer state.Unlock()

	com.ColorLog("[INFO] Start building...\n")
	path, _ := os.Getwd()
	os.Chdir(path)

	var err error
	// For applications use full import path like "github.com/.../.."
	// are able to use "go install" to reduce build time.
	if conf.GoInstall {
		icmd := exec.Command("go", "install")
		icmd.Stdout = os.Stdout
		icmd.Stderr = os.Stderr
		err = icmd.Run()
	}

	if err == nil {
		bcmd := exec.Command("go", "build")
		bcmd.Stdout = os.Stdout
		bcmd.Stderr = os.Stderr
		err = bcmd.Run()
	}

	if err != nil {
		com.ColorLog("[ERRO] ============== Build failed ===================\n")
		return
	}
	com.ColorLog("[SUCC] Build was successful\n")
	Restart(appname)
}

func Kill() {
	defer func() {
		if e := recover(); e != nil {
			fmt.Println("Kill -> ", e)
		}
	}()
	if cmd != nil {
		cmd.Process.Kill()
	}
}

func Restart(appname string) {
	Debugf("kill running process")
	Kill()
	go Start(appname)
}

func Start(appname string) {
	com.ColorLog("[INFO] Restarting %s ...\n", appname)
	if strings.Index(appname, "./") == -1 {
		appname = "./" + appname
	}

	cmd = exec.Command(appname)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	go cmd.Run()
	started <- true
	com.ColorLog("[INFO] %s is running...\n", appname)
}

// checkTMPFile returns true if the event was for TMP files.
func checkTMPFile(name string) bool {
	if strings.HasSuffix(strings.ToLower(name), ".tmp") {
		return true
	}
	return false
}

// checkIsGoFile returns true if the name HasSuffix ".go".
func checkIsGoFile(name string) bool {
	if strings.HasSuffix(name, ".go") {
		return true
	}
	return false
}
<commit_msg>Update<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/Unknwon/com"
	"github.com/howeyc/fsnotify"
)

var (
	cmd       *exec.Cmd
	state     sync.Mutex
	eventTime = make(map[string]int64)
)

func NewWatcher(paths []string) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		com.ColorLog("[ERRO] Fail to create new Watcher[ %s ]\n", err)
		os.Exit(2)
	}

	go func() {
		for {
			select {
			case e := <-watcher.Event:
				isbuild := true

				// Skip TMP files for Sublime Text.
				if checkTMPFile(e.Name) {
					continue
				}
				if !checkIsGoFile(e.Name) {
					continue
				}

				mt := getFileModTime(e.Name)
				if t := eventTime[e.Name]; mt == t {
					com.ColorLog("[SKIP] # %s #\n", e.String())
					isbuild = false
				}

				eventTime[e.Name] = mt

				if isbuild {
					com.ColorLog("[EVEN] %s\n", e)
					go Autobuild()
				}
			case err := <-watcher.Error:
				log.Fatal("error:", err)
			}
		}

		time.Sleep(500 * time.Millisecond)
	}()

	com.ColorLog("[INFO] Initializing watcher...\n")
	for _, path := range paths {
		com.ColorLog("[TRAC] Directory( %s )\n", path)
		err = watcher.Watch(path)
		if err != nil {
			com.ColorLog("[ERRO] Fail to watch directory[ %s ]\n", err)
			os.Exit(2)
		}
	}

}

// getFileModTime retuens unix timestamp of `os.File.ModTime` by given path.
func getFileModTime(path string) int64 {
	path = strings.Replace(path, "\\", "/", -1)
	f, err := os.Open(path)
	if err != nil {
		com.ColorLog("[ERRO] Fail to open file[ %s ]\n", err)
		return time.Now().Unix()
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		com.ColorLog("[ERRO] Fail to get file information[ %s ]\n", err)
		return time.Now().Unix()
	}

	return fi.ModTime().Unix()
}

func Autobuild() {
	state.Lock()
	defer state.Unlock()

	com.ColorLog("[INFO] Start building...\n")
	path, _ := os.Getwd()
	os.Chdir(path)

	var err error
	// For applications use full import path like "github.com/.../.."
	// are able to use "go install" to reduce build time.
	if conf.GoInstall {
		icmd := exec.Command("go", "install")
		icmd.Stdout = os.Stdout
		icmd.Stderr = os.Stderr
		err = icmd.Run()
	}

	if err == nil {
		bcmd := exec.Command("go", "build")
		bcmd.Stdout = os.Stdout
		bcmd.Stderr = os.Stderr
		err = bcmd.Run()
	}

	if err != nil {
		com.ColorLog("[ERRO] ============== Build failed ===================\n")
		return
	}
	com.ColorLog("[SUCC] Build was successful\n")
	Restart(appname)
}

func Kill() {
	defer func() {
		if e := recover(); e != nil {
			fmt.Println("Kill -> ", e)
		}
	}()
	if cmd != nil {
		cmd.Process.Kill()
	}
}

func Restart(appname string) {
	Debugf("kill running process")
	Kill()
	go Start(appname)
}

func Start(appname string) {
	com.ColorLog("[INFO] Restarting %s ...\n", appname)
	if strings.Index(appname, "./") == -1 {
		appname = "./" + appname
	}

	cmd = exec.Command(appname)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	go cmd.Run()
	com.ColorLog("[INFO] %s is running...\n", appname)
	started <- true
}

// checkTMPFile returns true if the event was for TMP files.
func checkTMPFile(name string) bool {
	if strings.HasSuffix(strings.ToLower(name), ".tmp") {
		return true
	}
	return false
}

// checkIsGoFile returns true if the name HasSuffix ".go".
func checkIsGoFile(name string) bool {
	if strings.HasSuffix(name, ".go") {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>// All UDP functions
package network
<commit_msg>Updated udp.go<commit_after>// All UDP functions
package network
package main

import (
	"fmt"
	"log"
	"net"
	"strconv"
	"time"
)

type ID string

func getSenderID(sender *net.UDPAddr) ID {
	return ID(sender.IP.String())
}

func listen(socket *net.UDPConn) {
	for {
		buffer := make([]byte, 1024)
		read_bytes, _, err := socket.ReadFromUDP(buffer)
		if err == nil {
			fmt.Println("Received: " + string(buffer[0:read_bytes]))
		} else {
			log.Println(err)
		}
	}
}
func transmit(socket *net.UDPConn) {
	for {
		time.Sleep(2000 * time.Millisecond)

		message := "From server: Hello I got your mesage"
		socket.Write([]byte(message))
		fmt.Println("Sendte: " + message)
	}
}

/*func CheckError(err error) {
	if err != nil {
		fmt.Println("Error:", err)
	}
}*/

func main() {
	broadcastIP := "129.241.187.255" //HUSK!!! riktig IP
	localPort := 30000
	localAddress, _ := net.ResolveUDPAddr("udp", ":"+strconv.Itoa(localPort))

	serverPort := 20005
	serverAddress, _ := net.ResolveUDPAddr("udp", broadcastIP+":"+strconv.Itoa(serverPort))

	fmt.Println("Local adress:", localAddress)
	fmt.Println("Server adress", serverAddress)

	listenSocket, _ := net.ListenUDP("udp", localAddress)
	transmitSocket, _ := net.DialUDP("udp", nil, serverAddress)

	listen(listenSocket)
	transmit(transmitSocket)
}<|endoftext|>


<commit_before>// Copyright 2014 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package github

import (
	"fmt"
	"time"
)

// IssueEvent represents an event that occurred around an Issue or Pull Request.
type IssueEvent struct {
	ID  *int    `json:"id,omitempty"`
	URL *string `json:"url,omitempty"`

	// The User that generated this event.
	Actor *User `json:"actor,omitempty"`

	// Event identifies the actual type of Event that occurred.  Possible
	// values are:
	//
	//     closed
	//       The issue was closed by the actor. When the commit_id is
	//       present, it identifies the commit that closed the issue using
	//       “closes / fixes #NN” syntax.
	//
	//     reopened
	//       The issue was reopened by the actor.
	//
	//     subscribed
	//       The actor subscribed to receive notifications for an issue.
	//
	//     merged
	//       The issue was merged by the actor. The commit_id attribute is the SHA1 of the HEAD commit that was merged.
	//
	//     referenced
	//       The issue was referenced from a commit message. The commit_id attribute is the commit SHA1 of where that happened.
	//
	//     mentioned
	//       The actor was @mentioned in an issue body.
	//
	//     assigned
	//       The issue was assigned to the actor.
	//
	//     head_ref_deleted
	//       The pull request’s branch was deleted.
	//
	//     head_ref_restored
	//       The pull request’s branch was restored.
	Event *string `json:"event,omitempty"`

	// The SHA of the commit that referenced this commit, if applicable.
	CommitID *string `json:"commit_id,omitempty"`

	CreatedAt *time.Time `json:"created_at,omitempty"`
	Issue     *Issue     `json:"issue,omitempty"`
}

// ListIssueEvents lists events for the specified issue.
//
// GitHub API docs: https://developer.github.com/v3/issues/events/#list-events-for-an-issue
func (s *IssuesService) ListIssueEvents(owner, repo string, number int, opt *ListOptions) ([]IssueEvent, *Response, error) {
	u := fmt.Sprintf("repos/%v/%v/issues/%v/events", owner, repo, number)
	u, err := addOptions(u, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	var events []IssueEvent
	resp, err := s.client.Do(req, &events)
	if err != nil {
		return nil, resp, err
	}

	return events, resp, err
}

// ListRepositoryEvents lists events for the specified repository.
//
// GitHub API docs: https://developer.github.com/v3/issues/events/#list-events-for-a-repository
func (s *IssuesService) ListRepositoryEvents(owner, repo string, opt *ListOptions) ([]IssueEvent, *Response, error) {
	u := fmt.Sprintf("repos/%v/%v/issues/events", owner, repo)
	u, err := addOptions(u, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	var events []IssueEvent
	resp, err := s.client.Do(req, &events)
	if err != nil {
		return nil, resp, err
	}

	return events, resp, err
}

// GetEvent returns the specified issue event.
//
// GitHub API docs: https://developer.github.com/v3/issues/events/#get-a-single-event
func (s *IssuesService) GetEvent(owner, repo string, id int) (*IssueEvent, *Response, error) {
	u := fmt.Sprintf("repos/%v/%v/issues/events/%v", owner, repo, id)

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	event := new(IssueEvent)
	resp, err := s.client.Do(req, event)
	if err != nil {
		return nil, resp, err
	}

	return event, resp, err
}
<commit_msg>Add 'Label' field to IssueEvent<commit_after>// Copyright 2014 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package github

import (
	"fmt"
	"time"
)

// IssueEvent represents an event that occurred around an Issue or Pull Request.
type IssueEvent struct {
	ID  *int    `json:"id,omitempty"`
	URL *string `json:"url,omitempty"`

	// The User that generated this event.
	Actor *User `json:"actor,omitempty"`

	// Event identifies the actual type of Event that occurred.  Possible
	// values are:
	//
	//     closed
	//       The issue was closed by the actor. When the commit_id is
	//       present, it identifies the commit that closed the issue using
	//       “closes / fixes #NN” syntax.
	//
	//     reopened
	//       The issue was reopened by the actor.
	//
	//     subscribed
	//       The actor subscribed to receive notifications for an issue.
	//
	//     merged
	//       The issue was merged by the actor. The commit_id attribute is the SHA1 of the HEAD commit that was merged.
	//
	//     referenced
	//       The issue was referenced from a commit message. The commit_id attribute is the commit SHA1 of where that happened.
	//
	//     mentioned
	//       The actor was @mentioned in an issue body.
	//
	//     assigned
	//       The issue was assigned to the actor.
	//
	//     head_ref_deleted
	//       The pull request’s branch was deleted.
	//
	//     head_ref_restored
	//       The pull request’s branch was restored.
	//
	//     labeled
	//       A label was added.
	Event *string `json:"event,omitempty"`

	// The SHA of the commit that referenced this commit, if applicable.
	CommitID *string `json:"commit_id,omitempty"`

	CreatedAt *time.Time `json:"created_at,omitempty"`
	Issue     *Issue     `json:"issue,omitempty"`

	// Only present on 'labeled' events
	Label *Label `json:"label,omitempty"`
}

// ListIssueEvents lists events for the specified issue.
//
// GitHub API docs: https://developer.github.com/v3/issues/events/#list-events-for-an-issue
func (s *IssuesService) ListIssueEvents(owner, repo string, number int, opt *ListOptions) ([]IssueEvent, *Response, error) {
	u := fmt.Sprintf("repos/%v/%v/issues/%v/events", owner, repo, number)
	u, err := addOptions(u, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	var events []IssueEvent
	resp, err := s.client.Do(req, &events)
	if err != nil {
		return nil, resp, err
	}

	return events, resp, err
}

// ListRepositoryEvents lists events for the specified repository.
//
// GitHub API docs: https://developer.github.com/v3/issues/events/#list-events-for-a-repository
func (s *IssuesService) ListRepositoryEvents(owner, repo string, opt *ListOptions) ([]IssueEvent, *Response, error) {
	u := fmt.Sprintf("repos/%v/%v/issues/events", owner, repo)
	u, err := addOptions(u, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	var events []IssueEvent
	resp, err := s.client.Do(req, &events)
	if err != nil {
		return nil, resp, err
	}

	return events, resp, err
}

// GetEvent returns the specified issue event.
//
// GitHub API docs: https://developer.github.com/v3/issues/events/#get-a-single-event
func (s *IssuesService) GetEvent(owner, repo string, id int) (*IssueEvent, *Response, error) {
	u := fmt.Sprintf("repos/%v/%v/issues/events/%v", owner, repo, id)

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	event := new(IssueEvent)
	resp, err := s.client.Do(req, event)
	if err != nil {
		return nil, resp, err
	}

	return event, resp, err
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"encoding/json"
	"fmt"
	"io"
	"strconv"

	"golang.org/x/net/context"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
)

type CmdConfigGet struct {
	key    string
	writer io.Writer
}

type CmdConfigSet struct {
	key    string
	value  string
	writer io.Writer
}

type CmdConfigReset struct {
	writer io.Writer
}

type CmdConfigInfo struct {
	writer io.Writer
}

func (v *CmdConfigGet) ParseArgv(ctx *cli.Context) error {
	if len(ctx.Args()) < 1 {
		return fmt.Errorf("Not enough arguments.")
	}
	v.key = ctx.Args()[0]
	if v.writer == nil {
		v.writer = GlobUI.OutputWriter()
	}
	return nil
}

func (v *CmdConfigSet) ParseArgv(ctx *cli.Context) error {
	if len(ctx.Args()) < 1 {
		return fmt.Errorf("Not enough arguments.")
	}
	v.key = ctx.Args()[0]
	if len(ctx.Args()) > 1 {
		v.value = ctx.Args()[1]
	}
	if v.writer == nil {
		v.writer = GlobUI.OutputWriter()
	}
	return nil
}

func (v *CmdConfigReset) ParseArgv(ctx *cli.Context) error {
	if v.writer == nil {
		v.writer = GlobUI.OutputWriter()
	}
	return nil
}

func (v *CmdConfigInfo) ParseArgv(ctx *cli.Context) error {
	if v.writer == nil {
		v.writer = GlobUI.OutputWriter()
	}
	return nil
}

func (v *CmdConfigGet) Run() error {
	cr := G.Env.GetConfig()
	// TODO: print dictionaries?
	if s, isSet := cr.GetStringAtPath(v.key); isSet {
		fmt.Fprintf(v.writer, "%s: %s\n", v.key, s)
	} else if b, isSet := cr.GetBoolAtPath(v.key); isSet {
		fmt.Fprintf(v.writer, "%s: %t\n", v.key, b)
	} else if i, isSet := cr.GetIntAtPath(v.key); isSet {
		fmt.Fprintf(v.writer, "%s: %d\n", v.key, i)
	} else if isSet := cr.GetNullAtPath(v.key); isSet {
		fmt.Fprintf(v.writer, "%s: null\n", v.key)
	}
	return nil
}

func (v *CmdConfigSet) Run() error {
	if v.value != "" {
		cw := G.Env.GetConfigWriter()
		// try to convert the value to an int, and then to a bool
		// if those don't work, use a string
		if val, e := strconv.Atoi(v.value); e == nil {
			cw.SetIntAtPath(v.key, val)
		} else if val, e := strconv.ParseBool(v.value); e == nil {
			// NOTE: this will also convert strings like 't' and 'F' to
			// a bool, which could potentially cause strange errors for
			// e.g. a user named "f"
			cw.SetBoolAtPath(v.key, val)
		} else if v.value == "null" {
			cw.SetNullAtPath(v.key)
		} else {
			cw.SetStringAtPath(v.key, v.value)
		}
		cw.Write()
	} else {
		cw := G.Env.GetConfigWriter()
		cw.DeleteAtPath(v.key)
		cw.Write()
	}
	return nil
}

func (v *CmdConfigReset) Run() error {
	// Clear out file
	cw := G.Env.GetConfigWriter()
	cw.Reset()
	cw.Write()
	return nil
}

func (v *CmdConfigInfo) Run() error {
	configFile := G.Env.GetConfigFilename()
	fmt.Fprintf(v.writer, "File: %s\n\n", configFile)

	cli, err := GetConfigClient(G)
	if err != nil {
		return err
	}
	if err := RegisterProtocols(nil); err != nil {
		return err
	}

	config, err := cli.GetConfig(context.TODO(), 0)
	if err != nil {
		return err
	}
	out, err := json.MarshalIndent(config, "", "  ")
	if err != nil {
		return err
	}
	fmt.Fprintf(v.writer, "%s", out)
	return nil
}

func NewCmdConfig(cl *libcmdline.CommandLine) cli.Command {
	return cli.Command{
		Name:         "config",
		Usage:        "Get and set configuration options",
		ArgumentHelp: "[arguments...]",
		Subcommands: []cli.Command{
			NewCmdConfigGet(cl),
			NewCmdConfigSet(cl),
			NewCmdConfigReset(cl),
			NewCmdConfigInfo(cl),
		},
	}
}

func NewCmdConfigGet(cl *libcmdline.CommandLine) cli.Command {
	return cli.Command{
		Name:         "get",
		Usage:        "Get a config value",
		ArgumentHelp: "<key>",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdConfigGet{}, "get", c)
		},
	}
}

func NewCmdConfigSet(cl *libcmdline.CommandLine) cli.Command {
	return cli.Command{
		Name:         "set",
		Usage:        "Set a config value",
		ArgumentHelp: "<key> <value>",
		Description:  "Set a config value. Specify an empty value to clear it.",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdConfigSet{}, "set", c)
		},
	}
}

func NewCmdConfigReset(cl *libcmdline.CommandLine) cli.Command {
	return cli.Command{
		Name:  "reset",
		Usage: "Reset the config",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdConfigReset{}, "reset", c)
		},
	}
}

func NewCmdConfigInfo(cl *libcmdline.CommandLine) cli.Command {
	return cli.Command{
		Name:  "info",
		Usage: "Show config file path",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdConfigInfo{}, "info", c)
		},
	}
}

func (v *CmdConfigGet) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config: true,
	}
}

func (v *CmdConfigSet) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config: true,
	}
}

func (v *CmdConfigReset) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config: true,
	}
}

func (v *CmdConfigInfo) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config: true,
	}
}
<commit_msg>Remove command 'config info'<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"fmt"
	"io"
	"strconv"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
)

type CmdConfigGet struct {
	key    string
	writer io.Writer
}

type CmdConfigSet struct {
	key    string
	value  string
	writer io.Writer
}

type CmdConfigReset struct {
	writer io.Writer
}

func (v *CmdConfigGet) ParseArgv(ctx *cli.Context) error {
	if len(ctx.Args()) < 1 {
		return fmt.Errorf("Not enough arguments.")
	}
	v.key = ctx.Args()[0]
	if v.writer == nil {
		v.writer = GlobUI.OutputWriter()
	}
	return nil
}

func (v *CmdConfigSet) ParseArgv(ctx *cli.Context) error {
	if len(ctx.Args()) < 1 {
		return fmt.Errorf("Not enough arguments.")
	}
	v.key = ctx.Args()[0]
	if len(ctx.Args()) > 1 {
		v.value = ctx.Args()[1]
	}
	if v.writer == nil {
		v.writer = GlobUI.OutputWriter()
	}
	return nil
}

func (v *CmdConfigReset) ParseArgv(ctx *cli.Context) error {
	if v.writer == nil {
		v.writer = GlobUI.OutputWriter()
	}
	return nil
}

func (v *CmdConfigGet) Run() error {
	cr := G.Env.GetConfig()
	// TODO: print dictionaries?
	if s, isSet := cr.GetStringAtPath(v.key); isSet {
		fmt.Fprintf(v.writer, "%s: %s\n", v.key, s)
	} else if b, isSet := cr.GetBoolAtPath(v.key); isSet {
		fmt.Fprintf(v.writer, "%s: %t\n", v.key, b)
	} else if i, isSet := cr.GetIntAtPath(v.key); isSet {
		fmt.Fprintf(v.writer, "%s: %d\n", v.key, i)
	} else if isSet := cr.GetNullAtPath(v.key); isSet {
		fmt.Fprintf(v.writer, "%s: null\n", v.key)
	}
	return nil
}

func (v *CmdConfigSet) Run() error {
	if v.value != "" {
		cw := G.Env.GetConfigWriter()
		// try to convert the value to an int, and then to a bool
		// if those don't work, use a string
		if val, e := strconv.Atoi(v.value); e == nil {
			cw.SetIntAtPath(v.key, val)
		} else if val, e := strconv.ParseBool(v.value); e == nil {
			// NOTE: this will also convert strings like 't' and 'F' to
			// a bool, which could potentially cause strange errors for
			// e.g. a user named "f"
			cw.SetBoolAtPath(v.key, val)
		} else if v.value == "null" {
			cw.SetNullAtPath(v.key)
		} else {
			cw.SetStringAtPath(v.key, v.value)
		}
		cw.Write()
	} else {
		cw := G.Env.GetConfigWriter()
		cw.DeleteAtPath(v.key)
		cw.Write()
	}
	return nil
}

func (v *CmdConfigReset) Run() error {
	// Clear out file
	cw := G.Env.GetConfigWriter()
	cw.Reset()
	cw.Write()
	return nil
}

func NewCmdConfig(cl *libcmdline.CommandLine) cli.Command {
	return cli.Command{
		Name:         "config",
		Usage:        "Get and set configuration options",
		ArgumentHelp: "[arguments...]",
		Subcommands: []cli.Command{
			NewCmdConfigGet(cl),
			NewCmdConfigSet(cl),
			NewCmdConfigReset(cl),
		},
	}
}

func NewCmdConfigGet(cl *libcmdline.CommandLine) cli.Command {
	return cli.Command{
		Name:         "get",
		Usage:        "Get a config value",
		ArgumentHelp: "<key>",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdConfigGet{}, "get", c)
		},
	}
}

func NewCmdConfigSet(cl *libcmdline.CommandLine) cli.Command {
	return cli.Command{
		Name:         "set",
		Usage:        "Set a config value",
		ArgumentHelp: "<key> <value>",
		Description:  "Set a config value. Specify an empty value to clear it.",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdConfigSet{}, "set", c)
		},
	}
}

func NewCmdConfigReset(cl *libcmdline.CommandLine) cli.Command {
	return cli.Command{
		Name:  "reset",
		Usage: "Reset the config",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdConfigReset{}, "reset", c)
		},
	}
}

func (v *CmdConfigGet) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config: true,
	}
}

func (v *CmdConfigSet) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config: true,
	}
}

func (v *CmdConfigReset) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config: true,
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"flag"
	"math/rand"
	"strings"
	"time"

	"vitess.io/vitess/go/vt/proto/vtrpc"
	"vitess.io/vitess/go/vt/vterrors"

	"vitess.io/vitess/go/exit"
	"vitess.io/vitess/go/vt/discovery"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/servenv"
	"vitess.io/vitess/go/vt/srvtopo"
	"vitess.io/vitess/go/vt/topo"
	"vitess.io/vitess/go/vt/topo/topoproto"
	"vitess.io/vitess/go/vt/vtgate"
	"vitess.io/vitess/go/vt/vttablet/tabletserver"

	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
)

var (
	cell              = flag.String("cell", "test_nj", "cell to use")
	tabletTypesToWait = flag.String("tablet_types_to_wait", "", "wait till connected for specified tablet types during Gateway initialization")
)

var resilientServer *srvtopo.ResilientServer
var legacyHealthCheck discovery.LegacyHealthCheck

func init() {
	rand.Seed(time.Now().UnixNano())
	servenv.RegisterDefaultFlags()
}

// CheckCellFlags will check validation of cell and cells_to_watch flag
// it will help to avoid strange behaviors when vtgate runs but actually does not work
func CheckCellFlags(ctx context.Context, serv srvtopo.Server, cell string, cellsToWatch string) error {
	//topo check
	var topoServer *topo.Server
	if serv != nil {
		var err error
		topoServer, err = serv.GetTopoServer()
		if err != nil {
			log.Exitf("Unable to create gateway: %v", err)
		}
	} else {
		log.Exitf("topo server cannot be nil")
	}
	cellsInTopo, err := topoServer.GetKnownCells(ctx)
	if err != nil {
		return err
	}
	if len(cellsInTopo) == 0 {
		return vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "topo server should have at least one cell")
	}

	//cell valid check
	if cell == "" || cell == "test_nj" {
		return vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "cell flag must be set")
	}
	hasCell := false
	for _, v := range cellsInTopo {
		if v == cell {
			hasCell = true
		}
	}
	if !hasCell {
		return vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "cell:[%v] does not exist in topo", cell)
	}

	//cells_to_watch valid check
	cells := make([]string, 0, 1)
	for _, c := range strings.Split(cellsToWatch, ",") {
		if c == "" {
			continue
		}
		//cell should be contained in cellsInTopo
		exists := topo.InCellList(c, cellsInTopo)
		if exists {
			cells = append(cells, c)
		} else {
			return vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "cell: [%v] is not valid. Available cells: [%v]", c, strings.Join(cellsInTopo, ","))
		}
	}
	if len(cells) == 0 {
		return vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "cells_to_watch flag cannot be empty")
	}

	return nil
}

func main() {
	defer exit.Recover()

	servenv.ParseFlags("vtgate")
	servenv.Init()

	ts := topo.Open()
	defer ts.Close()

	resilientServer = srvtopo.NewResilientServer(ts, "ResilientSrvTopoServer")

	tabletTypes := make([]topodatapb.TabletType, 0, 1)
	if len(*tabletTypesToWait) != 0 {
		for _, ttStr := range strings.Split(*tabletTypesToWait, ",") {
			tt, err := topoproto.ParseTabletType(ttStr)
			if err != nil {
				log.Errorf("unknown tablet type: %v", ttStr)
				continue
			}
			if tabletserver.IsServingType(tt) {
				tabletTypes = append(tabletTypes, tt)
			}
		}
	} else {
		log.Exitf("tablet_types_to_wait flag must be set")
	}

	if len(tabletTypes) == 0 {
		log.Exitf("tablet_types_to_wait should contain at least one serving tablet type")
	}

	err := CheckCellFlags(context.Background(), resilientServer, *cell, *vtgate.CellsToWatch)
	if err != nil {
		log.Exitf("cells_to_watch validation failed: %v", err)
	}

	var vtg *vtgate.VTGate
	if *vtgate.GatewayImplementation == vtgate.GatewayImplementationDiscovery {
		// default value
		legacyHealthCheck = discovery.NewLegacyHealthCheck(*vtgate.HealthCheckRetryDelay, *vtgate.HealthCheckTimeout)
		legacyHealthCheck.RegisterStats()

		vtg = vtgate.LegacyInit(context.Background(), legacyHealthCheck, resilientServer, *cell, *vtgate.RetryCount, tabletTypes)
	} else {
		// use new Init otherwise
		vtg = vtgate.Init(context.Background(), resilientServer, *cell, tabletTypes)
	}

	servenv.OnRun(func() {
		// Flags are parsed now. Parse the template using the actual flag value and overwrite the current template.
		discovery.ParseTabletURLTemplateFromFlag()
		addStatusParts(vtg)
	})
	servenv.OnClose(func() {
		_ = vtg.Gateway().Close(context.Background())
		if legacyHealthCheck != nil {
			_ = legacyHealthCheck.Close()
		}
	})
	servenv.RunDefault()
}
<commit_msg>Fixing nit code tweaks<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"flag"
	"math/rand"
	"strings"
	"time"

	"vitess.io/vitess/go/vt/proto/vtrpc"
	"vitess.io/vitess/go/vt/vterrors"

	"vitess.io/vitess/go/exit"
	"vitess.io/vitess/go/vt/discovery"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/servenv"
	"vitess.io/vitess/go/vt/srvtopo"
	"vitess.io/vitess/go/vt/topo"
	"vitess.io/vitess/go/vt/topo/topoproto"
	"vitess.io/vitess/go/vt/vtgate"
	"vitess.io/vitess/go/vt/vttablet/tabletserver"

	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
)

var (
	cell              = flag.String("cell", "test_nj", "cell to use")
	tabletTypesToWait = flag.String("tablet_types_to_wait", "", "wait till connected for specified tablet types during Gateway initialization")
)

var resilientServer *srvtopo.ResilientServer
var legacyHealthCheck discovery.LegacyHealthCheck

func init() {
	rand.Seed(time.Now().UnixNano())
	servenv.RegisterDefaultFlags()
}

// CheckCellFlags will check validation of cell and cells_to_watch flag
// it will help to avoid strange behaviors when vtgate runs but actually does not work
func CheckCellFlags(ctx context.Context, serv srvtopo.Server, cell string, cellsToWatch string) error {
	// topo check
	var topoServer *topo.Server
	if serv != nil {
		var err error
		topoServer, err = serv.GetTopoServer()
		if err != nil {
			log.Exitf("Unable to create gateway: %v", err)
		}
	} else {
		log.Exitf("topo server cannot be nil")
	}
	cellsInTopo, err := topoServer.GetKnownCells(ctx)
	if err != nil {
		return err
	}
	if len(cellsInTopo) == 0 {
		return vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "topo server should have at least one cell")
	}

	// cell valid check
	if cell == "" || cell == "test_nj" {
		return vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "cell flag must be set")
	}
	hasCell := false
	for _, v := range cellsInTopo {
		if v == cell {
			hasCell = true
			break
		}
	}
	if !hasCell {
		return vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "cell:[%v] does not exist in topo", cell)
	}

	// cells_to_watch valid check
	cells := make([]string, 0, 1)
	for _, c := range strings.Split(cellsToWatch, ",") {
		if c == "" {
			continue
		}
		// cell should contained in cellsInTopo
		if exists := topo.InCellList(c, cellsInTopo); !exists {
			return vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "cell: [%v] is not valid. Available cells: [%v]", c, strings.Join(cellsInTopo, ","))
		}
		cells = append(cells, c)
	}
	if len(cells) == 0 {
		return vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "cells_to_watch flag cannot be empty")
	}

	return nil
}

func main() {
	defer exit.Recover()

	servenv.ParseFlags("vtgate")
	servenv.Init()

	ts := topo.Open()
	defer ts.Close()

	resilientServer = srvtopo.NewResilientServer(ts, "ResilientSrvTopoServer")

	tabletTypes := make([]topodatapb.TabletType, 0, 1)
	if len(*tabletTypesToWait) != 0 {
		for _, ttStr := range strings.Split(*tabletTypesToWait, ",") {
			tt, err := topoproto.ParseTabletType(ttStr)
			if err != nil {
				log.Errorf("unknown tablet type: %v", ttStr)
				continue
			}
			if tabletserver.IsServingType(tt) {
				tabletTypes = append(tabletTypes, tt)
			}
		}
	} else {
		log.Exitf("tablet_types_to_wait flag must be set")
	}

	if len(tabletTypes) == 0 {
		log.Exitf("tablet_types_to_wait should contain at least one serving tablet type")
	}

	err := CheckCellFlags(context.Background(), resilientServer, *cell, *vtgate.CellsToWatch)
	if err != nil {
		log.Exitf("cells_to_watch validation failed: %v", err)
	}

	var vtg *vtgate.VTGate
	if *vtgate.GatewayImplementation == vtgate.GatewayImplementationDiscovery {
		// default value
		legacyHealthCheck = discovery.NewLegacyHealthCheck(*vtgate.HealthCheckRetryDelay, *vtgate.HealthCheckTimeout)
		legacyHealthCheck.RegisterStats()

		vtg = vtgate.LegacyInit(context.Background(), legacyHealthCheck, resilientServer, *cell, *vtgate.RetryCount, tabletTypes)
	} else {
		// use new Init otherwise
		vtg = vtgate.Init(context.Background(), resilientServer, *cell, tabletTypes)
	}

	servenv.OnRun(func() {
		// Flags are parsed now. Parse the template using the actual flag value and overwrite the current template.
		discovery.ParseTabletURLTemplateFromFlag()
		addStatusParts(vtg)
	})
	servenv.OnClose(func() {
		_ = vtg.Gateway().Close(context.Background())
		if legacyHealthCheck != nil {
			_ = legacyHealthCheck.Close()
		}
	})
	servenv.RunDefault()
}
<|endoftext|>


<commit_before>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file enables an external tool to intercept package requests.
// If the tool is present then its results are used in preference to
// the go list command.

package packages

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os/exec"
	"strings"
)

// The Driver Protocol
//
// The driver, given the inputs to a call to Load, returns metadata about the packages specified.
// This allows for different build systems to support go/packages by telling go/packages how the
// packages' source is organized.
// The driver is a binary, either specified by the GOPACKAGESDRIVER environment variable or in
// the path as gopackagesdriver. It's given the inputs to load in its argv. See the package
// documentation in doc.go for the full description of the patterns that need to be supported.
// A driver receives as a JSON-serialized driverRequest struct in standard input and will
// produce a JSON-serialized driverResponse (see definition in packages.go) in its standard output.

// driverRequest is used to provide the portion of Load's Config that is needed by a driver.
type driverRequest struct {
	Mode LoadMode `json:"mode"`
	// Env specifies the environment the underlying build system should be run in.
	Env []string `json:"env"`
	// BuildFlags are flags that should be passed to the underlying build system.
	BuildFlags []string `json:"build_flags"`
	// Tests specifies whether the patterns should also return test packages.
	Tests bool `json:"tests"`
	// Overlay maps file paths (relative to the driver's working directory) to the byte contents
	// of overlay files.
	Overlay map[string][]byte `json:"overlay"`
}

// findExternalDriver returns the file path of a tool that supplies
// the build system package structure, or "" if not found."
// If GOPACKAGESDRIVER is set in the environment findExternalTool returns its
// value, otherwise it searches for a binary named gopackagesdriver on the PATH.
func findExternalDriver(cfg *Config) driver {
	const toolPrefix = "GOPACKAGESDRIVER="
	tool := ""
	for _, env := range cfg.Env {
		if val := strings.TrimPrefix(env, toolPrefix); val != env {
			tool = val
		}
	}
	if tool != "" && tool == "off" {
		return nil
	}
	if tool == "" {
		var err error
		tool, err = exec.LookPath("gopackagesdriver")
		if err != nil {
			return nil
		}
	}
	return func(cfg *Config, words ...string) (*driverResponse, error) {
		req, err := json.Marshal(driverRequest{
			Mode:       cfg.Mode,
			Env:        cfg.Env,
			BuildFlags: cfg.BuildFlags,
			Tests:      cfg.Tests,
			Overlay:    cfg.Overlay,
		})
		if err != nil {
			return nil, fmt.Errorf("failed to encode message to driver tool: %v", err)
		}

		buf := new(bytes.Buffer)
		cmd := exec.CommandContext(cfg.Context, tool, words...)
		cmd.Dir = cfg.Dir
		cmd.Env = cfg.Env
		cmd.Stdin = bytes.NewReader(req)
		cmd.Stdout = buf
		cmd.Stderr = new(bytes.Buffer)
		if err := cmd.Run(); err != nil {
			return nil, fmt.Errorf("%v: %v: %s", tool, err, cmd.Stderr)
		}
		var response driverResponse
		if err := json.Unmarshal(buf.Bytes(), &response); err != nil {
			return nil, err
		}
		return &response, nil
	}
}
<commit_msg>go/packages: print gopackagesdriver errors when GOPACKAGESPRINTDRIVERERRORS is set<commit_after>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file enables an external tool to intercept package requests.
// If the tool is present then its results are used in preference to
// the go list command.

package packages

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"strings"
)

// The Driver Protocol
//
// The driver, given the inputs to a call to Load, returns metadata about the packages specified.
// This allows for different build systems to support go/packages by telling go/packages how the
// packages' source is organized.
// The driver is a binary, either specified by the GOPACKAGESDRIVER environment variable or in
// the path as gopackagesdriver. It's given the inputs to load in its argv. See the package
// documentation in doc.go for the full description of the patterns that need to be supported.
// A driver receives as a JSON-serialized driverRequest struct in standard input and will
// produce a JSON-serialized driverResponse (see definition in packages.go) in its standard output.

// driverRequest is used to provide the portion of Load's Config that is needed by a driver.
type driverRequest struct {
	Mode LoadMode `json:"mode"`
	// Env specifies the environment the underlying build system should be run in.
	Env []string `json:"env"`
	// BuildFlags are flags that should be passed to the underlying build system.
	BuildFlags []string `json:"build_flags"`
	// Tests specifies whether the patterns should also return test packages.
	Tests bool `json:"tests"`
	// Overlay maps file paths (relative to the driver's working directory) to the byte contents
	// of overlay files.
	Overlay map[string][]byte `json:"overlay"`
}

// findExternalDriver returns the file path of a tool that supplies
// the build system package structure, or "" if not found."
// If GOPACKAGESDRIVER is set in the environment findExternalTool returns its
// value, otherwise it searches for a binary named gopackagesdriver on the PATH.
func findExternalDriver(cfg *Config) driver {
	const toolPrefix = "GOPACKAGESDRIVER="
	tool := ""
	for _, env := range cfg.Env {
		if val := strings.TrimPrefix(env, toolPrefix); val != env {
			tool = val
		}
	}
	if tool != "" && tool == "off" {
		return nil
	}
	if tool == "" {
		var err error
		tool, err = exec.LookPath("gopackagesdriver")
		if err != nil {
			return nil
		}
	}
	return func(cfg *Config, words ...string) (*driverResponse, error) {
		req, err := json.Marshal(driverRequest{
			Mode:       cfg.Mode,
			Env:        cfg.Env,
			BuildFlags: cfg.BuildFlags,
			Tests:      cfg.Tests,
			Overlay:    cfg.Overlay,
		})
		if err != nil {
			return nil, fmt.Errorf("failed to encode message to driver tool: %v", err)
		}

		buf := new(bytes.Buffer)
		stderr := new(bytes.Buffer)
		cmd := exec.CommandContext(cfg.Context, tool, words...)
		cmd.Dir = cfg.Dir
		cmd.Env = cfg.Env
		cmd.Stdin = bytes.NewReader(req)
		cmd.Stdout = buf
		cmd.Stderr = stderr
		if len(stderr.Bytes()) != 0 && os.Getenv("GOPACKAGESPRINTDRIVERERRORS") != "" {
			fmt.Fprintf(os.Stderr, "%s stderr: <<%s>>\n", cmdDebugStr(cmd, words...), stderr)
		}

		if err := cmd.Run(); err != nil {
			return nil, fmt.Errorf("%v: %v: %s", tool, err, cmd.Stderr)
		}
		var response driverResponse
		if err := json.Unmarshal(buf.Bytes(), &response); err != nil {
			return nil, err
		}
		return &response, nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mysqlctl

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"code.google.com/p/vitess/go/ioutil2"
	"code.google.com/p/vitess/go/relog"
	"code.google.com/p/vitess/go/vt/hook"
)

// These methods deal with cloning a running instance of mysql.

const (
	maxLagSeconds = 5
)

const (
	SnapshotManifestFile = "snapshot_manifest.json"
)

// Validate that this instance is a reasonable source of data.
func (mysqld *Mysqld) validateCloneSource(serverMode bool) error {
	// NOTE(msolomon) Removing this check for now - I don't see the value of validating this.
	// // needs to be master, or slave that's not too far behind
	// slaveStatus, err := mysqld.slaveStatus()
	// if err != nil {
	// 	if err != ErrNotSlave {
	// 		return fmt.Errorf("mysqlctl: validateCloneSource failed, %v", err)
	// 	}
	// } else {
	// 	lagSeconds, _ := strconv.Atoi(slaveStatus["seconds_behind_master"])
	// 	if lagSeconds > maxLagSeconds {
	// 		return fmt.Errorf("mysqlctl: validateCloneSource failed, lag_seconds exceed maximum tolerance (%v)", lagSeconds)
	// 	}
	// }

	// make sure we can write locally
	if err := mysqld.ValidateSnapshotPath(); err != nil {
		return err
	}

	// run a hook to check local things
	// FIXME(alainjobart) What other parameters do we have to
	// provide? dbname, host, socket?
	params := make(map[string]string)
	if serverMode {
		params["server-mode"] = ""
	}
	if err := hook.NewHook("preflight_snapshot", params).ExecuteOptional(); err != nil {
		return err
	}

	// FIXME(msolomon) check free space based on an estimate of the current
	// size of the db files.
	// Also, check that we aren't already cloning/compressing or acting as a
	// source. Mysqld being down isn't enough, presumably that will be
	// restarted as soon as the snapshot is taken.
	return nil
}

func (mysqld *Mysqld) validateCloneTarget() error {
	// run a hook to check local things
	// FIXME(alainjobart) What other parameters do we have to
	// provide? dbname, host, socket?
	if err := hook.NewSimpleHook("preflight_restore").ExecuteOptional(); err != nil {
		return err
	}

	rows, err := mysqld.fetchSuperQuery("SHOW DATABASES")
	if err != nil {
		return fmt.Errorf("mysqlctl: validateCloneTarget failed, %v", err)
	}

	for _, row := range rows {
		if strings.HasPrefix(row[0].String(), "vt_") {
			dbName := row[0].String()
			tableRows, err := mysqld.fetchSuperQuery("SHOW TABLES FROM " + dbName)
			if err != nil {
				return fmt.Errorf("mysqlctl: validateCloneTarget failed, %v", err)
			} else if len(tableRows) == 0 {
				// no tables == empty db, all is well
				continue
			}
			return fmt.Errorf("mysqlctl: validateCloneTarget failed, found active db %v", dbName)
		}
	}

	return nil
}

func findFilesToServe(srcDir, dstDir string, compress bool) ([]string, []string, error) {
	fiList, err := ioutil.ReadDir(srcDir)
	if err != nil {
		return nil, nil, err
	}
	sources := make([]string, 0, len(fiList))
	destinations := make([]string, 0, len(fiList))
	for _, fi := range fiList {
		if !fi.IsDir() {
			srcPath := path.Join(srcDir, fi.Name())
			var dstPath string
			if compress {
				dstPath = path.Join(dstDir, fi.Name()+".gz")
			} else {
				dstPath = path.Join(dstDir, fi.Name())
			}
			sources = append(sources, srcPath)
			destinations = append(destinations, dstPath)
		}
	}
	return sources, destinations, nil
}

func (mysqld *Mysqld) FindVtDatabases() ([]string, error) {
	fiList, err := ioutil.ReadDir(mysqld.config.DataDir)
	if err != nil {
		return nil, err
	}

	dbNames := make([]string, 0, 16)
	for _, fi := range fiList {
		if strings.HasSuffix(fi.Name(), "vt_") {
			dbNames = append(dbNames, fi.Name())
		}
	}
	return dbNames, nil
}

func (mysqld *Mysqld) createSnapshot(dbName, snapshotPath string, concurrency int, serverMode bool) ([]SnapshotFile, error) {
	sources := make([]string, 0, 128)
	destinations := make([]string, 0, 128)

	// clean out and start fresh
	relog.Info("removing previous snapshots: %v", snapshotPath)
	if err := os.RemoveAll(snapshotPath); err != nil {
		return nil, err
	}

	// FIXME(msolomon) innodb paths must match patterns in mycnf -
	// probably belongs as a derived path.
	dps := []struct{ srcDir, dstDir string }{
		{path.Join(mysqld.config.DataDir, dbName), path.Join(snapshotPath, dataDir, dbName)},
		{path.Join(mysqld.config.DataDir, "_vt"), path.Join(snapshotPath, dataDir, "_vt")},
		{path.Join(mysqld.config.DataDir, "mysql"), path.Join(snapshotPath, dataDir, "mysql")},
		{mysqld.config.InnodbDataHomeDir, path.Join(snapshotPath, innodbDataSubdir)},
		{mysqld.config.InnodbLogGroupHomeDir, path.Join(snapshotPath, innodbLogSubdir)},
	}

	for _, dp := range dps {
		if err := os.MkdirAll(dp.dstDir, 0775); err != nil {
			return nil, err
		}
		if s, d, err := findFilesToServe(dp.srcDir, dp.dstDir, !serverMode); err != nil {
			return nil, err
		} else {
			sources = append(sources, s...)
			destinations = append(destinations, d...)
		}
	}

	return newSnapshotFiles(sources, destinations, mysqld.SnapshotDir, concurrency, !serverMode)
}

// This function runs on the machine acting as the source for the clone.
//
// Check master/slave status and determine restore needs.
// If this instance is a slave, stop replication, otherwise place in read-only mode.
// Record replication position.
// Shutdown mysql
// Check paths for storing data
//
// Depending on the serverMode flag, we do the following:
// serverMode = false:
//   Compress /vt/vt_[0-9a-f]+/data/vt_.+
//   Compute md5() sums (of compressed files)
//   Place in /vt/clone_src where they will be served by http server (not rpc)
//   Restart mysql
// serverMode = true:
//   Make symlinks for /vt/vt_[0-9a-f]+/data/vt_.+ to innodb files
//   Compute md5() sums (of uncompressed files)
//   Place symlinks in /vt/clone_src where they will be served by http server
//   Leave mysql stopped, return slaveStartRequired, readOnly
func (mysqld *Mysqld) CreateSnapshot(dbName, sourceAddr string, allowHierarchicalReplication bool, concurrency int, serverMode bool) (snapshotManifestUrlPath string, slaveStartRequired, readOnly bool, err error) {
	if dbName == "" {
		return "", false, false, errors.New("CreateSnapshot failed: no database name provided")
	}

	if err = mysqld.validateCloneSource(serverMode); err != nil {
		return
	}

	// save initial state so we can restore on Start()
	slaveStartRequired = false
	sourceIsMaster := false
	readOnly = true

	slaveStatus, slaveErr := mysqld.slaveStatus()
	if slaveErr == nil {
		slaveStartRequired = (slaveStatus["Slave_IO_Running"] == "Yes" && slaveStatus["Slave_SQL_Running"] == "Yes")
	} else if slaveErr == ErrNotSlave {
		sourceIsMaster = true
	} else {
		// If we can't get any data, just fail.
		return
	}

	readOnly, err = mysqld.IsReadOnly()
	if err != nil {
		return
	}

	// Stop sources of writes so we can get a consistent replication position.
	// If the source is a slave use the master replication position
	// unless we are allowing hierachical replicas.
	masterAddr := ""
	var replicationPosition *ReplicationPosition
	if sourceIsMaster {
		if err = mysqld.SetReadOnly(true); err != nil {
			return
		}
		replicationPosition, err = mysqld.MasterStatus()
		if err != nil {
			return
		}
		masterAddr = mysqld.Addr()
	} else {
		if err = mysqld.StopSlave(); err != nil {
			return
		}
		replicationPosition, err = mysqld.SlaveStatus()
		if err != nil {
			return
		}
		// We are a slave, check our replication strategy before
		// choosing the master address.
		if allowHierarchicalReplication {
			masterAddr = mysqld.Addr()
		} else {
			masterAddr, err = mysqld.GetMasterAddr()
			if err != nil {
				return
			}
		}
	}

	if err = Shutdown(mysqld, true, MysqlWaitTime); err != nil {
		return
	}

	var smFile string
	dataFiles, snapshotErr := mysqld.createSnapshot(dbName, mysqld.SnapshotDir, concurrency, serverMode)
	if snapshotErr != nil {
		relog.Error("CreateSnapshot failed: %v", snapshotErr)
	} else {

		sm := NewSnapshotManifest(sourceAddr, masterAddr, mysqld.replParams.Uname, mysqld.replParams.Pass,
			dbName, dataFiles, replicationPosition)
		smFile = path.Join(mysqld.SnapshotDir, SnapshotManifestFile)
		if snapshotErr = writeJson(smFile, sm); snapshotErr != nil {
			relog.Error("CreateSnapshot failed: %v", snapshotErr)
		}
	}

	// restore our state if required
	if serverMode && snapshotErr == nil {
		relog.Info("server mode snapshot worked, not restarting mysql")
	} else {
		if err = mysqld.SnapshotSourceEnd(slaveStartRequired, readOnly); err != nil {
			return
		}
	}

	if snapshotErr != nil {
		return "", slaveStartRequired, readOnly, snapshotErr
	}
	relative, err := filepath.Rel(mysqld.SnapshotDir, smFile)
	if err != nil {
		return "", slaveStartRequired, readOnly, nil
	}
	return path.Join(SnapshotURLPath, relative), slaveStartRequired, readOnly, nil
}

func (mysqld *Mysqld) SnapshotSourceEnd(slaveStartRequired, readOnly bool) error {
	// Try to restart mysqld
	if err := Start(mysqld, MysqlWaitTime); err != nil {
		return err
	}

	// Restore original mysqld state that we saved above.
	if slaveStartRequired {
		if err := mysqld.StartSlave(); err != nil {
			return err
		}

		// this should be quick, but we might as well just wait
		if err := mysqld.WaitForSlaveStart(SlaveStartDeadline); err != nil {
			return err
		}
	}

	// And set read-only mode
	if err := mysqld.SetReadOnly(readOnly); err != nil {
		return err
	}

	return nil
}

func writeJson(filename string, x interface{}) error {
	data, err := json.MarshalIndent(x, "  ", "  ")
	if err != nil {
		return err
	}
	return ioutil2.WriteFileAtomic(filename, data, 0660)
}

func ReadSnapshotManifest(filename string) (*SnapshotManifest, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	sm := new(SnapshotManifest)
	if err = json.Unmarshal(data, sm); err != nil {
		return nil, fmt.Errorf("ReadSnapshotManifest failed: %v %v", filename, err)
	}
	return sm, nil
}

// This piece runs on the presumably empty machine acting as the target in the
// create replica action.
//
// validate target (self)
// shutdown_mysql()
// create temp data directory /vt/target/vt_<keyspace>
// copy compressed data files via HTTP
// verify md5sum of compressed files
// uncompress into /vt/vt_<target-uid>/data/vt_<keyspace>
// start_mysql()
// clean up compressed files
func (mysqld *Mysqld) RestoreFromSnapshot(snapshotManifest *SnapshotManifest, fetchConcurrency, fetchRetryCount int, encoding string, dontWaitForSlaveStart bool) error {
	if snapshotManifest == nil {
		return errors.New("RestoreFromSnapshot: nil snapshotManifest")
	}

	relog.Debug("validateCloneTarget")
	if err := mysqld.validateCloneTarget(); err != nil {
		return err
	}

	relog.Debug("Shutdown mysqld")
	if err := Shutdown(mysqld, true, MysqlWaitTime); err != nil {
		return err
	}

	relog.Debug("Fetch snapshot")
	if err := mysqld.fetchSnapshot(snapshotManifest, fetchConcurrency, fetchRetryCount, encoding); err != nil {
		return err
	}

	relog.Debug("Restart mysqld")
	if err := Start(mysqld, MysqlWaitTime); err != nil {
		return err
	}

	cmdList := StartReplicationCommands(snapshotManifest.ReplicationState)
	relog.Info("StartReplicationCommands %#v", cmdList)
	if err := mysqld.executeSuperQueryList(cmdList); err != nil {
		return err
	}

	if dontWaitForSlaveStart {
		return nil
	}
	return mysqld.WaitForSlaveStart(SlaveStartDeadline)
}

func (mysqld *Mysqld) fetchSnapshot(snapshotManifest *SnapshotManifest, fetchConcurrency, fetchRetryCount int, encoding string) error {
	replicaDbPath := path.Join(mysqld.config.DataDir, snapshotManifest.DbName)

	cleanDirs := []string{mysqld.SnapshotDir, replicaDbPath,
		mysqld.config.InnodbDataHomeDir, mysqld.config.InnodbLogGroupHomeDir}

	// clean out and start fresh
	// FIXME(msolomon) this might be changed to allow partial recovery, but at that point
	// we are starting to reimplement rsync.
	for _, dir := range cleanDirs {
		if err := os.RemoveAll(dir); err != nil {
			return err
		}
		if err := os.MkdirAll(dir, 0775); err != nil {
			return err
		}
	}

	return fetchFiles(snapshotManifest, mysqld.TabletDir, fetchConcurrency, fetchRetryCount, encoding)
}
<commit_msg>change clone to probe for databases during a cold snapshort<commit_after>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mysqlctl

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"code.google.com/p/vitess/go/ioutil2"
	"code.google.com/p/vitess/go/relog"
	"code.google.com/p/vitess/go/vt/hook"
)

// These methods deal with cloning a running instance of mysql.

const (
	maxLagSeconds = 5
)

const (
	SnapshotManifestFile = "snapshot_manifest.json"
)

// Validate that this instance is a reasonable source of data.
func (mysqld *Mysqld) validateCloneSource(serverMode bool) error {
	// NOTE(msolomon) Removing this check for now - I don't see the value of validating this.
	// // needs to be master, or slave that's not too far behind
	// slaveStatus, err := mysqld.slaveStatus()
	// if err != nil {
	// 	if err != ErrNotSlave {
	// 		return fmt.Errorf("mysqlctl: validateCloneSource failed, %v", err)
	// 	}
	// } else {
	// 	lagSeconds, _ := strconv.Atoi(slaveStatus["seconds_behind_master"])
	// 	if lagSeconds > maxLagSeconds {
	// 		return fmt.Errorf("mysqlctl: validateCloneSource failed, lag_seconds exceed maximum tolerance (%v)", lagSeconds)
	// 	}
	// }

	// make sure we can write locally
	if err := mysqld.ValidateSnapshotPath(); err != nil {
		return err
	}

	// run a hook to check local things
	// FIXME(alainjobart) What other parameters do we have to
	// provide? dbname, host, socket?
	params := make(map[string]string)
	if serverMode {
		params["server-mode"] = ""
	}
	if err := hook.NewHook("preflight_snapshot", params).ExecuteOptional(); err != nil {
		return err
	}

	// FIXME(msolomon) check free space based on an estimate of the current
	// size of the db files.
	// Also, check that we aren't already cloning/compressing or acting as a
	// source. Mysqld being down isn't enough, presumably that will be
	// restarted as soon as the snapshot is taken.
	return nil
}

func (mysqld *Mysqld) validateCloneTarget() error {
	// run a hook to check local things
	// FIXME(alainjobart) What other parameters do we have to
	// provide? dbname, host, socket?
	if err := hook.NewSimpleHook("preflight_restore").ExecuteOptional(); err != nil {
		return err
	}

	rows, err := mysqld.fetchSuperQuery("SHOW DATABASES")
	if err != nil {
		return fmt.Errorf("mysqlctl: validateCloneTarget failed, %v", err)
	}

	for _, row := range rows {
		if strings.HasPrefix(row[0].String(), "vt_") {
			dbName := row[0].String()
			tableRows, err := mysqld.fetchSuperQuery("SHOW TABLES FROM " + dbName)
			if err != nil {
				return fmt.Errorf("mysqlctl: validateCloneTarget failed, %v", err)
			} else if len(tableRows) == 0 {
				// no tables == empty db, all is well
				continue
			}
			return fmt.Errorf("mysqlctl: validateCloneTarget failed, found active db %v", dbName)
		}
	}

	return nil
}

func findFilesToServe(srcDir, dstDir string, compress bool) ([]string, []string, error) {
	fiList, err := ioutil.ReadDir(srcDir)
	if err != nil {
		return nil, nil, err
	}
	sources := make([]string, 0, len(fiList))
	destinations := make([]string, 0, len(fiList))
	for _, fi := range fiList {
		if !fi.IsDir() {
			srcPath := path.Join(srcDir, fi.Name())
			var dstPath string
			if compress {
				dstPath = path.Join(dstDir, fi.Name()+".gz")
			} else {
				dstPath = path.Join(dstDir, fi.Name())
			}
			sources = append(sources, srcPath)
			destinations = append(destinations, dstPath)
		}
	}
	return sources, destinations, nil
}

func (mysqld *Mysqld) FindVtDatabases() ([]string, error) {
	fiList, err := ioutil.ReadDir(mysqld.config.DataDir)
	if err != nil {
		return nil, err
	}

	dbNames := make([]string, 0, 16)
	for _, fi := range fiList {
		if strings.HasSuffix(fi.Name(), "vt_") {
			dbNames = append(dbNames, fi.Name())
		}
	}
	return dbNames, nil
}

func (mysqld *Mysqld) createSnapshot(snapshotPath string, concurrency int, serverMode bool) ([]SnapshotFile, error) {
	sources := make([]string, 0, 128)
	destinations := make([]string, 0, 128)

	// clean out and start fresh
	relog.Info("removing previous snapshots: %v", snapshotPath)
	if err := os.RemoveAll(snapshotPath); err != nil {
		return nil, err
	}

	// FIXME(msolomon) innodb paths must match patterns in mycnf -
	// probably belongs as a derived path.
	type snapPair struct{ srcDir, dstDir string }
	dps := []snapPair{
		{path.Join(mysqld.config.DataDir, "mysql"), path.Join(snapshotPath, dataDir, "mysql")},
		{mysqld.config.InnodbDataHomeDir, path.Join(snapshotPath, innodbDataSubdir)},
		{mysqld.config.InnodbLogGroupHomeDir, path.Join(snapshotPath, innodbLogSubdir)},
	}

	dirEntries, err := ioutil.ReadDir(mysqld.config.DataDir)
	if err != nil {
		return nil, err
	}

	// Copy anything that defines a db.opt file - that includes empty databases.
	for _, de := range dirEntries {
		if de.IsDir() {
			_, err := os.Stat(path.Join(mysqld.config.DataDir, de.Name(), "db.opt"))
			if err == nil {
				dps = append(dps, snapPair{path.Join(mysqld.config.DataDir, de.Name()), path.Join(snapshotPath, dataDir, de.Name())})
			}
		}
	}

	for _, dp := range dps {
		if err := os.MkdirAll(dp.dstDir, 0775); err != nil {
			return nil, err
		}
		if s, d, err := findFilesToServe(dp.srcDir, dp.dstDir, !serverMode); err != nil {
			return nil, err
		} else {
			sources = append(sources, s...)
			destinations = append(destinations, d...)
		}
	}

	return newSnapshotFiles(sources, destinations, mysqld.SnapshotDir, concurrency, !serverMode)
}

// This function runs on the machine acting as the source for the clone.
//
// Check master/slave status and determine restore needs.
// If this instance is a slave, stop replication, otherwise place in read-only mode.
// Record replication position.
// Shutdown mysql
// Check paths for storing data
//
// Depending on the serverMode flag, we do the following:
// serverMode = false:
//   Compress /vt/vt_[0-9a-f]+/data/vt_.+
//   Compute md5() sums (of compressed files)
//   Place in /vt/clone_src where they will be served by http server (not rpc)
//   Restart mysql
// serverMode = true:
//   Make symlinks for /vt/vt_[0-9a-f]+/data/vt_.+ to innodb files
//   Compute md5() sums (of uncompressed files)
//   Place symlinks in /vt/clone_src where they will be served by http server
//   Leave mysql stopped, return slaveStartRequired, readOnly
func (mysqld *Mysqld) CreateSnapshot(dbName, sourceAddr string, allowHierarchicalReplication bool, concurrency int, serverMode bool) (snapshotManifestUrlPath string, slaveStartRequired, readOnly bool, err error) {
	if dbName == "" {
		return "", false, false, errors.New("CreateSnapshot failed: no database name provided")
	}

	if err = mysqld.validateCloneSource(serverMode); err != nil {
		return
	}

	// save initial state so we can restore on Start()
	slaveStartRequired = false
	sourceIsMaster := false
	readOnly = true

	slaveStatus, slaveErr := mysqld.slaveStatus()
	if slaveErr == nil {
		slaveStartRequired = (slaveStatus["Slave_IO_Running"] == "Yes" && slaveStatus["Slave_SQL_Running"] == "Yes")
	} else if slaveErr == ErrNotSlave {
		sourceIsMaster = true
	} else {
		// If we can't get any data, just fail.
		return
	}

	readOnly, err = mysqld.IsReadOnly()
	if err != nil {
		return
	}

	// Stop sources of writes so we can get a consistent replication position.
	// If the source is a slave use the master replication position
	// unless we are allowing hierachical replicas.
	masterAddr := ""
	var replicationPosition *ReplicationPosition
	if sourceIsMaster {
		if err = mysqld.SetReadOnly(true); err != nil {
			return
		}
		replicationPosition, err = mysqld.MasterStatus()
		if err != nil {
			return
		}
		masterAddr = mysqld.Addr()
	} else {
		if err = mysqld.StopSlave(); err != nil {
			return
		}
		replicationPosition, err = mysqld.SlaveStatus()
		if err != nil {
			return
		}
		// We are a slave, check our replication strategy before
		// choosing the master address.
		if allowHierarchicalReplication {
			masterAddr = mysqld.Addr()
		} else {
			masterAddr, err = mysqld.GetMasterAddr()
			if err != nil {
				return
			}
		}
	}

	if err = Shutdown(mysqld, true, MysqlWaitTime); err != nil {
		return
	}

	var smFile string
	dataFiles, snapshotErr := mysqld.createSnapshot(mysqld.SnapshotDir, concurrency, serverMode)
	if snapshotErr != nil {
		relog.Error("CreateSnapshot failed: %v", snapshotErr)
	} else {

		sm := NewSnapshotManifest(sourceAddr, masterAddr, mysqld.replParams.Uname, mysqld.replParams.Pass,
			dbName, dataFiles, replicationPosition)
		smFile = path.Join(mysqld.SnapshotDir, SnapshotManifestFile)
		if snapshotErr = writeJson(smFile, sm); snapshotErr != nil {
			relog.Error("CreateSnapshot failed: %v", snapshotErr)
		}
	}

	// restore our state if required
	if serverMode && snapshotErr == nil {
		relog.Info("server mode snapshot worked, not restarting mysql")
	} else {
		if err = mysqld.SnapshotSourceEnd(slaveStartRequired, readOnly); err != nil {
			return
		}
	}

	if snapshotErr != nil {
		return "", slaveStartRequired, readOnly, snapshotErr
	}
	relative, err := filepath.Rel(mysqld.SnapshotDir, smFile)
	if err != nil {
		return "", slaveStartRequired, readOnly, nil
	}
	return path.Join(SnapshotURLPath, relative), slaveStartRequired, readOnly, nil
}

func (mysqld *Mysqld) SnapshotSourceEnd(slaveStartRequired, readOnly bool) error {
	// Try to restart mysqld
	if err := Start(mysqld, MysqlWaitTime); err != nil {
		return err
	}

	// Restore original mysqld state that we saved above.
	if slaveStartRequired {
		if err := mysqld.StartSlave(); err != nil {
			return err
		}

		// this should be quick, but we might as well just wait
		if err := mysqld.WaitForSlaveStart(SlaveStartDeadline); err != nil {
			return err
		}
	}

	// And set read-only mode
	if err := mysqld.SetReadOnly(readOnly); err != nil {
		return err
	}

	return nil
}

func writeJson(filename string, x interface{}) error {
	data, err := json.MarshalIndent(x, "  ", "  ")
	if err != nil {
		return err
	}
	return ioutil2.WriteFileAtomic(filename, data, 0660)
}

func ReadSnapshotManifest(filename string) (*SnapshotManifest, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	sm := new(SnapshotManifest)
	if err = json.Unmarshal(data, sm); err != nil {
		return nil, fmt.Errorf("ReadSnapshotManifest failed: %v %v", filename, err)
	}
	return sm, nil
}

// This piece runs on the presumably empty machine acting as the target in the
// create replica action.
//
// validate target (self)
// shutdown_mysql()
// create temp data directory /vt/target/vt_<keyspace>
// copy compressed data files via HTTP
// verify md5sum of compressed files
// uncompress into /vt/vt_<target-uid>/data/vt_<keyspace>
// start_mysql()
// clean up compressed files
func (mysqld *Mysqld) RestoreFromSnapshot(snapshotManifest *SnapshotManifest, fetchConcurrency, fetchRetryCount int, encoding string, dontWaitForSlaveStart bool) error {
	if snapshotManifest == nil {
		return errors.New("RestoreFromSnapshot: nil snapshotManifest")
	}

	relog.Debug("validateCloneTarget")
	if err := mysqld.validateCloneTarget(); err != nil {
		return err
	}

	relog.Debug("Shutdown mysqld")
	if err := Shutdown(mysqld, true, MysqlWaitTime); err != nil {
		return err
	}

	relog.Debug("Fetch snapshot")
	if err := mysqld.fetchSnapshot(snapshotManifest, fetchConcurrency, fetchRetryCount, encoding); err != nil {
		return err
	}

	relog.Debug("Restart mysqld")
	if err := Start(mysqld, MysqlWaitTime); err != nil {
		return err
	}

	cmdList := StartReplicationCommands(snapshotManifest.ReplicationState)
	relog.Info("StartReplicationCommands %#v", cmdList)
	if err := mysqld.executeSuperQueryList(cmdList); err != nil {
		return err
	}

	if dontWaitForSlaveStart {
		return nil
	}
	return mysqld.WaitForSlaveStart(SlaveStartDeadline)
}

func (mysqld *Mysqld) fetchSnapshot(snapshotManifest *SnapshotManifest, fetchConcurrency, fetchRetryCount int, encoding string) error {
	replicaDbPath := path.Join(mysqld.config.DataDir, snapshotManifest.DbName)

	cleanDirs := []string{mysqld.SnapshotDir, replicaDbPath,
		mysqld.config.InnodbDataHomeDir, mysqld.config.InnodbLogGroupHomeDir}

	// clean out and start fresh
	// FIXME(msolomon) this might be changed to allow partial recovery, but at that point
	// we are starting to reimplement rsync.
	for _, dir := range cleanDirs {
		if err := os.RemoveAll(dir); err != nil {
			return err
		}
		if err := os.MkdirAll(dir, 0775); err != nil {
			return err
		}
	}

	return fetchFiles(snapshotManifest, mysqld.TabletDir, fetchConcurrency, fetchRetryCount, encoding)
}
<|endoftext|>