<commit_before>package geohash

import (
	"fmt"
	"math"
)

const (
	// MaxBitDepth defines both the maximum and default geohash accuracy.
	MaxBitDepth int64 = 52
)

// bearing defines the compass bearing/direction in matrix form relative to a center point of 0,0
//  |----------------------|
// 	|   NW  |   N   |  NE  |
// 	|  1,-1 |  1,0  |  1,1 |
//  |----------------------|
// 	|   W   |   X   |   E  |
// 	|  0,-1 |  0,0  |  0,1 |
//  |----------------------|
// 	|   SW  |   S   |  NE  |
// 	| -1,-1 |  -1,0 | -1,1 |
//  |----------------------|
type bearing struct {
	x, y int
}

// North bearing from reference point X
var North = bearing{1, 0}

// NorthEast bearing from reference point X
var NorthEast = bearing{1, 1}

// East bearing from reference point X
var East = bearing{0, 1}

// SouthEast bearing from reference point X
var SouthEast = bearing{-1, 1}

// South bearing from reference point X
var South = bearing{-1, 0}

// SouthWest bearing from reference point X
var SouthWest = bearing{-1, -1}

// West bearing from reference point X
var West = bearing{0, -1}

// NorthWest bearing from reference point X
var NorthWest = bearing{1, -1}

// bitsToDistanceInMeters provides a mapping between bitDepth values and distances
var bitsToDistanceInMeters map[int64]float64

func init() {
	// Reference: https://github.com/yinqiwen/ardb/blob/master/doc/spatial-index.md
	bitsToDistanceInMeters = map[int64]float64{
		52: 0.5971,
		50: 1.1943,
		48: 2.3889,
		46: 4.7774,
		44: 9.5547,
		42: 19.1095,
		40: 38.2189,
		38: 76.4378,
		36: 152.8757,
		34: 305.751,
		32: 611.5028,
		30: 1223.0056,
		28: 2446.0112,
		26: 4892.0224,
		24: 9784.0449,
		22: 19568.0898,
		20: 39136.1797,
		18: 78272.35938,
		16: 156544.7188,
		14: 313089.4375,
		12: 626178.875,
		10: 1252357.75,
		8:  2504715.5,
		6:  5009431,
		4:  10018863,
	}
}

// EncodeInt will encode a pair of latitude and longitude values into a geohash integer.
//
// The third argument is the bitDepth of this number, which affects the precision of the geohash
// but also must be used consistently when decoding. Bit depth must be even.
func EncodeInt(latitude float64, longitude float64, bitDepth int64) int64 {
	// input validation
	validateBitDepth(bitDepth)

	// initialize the calculation
	var bitsTotal int64
	var mid float64
	var maxLat float64 = 90.0
	var minLat float64 = -90.0
	var maxLng float64 = 180.0
	var minLng float64 = -180.0

	var geohash int64
	for bitsTotal < bitDepth {
		geohash *= 2

		if bitsTotal%2 == 0 {
			mid = (maxLng + minLng) / 2

			if longitude > mid {
				geohash += 1
				minLng = mid
			} else {
				maxLng = mid
			}
		} else {
			mid = (maxLat + minLat) / 2
			if latitude > mid {
				geohash += 1
				minLat = mid
			} else {
				maxLat = mid
			}
		}
		bitsTotal++
	}
	return geohash
}

// DecodeInt with decode a integer geohashed number into pair of latitude and longitude value approximations.
//
// Returned values include a latitude and longitude along with the maximum error of the calculation.
// This effectively means that a geohash integer will not return a location but an "area".
// The size of the area returned will be vary with different bitDepth settings.
//
// Note: You should provide the same bitDepth to decode the number as was used to produce the geohash originally.
func DecodeInt(geohash int64, bitDepth int64) (lat float64, lng float64, latErr float64, lngErr float64) {
	// input validation
	validateBitDepth(bitDepth)

	minLat, minLng, maxLat, maxLng := DecodeBboxInt(geohash, bitDepth)
	lat = (minLat + maxLat) / 2
	lng = (minLng + maxLng) / 2
	latErr = maxLat - lat
	lngErr = maxLng - lng
	return
}

// DecodeBboxInt will decode a geohash integer into the bounding box that matches it.
//
// Returned as a four corners of a square region.
func DecodeBboxInt(geohash int64, bitDepth int64) (minLat float64, minLng float64, maxLat float64, maxLng float64) {
	// input validation
	validateBitDepth(bitDepth)

	// initialize the calculation
	maxLat = 90
	minLat = -90
	maxLng = 180
	minLng = -180

	var latBit int64
	var lonBit int64
	var steps int64 = bitDepth / 2

	for thisStep := int64(0); thisStep < steps; thisStep++ {
		lonBit = getBit(geohash, ((steps-thisStep)*2)-1)
		latBit = getBit(geohash, ((steps-thisStep)*2)-2)

		if latBit == 0 {
			maxLat = (maxLat + minLat) / 2
		} else {
			minLat = (maxLat + minLat) / 2
		}

		if lonBit == 0 {
			maxLng = (maxLng + minLng) / 2
		} else {
			minLng = (maxLng + minLng) / 2
		}
	}

	return
}

// NeighborInt will find the neighbor of a integer geohash in certain bearing/direction.
//
// The bitDepth should be specified and the same as the value used to generate the hash.
func NeighborInt(geohash int64, bearing bearing, bitDepth int64) int64 {
	// input validation
	validateBitDepth(bitDepth)

	lat, lng, latErr, lngErr := DecodeInt(geohash, bitDepth)
	neighborLat := lat + float64(bearing.x)*latErr*2
	neighborLng := lng + float64(bearing.y)*lngErr*2
	return EncodeInt(neighborLat, neighborLng, bitDepth)
}

// NeighborsInt is the same as calling NeighborInt for each direction and will return all 8 neighbors and the center location.
func NeighborsInt(geohash int64, bitDepth int64) []int64 {
	// input validation
	validateBitDepth(bitDepth)

	var output []int64
	output = append(output, NeighborInt(geohash, North, bitDepth))
	output = append(output, NeighborInt(geohash, NorthEast, bitDepth))
	output = append(output, NeighborInt(geohash, East, bitDepth))
	output = append(output, NeighborInt(geohash, SouthEast, bitDepth))
	output = append(output, NeighborInt(geohash, South, bitDepth))
	output = append(output, NeighborInt(geohash, SouthWest, bitDepth))
	output = append(output, NeighborInt(geohash, West, bitDepth))
	output = append(output, NeighborInt(geohash, NorthWest, bitDepth))
	output = append(output, geohash)
	return output
}

// BboxesInt will return all the hash integers between minLat, minLon, maxLat, maxLon at the requested bitDepth
func BboxesInt(minLat float64, minLon float64, maxLat float64, maxLon float64, bitDepth int64) []int64 {
	// input validation
	validateBitDepth(bitDepth)

	// find the corners
	hashSouthWest := EncodeInt(minLat, minLon, bitDepth)
	hashNorthEast := EncodeInt(maxLat, maxLon, bitDepth)

	_, _, latErr, lngErr := DecodeInt(hashSouthWest, bitDepth)
	perLat := latErr * 2
	perLng := lngErr * 2

	swMinLat, _, _, swMaxLng := DecodeBboxInt(hashSouthWest, bitDepth)
	neMinLat, _, _, neMaxLng := DecodeBboxInt(hashNorthEast, bitDepth)

	latStep := round((neMinLat-swMinLat)/perLat, 0.5, 0)
	lngStep := round((neMaxLng-swMaxLng)/perLng, 0.5, 0)

	var output []int64
	for lat := 0; lat <= int(latStep); lat++ {
		for lng := 0; lng <= int(lngStep); lng++ {
			output = append(output, NeighborInt(hashSouthWest, bearing{lat, lng}, bitDepth))
		}
	}
	return output
}

// getBit returns the bit at the requested location
func getBit(geohash int64, position int64) int64 {
	return int64(int((float64(geohash) / math.Pow(float64(2), float64(position)))) & 0x01)
}

// FindBitDepth will attempt to find the maximum bitdepth which contains the supplied distance
func FindBitDepth(distanceMeters float64) int64 {
	for key, value := range bitsToDistanceInMeters {
		if value > distanceMeters {
			return MaxBitDepth - key
		}
	}
	return 0
}

// Shift provides a convenient way to convert from MaxBitDepth to another
func Shift(value int64, bitDepth int64) int64 {
	// input validation
	validateBitDepth(bitDepth)

	return value << uint64(MaxBitDepth-bitDepth)
}

// validateBitDepth will ensure the supplied bitDepth is valid or cause panic() otherwise.
func validateBitDepth(bitDepth int64) {
	if bitDepth > MaxBitDepth || bitDepth <= 0 {
		panic(fmt.Sprintf("bitDepth must be greater than 0 and less than or equal to %d, was %d", MaxBitDepth, bitDepth))
	}
	if math.Mod(float64(bitDepth), float64(2)) != 0 {
		panic(fmt.Sprintf("bitDepth must be even, was %d", bitDepth))
	}
}

// round is the "missing" round function from the math lib
func round(val float64, roundOn float64, places int) float64 {
	var round float64
	pow := math.Pow(10, float64(places))
	digit := pow * val
	_, div := math.Modf(digit)
	_div := math.Copysign(div, val)
	_roundOn := math.Copysign(roundOn, val)
	if _div >= _roundOn {
		round = math.Ceil(digit)
	} else {
		round = math.Floor(digit)
	}
	return round / pow
}
<commit_msg>revert previous change<commit_after>package geohash

import (
	"fmt"
	"math"
)

const (
	// MaxBitDepth defines both the maximum and default geohash accuracy.
	MaxBitDepth int64 = 52
)

// bearing defines the compass bearing/direction in matrix form relative to a center point of 0,0
//  |----------------------|
// 	|   NW  |   N   |  NE  |
// 	|  1,-1 |  1,0  |  1,1 |
//  |----------------------|
// 	|   W   |   X   |   E  |
// 	|  0,-1 |  0,0  |  0,1 |
//  |----------------------|
// 	|   SW  |   S   |  NE  |
// 	| -1,-1 |  -1,0 | -1,1 |
//  |----------------------|
type bearing struct {
	x, y int
}

// North bearing from reference point X
var North = bearing{1, 0}

// NorthEast bearing from reference point X
var NorthEast = bearing{1, 1}

// East bearing from reference point X
var East = bearing{0, 1}

// SouthEast bearing from reference point X
var SouthEast = bearing{-1, 1}

// South bearing from reference point X
var South = bearing{-1, 0}

// SouthWest bearing from reference point X
var SouthWest = bearing{-1, -1}

// West bearing from reference point X
var West = bearing{0, -1}

// NorthWest bearing from reference point X
var NorthWest = bearing{1, -1}

// bitsToDistanceInMeters provides a mapping between bitDepth values and distances
var bitsToDistanceInMeters []float64

func init() {
	bitsToDistanceInMeters = make([]float64, 25)
	bitsToDistanceInMeters[0] = 0.5971
	bitsToDistanceInMeters[1] = 1.1943
	bitsToDistanceInMeters[2] = 2.3889
	bitsToDistanceInMeters[3] = 4.7774
	bitsToDistanceInMeters[4] = 9.5547
	bitsToDistanceInMeters[5] = 19.1095
	bitsToDistanceInMeters[6] = 38.2189
	bitsToDistanceInMeters[7] = 76.4378
	bitsToDistanceInMeters[8] = 152.8757
	bitsToDistanceInMeters[9] = 305.751
	bitsToDistanceInMeters[10] = 611.5028
	bitsToDistanceInMeters[11] = 1223.0056
	bitsToDistanceInMeters[12] = 2446.0112
	bitsToDistanceInMeters[13] = 4892.0224
	bitsToDistanceInMeters[14] = 9784.0449
	bitsToDistanceInMeters[15] = 19568.0898
	bitsToDistanceInMeters[16] = 39136.1797
	bitsToDistanceInMeters[17] = 78272.35938
	bitsToDistanceInMeters[18] = 156544.7188
	bitsToDistanceInMeters[19] = 313089.4375
	bitsToDistanceInMeters[20] = 626178.875
	bitsToDistanceInMeters[21] = 1252357.75
	bitsToDistanceInMeters[22] = 2504715.5
	bitsToDistanceInMeters[23] = 5009431
	bitsToDistanceInMeters[24] = 10018863
}

// EncodeInt will encode a pair of latitude and longitude values into a geohash integer.
//
// The third argument is the bitDepth of this number, which affects the precision of the geohash
// but also must be used consistently when decoding. Bit depth must be even.
func EncodeInt(latitude float64, longitude float64, bitDepth int64) int64 {
	// input validation
	validateBitDepth(bitDepth)

	// initialize the calculation
	var bitsTotal int64
	var mid float64
	var maxLat float64 = 90.0
	var minLat float64 = -90.0
	var maxLng float64 = 180.0
	var minLng float64 = -180.0

	var geohash int64
	for bitsTotal < bitDepth {
		geohash *= 2

		if bitsTotal%2 == 0 {
			mid = (maxLng + minLng) / 2

			if longitude > mid {
				geohash += 1
				minLng = mid
			} else {
				maxLng = mid
			}
		} else {
			mid = (maxLat + minLat) / 2
			if latitude > mid {
				geohash += 1
				minLat = mid
			} else {
				maxLat = mid
			}
		}
		bitsTotal++
	}
	return geohash
}

// DecodeInt with decode a integer geohashed number into pair of latitude and longitude value approximations.
//
// Returned values include a latitude and longitude along with the maximum error of the calculation.
// This effectively means that a geohash integer will not return a location but an "area".
// The size of the area returned will be vary with different bitDepth settings.
//
// Note: You should provide the same bitDepth to decode the number as was used to produce the geohash originally.
func DecodeInt(geohash int64, bitDepth int64) (lat float64, lng float64, latErr float64, lngErr float64) {
	// input validation
	validateBitDepth(bitDepth)

	minLat, minLng, maxLat, maxLng := DecodeBboxInt(geohash, bitDepth)
	lat = (minLat + maxLat) / 2
	lng = (minLng + maxLng) / 2
	latErr = maxLat - lat
	lngErr = maxLng - lng
	return
}

// DecodeBboxInt will decode a geohash integer into the bounding box that matches it.
//
// Returned as a four corners of a square region.
func DecodeBboxInt(geohash int64, bitDepth int64) (minLat float64, minLng float64, maxLat float64, maxLng float64) {
	// input validation
	validateBitDepth(bitDepth)

	// initialize the calculation
	maxLat = 90
	minLat = -90
	maxLng = 180
	minLng = -180

	var latBit int64
	var lonBit int64
	var steps int64 = bitDepth / 2

	for thisStep := int64(0); thisStep < steps; thisStep++ {
		lonBit = getBit(geohash, ((steps-thisStep)*2)-1)
		latBit = getBit(geohash, ((steps-thisStep)*2)-2)

		if latBit == 0 {
			maxLat = (maxLat + minLat) / 2
		} else {
			minLat = (maxLat + minLat) / 2
		}

		if lonBit == 0 {
			maxLng = (maxLng + minLng) / 2
		} else {
			minLng = (maxLng + minLng) / 2
		}
	}

	return
}

// NeighborInt will find the neighbor of a integer geohash in certain bearing/direction.
//
// The bitDepth should be specified and the same as the value used to generate the hash.
func NeighborInt(geohash int64, bearing bearing, bitDepth int64) int64 {
	// input validation
	validateBitDepth(bitDepth)

	lat, lng, latErr, lngErr := DecodeInt(geohash, bitDepth)
	neighborLat := lat + float64(bearing.x)*latErr*2
	neighborLng := lng + float64(bearing.y)*lngErr*2
	return EncodeInt(neighborLat, neighborLng, bitDepth)
}

// NeighborsInt is the same as calling NeighborInt for each direction and will return all 8 neighbors and the center location.
func NeighborsInt(geohash int64, bitDepth int64) []int64 {
	// input validation
	validateBitDepth(bitDepth)

	var output []int64
	output = append(output, NeighborInt(geohash, North, bitDepth))
	output = append(output, NeighborInt(geohash, NorthEast, bitDepth))
	output = append(output, NeighborInt(geohash, East, bitDepth))
	output = append(output, NeighborInt(geohash, SouthEast, bitDepth))
	output = append(output, NeighborInt(geohash, South, bitDepth))
	output = append(output, NeighborInt(geohash, SouthWest, bitDepth))
	output = append(output, NeighborInt(geohash, West, bitDepth))
	output = append(output, NeighborInt(geohash, NorthWest, bitDepth))
	output = append(output, geohash)
	return output
}

// BboxesInt will return all the hash integers between minLat, minLon, maxLat, maxLon at the requested bitDepth
func BboxesInt(minLat float64, minLon float64, maxLat float64, maxLon float64, bitDepth int64) []int64 {
	// input validation
	validateBitDepth(bitDepth)

	// find the corners
	hashSouthWest := EncodeInt(minLat, minLon, bitDepth)
	hashNorthEast := EncodeInt(maxLat, maxLon, bitDepth)

	_, _, latErr, lngErr := DecodeInt(hashSouthWest, bitDepth)
	perLat := latErr * 2
	perLng := lngErr * 2

	swMinLat, _, _, swMaxLng := DecodeBboxInt(hashSouthWest, bitDepth)
	neMinLat, _, _, neMaxLng := DecodeBboxInt(hashNorthEast, bitDepth)

	latStep := round((neMinLat-swMinLat)/perLat, 0.5, 0)
	lngStep := round((neMaxLng-swMaxLng)/perLng, 0.5, 0)

	var output []int64
	for lat := 0; lat <= int(latStep); lat++ {
		for lng := 0; lng <= int(lngStep); lng++ {
			output = append(output, NeighborInt(hashSouthWest, bearing{lat, lng}, bitDepth))
		}
	}
	return output
}

// getBit returns the bit at the requested location
func getBit(geohash int64, position int64) int64 {
	return int64(int((float64(geohash) / math.Pow(float64(2), float64(position)))) & 0x01)
}

// FindBitDepth will attempt to find the maximum bitdepth which contains the supplied distance
func FindBitDepth(distanceMeters float64) int64 {
	for key, value := range bitsToDistanceInMeters {
		if value > distance {
			return MaxBitDepth - (int64(key) * 2)
		}
	}
	return 0
}

// Shift provides a convenient way to convert from MaxBitDepth to another
func Shift(value int64, bitDepth int64) int64 {
	// input validation
	validateBitDepth(bitDepth)

	return value << uint64(MaxBitDepth-bitDepth)
}

// validateBitDepth will ensure the supplied bitDepth is valid or cause panic() otherwise.
func validateBitDepth(bitDepth int64) {
	if bitDepth > MaxBitDepth || bitDepth <= 0 {
		panic(fmt.Sprintf("bitDepth must be greater than 0 and less than or equal to %d, was %d", MaxBitDepth, bitDepth))
	}
	if math.Mod(float64(bitDepth), float64(2)) != 0 {
		panic(fmt.Sprintf("bitDepth must be even, was %d", bitDepth))
	}
}

// round is the "missing" round function from the math lib
func round(val float64, roundOn float64, places int) float64 {
	var round float64
	pow := math.Pow(10, float64(places))
	digit := pow * val
	_, div := math.Modf(digit)
	_div := math.Copysign(div, val)
	_roundOn := math.Copysign(roundOn, val)
	if _div >= _roundOn {
		round = math.Ceil(digit)
	} else {
		round = math.Floor(digit)
	}
	return round / pow
}
<|endoftext|>


<commit_before>// Copyright (C) 2015 Thomas de Zeeuw.
//
// Licensed onder the MIT license that can be found in the LICENSE file.

package logger

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"
	"time"
)

type msgWriter struct {
	msgs []Msg
}

func (mw *msgWriter) Write(msg Msg) error {
	mw.msgs = append(mw.msgs, msg)
	return nil
}

func (mw *msgWriter) Close() error {
	return nil
}

func TestNew(t *testing.T) {
	t.Parallel()

	const logName = "TestNew"
	mw := &msgWriter{}
	log, err := New(logName, mw)
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}

	t1, err := sendMessages(log)
	if err != nil {
		t.Fatal(err)
	}
	if err := checkMessages(t1, mw, true); err != nil {
		t.Fatal(err)
	}
}

func TestNewExistingName(t *testing.T) {
	t.Parallel()

	const logName = "TestNewExistingName"
	_, err := New(logName, &msgWriter{})
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}

	expectedErr := "logger: name " + logName + " already taken"
	_, err = New(logName, &msgWriter{})
	if err == nil {
		t.Fatal("Expected an error when creating a logger with the same name")
	} else if err.Error() != expectedErr {
		t.Fatalf("Expected the error to be %q, but got %q",
			expectedErr, err.Error())
	}
}

func TestNewFile(t *testing.T) {
	t.Parallel()

	const logName = "TestNewFile"
	path := filepath.Join(os.TempDir(), "LOGGER_TEST.log")
	defer os.Remove(path)
	log, err := NewFile(logName, path)
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}

	t1, err := sendMessages(log)
	if err != nil {
		t.Fatal(err)
	}

	b, err := ioutil.ReadFile(path)
	if err != nil {
		t.Fatal("Unexpected error opening log file: " + err.Error())
	}

	if err := checkMessagesString(t1, b); err != nil {
		t.Fatal(err)
	}
}

func TestNewConsole(t *testing.T) {
	t.Parallel()

	var buf bytes.Buffer
	oldStderr := stderr
	stderr = &buf

	const logName = "TestNewConsole"
	log, err := NewConsole(logName)
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}

	t1, err := sendMessages(log)
	if err != nil {
		t.Fatal(err)
	}
	if err := checkMessagesString(t1, buf.Bytes()); err != nil {
		t.Fatal(err)
	}

	stderr = oldStderr
}

func TestGet(t *testing.T) {
	t.Parallel()

	const logName = "TestGet"
	log1, err := New(logName, &msgWriter{})
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}

	log2, err := Get(logName)
	if err != nil {
		t.Fatal("Unexpected error, getting the logger: " + err.Error())
	}

	if log1 != log2 {
		t.Fatal("Expected the created logger to be the same as the gotten logger")
	}
}

func TestGetNotFound(t *testing.T) {
	t.Parallel()

	const notLogName = "A logger which doesn't exists"
	expectedErr := "logger: no logger found with name " + notLogName
	_, err := Get(notLogName)
	if err == nil {
		t.Fatal("Expected an error when creating a logger with the same name")
	} else if err.Error() != expectedErr {
		t.Fatalf("Expected the error to be %q, but got %q",
			expectedErr, err.Error())
	}
}

func TestCombine(t *testing.T) {
	t.Parallel()

	const logName = "TestCombine"
	mw1 := &msgWriter{}
	log1, err := New(logName+"1", mw1)
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}

	mw2 := &msgWriter{}
	log2, err := New(logName+"2", mw2)
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}
	log2.ShowDebug = true

	log, err := Combine(logName, log1, log2)
	if err != nil {
		t.Fatal("Unexpected error, combining loggers: " + err.Error())
	}

	t1, err := sendMessages(log)
	if err != nil {
		t.Fatal(err)
	}
	if err := checkMessages(t1, mw1, false); err != nil {
		t.Fatal(err)
	} else if err := checkMessages(t1, mw2, true); err != nil {
		t.Fatal(err)
	}
}

func TestCombineNone(t *testing.T) {
	t.Parallel()

	const logName = "TestCombineNone"
	expectedErr := "logger: Combine requires atleast one logger"
	_, err := Combine(logName)
	if err == nil {
		t.Fatal("Expected an error when creating a logger with the same name")
	} else if err.Error() != expectedErr {
		t.Fatalf("Expected the error to be %q, but got %q",
			expectedErr, err.Error())
	}
}

func TestCombineExistingName(t *testing.T) {
	t.Parallel()

	const logName = "TestCombineExistingName"
	log, err := New(logName, &msgWriter{})
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}

	expectedErr := "logger: name " + logName + " already taken"
	_, err = Combine(logName, log)
	if err == nil {
		t.Fatal("Expected an error when creating a logger with the same name")
	} else if err.Error() != expectedErr {
		t.Fatalf("Expected the error to be %q, but got %q",
			expectedErr, err.Error())
	}
}

// sendMessages is linked with checkMessages and checkMessagesString, any
// changes most be checked in those functions aswell.
func sendMessages(log *Logger) (time.Time, error) {
	tags := Tags{"test"}
	t1 := time.Now().UTC().Truncate(time.Second)
	log.Fatal(tags, errors.New("Fatal message1"))
	log.Fatal(tags, errors.New("Fatal message2"))
	log.Fatal(tags, "Fatal message3")
	log.Error(tags, errors.New("Error message1"))
	log.Error(tags, errors.New("Error message2"))
	log.Error(tags, errors.New("Error message3"))
	log.Info(tags, "Info message1")
	log.Info(tags, "Info message2")
	log.Info(tags, "Info message3")
	log.Thumbstone(tags, "Thumb message1")
	log.Thumbstone(tags, "Thumb message2")
	log.Thumbstone(tags, "Thumb message3")
	log.ShowDebug = true
	log.Debug(tags, "Debug message1")
	log.Debug(tags, "Debug message2")
	log.Debug(tags, "Debug message3")
	log.ShowDebug = false
	log.Debug(tags, "Debug message4")

	if err := log.Close(); err != nil {
		return t1, errors.New("Unexpected error, closing logger: " + err.Error())
	}

	return t1, nil
}

// checkMessages is linked to sendMessages.
func checkMessages(t1 time.Time, mw *msgWriter, debugEnabled bool) error {
	if nMsgs := 15; !debugEnabled {
		nMsgs = 12
	} else if len(mw.msgs) != nMsgs {
		return fmt.Errorf("Expected %d messages, but got %d", nMsgs, len(mw.msgs))
	}

	for i, msg := range mw.msgs {
		var expectedLevel = Debug
		if i < 3 {
			expectedLevel = Fatal
		} else if i >= 3 && i < 6 {
			expectedLevel = Error
		} else if i >= 6 && i < 9 {
			expectedLevel = Info
		} else if i >= 9 && i < 12 {
			expectedLevel = Thumb
		}

		expectedMsg := expectedLevel.String()
		expectedMsg = strings.TrimSpace(expectedMsg) + " message"
		expectedMsg += fmt.Sprintf("%d", i%3+1)

		if expectedLevel == Fatal {
			msg.Msg = msg.Msg[:14] // trim stack trace from message.
		}

		tags := Tags{"test"}
		if expectedLevel == Thumb {
			tags = Tags{"thumbstone", "test"}
		}

		if msg.Level != expectedLevel {
			return fmt.Errorf("Expected msg.Level to be %q, but got %q",
				expectedLevel, msg.Level)
		} else if msg.Msg != expectedMsg {
			return fmt.Errorf("Expected msg.Msg to be %q, but got %q",
				expectedMsg, msg.Msg)
		} else if !reflect.DeepEqual(msg.Tags, tags) {
			return fmt.Errorf("Expected msg.Tags to be %q, but got %q",
				tags.String(), msg.Tags.String())
		} else if !msg.Timestamp.Truncate(time.Second).Equal(t1) {
			return fmt.Errorf("Expected msg.Timestamp to be %v, but got %v",
				t1, msg.Timestamp)
		}
	}

	return nil
}

// checkMessagesString is linked to sendMessages.
func checkMessagesString(t1 time.Time, gotBytes []byte) error {
	t1Str := t1.Format("2006-01-02 15:04:05")

	// not the prettiest solution, but good enough...
	expectedLines := []string{
		t1Str + " [Fatal] test: Fatal message1",
		t1Str + " [Fatal] test: Fatal message2",
		t1Str + " [Fatal] test: Fatal message3",
		t1Str + " [Error] test: Error message1",
		t1Str + " [Error] test: Error message2",
		t1Str + " [Error] test: Error message3",
		t1Str + " [Info] test: Info message1",
		t1Str + " [Info] test: Info message2",
		t1Str + " [Info] test: Info message3",
		t1Str + " [Thumb] thumbstone, test: Thumb message1",
		t1Str + " [Thumb] thumbstone, test: Thumb message2",
		t1Str + " [Thumb] thumbstone, test: Thumb message3",
		t1Str + " [Debug] test: Debug message1",
		t1Str + " [Debug] test: Debug message2",
		t1Str + " [Debug] test: Debug message3",
	}

	i := 0
	s := bufio.NewScanner(bytes.NewReader(gotBytes))
	for s.Scan() {
		got := s.Text()
		expected := expectedLines[i]

		if !strings.HasPrefix(got, t1Str) {
			continue
		}

		if got != expected {
			return fmt.Errorf("Error comparing line %d\nExpected: %q\nbut got:  %q",
				i, expected, got)
		}
		i++
	}

	if i != len(expectedLines) {
		return errors.New("Didn't get the same amount of lines as expected")
	} else if err := s.Err(); err != nil {
		return fmt.Errorf("Unexpected scanning error: %s", err.Error())
	}
	return nil
}
<commit_msg>Add tests for Logger.Warn<commit_after>// Copyright (C) 2015 Thomas de Zeeuw.
//
// Licensed onder the MIT license that can be found in the LICENSE file.

package logger

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"
	"time"
)

type msgWriter struct {
	msgs []Msg
}

func (mw *msgWriter) Write(msg Msg) error {
	mw.msgs = append(mw.msgs, msg)
	return nil
}

func (mw *msgWriter) Close() error {
	return nil
}

func TestNew(t *testing.T) {
	t.Parallel()

	const logName = "TestNew"
	mw := &msgWriter{}
	log, err := New(logName, mw)
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}

	t1, err := sendMessages(log)
	if err != nil {
		t.Fatal(err)
	}
	if err := checkMessages(t1, mw, true); err != nil {
		t.Fatal(err)
	}
}

func TestNewExistingName(t *testing.T) {
	t.Parallel()

	const logName = "TestNewExistingName"
	_, err := New(logName, &msgWriter{})
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}

	expectedErr := "logger: name " + logName + " already taken"
	_, err = New(logName, &msgWriter{})
	if err == nil {
		t.Fatal("Expected an error when creating a logger with the same name")
	} else if err.Error() != expectedErr {
		t.Fatalf("Expected the error to be %q, but got %q",
			expectedErr, err.Error())
	}
}

func TestNewFile(t *testing.T) {
	t.Parallel()

	const logName = "TestNewFile"
	path := filepath.Join(os.TempDir(), "LOGGER_TEST.log")
	defer os.Remove(path)
	log, err := NewFile(logName, path)
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}

	t1, err := sendMessages(log)
	if err != nil {
		t.Fatal(err)
	}

	b, err := ioutil.ReadFile(path)
	if err != nil {
		t.Fatal("Unexpected error opening log file: " + err.Error())
	}

	if err := checkMessagesString(t1, b); err != nil {
		t.Fatal(err)
	}
}

func TestNewConsole(t *testing.T) {
	t.Parallel()

	var buf bytes.Buffer
	oldStderr := stderr
	stderr = &buf

	const logName = "TestNewConsole"
	log, err := NewConsole(logName)
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}

	t1, err := sendMessages(log)
	if err != nil {
		t.Fatal(err)
	}
	if err := checkMessagesString(t1, buf.Bytes()); err != nil {
		t.Fatal(err)
	}

	stderr = oldStderr
}

func TestGet(t *testing.T) {
	t.Parallel()

	const logName = "TestGet"
	log1, err := New(logName, &msgWriter{})
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}

	log2, err := Get(logName)
	if err != nil {
		t.Fatal("Unexpected error, getting the logger: " + err.Error())
	}

	if log1 != log2 {
		t.Fatal("Expected the created logger to be the same as the gotten logger")
	}
}

func TestGetNotFound(t *testing.T) {
	t.Parallel()

	const notLogName = "A logger which doesn't exists"
	expectedErr := "logger: no logger found with name " + notLogName
	_, err := Get(notLogName)
	if err == nil {
		t.Fatal("Expected an error when creating a logger with the same name")
	} else if err.Error() != expectedErr {
		t.Fatalf("Expected the error to be %q, but got %q",
			expectedErr, err.Error())
	}
}

func TestCombine(t *testing.T) {
	t.Parallel()

	const logName = "TestCombine"
	mw1 := &msgWriter{}
	log1, err := New(logName+"1", mw1)
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}

	mw2 := &msgWriter{}
	log2, err := New(logName+"2", mw2)
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}
	log2.ShowDebug = true

	log, err := Combine(logName, log1, log2)
	if err != nil {
		t.Fatal("Unexpected error, combining loggers: " + err.Error())
	}

	t1, err := sendMessages(log)
	if err != nil {
		t.Fatal(err)
	}
	if err := checkMessages(t1, mw1, false); err != nil {
		t.Fatal(err)
	} else if err := checkMessages(t1, mw2, true); err != nil {
		t.Fatal(err)
	}
}

func TestCombineNone(t *testing.T) {
	t.Parallel()

	const logName = "TestCombineNone"
	expectedErr := "logger: Combine requires atleast one logger"
	_, err := Combine(logName)
	if err == nil {
		t.Fatal("Expected an error when creating a logger with the same name")
	} else if err.Error() != expectedErr {
		t.Fatalf("Expected the error to be %q, but got %q",
			expectedErr, err.Error())
	}
}

func TestCombineExistingName(t *testing.T) {
	t.Parallel()

	const logName = "TestCombineExistingName"
	log, err := New(logName, &msgWriter{})
	if err != nil {
		t.Fatal("Unexpected error, creating a new logger: " + err.Error())
	}

	expectedErr := "logger: name " + logName + " already taken"
	_, err = Combine(logName, log)
	if err == nil {
		t.Fatal("Expected an error when creating a logger with the same name")
	} else if err.Error() != expectedErr {
		t.Fatalf("Expected the error to be %q, but got %q",
			expectedErr, err.Error())
	}
}

// sendMessages is linked with checkMessages and checkMessagesString, any
// changes most be checked in those functions aswell.
func sendMessages(log *Logger) (time.Time, error) {
	tags := Tags{"test"}
	t1 := time.Now().UTC().Truncate(time.Second)
	log.Fatal(tags, errors.New("Fatal message1"))
	log.Fatal(tags, errors.New("Fatal message2"))
	log.Fatal(tags, "Fatal message3")
	log.Error(tags, errors.New("Error message1"))
	log.Error(tags, errors.New("Error message2"))
	log.Error(tags, errors.New("Error message3"))
	log.Warn(tags, "Warn message1")
	log.Warn(tags, "Warn message2")
	log.Warn(tags, "Warn message3")
	log.Info(tags, "Info message1")
	log.Info(tags, "Info message2")
	log.Info(tags, "Info message3")
	log.Thumbstone(tags, "Thumb message1")
	log.Thumbstone(tags, "Thumb message2")
	log.Thumbstone(tags, "Thumb message3")
	log.ShowDebug = true
	log.Debug(tags, "Debug message1")
	log.Debug(tags, "Debug message2")
	log.Debug(tags, "Debug message3")
	log.ShowDebug = false
	log.Debug(tags, "Debug message4")

	if err := log.Close(); err != nil {
		return t1, errors.New("Unexpected error, closing logger: " + err.Error())
	}

	return t1, nil
}

// checkMessages is linked to sendMessages.
func checkMessages(t1 time.Time, mw *msgWriter, debugEnabled bool) error {
	if nMsgs := 18; !debugEnabled {
		nMsgs = 12
	} else if len(mw.msgs) != nMsgs {
		return fmt.Errorf("Expected %d messages, but got %d", nMsgs, len(mw.msgs))
	}

	for i, msg := range mw.msgs {
		var expectedLevel = Debug
		if i < 3 {
			expectedLevel = Fatal
		} else if i >= 3 && i < 6 {
			expectedLevel = Error
		} else if i >= 6 && i < 9 {
			expectedLevel = Warn
		} else if i >= 9 && i < 12 {
			expectedLevel = Info
		} else if i >= 12 && i < 15 {
			expectedLevel = Thumb
		}

		expectedMsg := expectedLevel.String()
		expectedMsg = strings.TrimSpace(expectedMsg) + " message"
		expectedMsg += fmt.Sprintf("%d", i%3+1)

		if expectedLevel == Fatal {
			msg.Msg = msg.Msg[:14] // trim stack trace from message.
		}

		tags := Tags{"test"}
		if expectedLevel == Thumb {
			tags = Tags{"thumbstone", "test"}
		}

		if msg.Level != expectedLevel {
			return fmt.Errorf("Expected msg.Level to be %q, but got %q",
				expectedLevel, msg.Level)
		} else if msg.Msg != expectedMsg {
			return fmt.Errorf("Expected msg.Msg to be %q, but got %q",
				expectedMsg, msg.Msg)
		} else if !reflect.DeepEqual(msg.Tags, tags) {
			return fmt.Errorf("Expected msg.Tags to be %q, but got %q",
				tags.String(), msg.Tags.String())
		} else if !msg.Timestamp.Truncate(time.Second).Equal(t1) {
			return fmt.Errorf("Expected msg.Timestamp to be %v, but got %v",
				t1, msg.Timestamp)
		}
	}

	return nil
}

// checkMessagesString is linked to sendMessages.
func checkMessagesString(t1 time.Time, gotBytes []byte) error {
	t1Str := t1.Format("2006-01-02 15:04:05")

	// not the prettiest solution, but good enough...
	expectedLines := []string{
		t1Str + " [Fatal] test: Fatal message1",
		t1Str + " [Fatal] test: Fatal message2",
		t1Str + " [Fatal] test: Fatal message3",
		t1Str + " [Error] test: Error message1",
		t1Str + " [Error] test: Error message2",
		t1Str + " [Error] test: Error message3",
		t1Str + " [Warn] test: Warn message1",
		t1Str + " [Warn] test: Warn message2",
		t1Str + " [Warn] test: Warn message3",
		t1Str + " [Info] test: Info message1",
		t1Str + " [Info] test: Info message2",
		t1Str + " [Info] test: Info message3",
		t1Str + " [Thumb] thumbstone, test: Thumb message1",
		t1Str + " [Thumb] thumbstone, test: Thumb message2",
		t1Str + " [Thumb] thumbstone, test: Thumb message3",
		t1Str + " [Debug] test: Debug message1",
		t1Str + " [Debug] test: Debug message2",
		t1Str + " [Debug] test: Debug message3",
	}

	i := 0
	s := bufio.NewScanner(bytes.NewReader(gotBytes))
	for s.Scan() {
		got := s.Text()
		expected := expectedLines[i]

		if !strings.HasPrefix(got, t1Str) {
			continue
		}

		if got != expected {
			return fmt.Errorf("Error comparing line %d\nExpected: %q\nbut got:  %q",
				i, expected, got)
		}
		i++
	}

	if i != len(expectedLines) {
		return errors.New("Didn't get the same amount of lines as expected")
	} else if err := s.Err(); err != nil {
		return fmt.Errorf("Unexpected scanning error: %s", err.Error())
	}
	return nil
}
<|endoftext|>


<commit_before>/* Copyright (c) 2016 PaddlePaddle Authors All Rights Reserve.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
	 limitations under the License. */

package edl

import (
	"fmt"

	edlresource "github.com/PaddlePaddle/cloud/go/edl/resource"
	batchv1 "k8s.io/api/batch/v1"
	"k8s.io/api/core/v1"
	"k8s.io/api/extensions/v1beta1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/pkg/api"
)

// ClusterResource is the resource of a cluster
type ClusterResource struct {
	NodeCount int // The total number of nodes in the cluster.

	// Each Kubernetes job could require some number of GPUs in
	// the range of [request, limit].
	GPURequest int // \sum_job num_gpu_request(job)
	GPULimit   int // \sum_job num_gpu_limit(job)
	GPUTotal   int // The total number of GPUs in the cluster

	// Each Kubernetes job could require some CPU timeslices in
	// the unit of *milli*.
	CPURequestMilli int64 // \sum_job cpu_request_in_milli(job)
	CPULimitMilli   int64 // \sum_job cpu_request_in_milli(job)
	CPUTotalMilli   int64 // The total amount of CPUs in the cluster in milli.

	// Each Kubernetes job could require some amount of memory in
	// the unit of *mega*.
	MemoryRequestMega int64 // \sum_job memory_request_in_mega(job)
	MemoryLimitMega   int64 // \sum_job memory_limit_in_mega(job)
	MemoryTotalMega   int64 // The total amount of memory in the cluster in mega.

	Nodes Nodes
}

// Nodes records the amount of idle CPU and free memory of each node
// in the cluster.
type Nodes struct {
	NodesCPUIdleMilli   map[string]int64 // node id -> idle CPU
	NodesMemoryFreeMega map[string]int64 // node id -> free memory
}

func (ns *Nodes) Len() int {
	if len(ns.NodesCPUIdleMilli) != len(ns.NodesMemoryFreeMega) {
		panic("Inconsistent length in Nodes")
	}
	return len(ns.NodesCPUIdleMilli)
}

// Cluster is our interface to the Kubernetes cluster. It can inquiry
// the cluster's overall status and the status of a specific
// PaddlePaddle trainning job.  It can also create training jobs and
// replica.
//
// TODO(yi): The above functionalities are NOT logically related with
// each other.  I am not sure if it is a good idea to group them in
// this source file.
type Cluster struct {
	clientset *kubernetes.Clientset
}

// newCluster create a new instance of K8sCluster.
func newCluster(clientset *kubernetes.Clientset) *Cluster {
	return &Cluster{
		clientset: clientset,
	}
}

// GetTrainerJob gets the trainer job spec.
func (c Cluster) GetTrainerJob(job *edlresource.TrainingJob) (*batchv1.Job, error) {
	namespace := job.ObjectMeta.Namespace
	jobname := job.ObjectMeta.Name
	return c.clientset.
		BatchV1().
		Jobs(namespace).
		Get(fmt.Sprintf("%s-trainer", jobname), metav1.GetOptions{})
}

// GetTrainerJobByName gets the trainer job spec.
func (c Cluster) GetTrainerJobByName(namespace, name string) (*batchv1.Job, error) {
	return c.clientset.
		BatchV1().
		Jobs(namespace).
		Get((name), metav1.GetOptions{})
}

// UpdateTrainerJob updates the trainer job spec
// this will do the actual scale up/down.
func (c Cluster) UpdateTrainerJob(job *batchv1.Job) error {
	_, err := c.clientset.BatchV1().Jobs(job.ObjectMeta.Namespace).Update(job)
	return err
}

// JobPods returns the number total desired pods and the number of
// running pods of a job.
func (c Cluster) JobPods(job *edlresource.TrainingJob) (total, running, pending int, err error) {
	if err != nil {
		return
	}
	// get pods of the job
	jobPods, err := c.clientset.CoreV1().
		Pods(job.ObjectMeta.Namespace).
		List(metav1.ListOptions{LabelSelector: "paddle-job=" + job.ObjectMeta.Name})
	for _, pod := range jobPods.Items {
		total++
		// pod.ObjectMeta.DeletionTimestamp means pod is terminating
		if pod.ObjectMeta.DeletionTimestamp == nil && pod.Status.Phase == v1.PodRunning {
			running++
		}
		if pod.ObjectMeta.DeletionTimestamp == nil && pod.Status.Phase == v1.PodPending {
			pending++
		}
	}
	return
}

// getPodsTotalRequestsAndLimits accumulate resource requests and
// limits from all pods containers.
func getPodsTotalRequestsAndLimits(podList *v1.PodList) (reqs v1.ResourceList, limits v1.ResourceList, err error) {
	reqs, limits = v1.ResourceList{}, v1.ResourceList{}
	for _, pod := range podList.Items {
		for _, container := range pod.Spec.Containers {
			AddResourceList(reqs, container.Resources.Requests)
			AddResourceList(limits, container.Resources.Limits)
		}

		for _, container := range pod.Spec.InitContainers {
			AddResourceList(reqs, container.Resources.Requests)
			AddResourceList(limits, container.Resources.Limits)
		}
	}
	return
}

func updateNodesIdleResource(podList *v1.PodList, nodesCPUIdleMilli map[string]int64, nodesMemoryFreeMega map[string]int64) (err error) {
	for _, pod := range podList.Items {
		nodeName := pod.Spec.NodeName
		if nodeName == "" {
			continue
		}
		for _, container := range pod.Spec.Containers {
			nodesCPUIdleMilli[nodeName] -= container.Resources.Requests.Cpu().ScaledValue(resource.Milli)
			nodesMemoryFreeMega[nodeName] -= container.Resources.Requests.Memory().ScaledValue(resource.Mega)
		}

		for _, container := range pod.Spec.InitContainers {
			nodesCPUIdleMilli[nodeName] -= container.Resources.Requests.Cpu().ScaledValue(resource.Milli)
			nodesMemoryFreeMega[nodeName] -= container.Resources.Requests.Memory().ScaledValue(resource.Mega)
		}
	}
	return
}

// InquiryResource returns the idle and total resources of the Kubernetes cluster.
func (c *Cluster) InquiryResource() (res ClusterResource, err error) {
	nodes := c.clientset.CoreV1().Nodes()
	nodeList, err := nodes.List(metav1.ListOptions{})
	if err != nil {
		return ClusterResource{}, err
	}
	allocatable := make(v1.ResourceList)
	nodesCPUIdleMilli := make(map[string]int64)
	nodesMemoryFreeMega := make(map[string]int64)

	for _, node := range nodeList.Items {
		nodesCPUIdleMilli[node.GetObjectMeta().GetName()] =
			node.Status.Allocatable.Cpu().ScaledValue(resource.Milli)
		nodesMemoryFreeMega[node.GetObjectMeta().GetName()] =
			node.Status.Allocatable.Memory().ScaledValue(resource.Mega)
		AddResourceList(allocatable, node.Status.Allocatable)
	}

	// Get non-terminated pods from all namespaces.
	namespace := ""

	// FIXME(typhoonzero): scan all pods is not a efficient way.
	// NOTE: pending pods need to be caculated for scale down.
	// NOTE: "terminating" pods' status is still running, do not
	// scale up/down the job if job is still at last scaling
	// process.
	fieldSelector, err := fields.ParseSelector("status.phase!=" + string(api.PodSucceeded) + ",status.phase!=" + string(api.PodFailed))
	if err != nil {
		return ClusterResource{}, err
	}

	allPodsList, err := c.clientset.CoreV1().Pods(namespace).List(metav1.ListOptions{FieldSelector: fieldSelector.String()})
	if err != nil {
		return ClusterResource{}, err
	}

	allReqs, allLimits, err := getPodsTotalRequestsAndLimits(allPodsList)
	if err != nil {
		return ClusterResource{}, err
	}

	err = updateNodesIdleResource(allPodsList, nodesCPUIdleMilli, nodesMemoryFreeMega)
	if err != nil {
		return ClusterResource{}, err
	}

	res = ClusterResource{
		NodeCount:       len(nodeList.Items),
		GPUTotal:        int(allocatable.NvidiaGPU().Value()),
		CPUTotalMilli:   allocatable.Cpu().ScaledValue(resource.Milli),
		MemoryTotalMega: allocatable.Memory().ScaledValue(resource.Mega),

		GPURequest:        int(allReqs.NvidiaGPU().Value()),
		CPURequestMilli:   allReqs.Cpu().ScaledValue(resource.Milli),
		MemoryRequestMega: allReqs.Memory().ScaledValue(resource.Mega),

		GPULimit:        int(allLimits.NvidiaGPU().Value()),
		CPULimitMilli:   allLimits.Cpu().ScaledValue(resource.Milli),
		MemoryLimitMega: allLimits.Memory().ScaledValue(resource.Mega),

		Nodes: Nodes{
			NodesCPUIdleMilli:   nodesCPUIdleMilli,
			NodesMemoryFreeMega: nodesMemoryFreeMega,
		},
	}
	return
}

// CreateJob creates a Job.
func (c *Cluster) CreateJob(j *batchv1.Job) (*batchv1.Job, error) {
	return c.clientset.
		BatchV1().
		Jobs(j.ObjectMeta.Namespace).
		Create(j)
}

// CreateReplicaSet creates a ReplicaSet.
func (c *Cluster) CreateReplicaSet(r *v1beta1.ReplicaSet) (*v1beta1.ReplicaSet, error) {
	return c.clientset.
		ExtensionsV1beta1().
		ReplicaSets(r.ObjectMeta.Namespace).
		Create(r)
}

// GetReplicaSet gets a ReplicaSet.
func (c *Cluster) GetReplicaSet(namespace, name string) (*v1beta1.ReplicaSet, error) {
	return c.clientset.
		ExtensionsV1beta1().
		ReplicaSets(namespace).
		Get(name, metav1.GetOptions{})
}

// DeleteTrainerJob deletes a trainerjob and their pods.
// see: https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/
func (c *Cluster) DeleteTrainerJob(namespace, name string) error {
	deletePolicy := metav1.DeletePropagationForeground
	options := metav1.DeleteOptions{
		PropagationPolicy: &deletePolicy,
	}
	return c.clientset.
		BatchV1().
		Jobs(namespace).
		Delete(name, &options)
}

// DeleteReplicaSet delete a ReplicaSet and their pods.
func (c *Cluster) DeleteReplicaSet(namespace, name string) error {
	deletePolicy := metav1.DeletePropagationForeground
	options := metav1.DeleteOptions{
		PropagationPolicy: &deletePolicy,
	}
	return c.clientset.
		ExtensionsV1beta1().
		ReplicaSets(namespace).
		Delete(name, &options)
}
<commit_msg>create custom String() for Nodes, remove printing clutter<commit_after>/* Copyright (c) 2016 PaddlePaddle Authors All Rights Reserve.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
	 limitations under the License. */

package edl

import (
	"fmt"

	edlresource "github.com/PaddlePaddle/cloud/go/edl/resource"
	batchv1 "k8s.io/api/batch/v1"
	"k8s.io/api/core/v1"
	"k8s.io/api/extensions/v1beta1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/pkg/api"
)

// ClusterResource is the resource of a cluster
type ClusterResource struct {
	NodeCount int // The total number of nodes in the cluster.

	// Each Kubernetes job could require some number of GPUs in
	// the range of [request, limit].
	GPURequest int // \sum_job num_gpu_request(job)
	GPULimit   int // \sum_job num_gpu_limit(job)
	GPUTotal   int // The total number of GPUs in the cluster

	// Each Kubernetes job could require some CPU timeslices in
	// the unit of *milli*.
	CPURequestMilli int64 // \sum_job cpu_request_in_milli(job)
	CPULimitMilli   int64 // \sum_job cpu_request_in_milli(job)
	CPUTotalMilli   int64 // The total amount of CPUs in the cluster in milli.

	// Each Kubernetes job could require some amount of memory in
	// the unit of *mega*.
	MemoryRequestMega int64 // \sum_job memory_request_in_mega(job)
	MemoryLimitMega   int64 // \sum_job memory_limit_in_mega(job)
	MemoryTotalMega   int64 // The total amount of memory in the cluster in mega.

	Nodes Nodes
}

// Nodes records the amount of idle CPU and free memory of each node
// in the cluster.
type Nodes struct {
	NodesCPUIdleMilli   map[string]int64 // node id -> idle CPU
	NodesMemoryFreeMega map[string]int64 // node id -> free memory
}

func (ns *Nodes) String() string {
	if len(ns.NodesCPUIdleMilli) != len(ns.NodesMemoryFreeMega) {
		panic("Inconsistent length in Nodes")
	}

	return fmt.Sprintf("%d Nodes", len(ns.NodesCPUIdleMilli))
}

// Cluster is our interface to the Kubernetes cluster. It can inquiry
// the cluster's overall status and the status of a specific
// PaddlePaddle trainning job.  It can also create training jobs and
// replica.
//
// TODO(yi): The above functionalities are NOT logically related with
// each other.  I am not sure if it is a good idea to group them in
// this source file.
type Cluster struct {
	clientset *kubernetes.Clientset
}

// newCluster create a new instance of K8sCluster.
func newCluster(clientset *kubernetes.Clientset) *Cluster {
	return &Cluster{
		clientset: clientset,
	}
}

// GetTrainerJob gets the trainer job spec.
func (c Cluster) GetTrainerJob(job *edlresource.TrainingJob) (*batchv1.Job, error) {
	namespace := job.ObjectMeta.Namespace
	jobname := job.ObjectMeta.Name
	return c.clientset.
		BatchV1().
		Jobs(namespace).
		Get(fmt.Sprintf("%s-trainer", jobname), metav1.GetOptions{})
}

// GetTrainerJobByName gets the trainer job spec.
func (c Cluster) GetTrainerJobByName(namespace, name string) (*batchv1.Job, error) {
	return c.clientset.
		BatchV1().
		Jobs(namespace).
		Get((name), metav1.GetOptions{})
}

// UpdateTrainerJob updates the trainer job spec
// this will do the actual scale up/down.
func (c Cluster) UpdateTrainerJob(job *batchv1.Job) error {
	_, err := c.clientset.BatchV1().Jobs(job.ObjectMeta.Namespace).Update(job)
	return err
}

// JobPods returns the number total desired pods and the number of
// running pods of a job.
func (c Cluster) JobPods(job *edlresource.TrainingJob) (total, running, pending int, err error) {
	if err != nil {
		return
	}
	// get pods of the job
	jobPods, err := c.clientset.CoreV1().
		Pods(job.ObjectMeta.Namespace).
		List(metav1.ListOptions{LabelSelector: "paddle-job=" + job.ObjectMeta.Name})
	for _, pod := range jobPods.Items {
		total++
		// pod.ObjectMeta.DeletionTimestamp means pod is terminating
		if pod.ObjectMeta.DeletionTimestamp == nil && pod.Status.Phase == v1.PodRunning {
			running++
		}
		if pod.ObjectMeta.DeletionTimestamp == nil && pod.Status.Phase == v1.PodPending {
			pending++
		}
	}
	return
}

// getPodsTotalRequestsAndLimits accumulate resource requests and
// limits from all pods containers.
func getPodsTotalRequestsAndLimits(podList *v1.PodList) (reqs v1.ResourceList, limits v1.ResourceList, err error) {
	reqs, limits = v1.ResourceList{}, v1.ResourceList{}
	for _, pod := range podList.Items {
		for _, container := range pod.Spec.Containers {
			AddResourceList(reqs, container.Resources.Requests)
			AddResourceList(limits, container.Resources.Limits)
		}

		for _, container := range pod.Spec.InitContainers {
			AddResourceList(reqs, container.Resources.Requests)
			AddResourceList(limits, container.Resources.Limits)
		}
	}
	return
}

func updateNodesIdleResource(podList *v1.PodList, nodesCPUIdleMilli map[string]int64, nodesMemoryFreeMega map[string]int64) (err error) {
	for _, pod := range podList.Items {
		nodeName := pod.Spec.NodeName
		if nodeName == "" {
			continue
		}
		for _, container := range pod.Spec.Containers {
			nodesCPUIdleMilli[nodeName] -= container.Resources.Requests.Cpu().ScaledValue(resource.Milli)
			nodesMemoryFreeMega[nodeName] -= container.Resources.Requests.Memory().ScaledValue(resource.Mega)
		}

		for _, container := range pod.Spec.InitContainers {
			nodesCPUIdleMilli[nodeName] -= container.Resources.Requests.Cpu().ScaledValue(resource.Milli)
			nodesMemoryFreeMega[nodeName] -= container.Resources.Requests.Memory().ScaledValue(resource.Mega)
		}
	}
	return
}

// InquiryResource returns the idle and total resources of the Kubernetes cluster.
func (c *Cluster) InquiryResource() (res ClusterResource, err error) {
	nodes := c.clientset.CoreV1().Nodes()
	nodeList, err := nodes.List(metav1.ListOptions{})
	if err != nil {
		return ClusterResource{}, err
	}
	allocatable := make(v1.ResourceList)
	nodesCPUIdleMilli := make(map[string]int64)
	nodesMemoryFreeMega := make(map[string]int64)

	for _, node := range nodeList.Items {
		nodesCPUIdleMilli[node.GetObjectMeta().GetName()] =
			node.Status.Allocatable.Cpu().ScaledValue(resource.Milli)
		nodesMemoryFreeMega[node.GetObjectMeta().GetName()] =
			node.Status.Allocatable.Memory().ScaledValue(resource.Mega)
		AddResourceList(allocatable, node.Status.Allocatable)
	}

	// Get non-terminated pods from all namespaces.
	namespace := ""

	// FIXME(typhoonzero): scan all pods is not a efficient way.
	// NOTE: pending pods need to be caculated for scale down.
	// NOTE: "terminating" pods' status is still running, do not
	// scale up/down the job if job is still at last scaling
	// process.
	fieldSelector, err := fields.ParseSelector("status.phase!=" + string(api.PodSucceeded) + ",status.phase!=" + string(api.PodFailed))
	if err != nil {
		return ClusterResource{}, err
	}

	allPodsList, err := c.clientset.CoreV1().Pods(namespace).List(metav1.ListOptions{FieldSelector: fieldSelector.String()})
	if err != nil {
		return ClusterResource{}, err
	}

	allReqs, allLimits, err := getPodsTotalRequestsAndLimits(allPodsList)
	if err != nil {
		return ClusterResource{}, err
	}

	err = updateNodesIdleResource(allPodsList, nodesCPUIdleMilli, nodesMemoryFreeMega)
	if err != nil {
		return ClusterResource{}, err
	}

	res = ClusterResource{
		NodeCount:       len(nodeList.Items),
		GPUTotal:        int(allocatable.NvidiaGPU().Value()),
		CPUTotalMilli:   allocatable.Cpu().ScaledValue(resource.Milli),
		MemoryTotalMega: allocatable.Memory().ScaledValue(resource.Mega),

		GPURequest:        int(allReqs.NvidiaGPU().Value()),
		CPURequestMilli:   allReqs.Cpu().ScaledValue(resource.Milli),
		MemoryRequestMega: allReqs.Memory().ScaledValue(resource.Mega),

		GPULimit:        int(allLimits.NvidiaGPU().Value()),
		CPULimitMilli:   allLimits.Cpu().ScaledValue(resource.Milli),
		MemoryLimitMega: allLimits.Memory().ScaledValue(resource.Mega),

		Nodes: Nodes{
			NodesCPUIdleMilli:   nodesCPUIdleMilli,
			NodesMemoryFreeMega: nodesMemoryFreeMega,
		},
	}
	return
}

// CreateJob creates a Job.
func (c *Cluster) CreateJob(j *batchv1.Job) (*batchv1.Job, error) {
	return c.clientset.
		BatchV1().
		Jobs(j.ObjectMeta.Namespace).
		Create(j)
}

// CreateReplicaSet creates a ReplicaSet.
func (c *Cluster) CreateReplicaSet(r *v1beta1.ReplicaSet) (*v1beta1.ReplicaSet, error) {
	return c.clientset.
		ExtensionsV1beta1().
		ReplicaSets(r.ObjectMeta.Namespace).
		Create(r)
}

// GetReplicaSet gets a ReplicaSet.
func (c *Cluster) GetReplicaSet(namespace, name string) (*v1beta1.ReplicaSet, error) {
	return c.clientset.
		ExtensionsV1beta1().
		ReplicaSets(namespace).
		Get(name, metav1.GetOptions{})
}

// DeleteTrainerJob deletes a trainerjob and their pods.
// see: https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/
func (c *Cluster) DeleteTrainerJob(namespace, name string) error {
	deletePolicy := metav1.DeletePropagationForeground
	options := metav1.DeleteOptions{
		PropagationPolicy: &deletePolicy,
	}
	return c.clientset.
		BatchV1().
		Jobs(namespace).
		Delete(name, &options)
}

// DeleteReplicaSet delete a ReplicaSet and their pods.
func (c *Cluster) DeleteReplicaSet(namespace, name string) error {
	deletePolicy := metav1.DeletePropagationForeground
	options := metav1.DeleteOptions{
		PropagationPolicy: &deletePolicy,
	}
	return c.clientset.
		ExtensionsV1beta1().
		ReplicaSets(namespace).
		Delete(name, &options)
}
<|endoftext|>


<commit_before>package cluster

import (
	"errors"
	"fmt"
	"testing"

	"github.com/samalba/dockerclient"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
)

var (
	mockInfo = &dockerclient.Info{
		ID:              "id",
		Name:            "name",
		NCPU:            10,
		MemTotal:        20,
		Driver:          "driver-test",
		ExecutionDriver: "execution-driver-test",
		KernelVersion:   "1.2.3",
		OperatingSystem: "golang",
	}
)

func TestNodeConnectionFailure(t *testing.T) {
	node := NewNode("test")
	assert.False(t, node.IsConnected())

	// Always fail.
	client := dockerclient.NewMockClient()
	client.On("Info").Return(&dockerclient.Info{}, errors.New("fail"))

	// Connect() should fail and IsConnected() return false.
	assert.Error(t, node.connectClient(client))
	assert.False(t, node.IsConnected())

	client.Mock.AssertExpectations(t)
}

func TestOutdatedNode(t *testing.T) {
	node := NewNode("test")
	client := dockerclient.NewMockClient()
	client.On("Info").Return(&dockerclient.Info{}, nil)

	assert.Error(t, node.connectClient(client))
	assert.False(t, node.IsConnected())

	client.Mock.AssertExpectations(t)
}

func TestNodeSpecs(t *testing.T) {
	node := NewNode("test")
	assert.False(t, node.IsConnected())

	client := dockerclient.NewMockClient()
	client.On("Info").Return(mockInfo, nil)
	client.On("ListContainers", true, false, "").Return([]dockerclient.Container{}, nil)
	client.On("StartMonitorEvents", mock.Anything, mock.Anything).Return()

	assert.NoError(t, node.connectClient(client))
	assert.True(t, node.IsConnected())

	assert.Equal(t, node.Cpus, mockInfo.NCPU)
	assert.Equal(t, node.Memory, mockInfo.MemTotal)
	assert.Equal(t, node.Labels["storagedriver"], mockInfo.Driver)
	assert.Equal(t, node.Labels["executiondriver"], mockInfo.ExecutionDriver)
	assert.Equal(t, node.Labels["kernelversion"], mockInfo.KernelVersion)
	assert.Equal(t, node.Labels["operatingsystem"], mockInfo.OperatingSystem)

	client.Mock.AssertExpectations(t)
}

func TestNodeState(t *testing.T) {
	node := NewNode("test")
	assert.False(t, node.IsConnected())

	client := dockerclient.NewMockClient()
	client.On("Info").Return(mockInfo, nil)
	client.On("StartMonitorEvents", mock.Anything, mock.Anything).Return()

	// The client will return one container at first, then a second one will appear.
	client.On("ListContainers", true, false, "").Return([]dockerclient.Container{{Id: "one"}}, nil).Once()
	client.On("InspectContainer", "one").Return(&dockerclient.ContainerInfo{Config: &dockerclient.ContainerConfig{CpuShares: 100}}, nil).Once()
	client.On("ListContainers", true, false, fmt.Sprintf("{%q:[%q]}", "id", "two")).Return([]dockerclient.Container{{Id: "two"}}, nil).Once()
	client.On("InspectContainer", "two").Return(&dockerclient.ContainerInfo{Config: &dockerclient.ContainerConfig{CpuShares: 100}}, nil).Once()

	assert.NoError(t, node.connectClient(client))
	assert.True(t, node.IsConnected())

	// The node should only have a single container at this point.
	containers := node.Containers()
	assert.Len(t, containers, 1)
	if containers[0].Id != "one" {
		t.Fatalf("Missing container: one")
	}

	// Fake an event which will trigger a refresh. The second container will appear.
	node.handler(&dockerclient.Event{Id: "two", Status: "created"})
	containers = node.Containers()
	assert.Len(t, containers, 2)
	if containers[0].Id != "one" && containers[1].Id != "one" {
		t.Fatalf("Missing container: one")
	}
	if containers[0].Id != "two" && containers[1].Id != "two" {
		t.Fatalf("Missing container: two")
	}

	client.Mock.AssertExpectations(t)
}
<commit_msg>add a few tests<commit_after>package cluster

import (
	"errors"
	"fmt"
	"testing"

	"github.com/samalba/dockerclient"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
)

var (
	mockInfo = &dockerclient.Info{
		ID:              "id",
		Name:            "name",
		NCPU:            10,
		MemTotal:        20,
		Driver:          "driver-test",
		ExecutionDriver: "execution-driver-test",
		KernelVersion:   "1.2.3",
		OperatingSystem: "golang",
		Labels:          []string{"foo=bar"},
	}
)

func TestNodeConnectionFailure(t *testing.T) {
	node := NewNode("test")
	assert.False(t, node.IsConnected())

	// Always fail.
	client := dockerclient.NewMockClient()
	client.On("Info").Return(&dockerclient.Info{}, errors.New("fail"))

	// Connect() should fail and IsConnected() return false.
	assert.Error(t, node.connectClient(client))
	assert.False(t, node.IsConnected())

	client.Mock.AssertExpectations(t)
}

func TestOutdatedNode(t *testing.T) {
	node := NewNode("test")
	client := dockerclient.NewMockClient()
	client.On("Info").Return(&dockerclient.Info{}, nil)

	assert.Error(t, node.connectClient(client))
	assert.False(t, node.IsConnected())

	client.Mock.AssertExpectations(t)
}

func TestNodeSpecs(t *testing.T) {
	node := NewNode("test")
	assert.False(t, node.IsConnected())

	client := dockerclient.NewMockClient()
	client.On("Info").Return(mockInfo, nil)
	client.On("ListContainers", true, false, "").Return([]dockerclient.Container{}, nil)
	client.On("StartMonitorEvents", mock.Anything, mock.Anything).Return()

	assert.NoError(t, node.connectClient(client))
	assert.True(t, node.IsConnected())
	assert.True(t, node.IsHealthy())

	assert.Equal(t, node.Cpus, mockInfo.NCPU)
	assert.Equal(t, node.Memory, mockInfo.MemTotal)
	assert.Equal(t, node.Labels["storagedriver"], mockInfo.Driver)
	assert.Equal(t, node.Labels["executiondriver"], mockInfo.ExecutionDriver)
	assert.Equal(t, node.Labels["kernelversion"], mockInfo.KernelVersion)
	assert.Equal(t, node.Labels["operatingsystem"], mockInfo.OperatingSystem)
	assert.Equal(t, node.Labels["foo"], "bar")

	client.Mock.AssertExpectations(t)
}

func TestNodeState(t *testing.T) {
	node := NewNode("test")
	assert.False(t, node.IsConnected())

	client := dockerclient.NewMockClient()
	client.On("Info").Return(mockInfo, nil)
	client.On("StartMonitorEvents", mock.Anything, mock.Anything).Return()

	// The client will return one container at first, then a second one will appear.
	client.On("ListContainers", true, false, "").Return([]dockerclient.Container{{Id: "one"}}, nil).Once()
	client.On("InspectContainer", "one").Return(&dockerclient.ContainerInfo{Config: &dockerclient.ContainerConfig{CpuShares: 100}}, nil).Once()
	client.On("ListContainers", true, false, fmt.Sprintf("{%q:[%q]}", "id", "two")).Return([]dockerclient.Container{{Id: "two"}}, nil).Once()
	client.On("InspectContainer", "two").Return(&dockerclient.ContainerInfo{Config: &dockerclient.ContainerConfig{CpuShares: 100}}, nil).Once()

	assert.NoError(t, node.connectClient(client))
	assert.True(t, node.IsConnected())

	// The node should only have a single container at this point.
	containers := node.Containers()
	assert.Len(t, containers, 1)
	if containers[0].Id != "one" {
		t.Fatalf("Missing container: one")
	}

	// Fake an event which will trigger a refresh. The second container will appear.
	node.handler(&dockerclient.Event{Id: "two", Status: "created"})
	containers = node.Containers()
	assert.Len(t, containers, 2)
	if containers[0].Id != "one" && containers[1].Id != "one" {
		t.Fatalf("Missing container: one")
	}
	if containers[0].Id != "two" && containers[1].Id != "two" {
		t.Fatalf("Missing container: two")
	}

	client.Mock.AssertExpectations(t)
}
<|endoftext|>


<commit_before>package main

import (
	info "github.com/google/cadvisor/info/v1"

	"errors"
	"fmt"
	"log"
	"net/http"
	"strings"

	"github.com/google/cadvisor/client"
	"time"
)

var filter string = "dummy"                  //the name of the servers the clients want to contact
var monitor string = "http://127.0.0.1:8080" // URL where the cAdvisor container is
var port string = ":6666"                    // port we will listen on
var mapContainers = make(map[string]uint64)

func updateContainers() {

	listContainers()

	for _ = range time.Tick(5 * time.Second) {
		log.Println("Starting list of containers update...")
		listContainers()
		log.Println("Found", len(mapContainers), "containers")
		log.Println("Done.\n")
	}
}

func listContainers() {

	allContainers, _ := getAllContainerInfo(monitor)

	var alias string
	var kbFree uint64

	// resetting mapContainers
	for key, _ := range mapContainers {
		delete(mapContainers, key)
	}

	for _, item := range allContainers {
		alias = item.Aliases[0]
		kbFree = item.Stats[0].Memory.Usage

		if strings.Contains(alias, filter) {
			mapContainers[alias] = kbFree
		}
	}
}

// determine the container with the more available RAM
func getLessLoaded() (string, error) {

	var lessLoaded string

	for key, _ := range mapContainers {
		lessLoaded = key
		break
	}

	for key, _ := range mapContainers {

		// verifying only wanted containers
		if mapContainers[key] < mapContainers[lessLoaded] {
			lessLoaded = key
		}
	}

	if lessLoaded == "" {
		return "", errors.New("No server found...")
	}
	return lessLoaded, nil
}

func handleRoot(w http.ResponseWriter, r *http.Request) {
	log.Println("GET vue")
	server, err := getLessLoaded()
	log.Println("chose server : ", server)

	if detectError(err, false) {
		w.WriteHeader(500) //warn the user that the server encountered a problem
	} else {
		w.WriteHeader(200)
	}

	w.Write([]byte(server))
}

func getAllContainerInfo(cadvisor string) ([]info.ContainerInfo, error) {

	client, err := client.NewClient(cadvisor)
	if detectError(err, true) {
		return nil, err
	}

	request := info.DefaultContainerInfoRequest()
	allContainers, err := client.AllDockerContainers(&request)
	if detectError(err, true) {
		return nil, err
	}

	return allContainers, nil
}

func detectError(err error, doLog bool) bool {

	if err != nil {
		if doLog {
			log.Println(err)
		}
		return true
	} else {
		return false
	}
}

func main() {

	go updateContainers()

	http.HandleFunc("/", handleRoot)

	fmt.Println("Listening on http://127.0.0.1" + port)
	log.Fatal(http.ListenAndServe(port, nil))
}
<commit_msg>Optimisation globale<commit_after>package main

import (
	info "github.com/google/cadvisor/info/v1"

	"errors"
	"fmt"
	"log"
	"net/http"
	"strings"

	"github.com/google/cadvisor/client"
	"time"
)

var filter = "dummy"                  //the name of the servers the clients want to contact
var monitor = "http://127.0.0.1:8080" // URL where the cAdvisor container is
var port = ":6666"                    // port we will listen on
var mapContainers = make(map[string]uint64)

func updateContainers() {

	listContainers()

	for _ = range time.Tick(5 * time.Second) {
		log.Println("Starting list of containers update...")
		listContainers()
		log.Println("Found", len(mapContainers), "containers")
		log.Print("Done.\n")
	}
}

func listContainers() {

	allContainers, _ := getAllContainerInfo(monitor)

	var alias string
	var kbFree uint64

	// resetting mapContainers
	for key := range mapContainers {
		delete(mapContainers, key)
	}

	for _, item := range allContainers {
		alias = item.Aliases[0]
		kbFree = item.Stats[0].Memory.Usage

		if strings.Contains(alias, filter) {
			mapContainers[alias] = kbFree
		}
	}
}

// determine the container with the more available RAM
func getLessLoaded() (string, error) {

	var lessLoaded string

	for key := range mapContainers {

		if lessLoaded == "" {
			lessLoaded = key
		} else if mapContainers[key] < mapContainers[lessLoaded] {
			lessLoaded = key
		}
	}

	if lessLoaded == "" {
		return "", errors.New("No server found...")
	}
	return lessLoaded, nil
}

func handleRoot(w http.ResponseWriter, r *http.Request) {
	log.Println("GET vue")
	server, err := getLessLoaded()
	log.Println("chose server : ", server)

	if detectError(err, false) {
		w.WriteHeader(500) //warn the user that the server encountered a problem
	} else {
		w.WriteHeader(200)
	}

	w.Write([]byte(server))
}

func getAllContainerInfo(cadvisor string) ([]info.ContainerInfo, error) {

	client, err := client.NewClient(cadvisor)
	if detectError(err, true) {
		return nil, err
	}

	request := info.DefaultContainerInfoRequest()
	allContainers, err := client.AllDockerContainers(&request)
	if detectError(err, true) {
		return nil, err
	}

	return allContainers, nil
}

func detectError(err error, doLog bool) bool {

	if err != nil {
		if doLog {
			log.Println(err)
		}
		return true
	} else {
		return false
	}
}

func main() {

	go updateContainers()

	http.HandleFunc("/", handleRoot)

	fmt.Println("Listening on http://127.0.0.1" + port)
	log.Fatal(http.ListenAndServe(port, nil))
}
<|endoftext|>


<commit_before>package command

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"time"

	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	"github.com/coreos/etcd/client"
)

func NewClusterHealthCommand() cli.Command {
	return cli.Command{
		Name:  "cluster-health",
		Usage: "check the health of the etcd cluster",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "forever", Usage: "forever check the health every 10 second until CTRL+C"},
		},
		Action: handleClusterHealth,
	}
}

func handleClusterHealth(c *cli.Context) {
	forever := c.Bool("forever")
	if forever {
		sigch := make(chan os.Signal, 1)
		signal.Notify(sigch, os.Interrupt)

		go func() {
			<-sigch
			os.Exit(0)
		}()
	}

	tr, err := getTransport(c)
	if err != nil {
		handleError(ExitServerError, err)
	}

	hc := http.Client{
		Transport: tr,
	}

	cln := mustNewClientNoSync(c)
	mi := client.NewMembersAPI(cln)
	ms, err := mi.List(context.TODO())
	if err != nil {
		fmt.Println("cluster may be unhealthy: failed to list members")
		handleError(ExitServerError, err)
	}

	for {
		health := false
		for _, m := range ms {
			if len(m.ClientURLs) == 0 {
				fmt.Printf("member %s is unreachable: no available published client urls\n", m.ID)
				continue
			}

			checked := false
			for _, url := range m.ClientURLs {
				resp, err := hc.Get(url + "/health")
				if err != nil {
					fmt.Printf("failed to check the health of member %s on %s: %v\n", m.ID, url, err)
					continue
				}

				result := struct{ Health string }{}
				d := json.NewDecoder(resp.Body)
				err = d.Decode(&result)
				resp.Body.Close()
				if err != nil {
					fmt.Printf("failed to check the health of member %s on %s: %v\n", m.ID, url, err)
					continue
				}

				checked = true
				if result.Health == "true" {
					health = true
					fmt.Printf("member %s is healthy: got healthy result from %s\n", m.ID, url)
				} else {
					fmt.Printf("member %s is unhealthy: got unhealthy result from %s\n", m.ID, url)
				}
				break
			}
			if !checked {
				fmt.Printf("member %s is unreachable: %v are all unreachable\n", m.ID, m.ClientURLs)
			}
		}
		if health {
			fmt.Println("cluster is healthy")
		} else {
			fmt.Println("cluster is unhealthy")
		}

		if !forever {
			break
		}
		fmt.Printf("\nnext check after 10 second...\n\n")
		time.Sleep(10 * time.Second)
	}
}
<commit_msg>etcdctl: prepare for health endpoint change<commit_after>package command

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"time"

	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	"github.com/coreos/etcd/client"
)

func NewClusterHealthCommand() cli.Command {
	return cli.Command{
		Name:  "cluster-health",
		Usage: "check the health of the etcd cluster",
		Flags: []cli.Flag{
			cli.BoolFlag{Name: "forever", Usage: "forever check the health every 10 second until CTRL+C"},
		},
		Action: handleClusterHealth,
	}
}

func handleClusterHealth(c *cli.Context) {
	forever := c.Bool("forever")
	if forever {
		sigch := make(chan os.Signal, 1)
		signal.Notify(sigch, os.Interrupt)

		go func() {
			<-sigch
			os.Exit(0)
		}()
	}

	tr, err := getTransport(c)
	if err != nil {
		handleError(ExitServerError, err)
	}

	hc := http.Client{
		Transport: tr,
	}

	cln := mustNewClientNoSync(c)
	mi := client.NewMembersAPI(cln)
	ms, err := mi.List(context.TODO())
	if err != nil {
		fmt.Println("cluster may be unhealthy: failed to list members")
		handleError(ExitServerError, err)
	}

	for {
		health := false
		for _, m := range ms {
			if len(m.ClientURLs) == 0 {
				fmt.Printf("member %s is unreachable: no available published client urls\n", m.ID)
				continue
			}

			checked := false
			for _, url := range m.ClientURLs {
				resp, err := hc.Get(url + "/health")
				if err != nil {
					fmt.Printf("failed to check the health of member %s on %s: %v\n", m.ID, url, err)
					continue
				}

				result := struct{ Health string }{}
				nresult := struct{ Health bool }{}
				bytes, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					fmt.Printf("failed to check the health of member %s on %s: %v\n", m.ID, url, err)
					continue
				}
				resp.Body.Close()

				err = json.Unmarshal(bytes, &result)
				if err != nil {
					err = json.Unmarshal(bytes, &nresult)
				}
				if err != nil {
					fmt.Printf("failed to check the health of member %s on %s: %v\n", m.ID, url, err)
					continue
				}

				checked = true
				if result.Health == "true" {
					health = true
					fmt.Printf("member %s is healthy: got healthy result from %s\n", m.ID, url)
				} else {
					fmt.Printf("member %s is unhealthy: got unhealthy result from %s\n", m.ID, url)
				}
				break
			}
			if !checked {
				fmt.Printf("member %s is unreachable: %v are all unreachable\n", m.ID, m.ClientURLs)
			}
		}
		if health {
			fmt.Println("cluster is healthy")
		} else {
			fmt.Println("cluster is unhealthy")
		}

		if !forever {
			break
		}
		fmt.Printf("\nnext check after 10 second...\n\n")
		time.Sleep(10 * time.Second)
	}
}
<|endoftext|>


<commit_before>package shell

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/zetamatta/go-mbcs"

	"github.com/zetamatta/nyagos/dos"
)

func loadEnvFile(fname string, verbose io.Writer) error {
	fp, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer fp.Close()

	scan := bufio.NewScanner(fp)
	for scan.Scan() {
		line, err := mbcs.AtoU(scan.Bytes())
		if err != nil {
			continue
		}
		line = strings.TrimSpace(line)
		eqlPos := strings.Index(line, "=")
		if eqlPos > 0 {
			left := line[:eqlPos]
			right := line[eqlPos+1:]
			if left != "ERRORLEVEL_" {
				orig := os.Getenv(left)
				if verbose != nil {
					fmt.Fprintf(verbose, "%s=%s\n", left, right)
				}
				if orig != right {
					// fmt.Fprintf(os.Stderr, "%s:=%s\n", left, right)
					os.Setenv(left, right)
				}
			}
		}
	}
	return scan.Err()
}

func loadPwdFile(fname string, verbose io.Writer) error {
	fp, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer fp.Close()
	scan := bufio.NewScanner(fp)
	if !scan.Scan() {
		return fmt.Errorf("Could not load the new current directory from %s", fname)
	}
	if err := scan.Err(); err != nil {
		return err
	}
	line, err := mbcs.AtoU(scan.Bytes())
	if err != nil {
		return err
	}
	line = strings.TrimSpace(line)
	if verbose != nil {
		fmt.Fprintf(verbose, "cd \"%s\"\n", line)
	}
	os.Chdir(line)
	return nil
}

func callBatch(batch string,
	args []string,
	env string,
	pwd string,
	verbose io.Writer,
	stdin io.Reader,
	stdout io.Writer,
	stderr io.Writer) (int, error) {
	params := []string{
		os.Getenv("COMSPEC"),
		"/C",
		batch,
	}
	fd, err := os.Create(batch)
	if err != nil {
		return 1, err
	}
	var writer *bufio.Writer
	if verbose != nil {
		writer = bufio.NewWriter(io.MultiWriter(fd, verbose))
	} else {
		writer = bufio.NewWriter(fd)
	}
	fmt.Fprint(writer, "@call")
	for _, v := range args {
		if strings.ContainsRune(v, ' ') {
			fmt.Fprintf(writer, " \"%s\"", v)
		} else {
			fmt.Fprintf(writer, " %s", v)
		}
	}
	fmt.Fprintf(writer, "\n@set \"ERRORLEVEL_=%%ERRORLEVEL%%\"\n")
	fmt.Fprintf(writer, "@set > \"%s\"\n", env)
	fmt.Fprintf(writer, "@cd > \"%s\"\n", pwd)
	fmt.Fprintf(writer, "@exit /b \"%%ERRORLEVEL_%%\"\n")
	writer.Flush()
	if err := fd.Close(); err != nil {
		return 1, err
	}

	cmd2 := exec.Cmd{
		Path:   params[0],
		Args:   params,
		Stdin:  stdin,
		Stdout: stdout,
		Stderr: stderr,
	}
	if err := cmd2.Run(); err != nil {
		return 1, err
	}
	errorlevel, errorlevelOk := dos.GetErrorLevel(&cmd2)
	if !errorlevelOk {
		errorlevel = 255
	}
	return errorlevel, nil
}

func Source(args []string, verbose io.Writer, debug bool, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
	tempDir := os.TempDir()
	pid := os.Getpid()
	batch := filepath.Join(tempDir, fmt.Sprintf("nyagos-%d.cmd", pid))
	env := filepath.Join(tempDir, fmt.Sprintf("nyagos-%d.tmp", pid))
	pwd := filepath.Join(tempDir, fmt.Sprintf("nyagos_%d.tmp", pid))

	errorlevel, err := callBatch(
		batch,
		args,
		env,
		pwd,
		verbose,
		stdin,
		stdout,
		stderr)

	if !debug {
		defer os.Remove(env)
		defer os.Remove(pwd)
		defer os.Remove(batch)
	}

	if err != nil {
		return errorlevel, err
	}

	if err := loadEnvFile(env, verbose); err != nil {
		return 1, err
	}

	if err := loadPwdFile(pwd, verbose); err != nil {
		return 1, err
	}
	return errorlevel, err
}
<commit_msg>Fix: batch parameter written in MBCS disappeared<commit_after>package shell

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/zetamatta/go-mbcs"

	"github.com/zetamatta/nyagos/dos"
)

func loadEnvFile(fname string, verbose io.Writer) error {
	fp, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer fp.Close()

	scan := bufio.NewScanner(fp)
	for scan.Scan() {
		line, err := mbcs.AtoU(scan.Bytes())
		if err != nil {
			continue
		}
		line = strings.TrimSpace(line)
		eqlPos := strings.Index(line, "=")
		if eqlPos > 0 {
			left := line[:eqlPos]
			right := line[eqlPos+1:]
			if left != "ERRORLEVEL_" {
				orig := os.Getenv(left)
				if verbose != nil {
					fmt.Fprintf(verbose, "%s=%s\n", left, right)
				}
				if orig != right {
					// fmt.Fprintf(os.Stderr, "%s:=%s\n", left, right)
					os.Setenv(left, right)
				}
			}
		}
	}
	return scan.Err()
}

func loadPwdFile(fname string, verbose io.Writer) error {
	fp, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer fp.Close()
	scan := bufio.NewScanner(fp)
	if !scan.Scan() {
		return fmt.Errorf("Could not load the new current directory from %s", fname)
	}
	if err := scan.Err(); err != nil {
		return err
	}
	line, err := mbcs.AtoU(scan.Bytes())
	if err != nil {
		return err
	}
	line = strings.TrimSpace(line)
	if verbose != nil {
		fmt.Fprintf(verbose, "cd \"%s\"\n", line)
	}
	os.Chdir(line)
	return nil
}

func callBatch(batch string,
	args []string,
	env string,
	pwd string,
	verbose io.Writer,
	stdin io.Reader,
	stdout io.Writer,
	stderr io.Writer) (int, error) {
	params := []string{
		os.Getenv("COMSPEC"),
		"/C",
		batch,
	}
	fd, err := os.Create(batch)
	if err != nil {
		return 1, err
	}
	var writer *bufio.Writer
	if verbose != nil {
		writer = bufio.NewWriter(io.MultiWriter(fd, verbose))
	} else {
		writer = bufio.NewWriter(fd)
	}
	fmt.Fprint(writer, "@call")
	for _, arg1 := range args {
		// UTF8 parameter to ANSI
		ansi, err := mbcs.UtoA(arg1)
		if err != nil {
			// println("utoa: " + err.Error())
			fd.Close()
			return -1, err
		}
		// chop last '\0'
		if ansi[len(ansi)-1] == 0 {
			ansi = ansi[:len(ansi)-1]
		}
		if strings.ContainsRune(arg1, ' ') {
			_, err = fmt.Fprintf(writer, " \"%s\"", ansi)
		} else {
			_, err = fmt.Fprintf(writer, " %s", ansi)
		}
		if err != nil {
			fd.Close()
			return -1, err
		}
	}
	fmt.Fprintf(writer, "\n@set \"ERRORLEVEL_=%%ERRORLEVEL%%\"\n")
	fmt.Fprintf(writer, "@set > \"%s\"\n", env)
	fmt.Fprintf(writer, "@cd > \"%s\"\n", pwd)
	fmt.Fprintf(writer, "@exit /b \"%%ERRORLEVEL_%%\"\n")
	writer.Flush()
	if err := fd.Close(); err != nil {
		return 1, err
	}

	cmd2 := exec.Cmd{
		Path:   params[0],
		Args:   params,
		Stdin:  stdin,
		Stdout: stdout,
		Stderr: stderr,
	}
	if err := cmd2.Run(); err != nil {
		return 1, err
	}
	errorlevel, errorlevelOk := dos.GetErrorLevel(&cmd2)
	if !errorlevelOk {
		errorlevel = 255
	}
	return errorlevel, nil
}

func Source(args []string, verbose io.Writer, debug bool, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error) {
	tempDir := os.TempDir()
	pid := os.Getpid()
	batch := filepath.Join(tempDir, fmt.Sprintf("nyagos-%d.cmd", pid))
	env := filepath.Join(tempDir, fmt.Sprintf("nyagos-%d.tmp", pid))
	pwd := filepath.Join(tempDir, fmt.Sprintf("nyagos_%d.tmp", pid))

	errorlevel, err := callBatch(
		batch,
		args,
		env,
		pwd,
		verbose,
		stdin,
		stdout,
		stderr)

	if err != nil {
		return -1, err
	}

	if !debug {
		defer os.Remove(env)
		defer os.Remove(pwd)
		defer os.Remove(batch)
	}

	if err != nil {
		return errorlevel, err
	}

	if err := loadEnvFile(env, verbose); err != nil {
		return 1, err
	}

	if err := loadPwdFile(pwd, verbose); err != nil {
		return 1, err
	}
	return errorlevel, err
}
<|endoftext|>


<commit_before>package cli

import (
	"errors"
	"flag"
	"io/ioutil"

	"github.com/Jeffail/gabs"
	"github.com/pelletier/go-toml"
	log "github.com/sirupsen/logrus"
)

type ConfigVariable struct {
	*StringVariable
	Type   ParsingType
	config configGetter
}

type configGetter interface {
	GetByVariable(string) (interface{}, error)
}

var (
	ErrFailedToLoadToml = errors.New("Failed to load toml.")
	ErrFailedToLoadJson = errors.New("Failed to load json.")
)

func (c *ConfigVariable) GetName() string {
	return c.StringVariable.Name
}

func (c *ConfigVariable) apply(interface{}) {
	log.Fatal("Config Variable does not apply values.")
}

func (c *ConfigVariable) ParseConfig(set *flag.FlagSet) error {
	// ignore error, library should handle this.
	stringPointer, _ := c.StringVariable.getFlagValue(set)
	value := stringPointer.(string)
	data, err := ioutil.ReadFile(value)
	if err != nil {
		return err
	}

	switch c.Type {
	case TomlConfig:
		tree, err := toml.Load(string(data))
		if err != nil {
			log.WithFields(log.Fields{"err": err}).Fatal(ErrFailedToLoadToml)
			return ErrFailedToLoadToml
		}
		config := &tomlConfig{
			tree: tree,
		}
		c.config = config
	case JsonConfig:
		container, err := gabs.ParseJSON(data)
		if err != nil {
			log.WithFields(log.Fields{"err": err}).Fatal(ErrFailedToLoadJson)
			return ErrFailedToLoadJson
		}
		config := &jsonConfig{
			container: container,
		}
		c.config = config

	default:
		log.Fatal("Unimplemented config type")

	}
	return nil
}

func (c *ConfigVariable) getConfigValue(path string) (interface{}, error) {
	return c.config.GetByVariable(path)
}

type tomlConfig struct {
	tree *toml.TomlTree
}

func (t *tomlConfig) GetByVariable(path string) (interface{}, error) {
	return t.tree.Get(path), nil
}

type jsonConfig struct {
	container *gabs.Container
}

func (j *jsonConfig) GetByVariable(path string) (interface{}, error) {
	return j.container.Path(path).Data(), nil
}
<commit_msg>remove unneeded function<commit_after>package cli

import (
	"errors"
	"flag"
	"io/ioutil"

	"github.com/Jeffail/gabs"
	"github.com/pelletier/go-toml"
	log "github.com/sirupsen/logrus"
)

type ConfigVariable struct {
	*StringVariable
	Type   ParsingType
	config configGetter
}

type configGetter interface {
	GetByVariable(string) (interface{}, error)
}

var (
	ErrFailedToLoadToml = errors.New("Failed to load toml.")
	ErrFailedToLoadJson = errors.New("Failed to load json.")
)

func (c *ConfigVariable) apply(interface{}) {
	log.Fatal("Config Variable does not apply values.")
}

func (c *ConfigVariable) ParseConfig(set *flag.FlagSet) error {
	// ignore error, library should handle this.
	stringPointer, _ := c.StringVariable.getFlagValue(set)
	value := stringPointer.(string)
	data, err := ioutil.ReadFile(value)
	if err != nil {
		return err
	}

	switch c.Type {
	case TomlConfig:
		tree, err := toml.Load(string(data))
		if err != nil {
			log.WithFields(log.Fields{"err": err}).Fatal(ErrFailedToLoadToml)
			return ErrFailedToLoadToml
		}
		config := &tomlConfig{
			tree: tree,
		}
		c.config = config
	case JsonConfig:
		container, err := gabs.ParseJSON(data)
		if err != nil {
			log.WithFields(log.Fields{"err": err}).Fatal(ErrFailedToLoadJson)
			return ErrFailedToLoadJson
		}
		config := &jsonConfig{
			container: container,
		}
		c.config = config

	default:
		log.Fatal("Unimplemented config type")

	}
	return nil
}

func (c *ConfigVariable) getConfigValue(path string) (interface{}, error) {
	return c.config.GetByVariable(path)
}

type tomlConfig struct {
	tree *toml.TomlTree
}

func (t *tomlConfig) GetByVariable(path string) (interface{}, error) {
	return t.tree.Get(path), nil
}

type jsonConfig struct {
	container *gabs.Container
}

func (j *jsonConfig) GetByVariable(path string) (interface{}, error) {
	return j.container.Path(path).Data(), nil
}
<|endoftext|>


<commit_before>/*
copyright 2019 the knative authors

licensed under the apache license, version 2.0 (the "license");
you may not use this file except in compliance with the license.
you may obtain a copy of the license at

    http://www.apache.org/licenses/license-2.0

unless required by applicable law or agreed to in writing, software
distributed under the license is distributed on an "as is" basis,
without warranties or conditions of any kind, either express or implied.
see the license for the specific language governing permissions and
limitations under the license.
*/

package kmeta

import (
	"crypto/md5"
	"fmt"
	"strings"
)

// The longest name supported by the K8s is 63.
// These constants
const (
	longest = 63
	md5Len  = 32
	head    = longest - md5Len // How much to truncate to fit the hash.
)

// ChildName generates a name for the resource based upon the parent resource and suffix.
// If the concatenated name is longer than K8s permits the name is hashed and truncated to permit
// construction of the resource, but still keeps it unique.
// If the suffix itself is longer than 31 characters, then the whole string will be hashed
// and `parent|hash|suffix` will be returned, where parent and suffix will be trimmed to
// fit (prefix of parent at most of length 31, and prefix of suffix at most length 30).
func ChildName(parent, suffix string) string {
	n := parent
	if len(parent) > (longest - len(suffix)) {
		// If the suffix is longer than the longest allowed suffix, then
		// we hash the whole combined string and use that as the suffix.
		if head-len(suffix) <= 0 {
			h := md5.Sum([]byte(parent + suffix))
			// 1. trim parent, if needed
			if head < len(parent) {
				parent = parent[:head]
			}
			// Format the return string, if it's shorter than longest: pad with
			// beginning of the suffix. This happens, for example, when parent is
			// short, but the suffix is very long.
			ret := parent + fmt.Sprintf("%x", h)
			if d := longest - len(ret); d > 0 {
				ret += suffix[:d]
			}
			// If due to trumming above we're terminating the string with a `-`,
			// remove it.
			return strings.TrimRight(ret, "-")
		}
		n = fmt.Sprintf("%s%x", parent[:head-len(suffix)], md5.Sum([]byte(parent)))
	}
	return n + suffix
}
<commit_msg>Fix a typo in the comment (#962)<commit_after>/*
copyright 2019 the knative authors

licensed under the apache license, version 2.0 (the "license");
you may not use this file except in compliance with the license.
you may obtain a copy of the license at

    http://www.apache.org/licenses/license-2.0

unless required by applicable law or agreed to in writing, software
distributed under the license is distributed on an "as is" basis,
without warranties or conditions of any kind, either express or implied.
see the license for the specific language governing permissions and
limitations under the license.
*/

package kmeta

import (
	"crypto/md5"
	"fmt"
	"strings"
)

// The longest name supported by the K8s is 63.
// These constants
const (
	longest = 63
	md5Len  = 32
	head    = longest - md5Len // How much to truncate to fit the hash.
)

// ChildName generates a name for the resource based upon the parent resource and suffix.
// If the concatenated name is longer than K8s permits the name is hashed and truncated to permit
// construction of the resource, but still keeps it unique.
// If the suffix itself is longer than 31 characters, then the whole string will be hashed
// and `parent|hash|suffix` will be returned, where parent and suffix will be trimmed to
// fit (prefix of parent at most of length 31, and prefix of suffix at most length 30).
func ChildName(parent, suffix string) string {
	n := parent
	if len(parent) > (longest - len(suffix)) {
		// If the suffix is longer than the longest allowed suffix, then
		// we hash the whole combined string and use that as the suffix.
		if head-len(suffix) <= 0 {
			h := md5.Sum([]byte(parent + suffix))
			// 1. trim parent, if needed
			if head < len(parent) {
				parent = parent[:head]
			}
			// Format the return string, if it's shorter than longest: pad with
			// beginning of the suffix. This happens, for example, when parent is
			// short, but the suffix is very long.
			ret := parent + fmt.Sprintf("%x", h)
			if d := longest - len(ret); d > 0 {
				ret += suffix[:d]
			}
			// If due to trimming above we're terminating the string with a `-`,
			// remove it.
			return strings.TrimRight(ret, "-")
		}
		n = fmt.Sprintf("%s%x", parent[:head-len(suffix)], md5.Sum([]byte(parent)))
	}
	return n + suffix
}
<|endoftext|>


<commit_before>package main

import (
"fmt"
)

func main(){
fmt.Println("Write grpc client to interact with order service")
}
<commit_msg>Update client.go<commit_after>package main

import (
  "fmt"
)

func TestClient(){
  fmt.Println("Write grpc client to interact with order service")
}
<|endoftext|>


<commit_before>package broker

import (
	"errors"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"h12.me/kpax/model"
	"h12.me/wipro"
)

var (
	ErrCorrelationIDMismatch = errors.New("correlationID mismatch")
)

type B struct {
	Addr     string
	Timeout  time.Duration
	QueueLen int

	connection
	cid      int32
	recvChan chan *brokerJob
	mu       sync.Mutex
}

type connection struct {
	conn net.Conn
	mu   sync.Mutex
	*B
}

type brokerJob struct {
	req     model.Request
	resp    model.Response
	errChan chan error
}

func New(addr string) model.Broker {
	b := &B{
		Addr:     addr,
		Timeout:  30 * time.Second,
		QueueLen: 1000,
	}
	b.connection.B = b
	return b
}

func (b *B) Do(req model.Request, resp model.Response) error {
	job := &brokerJob{
		req:     req,
		resp:    resp,
		errChan: make(chan error),
	}
	if err := b.send(job); err != nil {
		return err
	}
	return <-job.errChan
}

func (b *B) send(job *brokerJob) error {
	conn, err := b.getConn()
	if err != nil {
		return err
	}

	b.mu.Lock()
	defer b.mu.Unlock()
	job.req.SetID(atomic.AddInt32(&b.cid, 1))
	if err := job.req.Send(conn); err != nil {
		if err == wipro.ErrConn {
			b.closeConn()
		}
		return err
	}
	if b.recvChan == nil {
		b.recvChan = make(chan *brokerJob, b.QueueLen)
		go b.receiveLoop()
	}
	if job.requireAck() {
		b.recvChan <- job
	}
	return nil
}
func (j *brokerJob) requireAck() bool { return j.resp != nil }

func (c *B) receiveLoop() {
	for job := range c.recvChan {
		conn, err := c.getConn()
		if err != nil {
			job.errChan <- err
			continue
		}
		if err := job.resp.Receive(conn); err != nil {
			if err == wipro.ErrConn {
				c.closeConn()
			}
			job.errChan <- err
			continue
		}
		if job.resp.ID() != job.req.ID() {
			job.errChan <- ErrCorrelationIDMismatch
			continue
		}
		job.errChan <- nil
	}
	c.closeConn()
}

func (b *B) Close() {
	b.mu.Lock()
	defer b.mu.Unlock()
	if b.recvChan != nil {
		close(b.recvChan)
		b.recvChan = nil
	}
}

func (c *connection) getConn() (net.Conn, error) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.conn == nil {
		conn, err := net.DialTimeout("tcp", c.Addr, c.Timeout)
		if err != nil {
			return nil, err
		}
		c.conn = conn
	}
	return c.conn, c.conn.SetDeadline(time.Now().Add(c.Timeout))
}

func (c *connection) closeConn() {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.conn != nil {
		c.conn.Close()
		c.conn = nil
	}
}
<commit_msg>discard the connection whenever an error returned.<commit_after>package broker

import (
	"errors"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"h12.me/kpax/model"
	"h12.me/wipro"
)

var (
	ErrCorrelationIDMismatch = errors.New("correlationID mismatch")
)

type B struct {
	Addr     string
	Timeout  time.Duration
	QueueLen int

	connection
	cid      int32
	recvChan chan *brokerJob
	mu       sync.Mutex
}

type connection struct {
	conn net.Conn
	mu   sync.Mutex
	*B
}

type brokerJob struct {
	req     model.Request
	resp    model.Response
	errChan chan error
}

func New(addr string) model.Broker {
	b := &B{
		Addr:     addr,
		Timeout:  30 * time.Second,
		QueueLen: 1000,
	}
	b.connection.B = b
	return b
}

func (b *B) Do(req model.Request, resp model.Response) error {
	job := &brokerJob{
		req:     req,
		resp:    resp,
		errChan: make(chan error),
	}
	if err := b.send(job); err != nil {
		return err
	}
	return <-job.errChan
}

func (b *B) send(job *brokerJob) error {
	conn, err := b.getConn()
	if err != nil {
		return err
	}

	b.mu.Lock()
	defer b.mu.Unlock()
	job.req.SetID(atomic.AddInt32(&b.cid, 1))
	if err := job.req.Send(conn); err != nil {
		if err == wipro.ErrConn {
			b.closeConn()
		}
		return err
	}
	if b.recvChan == nil {
		b.recvChan = make(chan *brokerJob, b.QueueLen)
		go b.receiveLoop()
	}
	if job.requireAck() {
		b.recvChan <- job
	}
	return nil
}
func (j *brokerJob) requireAck() bool { return j.resp != nil }

func (c *B) receiveLoop() {
	for job := range c.recvChan {
		conn, err := c.getConn()
		if err != nil {
			job.errChan <- err
			continue
		}
		if err := job.resp.Receive(conn); err != nil {
			if err == wipro.ErrConn {
				c.closeConn()
			}
			job.errChan <- err
			continue
		}
		if job.resp.ID() != job.req.ID() {
			if err == wipro.ErrConn {
				c.closeConn()
			}
			job.errChan <- ErrCorrelationIDMismatch
			continue
		}
		job.errChan <- nil
	}
	c.closeConn()
}

func (b *B) Close() {
	b.mu.Lock()
	defer b.mu.Unlock()
	if b.recvChan != nil {
		close(b.recvChan)
		b.recvChan = nil
	}
}

func (c *connection) getConn() (net.Conn, error) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.conn == nil {
		conn, err := net.DialTimeout("tcp", c.Addr, c.Timeout)
		if err != nil {
			return nil, err
		}
		c.conn = conn
	}
	return c.conn, c.conn.SetDeadline(time.Now().Add(c.Timeout))
}

func (c *connection) closeConn() {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.conn != nil {
		c.conn.Close()
		c.conn = nil
	}
}
<|endoftext|>