<commit_before>// Package fibvec provides a vector that can store unsigned integers by first
// converting them to their fibonacci encoded values before saving to a bit
// array. This can save memory space (especially for small values) in exchange
// for slower operations.
package fibvec

import (
	"bytes"
	"encoding/gob"
	"unsafe"

	"github.com/robskie/bit"
)

const (
	// These variables affects the size and
	// speed of the vector. Lower values means
	// larger size but faster Gets and vice versa.

	// sr is the rank sampling block size.
	// This represents the number of bits in
	// each rank sampling block.
	sr = 1024

	// ss is the number of 1s in each select
	// sampling block. Note that the number of
	// bits in each block varies.
	ss = 256
)

// Vector represents a container for unsigned integers.
type Vector struct {
	bits *bit.Array

	// ranks[i] is the number of 11s
	// from 0 to index (i*sr)-1
	ranks []int

	// indices[i] points to the
	// beginning of the uint64 (LSB)
	// that contains the (i*ss)+1th
	// pair of bits.
	indices []int

	popcount int

	length int
}

// NewVector creates a new vector.
func NewVector() *Vector {
	b := bit.NewArray(0)
	rs := make([]int, 1)
	idx := make([]int, 1)

	// Add terminating bits
	b.Add(0x3, 3)

	return &Vector{
		bits:    b,
		ranks:   rs,
		indices: idx,
	}
}

// Add adds an unsigned integer to the vector.
func (v *Vector) Add(n uint) {
	if n > MaxValue {
		panic("fibvec: input is greater than max encodable value")
	}

	v.length++
	idx := v.bits.Len() - 3
	fc, lfc := fibencode(n)
	size := lfc

	if lfc > 64 {
		v.bits.Insert(idx, fc[0], 64)
		lfc -= 64

		for _, f := range fc[1 : len(fc)-1] {
			v.bits.Add(f, 64)
			lfc -= 64
		}
		v.bits.Add(fc[len(fc)-1], lfc)
	} else {
		v.bits.Insert(idx, fc[0], lfc)
	}

	// Add bit padding so that pairs
	// of 1 (11s) don't get separated
	// by array boundaries.
	if (v.bits.Len()-1)&63 == 62 {
		v.bits.Add(0x3, 2)
	}

	v.popcount++
	vlen := v.bits.Len()

	lenranks := len(v.ranks)
	overflow := vlen - (lenranks * sr)
	if overflow > 0 {
		v.ranks = append(v.ranks, 0)
		v.ranks[lenranks] = v.popcount
		if size <= overflow {
			v.ranks[lenranks]--
		}
	}

	lenidx := len(v.indices)
	if v.popcount-(lenidx*ss) > 0 {
		v.indices = append(v.indices, 0)
		v.indices[lenidx] = idx ^ 0x3F
	}

	// Add terminating bits so that
	// the last value can be decoded
	v.bits.Add(0x3, 3)
}

// Get returns the value at index i.
func (v *Vector) Get(i int) uint {
	if i >= v.length {
		panic("fibvec: index out of bounds")
	} else if i < 0 {
		panic("fibvec: invalid index")
	}

	idx := v.select11(i + 1)
	bits := v.bits.Bits()

	// Temporary store and
	// zero out extra bits
	aidx := idx >> 6
	bidx := idx & 63
	temp := bits[aidx]
	bits[aidx] &= ^((1 << uint(bidx)) - 1)

	// Transform to bytes
	bytes := byteSliceFromUint64Slice(bits)
	bytes = bytes[idx>>3:]

	// This makes sure that the last number is decoded
	if len(bytes) < 16 {
		bytes = append(bytes, []byte{0, 0}...)
	}
	result := fibdecode(bytes, 1)

	// Restore bits
	bits[aidx] = temp

	return result[0]
}

// GetValues returns the values from start to end-1.
func (v *Vector) GetValues(start, end int) []uint {
	if end-start <= 0 {
		panic("fibvec: end must be greater than start")
	} else if start < 0 || end < 0 {
		panic("fibvec: invalid index")
	} else if end > v.length {
		panic("fibvec: index out of bounds")
	}

	idx := v.select11(start + 1)
	bits := v.bits.Bits()

	// Temporary store and
	// zero out extra bits
	aidx := idx >> 6
	bidx := idx & 63
	temp := bits[aidx]
	bits[aidx] &= ^((1 << uint(bidx)) - 1)

	// Transform to bytes
	bytes := byteSliceFromUint64Slice(bits)
	bytes = bytes[idx>>3:]

	// This makes sure that the last number is decoded
	if len(bytes) < 16 {
		bytes = append(bytes, []byte{0, 0}...)
	}
	results := fibdecode(bytes, end-start)

	// Restore bits
	bits[aidx] = temp

	return results
}

// Size returns the vector size in bytes.
func (v *Vector) Size() int {
	sizeofInt := int(unsafe.Sizeof(int(0)))

	size := v.bits.Size()
	size += len(v.ranks) * sizeofInt
	size += len(v.indices) * sizeofInt

	return size
}

// Len returns the number of values stored.
func (v *Vector) Len() int {
	return v.length
}

// GobEncode encodes this vector into gob streams.
func (v *Vector) GobEncode() ([]byte, error) {
	buf := &bytes.Buffer{}
	enc := gob.NewEncoder(buf)

	enc.Encode(v.bits)
	enc.Encode(v.ranks)
	enc.Encode(v.indices)
	enc.Encode(v.popcount)
	enc.Encode(v.length)

	return buf.Bytes(), nil
}

// GobDecode populates this vector from gob streams.
func (v *Vector) GobDecode(data []byte) error {
	buf := bytes.NewReader(data)
	dec := gob.NewDecoder(buf)

	dec.Decode(v.bits)
	dec.Decode(&v.ranks)
	dec.Decode(&v.indices)
	dec.Decode(&v.popcount)
	dec.Decode(&v.length)

	return nil
}

// select11 selects the ith 11 pair.
//
// Taken from "Fast, Small, Simple Rank/Select
// on Bitmaps" by Navarro et al., with some minor
// modifications.
func (v *Vector) select11(i int) int {
	const m = 0xC000000000000000

	j := (i - 1) / ss
	q := v.indices[j] / sr

	k := 0
	r := 0
	rq := v.ranks[q:]
	for k, r = range rq {
		if r >= i {
			k--
			break
		}
	}

	idx := 0
	rank := rq[k]
	vbits := v.bits.Bits()
	aidx := ((q + k) * sr) >> 6

	vbits = vbits[aidx:]
	for ii, b := range vbits {
		rank += popcount11_64(b)

		// If b ends with 11 and the next bits
		// starts with 1, then the 11 in b is
		// not the beginning of an encoded value,
		// but popcount11_64 has already counted
		// it so we need to subtract 1 to rank
		if b&m == m && vbits[ii+1]&1 == 1 {
			rank--
		}

		if rank >= i {
			idx = (aidx + ii) << 6
			overflow := rank - i
			popcnt := popcount11_64(b)
			if b&m == m && vbits[ii+1]&1 == 1 {
				popcnt--
			}

			idx += select11_64(b, popcnt-overflow)

			break
		}
	}

	return idx
}

// popcount11 counts the number of 11 pairs
// in v. This assumes that v doesn't contain
// more than 3 consecutive 1s. This assumption
// is satisfied since the minimum encoded value
// is 011.
func popcount11_64(v uint64) int {
	// Reduce cluster of 1s by 1.
	// This makes 11 to 01, 111 to 011,
	// and unsets all 1s.
	v &= v >> 1

	// Reduces all 11s to 10s
	// while maintaining all lone 1s.
	v &= ^(v >> 1)

	// Proceed to regular bit counting
	return bit.PopCount(v)
}

// select11 returns the index of the ith 11 pair.
func select11_64(v uint64, i int) int {
	// Same with popcount11
	v &= v >> 1
	v &= ^(v >> 1)

	// Perform regular select
	return bit.Select(v, i)
}
<commit_msg>Added lazy initialization<commit_after>// Package fibvec provides a vector that can store unsigned integers by first
// converting them to their fibonacci encoded values before saving to a bit
// array. This can save memory space (especially for small values) in exchange
// for slower operations.
package fibvec

import (
	"bytes"
	"encoding/gob"
	"unsafe"

	"github.com/robskie/bit"
)

const (
	// These variables affects the size and
	// speed of the vector. Lower values means
	// larger size but faster Gets and vice versa.

	// sr is the rank sampling block size.
	// This represents the number of bits in
	// each rank sampling block.
	sr = 1024

	// ss is the number of 1s in each select
	// sampling block. Note that the number of
	// bits in each block varies.
	ss = 256
)

// Vector represents a container for unsigned integers.
type Vector struct {
	bits *bit.Array

	// ranks[i] is the number of 11s
	// from 0 to index (i*sr)-1
	ranks []int

	// indices[i] points to the
	// beginning of the uint64 (LSB)
	// that contains the (i*ss)+1th
	// pair of bits.
	indices []int

	popcount int

	length      int
	initialized bool
}

// Initialize vector
func (v *Vector) init() {
	v.bits = bit.NewArray(0)
	v.ranks = make([]int, 1)
	v.indices = make([]int, 1)

	// Add terminating bits
	v.bits.Add(0x3, 3)

	v.initialized = true
}

// NewVector creates a new vector.
func NewVector() *Vector {
	vec := &Vector{}
	vec.init()
	return vec
}

// Add adds an unsigned integer to the vector.
func (v *Vector) Add(n uint) {
	if n > MaxValue {
		panic("fibvec: input is greater than max encodable value")
	} else if !v.initialized {
		v.init()
	}

	v.length++
	idx := v.bits.Len() - 3
	fc, lfc := fibencode(n)
	size := lfc

	if lfc > 64 {
		v.bits.Insert(idx, fc[0], 64)
		lfc -= 64

		for _, f := range fc[1 : len(fc)-1] {
			v.bits.Add(f, 64)
			lfc -= 64
		}
		v.bits.Add(fc[len(fc)-1], lfc)
	} else {
		v.bits.Insert(idx, fc[0], lfc)
	}

	// Add bit padding so that pairs
	// of 1 (11s) don't get separated
	// by array boundaries.
	if (v.bits.Len()-1)&63 == 62 {
		v.bits.Add(0x3, 2)
	}

	v.popcount++
	vlen := v.bits.Len()

	lenranks := len(v.ranks)
	overflow := vlen - (lenranks * sr)
	if overflow > 0 {
		v.ranks = append(v.ranks, 0)
		v.ranks[lenranks] = v.popcount
		if size <= overflow {
			v.ranks[lenranks]--
		}
	}

	lenidx := len(v.indices)
	if v.popcount-(lenidx*ss) > 0 {
		v.indices = append(v.indices, 0)
		v.indices[lenidx] = idx ^ 0x3F
	}

	// Add terminating bits so that
	// the last value can be decoded
	v.bits.Add(0x3, 3)
}

// Get returns the value at index i.
func (v *Vector) Get(i int) uint {
	if i >= v.length {
		panic("fibvec: index out of bounds")
	} else if i < 0 {
		panic("fibvec: invalid index")
	}

	idx := v.select11(i + 1)
	bits := v.bits.Bits()

	// Temporary store and
	// zero out extra bits
	aidx := idx >> 6
	bidx := idx & 63
	temp := bits[aidx]
	bits[aidx] &= ^((1 << uint(bidx)) - 1)

	// Transform to bytes
	bytes := byteSliceFromUint64Slice(bits)
	bytes = bytes[idx>>3:]

	// This makes sure that the last number is decoded
	if len(bytes) < 16 {
		bytes = append(bytes, []byte{0, 0}...)
	}
	result := fibdecode(bytes, 1)

	// Restore bits
	bits[aidx] = temp

	return result[0]
}

// GetValues returns the values from start to end-1.
func (v *Vector) GetValues(start, end int) []uint {
	if end-start <= 0 {
		panic("fibvec: end must be greater than start")
	} else if start < 0 || end < 0 {
		panic("fibvec: invalid index")
	} else if end > v.length {
		panic("fibvec: index out of bounds")
	}

	idx := v.select11(start + 1)
	bits := v.bits.Bits()

	// Temporary store and
	// zero out extra bits
	aidx := idx >> 6
	bidx := idx & 63
	temp := bits[aidx]
	bits[aidx] &= ^((1 << uint(bidx)) - 1)

	// Transform to bytes
	bytes := byteSliceFromUint64Slice(bits)
	bytes = bytes[idx>>3:]

	// This makes sure that the last number is decoded
	if len(bytes) < 16 {
		bytes = append(bytes, []byte{0, 0}...)
	}
	results := fibdecode(bytes, end-start)

	// Restore bits
	bits[aidx] = temp

	return results
}

// Size returns the vector size in bytes.
func (v *Vector) Size() int {
	sizeofInt := int(unsafe.Sizeof(int(0)))

	size := v.bits.Size()
	size += len(v.ranks) * sizeofInt
	size += len(v.indices) * sizeofInt

	return size
}

// Len returns the number of values stored.
func (v *Vector) Len() int {
	return v.length
}

// GobEncode encodes this vector into gob streams.
func (v *Vector) GobEncode() ([]byte, error) {
	buf := &bytes.Buffer{}
	enc := gob.NewEncoder(buf)

	enc.Encode(v.bits)
	enc.Encode(v.ranks)
	enc.Encode(v.indices)
	enc.Encode(v.popcount)
	enc.Encode(v.length)
	enc.Encode(v.initialized)

	return buf.Bytes(), nil
}

// GobDecode populates this vector from gob streams.
func (v *Vector) GobDecode(data []byte) error {
	buf := bytes.NewReader(data)
	dec := gob.NewDecoder(buf)

	dec.Decode(v.bits)
	dec.Decode(&v.ranks)
	dec.Decode(&v.indices)
	dec.Decode(&v.popcount)
	dec.Decode(&v.length)
	dec.Decode(&v.initialized)

	return nil
}

// select11 selects the ith 11 pair.
//
// Taken from "Fast, Small, Simple Rank/Select
// on Bitmaps" by Navarro et al., with some minor
// modifications.
func (v *Vector) select11(i int) int {
	const m = 0xC000000000000000

	j := (i - 1) / ss
	q := v.indices[j] / sr

	k := 0
	r := 0
	rq := v.ranks[q:]
	for k, r = range rq {
		if r >= i {
			k--
			break
		}
	}

	idx := 0
	rank := rq[k]
	vbits := v.bits.Bits()
	aidx := ((q + k) * sr) >> 6

	vbits = vbits[aidx:]
	for ii, b := range vbits {
		rank += popcount11_64(b)

		// If b ends with 11 and the next bits
		// starts with 1, then the 11 in b is
		// not the beginning of an encoded value,
		// but popcount11_64 has already counted
		// it so we need to subtract 1 to rank
		if b&m == m && vbits[ii+1]&1 == 1 {
			rank--
		}

		if rank >= i {
			idx = (aidx + ii) << 6
			overflow := rank - i
			popcnt := popcount11_64(b)
			if b&m == m && vbits[ii+1]&1 == 1 {
				popcnt--
			}

			idx += select11_64(b, popcnt-overflow)

			break
		}
	}

	return idx
}

// popcount11 counts the number of 11 pairs
// in v. This assumes that v doesn't contain
// more than 3 consecutive 1s. This assumption
// is satisfied since the minimum encoded value
// is 011.
func popcount11_64(v uint64) int {
	// Reduce cluster of 1s by 1.
	// This makes 11 to 01, 111 to 011,
	// and unsets all 1s.
	v &= v >> 1

	// Reduces all 11s to 10s
	// while maintaining all lone 1s.
	v &= ^(v >> 1)

	// Proceed to regular bit counting
	return bit.PopCount(v)
}

// select11 returns the index of the ith 11 pair.
func select11_64(v uint64, i int) int {
	// Same with popcount11
	v &= v >> 1
	v &= ^(v >> 1)

	// Perform regular select
	return bit.Select(v, i)
}
<|endoftext|>


<commit_before>package hepmc

type FourVector [4]float64

func (vec *FourVector) Px() float64 {
	return vec[0]
}

func (vec *FourVector) Py() float64 {
	return vec[1]
}

func (vec *FourVector) Pz() float64 {
	return vec[2]
}

func (vec *FourVector) E() float64 {
	return vec[3]
}

func (vec *FourVector) X() float64 {
	return vec[0]
}

func (vec *FourVector) Y() float64 {
	return vec[1]
}

func (vec *FourVector) Z() float64 {
	return vec[2]
}

func (vec *FourVector) T() float64 {
	return vec[3]
}

type ThreeVector [3]float64

func (vec *ThreeVector) X() float64 {
	return vec[0]
}

func (vec *ThreeVector) Y() float64 {
	return vec[1]
}

func (vec *ThreeVector) Z() float64 {
	return vec[2]
}

// EOF
<commit_msg>vec: docstrings<commit_after>package hepmc

// FourVector is a simple quadri-vector representation.
type FourVector [4]float64

// Px returns the x-component of the 4-momentum
func (vec *FourVector) Px() float64 {
	return vec[0]
}

// Py returns the y-component of the 4-momentum
func (vec *FourVector) Py() float64 {
	return vec[1]
}

// Pz returns the z-component of the 4-momentum
func (vec *FourVector) Pz() float64 {
	return vec[2]
}

// E returns the energy of the 4-momentum
func (vec *FourVector) E() float64 {
	return vec[3]
}

// X returns the x-component of the 4-momentum
func (vec *FourVector) X() float64 {
	return vec[0]
}

// Y returns the y-component of the 4-momentum
func (vec *FourVector) Y() float64 {
	return vec[1]
}

// Z returns the z-component of the 4-momentum
func (vec *FourVector) Z() float64 {
	return vec[2]
}

// T returns the t-component of the 4-momentum
func (vec *FourVector) T() float64 {
	return vec[3]
}

// ThreeVector is a simple 3d-vector representation.
type ThreeVector [3]float64

// X returns the x-component of the 3d-vector
func (vec *ThreeVector) X() float64 {
	return vec[0]
}

// Y returns the y-component of the 3d-vector
func (vec *ThreeVector) Y() float64 {
	return vec[1]
}

// Z returns the z-component of the 3d-vector
func (vec *ThreeVector) Z() float64 {
	return vec[2]
}

// EOF
<|endoftext|>


<commit_before>package tripeg
<commit_msg>don't think we need this<commit_after><|endoftext|>


<commit_before>package openstack

import (
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/config"
)

// resourceMap is the mapping of resources we support to their basic
// operations. This makes it easy to implement new resource types.
var resourceMap *resource.Map

func init() {
	resourceMap = &resource.Map{
		Mapping: map[string]resource.Resource{
			"openstack_compute": resource.Resource{
				ConfigValidator: &config.Validator{
					Required: []string{
						"vpc_id",
						"route.*.cidr_block",
					},
					Optional: []string{
						"route.*.gateway_id",
						"route.*.instance_id",
					},
				},
				Create:          resource_openstack_compute_create,
				Destroy:         resource_openstack_compute_destroy,
				Diff:            resource_openstack_compute_diff,
				Update:          resource_openstack_compute_update,
				Refresh:         resource_openstack_compute_refresh,
			},
		},
	}
}
<commit_msg>Add Validation attributes on resources<commit_after>package openstack

import (
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/config"
)

// resourceMap is the mapping of resources we support to their basic
// operations. This makes it easy to implement new resource types.
var resourceMap *resource.Map

func init() {
	resourceMap = &resource.Map{
		Mapping: map[string]resource.Resource{
			"openstack_compute": resource.Resource{
				ConfigValidator: &config.Validator{
					Required: []string{

					},
					Optional: []string{

					},
				},
				Create:          resource_openstack_compute_create,
				Destroy:         resource_openstack_compute_destroy,
				Diff:            resource_openstack_compute_diff,
				Update:          resource_openstack_compute_update,
				Refresh:         resource_openstack_compute_refresh,
			},
		},
	}
}
<|endoftext|>


<commit_before>package dev

import (
	"fmt"
	"net"
	"sort"
	"strconv"
	"time"
)

// ID returns the ID of n.
func (n *Node) ID() uint32 {
	if n != nil {
		return n.id
	}
	return 0
}

// Address returns network address of n.
func (n *Node) Address() string {
	if n != nil {
		return n.addr
	}
	return ""
}

// Port returns network port of n.
func (n *Node) Port() string {
	if n != nil {
		_, port, _ := net.SplitHostPort(n.addr)
		return port
	}
	return ""
}

func (n *Node) String() string {
	if n != nil {
		return fmt.Sprintf("addr: %s", n.addr)
	}
	return ""
}

func (n *Node) FullString() string {
	if n != nil {
		n.mu.Lock()
		defer n.mu.Unlock()
		return fmt.Sprintf(
			"node %d | addr: %s | latency: %v",
			n.id, n.addr, n.latency,
		)
	}
	return ""
}

func (n *Node) setLastErr(err error) {
	n.mu.Lock()
	defer n.mu.Unlock()
	n.lastErr = err
}

// LastErr returns the last error encountered (if any) when invoking a remote
// procedure call on this node.
func (n *Node) LastErr() error {
	n.mu.Lock()
	defer n.mu.Unlock()
	return n.lastErr
}

func (n *Node) setLatency(lat time.Duration) {
	n.mu.Lock()
	defer n.mu.Unlock()
	n.latency = lat
}

// Latency returns the latency of the last successful remote procedure call
// made to this node.
func (n *Node) Latency() time.Duration {
	n.mu.Lock()
	defer n.mu.Unlock()
	return n.latency
}

type lessFunc func(n1, n2 *Node) bool

// MultiSorter implements the Sort interface, sorting the nodes within.
type MultiSorter struct {
	nodes []*Node
	less  []lessFunc
}

// Sort sorts the argument slice according to the less functions passed to
// OrderedBy.
func (ms *MultiSorter) Sort(nodes []*Node) {
	ms.nodes = nodes
	sort.Sort(ms)
}

// OrderedBy returns a Sorter that sorts using the less functions, in order.
// Call its Sort method to sort the data.
func OrderedBy(less ...lessFunc) *MultiSorter {
	return &MultiSorter{
		less: less,
	}
}

// Len is part of sort.Interface.
func (ms *MultiSorter) Len() int {
	return len(ms.nodes)
}

// Swap is part of sort.Interface.
func (ms *MultiSorter) Swap(i, j int) {
	ms.nodes[i], ms.nodes[j] = ms.nodes[j], ms.nodes[i]
}

// Less is part of sort.Interface. It is implemented by looping along the
// less functions until it finds a comparison that is either Less or
// !Less. Note that it can call the less functions twice per call. We
// could change the functions to return -1, 0, 1 and reduce the
// number of calls for greater efficiency: an exercise for the reader.
func (ms *MultiSorter) Less(i, j int) bool {
	p, q := ms.nodes[i], ms.nodes[j]
	// Try all but the last comparison.
	var k int
	for k = 0; k < len(ms.less)-1; k++ {
		less := ms.less[k]
		switch {
		case less(p, q):
			// p < q, so we have a decision.
			return true
		case less(q, p):
			// p > q, so we have a decision.
			return false
		}
		// p == q; try the next comparison.
	}
	// All comparisons to here said "equal", so just return whatever
	// the final comparison reports.
	return ms.less[k](p, q)
}

// ID sorts nodes by their identifier in increasing order.
var ID = func(n1, n2 *Node) bool {
	return n1.id < n2.id
}

// Port sorts nodes by their port number in increasing order.
// Warning: This function may be removed in the future.
var Port = func(n1, n2 *Node) bool {
	p1, _ := strconv.Atoi(n1.Port())
	p2, _ := strconv.Atoi(n2.Port())
	return p1 < p2
}

// Latency sorts nodes by latency in increasing order. Latencies less then
// zero (sentinel value) are considered greater than any positive latency.
var Latency = func(n1, n2 *Node) bool {
	if n1.latency < 0 {
		return false
	}
	return n1.latency < n2.latency

}

// Error sorts nodes by their LastErr() status in increasing order. A
// node with LastErr() != nil is larger than a node with LastErr() == nil.
var Error = func(n1, n2 *Node) bool {
	if n1.lastErr != nil && n2.lastErr == nil {
		return false
	}
	return true
}
<commit_msg>return <nil> string instead of empty string<commit_after>package dev

import (
	"fmt"
	"net"
	"sort"
	"strconv"
	"time"
)

const nilAngleString = "<nil>"

// ID returns the ID of n.
func (n *Node) ID() uint32 {
	if n != nil {
		return n.id
	}
	return 0
}

// Address returns network address of n.
func (n *Node) Address() string {
	if n != nil {
		return n.addr
	}
	return nilAngleString
}

// Port returns network port of n.
func (n *Node) Port() string {
	if n != nil {
		_, port, _ := net.SplitHostPort(n.addr)
		return port
	}
	return nilAngleString
}

func (n *Node) String() string {
	if n != nil {
		return fmt.Sprintf("addr: %s", n.addr)
	}
	return nilAngleString
}

func (n *Node) FullString() string {
	if n != nil {
		n.mu.Lock()
		defer n.mu.Unlock()
		return fmt.Sprintf(
			"node %d | addr: %s | latency: %v",
			n.id, n.addr, n.latency,
		)
	}
	return nilAngleString
}

func (n *Node) setLastErr(err error) {
	n.mu.Lock()
	defer n.mu.Unlock()
	n.lastErr = err
}

// LastErr returns the last error encountered (if any) when invoking a remote
// procedure call on this node.
func (n *Node) LastErr() error {
	n.mu.Lock()
	defer n.mu.Unlock()
	return n.lastErr
}

func (n *Node) setLatency(lat time.Duration) {
	n.mu.Lock()
	defer n.mu.Unlock()
	n.latency = lat
}

// Latency returns the latency of the last successful remote procedure call
// made to this node.
func (n *Node) Latency() time.Duration {
	n.mu.Lock()
	defer n.mu.Unlock()
	return n.latency
}

type lessFunc func(n1, n2 *Node) bool

// MultiSorter implements the Sort interface, sorting the nodes within.
type MultiSorter struct {
	nodes []*Node
	less  []lessFunc
}

// Sort sorts the argument slice according to the less functions passed to
// OrderedBy.
func (ms *MultiSorter) Sort(nodes []*Node) {
	ms.nodes = nodes
	sort.Sort(ms)
}

// OrderedBy returns a Sorter that sorts using the less functions, in order.
// Call its Sort method to sort the data.
func OrderedBy(less ...lessFunc) *MultiSorter {
	return &MultiSorter{
		less: less,
	}
}

// Len is part of sort.Interface.
func (ms *MultiSorter) Len() int {
	return len(ms.nodes)
}

// Swap is part of sort.Interface.
func (ms *MultiSorter) Swap(i, j int) {
	ms.nodes[i], ms.nodes[j] = ms.nodes[j], ms.nodes[i]
}

// Less is part of sort.Interface. It is implemented by looping along the
// less functions until it finds a comparison that is either Less or
// !Less. Note that it can call the less functions twice per call. We
// could change the functions to return -1, 0, 1 and reduce the
// number of calls for greater efficiency: an exercise for the reader.
func (ms *MultiSorter) Less(i, j int) bool {
	p, q := ms.nodes[i], ms.nodes[j]
	// Try all but the last comparison.
	var k int
	for k = 0; k < len(ms.less)-1; k++ {
		less := ms.less[k]
		switch {
		case less(p, q):
			// p < q, so we have a decision.
			return true
		case less(q, p):
			// p > q, so we have a decision.
			return false
		}
		// p == q; try the next comparison.
	}
	// All comparisons to here said "equal", so just return whatever
	// the final comparison reports.
	return ms.less[k](p, q)
}

// ID sorts nodes by their identifier in increasing order.
var ID = func(n1, n2 *Node) bool {
	return n1.id < n2.id
}

// Port sorts nodes by their port number in increasing order.
// Warning: This function may be removed in the future.
var Port = func(n1, n2 *Node) bool {
	p1, _ := strconv.Atoi(n1.Port())
	p2, _ := strconv.Atoi(n2.Port())
	return p1 < p2
}

// Latency sorts nodes by latency in increasing order. Latencies less then
// zero (sentinel value) are considered greater than any positive latency.
var Latency = func(n1, n2 *Node) bool {
	if n1.latency < 0 {
		return false
	}
	return n1.latency < n2.latency

}

// Error sorts nodes by their LastErr() status in increasing order. A
// node with LastErr() != nil is larger than a node with LastErr() == nil.
var Error = func(n1, n2 *Node) bool {
	if n1.lastErr != nil && n2.lastErr == nil {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fi

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/golang/glog"
	"k8s.io/kops/upup/pkg/fi/utils"
	"k8s.io/kops/util/pkg/hashing"
)

type asset struct {
	Key       string
	AssetPath string
	resource  Resource
	source    *Source
}

type Source struct {
	Parent             *Source
	URL                string
	Hash               *hashing.Hash
	ExtractFromArchive string
}

// Builds a unique key for this source
func (s *Source) Key() string {
	var k string
	if s.Parent != nil {
		k = s.Parent.Key() + "/"
	}
	if s.URL != "" {
		k += s.URL
	} else if s.ExtractFromArchive != "" {
		k += s.ExtractFromArchive
	} else {
		glog.Fatalf("expected either URL or ExtractFromArchive to be set")
	}
	return k
}

func (s *Source) String() string {
	return "Source[" + s.Key() + "]"
}

type HasSource interface {
	GetSource() *Source
}

// assetResource implements Resource, but also implements HasFetchInstructions
type assetResource struct {
	asset *asset
}

var _ Resource = &assetResource{}
var _ HasSource = &assetResource{}

func (r *assetResource) Open() (io.Reader, error) {
	return r.asset.resource.Open()
}

func (r *assetResource) GetSource() *Source {
	return r.asset.source
}

type AssetStore struct {
	cacheDir string
	assets   []*asset
}

func NewAssetStore(cacheDir string) *AssetStore {
	a := &AssetStore{
		cacheDir: cacheDir,
	}
	return a
}
func (a *AssetStore) Find(key string, assetPath string) (Resource, error) {
	var matches []*asset
	for _, asset := range a.assets {
		if asset.Key != key {
			continue
		}

		if assetPath != "" {
			if !strings.HasSuffix(asset.AssetPath, assetPath) {
				continue
			}
		}

		matches = append(matches, asset)
	}

	if len(matches) == 0 {
		return nil, nil
	}
	if len(matches) == 1 {
		glog.Infof("Resolved asset %s:%s to %s", key, assetPath, matches[0].AssetPath)
		return &assetResource{asset: matches[0]}, nil
	}

	glog.Infof("Matching assets:")
	for _, match := range matches {
		glog.Infof("    %s %s", match.Key, match.AssetPath)
	}
	return nil, fmt.Errorf("found multiple matching assets for key: %q", key)
}

func hashFromHttpHeader(url string) (*hashing.Hash, error) {
	glog.Infof("Doing HTTP HEAD on %q", url)
	response, err := http.Head(url)
	if err != nil {
		return nil, fmt.Errorf("error doing HEAD on %q: %v", url, err)
	}
	defer response.Body.Close()

	etag := response.Header.Get("ETag")
	etag = strings.TrimSpace(etag)
	etag = strings.Trim(etag, "'\"")

	if etag != "" {
		if len(etag) == 32 {
			// Likely md5
			return hashing.HashAlgorithmMD5.FromString(etag)
		}
	}

	return nil, fmt.Errorf("unable to determine hash from HTTP HEAD: %q", url)
}

// Add an asset into the store, in one of the recognized formats (see Assets in types package)
func (a *AssetStore) Add(id string) error {
	if strings.HasPrefix(id, "http://") || strings.HasPrefix(id, "https://") {
		return a.addURL(id, nil)
	}
	i := strings.Index(id, "@http://")
	if i == -1 {
		i = strings.Index(id, "@https://")
	}
	if i != -1 {
		url := id[i+1:]
		hash, err := hashing.FromString(id[:i])
		if err != nil {
			return err
		}
		return a.addURL(url, hash)
	}
	// TODO: local files!
	return fmt.Errorf("unknown asset format: %q", id)
}

func (a *AssetStore) addURL(url string, hash *hashing.Hash) error {
	var err error

	if hash == nil {
		hash, err = hashFromHttpHeader(url)
		if err != nil {
			return err
		}
	}

	localFile := path.Join(a.cacheDir, hash.String()+"_"+utils.SanitizeString(url))
	_, err = DownloadURL(url, localFile, hash)
	if err != nil {
		return err
	}

	key := path.Base(url)
	assetPath := url
	r := NewFileResource(localFile)

	source := &Source{URL: url, Hash: hash}

	asset := &asset{
		Key:       key,
		AssetPath: assetPath,
		resource:  r,
		source:    source,
	}
	glog.V(2).Infof("added asset %q for %q", asset.Key, asset.resource)
	a.assets = append(a.assets, asset)

	if strings.HasSuffix(assetPath, ".tar.gz") {
		err = a.addArchive(source, localFile)
		if err != nil {
			return err
		}
	}

	return nil
}

//func (a *AssetStore) addFile(assetPath string, p string) error {
//	r := NewFileResource(p)
//	return a.addResource(assetPath, r)
//}

//func (a *AssetStore) addResource(assetPath string, r Resource) error {
//	hash, err := HashForResource(r, HashAlgorithmSHA256)
//	if err != nil {
//		return err
//	}
//
//	localFile := path.Join(a.assetDir, hash + "_" + utils.SanitizeString(assetPath))
//	hasHash, err := fileHasHash(localFile, hash)
//	if err != nil {
//		return err
//	}
//
//	if !hasHash {
//		err = WriteFile(localFile, r, 0644, 0755)
//		if err != nil {
//			return err
//		}
//	}
//
//	asset := &asset{
//		Key:       localFile,
//		AssetPath: assetPath,
//		resource:  r,
//	}
//	glog.V(2).Infof("added asset %q for %q", asset.Key, asset.resource)
//	a.assets = append(a.assets, asset)
//
//	if strings.HasSuffix(assetPath, ".tar.gz") {
//		err = a.addArchive(localFile)
//		if err != nil {
//			return err
//		}
//	}
//
//	return nil
//}

func (a *AssetStore) addArchive(archiveSource *Source, archiveFile string) error {
	extracted := path.Join(a.cacheDir, "extracted/"+path.Base(archiveFile))

	if _, err := os.Stat(extracted); os.IsNotExist(err) {
		// We extract to a temporary dir which we then rename so this is atomic
		// (untarring can be slow, and we might crash / be interrupted half-way through)
		extractedTemp := extracted + ".tmp-" + strconv.FormatInt(time.Now().UnixNano(), 10)
		err := os.MkdirAll(extractedTemp, 0755)
		if err != nil {
			return fmt.Errorf("error creating directories %q: %v", path.Dir(extractedTemp), err)
		}

		args := []string{"tar", "zxf", archiveFile, "-C", extractedTemp}
		glog.Infof("running extract command %s", args)
		cmd := exec.Command(args[0], args[1:]...)
		output, err := cmd.CombinedOutput()
		if err != nil {
			return fmt.Errorf("error expanding asset file %q %v: %s", archiveFile, err, string(output))
		}

		if err := os.Rename(extractedTemp, extracted); err != nil {
			return fmt.Errorf("error renaming extracted temp dir %s -> %s: %v", extractedTemp, extracted, err)
		}
	}

	localBase := extracted
	assetBase := ""

	walker := func(localPath string, info os.FileInfo, err error) error {
		if err != nil {
			return fmt.Errorf("error descending into path %q: %v", localPath, err)
		}

		if info.IsDir() {
			return nil
		}

		relativePath, err := filepath.Rel(localBase, localPath)
		if err != nil {
			return fmt.Errorf("error finding relative path for %q: %v", localPath, err)
		}

		assetPath := path.Join(assetBase, relativePath)
		key := info.Name()
		r := NewFileResource(localPath)

		asset := &asset{
			Key:       key,
			AssetPath: assetPath,
			resource:  r,
			source:    &Source{Parent: archiveSource, ExtractFromArchive: assetPath},
		}
		glog.V(2).Infof("added asset %q for %q", asset.Key, asset.resource)
		a.assets = append(a.assets, asset)

		return nil
	}

	err := filepath.Walk(localBase, walker)
	if err != nil {
		return fmt.Errorf("error adding expanded asset files in %q: %v", extracted, err)
	}
	return nil

}
<commit_msg>Adding capability to handle tgz files as archive files<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fi

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/golang/glog"
	"k8s.io/kops/upup/pkg/fi/utils"
	"k8s.io/kops/util/pkg/hashing"
)

type asset struct {
	Key       string
	AssetPath string
	resource  Resource
	source    *Source
}

type Source struct {
	Parent             *Source
	URL                string
	Hash               *hashing.Hash
	ExtractFromArchive string
}

// Builds a unique key for this source
func (s *Source) Key() string {
	var k string
	if s.Parent != nil {
		k = s.Parent.Key() + "/"
	}
	if s.URL != "" {
		k += s.URL
	} else if s.ExtractFromArchive != "" {
		k += s.ExtractFromArchive
	} else {
		glog.Fatalf("expected either URL or ExtractFromArchive to be set")
	}
	return k
}

func (s *Source) String() string {
	return "Source[" + s.Key() + "]"
}

type HasSource interface {
	GetSource() *Source
}

// assetResource implements Resource, but also implements HasFetchInstructions
type assetResource struct {
	asset *asset
}

var _ Resource = &assetResource{}
var _ HasSource = &assetResource{}

func (r *assetResource) Open() (io.Reader, error) {
	return r.asset.resource.Open()
}

func (r *assetResource) GetSource() *Source {
	return r.asset.source
}

type AssetStore struct {
	cacheDir string
	assets   []*asset
}

func NewAssetStore(cacheDir string) *AssetStore {
	a := &AssetStore{
		cacheDir: cacheDir,
	}
	return a
}
func (a *AssetStore) Find(key string, assetPath string) (Resource, error) {
	var matches []*asset
	for _, asset := range a.assets {
		if asset.Key != key {
			continue
		}

		if assetPath != "" {
			if !strings.HasSuffix(asset.AssetPath, assetPath) {
				continue
			}
		}

		matches = append(matches, asset)
	}

	if len(matches) == 0 {
		return nil, nil
	}
	if len(matches) == 1 {
		glog.Infof("Resolved asset %s:%s to %s", key, assetPath, matches[0].AssetPath)
		return &assetResource{asset: matches[0]}, nil
	}

	glog.Infof("Matching assets:")
	for _, match := range matches {
		glog.Infof("    %s %s", match.Key, match.AssetPath)
	}
	return nil, fmt.Errorf("found multiple matching assets for key: %q", key)
}

func hashFromHttpHeader(url string) (*hashing.Hash, error) {
	glog.Infof("Doing HTTP HEAD on %q", url)
	response, err := http.Head(url)
	if err != nil {
		return nil, fmt.Errorf("error doing HEAD on %q: %v", url, err)
	}
	defer response.Body.Close()

	etag := response.Header.Get("ETag")
	etag = strings.TrimSpace(etag)
	etag = strings.Trim(etag, "'\"")

	if etag != "" {
		if len(etag) == 32 {
			// Likely md5
			return hashing.HashAlgorithmMD5.FromString(etag)
		}
	}

	return nil, fmt.Errorf("unable to determine hash from HTTP HEAD: %q", url)
}

// Add an asset into the store, in one of the recognized formats (see Assets in types package)
func (a *AssetStore) Add(id string) error {
	if strings.HasPrefix(id, "http://") || strings.HasPrefix(id, "https://") {
		return a.addURL(id, nil)
	}
	i := strings.Index(id, "@http://")
	if i == -1 {
		i = strings.Index(id, "@https://")
	}
	if i != -1 {
		url := id[i+1:]
		hash, err := hashing.FromString(id[:i])
		if err != nil {
			return err
		}
		return a.addURL(url, hash)
	}
	// TODO: local files!
	return fmt.Errorf("unknown asset format: %q", id)
}

func (a *AssetStore) addURL(url string, hash *hashing.Hash) error {
	var err error

	if hash == nil {
		hash, err = hashFromHttpHeader(url)
		if err != nil {
			return err
		}
	}

	localFile := path.Join(a.cacheDir, hash.String()+"_"+utils.SanitizeString(url))
	_, err = DownloadURL(url, localFile, hash)
	if err != nil {
		return err
	}

	key := path.Base(url)
	assetPath := url
	r := NewFileResource(localFile)

	source := &Source{URL: url, Hash: hash}

	asset := &asset{
		Key:       key,
		AssetPath: assetPath,
		resource:  r,
		source:    source,
	}
	glog.V(2).Infof("added asset %q for %q", asset.Key, asset.resource)
	a.assets = append(a.assets, asset)

	// normalize filename suffix
	file := strings.ToLower(assetPath)
	// pickup both tar.gz and tgz files
	if strings.HasSuffix(file, ".tar.gz") || strings.HasSuffix(file, ".tgz") {
		err = a.addArchive(source, localFile)
		if err != nil {
			return err
		}
	}

	return nil
}

//func (a *AssetStore) addFile(assetPath string, p string) error {
//	r := NewFileResource(p)
//	return a.addResource(assetPath, r)
//}

//func (a *AssetStore) addResource(assetPath string, r Resource) error {
//	hash, err := HashForResource(r, HashAlgorithmSHA256)
//	if err != nil {
//		return err
//	}
//
//	localFile := path.Join(a.assetDir, hash + "_" + utils.SanitizeString(assetPath))
//	hasHash, err := fileHasHash(localFile, hash)
//	if err != nil {
//		return err
//	}
//
//	if !hasHash {
//		err = WriteFile(localFile, r, 0644, 0755)
//		if err != nil {
//			return err
//		}
//	}
//
//	asset := &asset{
//		Key:       localFile,
//		AssetPath: assetPath,
//		resource:  r,
//	}
//	glog.V(2).Infof("added asset %q for %q", asset.Key, asset.resource)
//	a.assets = append(a.assets, asset)
//
//	if strings.HasSuffix(assetPath, ".tar.gz") {
//		err = a.addArchive(localFile)
//		if err != nil {
//			return err
//		}
//	}
//
//	return nil
//}

func (a *AssetStore) addArchive(archiveSource *Source, archiveFile string) error {
	extracted := path.Join(a.cacheDir, "extracted/"+path.Base(archiveFile))

	if _, err := os.Stat(extracted); os.IsNotExist(err) {
		// We extract to a temporary dir which we then rename so this is atomic
		// (untarring can be slow, and we might crash / be interrupted half-way through)
		extractedTemp := extracted + ".tmp-" + strconv.FormatInt(time.Now().UnixNano(), 10)
		err := os.MkdirAll(extractedTemp, 0755)
		if err != nil {
			return fmt.Errorf("error creating directories %q: %v", path.Dir(extractedTemp), err)
		}

		args := []string{"tar", "zxf", archiveFile, "-C", extractedTemp}
		glog.Infof("running extract command %s", args)
		cmd := exec.Command(args[0], args[1:]...)
		output, err := cmd.CombinedOutput()
		if err != nil {
			return fmt.Errorf("error expanding asset file %q %v: %s", archiveFile, err, string(output))
		}

		if err := os.Rename(extractedTemp, extracted); err != nil {
			return fmt.Errorf("error renaming extracted temp dir %s -> %s: %v", extractedTemp, extracted, err)
		}
	}

	localBase := extracted
	assetBase := ""

	walker := func(localPath string, info os.FileInfo, err error) error {
		if err != nil {
			return fmt.Errorf("error descending into path %q: %v", localPath, err)
		}

		if info.IsDir() {
			return nil
		}

		relativePath, err := filepath.Rel(localBase, localPath)
		if err != nil {
			return fmt.Errorf("error finding relative path for %q: %v", localPath, err)
		}

		assetPath := path.Join(assetBase, relativePath)
		key := info.Name()
		r := NewFileResource(localPath)

		asset := &asset{
			Key:       key,
			AssetPath: assetPath,
			resource:  r,
			source:    &Source{Parent: archiveSource, ExtractFromArchive: assetPath},
		}
		glog.V(2).Infof("added asset %q for %q", asset.Key, asset.resource)
		a.assets = append(a.assets, asset)

		return nil
	}

	err := filepath.Walk(localBase, walker)
	if err != nil {
		return fmt.Errorf("error adding expanded asset files in %q: %v", extracted, err)
	}
	return nil

}
<|endoftext|>


<commit_before>// pgconfig.go provides methods for configuring a node corresponding to if it is
// running a 'master' or 'slave' instance of postgres.

package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
)

//
type pgConfig struct {
	listenAddr string
	master     bool
}

// configureHBAConf attempts to open the 'pg_hba.conf' file. Once open it will scan
// the file line by line looking for replication settings, and overwrite only those
// settings with the settings required for redundancy on Pagoda Box
func configureHBAConf() error {

	// get the role of the other node in the cluster that is running an instance
	// of postgresql
	self := Whoami()
	other, err := Whoisnot(self.CRole)
	if err != nil {
		log.Warn("[pg_config.configureHBAConf] Unable to find another!\n%s\n", err)
	}

	// open the pg_hba.conf
	file := conf.DataDir + "pg_hba.conf"
	f, err := os.Open(file)
	if err != nil {
		log.Error("[pg_config.configureHBAConf] Failed to open '%s'!\n%s\n", file, err)
		return err
	}

	defer f.Close()

	scanner := bufio.NewScanner(f)
	reFindConfigOption := regexp.MustCompile(`^\s*#?\s*(local|host)\s*(replication)`)
	entry := ""

	// scan the file line by line to build an 'entry' to be re-written back to the
	// file, skipping ('removing') any line that deals with redundancy.
	for scanner.Scan() {

		// stop scanning if a special prefix is encountered.
		if strings.HasPrefix(scanner.Text(), "#~") {
			break
		}

		// dont care about submatches, just if the string matches, 'skipping' any lines
		// that are custom configurations
		if reFindConfigOption.FindString(scanner.Text()) == "" {
			entry += fmt.Sprintf("%s\n", scanner.Text())
		}
	}

	// if the other node is present, write redundancy into the 'entry' (otherwise
	// just leave it out)

	if other != nil {
		entry += fmt.Sprintf(`#~-----------------------------------------------------------------------------
# YOKE CONFIG
#------------------------------------------------------------------------------

# these configuration options have been removed from their standard location and
# placed here so that Pagoda Box could override them with the neccessary values
# to configure redundancy.

# IMPORTANT: these settings will always be overriden when the server boots. They
# are set dynamically and so should never change.

host    replication     %s        %s/32            trust`, SystemUser(), other.Ip)
	}

	// write the 'entry' to the file
	err = ioutil.WriteFile(file, []byte(entry), 0644)
	if err != nil {
		log.Error("[pg_config.configureHBAConf] Failed to write to '%s'!\n%s\n", file, err)
		return err
	}

	return nil
}

// configurePGConf attempts to open the 'postgresql.conf' file. Once open it will
// scan the file line by line looking for replication settings, and overwrite only
// those settings with the settings required for redundancy on Pagoda Box
func configurePGConf(opts pgConfig) error {

	// open the postgresql.conf
	file := conf.DataDir + "postgresql.conf"
	f, err := os.Open(file)
	if err != nil {
		return err
	}

	defer f.Close()

	reFindConfigOption := regexp.MustCompile(`^\s*#?\s*(listen_addresses|port|wal_level|archive_mode|archive_command|max_wal_senders|wal_keep_segments|hot_standby|synchronous_standby_names)\s*=\s*`)
	scanner := bufio.NewScanner(f)
	entry := ""

	// scan the file line by line to build an 'entry' to be re-written back to the
	// file, skipping ('removing') any lines that need to be manually configured
	for scanner.Scan() {

		submatch := reFindConfigOption.FindStringSubmatch(scanner.Text())

		// stop scanning if a special prefix is encountered. This ensures there are
		// no duplicate Pagoda Box comment blocks
		if strings.HasPrefix(scanner.Text(), "#~") {
			break
		}

		// build the 'entry' from all lines that don't match the custom configurations
		if submatch == nil {
			entry += fmt.Sprintf("%s\n", scanner.Text())
		}
	}

	// write manual configurations into an 'entry'
	entry += fmt.Sprintf(`#~-----------------------------------------------------------------------------
# YOKE CONFIG
#------------------------------------------------------------------------------

# these configuration options have been removed from their standard location and
# placed here so that Pagoda Box could override them with the neccessary values
# to configure redundancy.

# IMPORTANT: these settings will always be overriden when the server boots. They
# are set dynamically and so should never change.

listen_addresses = '%s'      # what IP address(es) to listen on;
                                  # comma-separated list of addresses;
                                  # defaults to 'localhost'; use '*' for all
                                  # (change requires restart)
port = %d                     # (change requires restart)
wal_level = hot_standby           # minimal, archive, or hot_standby
                                  # (change requires restart)
archive_mode = on                 # allows archiving to be done
                                  # (change requires restart)
archive_command = 'exit 0'        # command to use to archive a logfile segment
                                  # placeholders: \%p = path of file to archive
                                  #               \%f = file name only
                                  # e.g. 'test ! -f /mnt/server/archivedir/\%f && cp \%p /mnt/server/archivedir/\%f'
max_wal_senders = 10              # max number of walsender processes
                                  # (change requires restart)
wal_keep_segments = 5000          # in logfile segments, 16MB each; 0 disables
hot_standby = on                  # "on" allows queries during recovery
                                  # (change requires restart)
`, opts.listenAddr, conf.PGPort)

	// if this node is currenty 'master' then write one additional configuration
	// into the 'entry'
	if opts.master {
		entry += `
# added only the the node running postgres as 'master'
synchronous_standby_names = slave # standby servers that provide sync rep
                                  # comma-separated list of application_name
                                  # from standby(s); '*' = all
`
	}

	// write 'entry' to the file
	err = ioutil.WriteFile(file, []byte(entry), 0644)
	if err != nil {
		log.Error("[pg_config.configurePGConf] Failed to write to '%s'!\n%s\n", file, err)
	}

	return nil
}

// createRecovery creates a 'recovery.conf' file with the necessary settings
// required for redundancy on Pagoda Box. This method is called on the node that
// is being configured to run the 'slave' instance of postgres
func createRecovery() error {

	file := conf.DataDir + "recovery.conf"

	self := Whoami()
	other, err := Whoisnot(self.CRole)
	if err != nil {
		log.Fatal("[pg_config.createRecovery] Unable to find another... Exiting!\n%s\n", err)
		os.Exit(1)
	}

	// open/truncate the recover.conf
	f, err := os.Create(file)
	if err != nil {
		log.Error("[pg_config.createRecovery] Failed to create '%s'!\n%s\n", file, err)
		return err
	}

	// write manual configuration an 'entry'
	entry := fmt.Sprintf(`#~-----------------------------------------------------------------------------
# YOKE CONFIG
#------------------------------------------------------------------------------

# IMPORTANT: this config file is dynamically generated by Pagoda Box for redundancy
# any changes made here will be overriden.

# When standby_mode is enabled, the PostgreSQL server will work as a standby. It
# tries to connect to the primary according to the connection settings
# primary_conninfo, and receives XLOG records continuously.
standby_mode = on
primary_conninfo = 'host=%s port=%d application_name=slave'

# restore_command specifies the shell command that is executed to copy log files
# back from archival storage. This parameter is *required* for an archive
# recovery, but optional for streaming replication. The given command satisfies
# the requirement without doing anything.
restore_command = 'exit 0'
`, other.Ip, other.PGPort)

	// write 'entry' to the file
	if _, err := f.WriteString(entry); err != nil {
		log.Error("[pg_config.createRecovery] Failed to write to '%s'!\n%s\n", file, err)
		return err
	}

	return nil
}

// destroyRecovery attempts to destroy the 'recovery.conf'. This method is called
// on a node that is being configured to run the 'master' instance of postgres
func destroyRecovery() {

	file := conf.DataDir + "recovery.conf"

	// remove 'recovery.conf'
	err := os.Remove(file)
	if err != nil {
		log.Warn("[pg_config.destroyRecovery] No recovery.conf found at '%s'", file)
	}
}
<commit_msg>lowering wal_keep_segments to 16 from 5000 (lowering the disk space from 80gb to 256mb<commit_after>// pgconfig.go provides methods for configuring a node corresponding to if it is
// running a 'master' or 'slave' instance of postgres.

package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
)

//
type pgConfig struct {
	listenAddr string
	master     bool
}

// configureHBAConf attempts to open the 'pg_hba.conf' file. Once open it will scan
// the file line by line looking for replication settings, and overwrite only those
// settings with the settings required for redundancy on Pagoda Box
func configureHBAConf() error {

	// get the role of the other node in the cluster that is running an instance
	// of postgresql
	self := Whoami()
	other, err := Whoisnot(self.CRole)
	if err != nil {
		log.Warn("[pg_config.configureHBAConf] Unable to find another!\n%s\n", err)
	}

	// open the pg_hba.conf
	file := conf.DataDir + "pg_hba.conf"
	f, err := os.Open(file)
	if err != nil {
		log.Error("[pg_config.configureHBAConf] Failed to open '%s'!\n%s\n", file, err)
		return err
	}

	defer f.Close()

	scanner := bufio.NewScanner(f)
	reFindConfigOption := regexp.MustCompile(`^\s*#?\s*(local|host)\s*(replication)`)
	entry := ""

	// scan the file line by line to build an 'entry' to be re-written back to the
	// file, skipping ('removing') any line that deals with redundancy.
	for scanner.Scan() {

		// stop scanning if a special prefix is encountered.
		if strings.HasPrefix(scanner.Text(), "#~") {
			break
		}

		// dont care about submatches, just if the string matches, 'skipping' any lines
		// that are custom configurations
		if reFindConfigOption.FindString(scanner.Text()) == "" {
			entry += fmt.Sprintf("%s\n", scanner.Text())
		}
	}

	// if the other node is present, write redundancy into the 'entry' (otherwise
	// just leave it out)

	if other != nil {
		entry += fmt.Sprintf(`#~-----------------------------------------------------------------------------
# YOKE CONFIG
#------------------------------------------------------------------------------

# these configuration options have been removed from their standard location and
# placed here so that Pagoda Box could override them with the neccessary values
# to configure redundancy.

# IMPORTANT: these settings will always be overriden when the server boots. They
# are set dynamically and so should never change.

host    replication     %s        %s/32            trust`, SystemUser(), other.Ip)
	}

	// write the 'entry' to the file
	err = ioutil.WriteFile(file, []byte(entry), 0644)
	if err != nil {
		log.Error("[pg_config.configureHBAConf] Failed to write to '%s'!\n%s\n", file, err)
		return err
	}

	return nil
}

// configurePGConf attempts to open the 'postgresql.conf' file. Once open it will
// scan the file line by line looking for replication settings, and overwrite only
// those settings with the settings required for redundancy on Pagoda Box
func configurePGConf(opts pgConfig) error {

	// open the postgresql.conf
	file := conf.DataDir + "postgresql.conf"
	f, err := os.Open(file)
	if err != nil {
		return err
	}

	defer f.Close()

	reFindConfigOption := regexp.MustCompile(`^\s*#?\s*(listen_addresses|port|wal_level|archive_mode|archive_command|max_wal_senders|wal_keep_segments|hot_standby|synchronous_standby_names)\s*=\s*`)
	scanner := bufio.NewScanner(f)
	entry := ""

	// scan the file line by line to build an 'entry' to be re-written back to the
	// file, skipping ('removing') any lines that need to be manually configured
	for scanner.Scan() {

		submatch := reFindConfigOption.FindStringSubmatch(scanner.Text())

		// stop scanning if a special prefix is encountered. This ensures there are
		// no duplicate Pagoda Box comment blocks
		if strings.HasPrefix(scanner.Text(), "#~") {
			break
		}

		// build the 'entry' from all lines that don't match the custom configurations
		if submatch == nil {
			entry += fmt.Sprintf("%s\n", scanner.Text())
		}
	}

	// write manual configurations into an 'entry'
	entry += fmt.Sprintf(`#~-----------------------------------------------------------------------------
# YOKE CONFIG
#------------------------------------------------------------------------------

# these configuration options have been removed from their standard location and
# placed here so that Pagoda Box could override them with the neccessary values
# to configure redundancy.

# IMPORTANT: these settings will always be overriden when the server boots. They
# are set dynamically and so should never change.

listen_addresses = '%s'      # what IP address(es) to listen on;
                                  # comma-separated list of addresses;
                                  # defaults to 'localhost'; use '*' for all
                                  # (change requires restart)
port = %d                     # (change requires restart)
wal_level = hot_standby           # minimal, archive, or hot_standby
                                  # (change requires restart)
archive_mode = on                 # allows archiving to be done
                                  # (change requires restart)
archive_command = 'exit 0'        # command to use to archive a logfile segment
                                  # placeholders: \%p = path of file to archive
                                  #               \%f = file name only
                                  # e.g. 'test ! -f /mnt/server/archivedir/\%f && cp \%p /mnt/server/archivedir/\%f'
max_wal_senders = 10              # max number of walsender processes
                                  # (change requires restart)
wal_keep_segments = 16          	# in logfile segments, 16MB each; 0 disables
hot_standby = on                  # "on" allows queries during recovery
                                  # (change requires restart)
`, opts.listenAddr, conf.PGPort)

	// if this node is currenty 'master' then write one additional configuration
	// into the 'entry'
	if opts.master {
		entry += `
# added only the the node running postgres as 'master'
synchronous_standby_names = slave # standby servers that provide sync rep
                                  # comma-separated list of application_name
                                  # from standby(s); '*' = all
`
	}

	// write 'entry' to the file
	err = ioutil.WriteFile(file, []byte(entry), 0644)
	if err != nil {
		log.Error("[pg_config.configurePGConf] Failed to write to '%s'!\n%s\n", file, err)
	}

	return nil
}

// createRecovery creates a 'recovery.conf' file with the necessary settings
// required for redundancy on Pagoda Box. This method is called on the node that
// is being configured to run the 'slave' instance of postgres
func createRecovery() error {

	file := conf.DataDir + "recovery.conf"

	self := Whoami()
	other, err := Whoisnot(self.CRole)
	if err != nil {
		log.Fatal("[pg_config.createRecovery] Unable to find another... Exiting!\n%s\n", err)
		os.Exit(1)
	}

	// open/truncate the recover.conf
	f, err := os.Create(file)
	if err != nil {
		log.Error("[pg_config.createRecovery] Failed to create '%s'!\n%s\n", file, err)
		return err
	}

	// write manual configuration an 'entry'
	entry := fmt.Sprintf(`#~-----------------------------------------------------------------------------
# YOKE CONFIG
#------------------------------------------------------------------------------

# IMPORTANT: this config file is dynamically generated by Pagoda Box for redundancy
# any changes made here will be overriden.

# When standby_mode is enabled, the PostgreSQL server will work as a standby. It
# tries to connect to the primary according to the connection settings
# primary_conninfo, and receives XLOG records continuously.
standby_mode = on
primary_conninfo = 'host=%s port=%d application_name=slave'

# restore_command specifies the shell command that is executed to copy log files
# back from archival storage. This parameter is *required* for an archive
# recovery, but optional for streaming replication. The given command satisfies
# the requirement without doing anything.
restore_command = 'exit 0'
`, other.Ip, other.PGPort)

	// write 'entry' to the file
	if _, err := f.WriteString(entry); err != nil {
		log.Error("[pg_config.createRecovery] Failed to write to '%s'!\n%s\n", file, err)
		return err
	}

	return nil
}

// destroyRecovery attempts to destroy the 'recovery.conf'. This method is called
// on a node that is being configured to run the 'master' instance of postgres
func destroyRecovery() {

	file := conf.DataDir + "recovery.conf"

	// remove 'recovery.conf'
	err := os.Remove(file)
	if err != nil {
		log.Warn("[pg_config.destroyRecovery] No recovery.conf found at '%s'", file)
	}
}
<|endoftext|>


<commit_before>package lib

import (
	"crypto/sha256"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"testing"
	"time"

	"golang.org/x/crypto/ed25519"
	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/packet"

	"github.com/stretchr/testify/require"
)

func TestPgpGolangSig(t *testing.T) {

}

func TestPgpValidSig(t *testing.T) {
	dirName, err := ioutil.TempDir("", "mulsigo")
	defer os.RemoveAll(dirName)
	require.Nil(t, err)
	var pubFileName = path.Join(dirName, "pub.pgp")
	var dataFileName = path.Join(dirName, "data")
	var sigFileName = path.Join(dirName, "data.sig")
	fmt.Println("Dirname: ", dirName)

	pub, priv, err := NewKeyPair(nil)
	require.Nil(t, err)
	privScalar := priv.Scalar()

	var msg = []byte("Hello World")
	h := sha256.New()
	HashMessage(h, msg)
	var preHashed = h.Sum(nil)
	sig := SchnorrSign(privScalar, preHashed, nil)

	require.True(t, SchnorrVerify(pub, preHashed, sig))

	pubFile, err := os.Create(pubFileName)
	require.Nil(t, err)
	sigFile, err := os.Create(sigFileName)
	require.Nil(t, err)
	dataFile, err := os.Create(dataFileName)
	require.Nil(t, err)

	pubCasted := ed25519.PublicKey([]byte(pub[:]))
	var pgpPub = packet.NewEDDSAPublicKey(time.Now().Add(-20*time.Hour), &pubCasted)

	entity, err := openpgp.NewEntity("Test Entity", " <yep> ", nil)
	require.Nil(t, err)

	pgpPub.Serialize(pubFile)

	require.Nil(t, SerializePubKey(pubFile, pub[:], "test@test.test"))
	r := sig[:32]
	s := sig[32:]
	require.Nil(t, SerializeSignature(sigFile, msg, pub[:], r, s))

	dataFile.Write(msg)

	pubFile.Close()
	sigFile.Close()
	dataFile.Close()

	cmd := exec.Command("gpg2", "--homedir", dirName, "--allow-non-selfsigned-uid", "--import", pubFileName)
	cmd.Stdout = os.Stdout
	require.Nil(t, cmd.Run())

	cmd = exec.Command("gpg2", "--homedir", dirName, "--allow-non-selfsigned-uid", "--ignore-time-conflict", "--verify", sigFileName)
	cmd.Stdout = os.Stdout
	require.Nil(t, cmd.Run())

	//cmd = exec.Command("rm", "-rf", dirName)
	//cmd.Run()
}

// exists returns whether the given file or directory exists or not
func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return true, err
}
<commit_msg> gpg2 accepts it all<commit_after>package lib

import (
	"bytes"
	"crypto"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"testing"
	"time"

	"golang.org/x/crypto/ed25519"
	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/packet"

	"github.com/stretchr/testify/require"
)

var name = "TestName"
var email = "TestEmail"
var comment = "TestComment"

func TestPgpGolangRSASig(t *testing.T) {
	dirName, err := ioutil.TempDir("", "mulsigo")
	defer os.RemoveAll(dirName)
	require.Nil(t, err)
	var pubFileName = path.Join(dirName, "pub.pgp")
	var privFileName = path.Join(dirName, "priv.pgp")
	var dataFileName = path.Join(dirName, "data")
	var sigFileName = dataFileName + ".sig"

	pubFile, err := os.Create(pubFileName)
	defer pubFile.Close()
	require.Nil(t, err)

	privFile, err := os.Create(privFileName)
	defer privFile.Close()
	require.Nil(t, err)

	dataFile, err := os.Create(dataFileName)
	defer dataFile.Close()
	require.Nil(t, err)

	sigFile, err := os.Create(sigFileName)
	defer sigFile.Close()
	require.Nil(t, err)

	_seed := "Hello World, I'm gonna be your seed during this test, would you?"
	var seed = bytes.NewBuffer([]byte(_seed))

	config := packet.Config{
		Rand: seed,
	}
	ent, err := openpgp.NewEDDSAEntity(name, email, comment, &config)
	require.Nil(t, err)

	p := ent.PrimaryKey.PublicKey.(*ed25519.PublicKey)
	fmt.Println("Public key created:", hex.EncodeToString([]byte(*p)))
	var id string
	var identity *openpgp.Identity
	for name, i := range ent.Identities {
		if id == "" {
			id = name
			identity = i
		}
	}

	err = identity.SelfSignature.SignUserId(id, ent.PrimaryKey, ent.PrivateKey, nil)
	require.Nil(t, err)

	err = ent.Serialize(pubFile)
	require.Nil(t, err)

	gpg2Import(dirName, pubFileName, t)

	/*err = ent.SerializePrivate(privFile, nil)*/
	//require.Nil(t, err)

	//gpg2Import(dirName, privFileName, t)

	sig := &packet.Signature{
		PubKeyAlgo: packet.PubKeyAlgoEDDSA,
		Hash:       crypto.SHA256,
	}

	sig.IssuerKeyId = &ent.PrimaryKey.KeyId
	msg := []byte("Hello World!")

	h := sha256.New()
	_, err = h.Write(msg)
	require.Nil(t, err)

	if err := sig.Sign(h, ent.PrivateKey, nil); err != nil {
		t.Fatal(err)
	}

	h = sha256.New()
	_, err = h.Write(msg)
	require.Nil(t, err)

	if err := ent.PrivateKey.VerifySignature(h, sig); err != nil {
		t.Fatal(err)
	}

	err = sig.Serialize(sigFile)
	require.Nil(t, err)

	_, err = dataFile.Write(msg)
	require.Nil(t, err)

	var out bytes.Buffer
	cmd := exec.Command("gpg2", "--homedir", dirName, "--verify", sigFileName)
	cmd.Stdout = &out
	cmd.Stderr = &out
	err = cmd.Run()
	fmt.Println(cmd.Args)
	fmt.Println(out.String())
	require.Nil(t, err)

}

func gpg2Import(dirName, fileName string, t *testing.T) {
	var out bytes.Buffer
	cmd := exec.Command("gpg2", "--homedir", dirName, "--import", fileName)
	cmd.Stdout = &out
	cmd.Stderr = &out
	err := cmd.Run()
	fmt.Println(cmd.Args)
	fmt.Println(out.String())
	require.Nil(t, err)
}

func TestPgpGolangSig(t *testing.T) {
	dirName, err := ioutil.TempDir("", "mulsigo")
	defer os.RemoveAll(dirName)
	require.Nil(t, err)
	var pubFileName = path.Join(dirName, "pub.pgp")
	fmt.Println("Dirname: ", dirName)

	pubFile, err := os.Create(pubFileName)
	require.Nil(t, err)

	var currentTime = time.Now().Add(-20 * time.Hour)

	_seed := "Hello World, I'm gonna be your seed during this test, would you?"
	var seed = bytes.NewBuffer([]byte(_seed))
	eddsaPub, eddsaPriv, err := ed25519.GenerateKey(seed)
	pointer := ed25519.PrivateKey(eddsaPriv)

	eddsaPrivateKey := packet.NewEDDSAPrivateKey(time.Now(), &pointer)
	pubCasted := ed25519.PublicKey([]byte(eddsaPub[:]))
	var pgpPub = packet.NewEDDSAPublicKey(currentTime, &pubCasted)

	uid := packet.NewUserId(name, email, comment)
	require.NotNil(t, uid)

	primary := true

	SelfSignature := &packet.Signature{
		CreationTime: currentTime,
		SigType:      packet.SigTypePositiveCert,
		PubKeyAlgo:   packet.PubKeyAlgoRSA,
		Hash:         crypto.SHA256,
		IsPrimaryId:  &primary,
		FlagsValid:   true,
		FlagSign:     true,
		FlagCertify:  true,
		IssuerKeyId:  &pgpPub.KeyId,
	}
	require.Nil(t, SelfSignature.SignUserId(uid.Id, pgpPub, eddsaPrivateKey, nil))

	// serialize
	require.Nil(t, pgpPub.Serialize(pubFile))
	require.Nil(t, uid.Serialize(pubFile))
	require.Nil(t, SelfSignature.Serialize(pubFile))

	require.Nil(t, pubFile.Close())
	var out bytes.Buffer
	cmd := exec.Command("gpg2", "--homedir", dirName, "--import", pubFileName)
	cmd.Stdout = &out
	cmd.Stderr = &out
	err = cmd.Run()
	fmt.Println(cmd.Args)
	fmt.Println(out.String())
	require.Nil(t, err)
}

func TestPgpValidSig(t *testing.T) {
	dirName, err := ioutil.TempDir("", "mulsigo")
	defer os.RemoveAll(dirName)
	require.Nil(t, err)
	var pubFileName = path.Join(dirName, "pub.pgp")
	var dataFileName = path.Join(dirName, "data")
	var sigFileName = path.Join(dirName, "data.sig")
	fmt.Println("Dirname: ", dirName)

	pub, priv, err := NewKeyPair(nil)
	require.Nil(t, err)
	privScalar := priv.Scalar()

	var msg = []byte("Hello World")
	h := sha256.New()
	HashMessage(h, msg)
	var preHashed = h.Sum(nil)
	sig := SchnorrSign(privScalar, preHashed, nil)

	require.True(t, SchnorrVerify(pub, preHashed, sig))

	pubFile, err := os.Create(pubFileName)
	require.Nil(t, err)
	sigFile, err := os.Create(sigFileName)
	require.Nil(t, err)
	dataFile, err := os.Create(dataFileName)
	require.Nil(t, err)

	pubCasted := ed25519.PublicKey([]byte(pub[:]))
	var pgpPub = packet.NewEDDSAPublicKey(time.Now().Add(-20*time.Hour), &pubCasted)

	pgpPub.Serialize(pubFile)

	require.Nil(t, SerializePubKey(pubFile, pub[:], "test@test.test"))
	r := sig[:32]
	s := sig[32:]
	require.Nil(t, SerializeSignature(sigFile, msg, pub[:], r, s))

	dataFile.Write(msg)

	pubFile.Close()
	sigFile.Close()
	dataFile.Close()

	cmd := exec.Command("gpg2", "--homedir", dirName, "--allow-non-selfsigned-uid", "--import", pubFileName)
	cmd.Stdout = os.Stdout
	require.Nil(t, cmd.Run())

	cmd = exec.Command("gpg2", "--homedir", dirName, "--allow-non-selfsigned-uid", "--ignore-time-conflict", "--verify", sigFileName)
	cmd.Stdout = os.Stdout
	require.Nil(t, cmd.Run())

	//cmd = exec.Command("rm", "-rf", dirName)
	//cmd.Run()
}

// exists returns whether the given file or directory exists or not
func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return true, err
}
<|endoftext|>


<commit_before>// $G $D/$F.go && $L $F.$A && ./$A.out

// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Tests verifying the semantics of the select statement
// for basic empty/non-empty cases.

package main

import "time"

const always = "function did not"
const never = "function did"

func unreachable() {
	panic("control flow shouldn't reach here")
}

// Calls f and verifies that f always/never panics depending on signal.
func testPanic(signal string, f func()) {
	defer func() {
		s := never
		if recover() != nil {
			s = always // f panicked
		}
		if s != signal {
			panic(signal + " panic")
		}
	}()
	f()
}

// Calls f and empirically verifies that f always/never blocks depending on signal.
func testBlock(signal string, f func()) {
	c := make(chan string)
	go func() {
		f()
		c <- never // f didn't block
	}()
	go func() {
		time.Sleep(1e8) // 0.1s seems plenty long
		c <- always     // f blocked always
	}()
	if <-c != signal {
		panic(signal + " block")
	}
}

func main() {
	const async = 1 // asynchronous channels
	var nilch chan int
	closedch := make(chan int)
	close(closedch)

	// sending/receiving from a nil channel blocks
	testBlock(always, func() {
		nilch <- 7
	})
	testBlock(always, func() {
		<-nilch
	})

	// sending/receiving from a nil channel inside a select is never selected
	testPanic(never, func() {
		select {
		case nilch <- 7:
			unreachable()
		default:
		}
	})
	testPanic(never, func() {
		select {
		case <-nilch:
			unreachable()
		default:
		}
	})

	// sending to an async channel with free buffer space never blocks
	testBlock(never, func() {
		ch := make(chan int, async)
		ch <- 7
	})

	// receiving from a closed channel never blocks
	testBlock(never, func() {
		for i := 0; i < 10; i++ {
			if <-closedch != 0 {
				panic("expected zero value when reading from closed channel")
			}
			if x, ok := <-closedch; x != 0 || ok {
				println("closedch:", x, ok)
				panic("expected 0, false from closed channel")
			}
		}
	})

	// sending to a closed channel panics.
	testPanic(always, func() {
		closedch <- 7
	})

	// receiving from a non-ready channel always blocks
	testBlock(always, func() {
		ch := make(chan int)
		<-ch
	})

	// empty selects always block
	testBlock(always, func() {
		select {
		}
	})

	// selects with only nil channels always block
	testBlock(always, func() {
		select {
		case <-nilch:
			unreachable()
		}
	})
	testBlock(always, func() {
		select {
		case nilch <- 7:
			unreachable()
		}
	})
	testBlock(always, func() {
		select {
		case <-nilch:
			unreachable()
		case nilch <- 7:
			unreachable()
		}
	})

	// selects with non-ready non-nil channels always block
	testBlock(always, func() {
		ch := make(chan int)
		select {
		case <-ch:
			unreachable()
		}
	})

	// selects with default cases don't block
	testBlock(never, func() {
		select {
		default:
		}
	})
	testBlock(never, func() {
		select {
		case <-nilch:
			unreachable()
		default:
		}
	})
	testBlock(never, func() {
		select {
		case nilch <- 7:
			unreachable()
		default:
		}
	})

	// selects with ready channels don't block
	testBlock(never, func() {
		ch := make(chan int, async)
		select {
		case ch <- 7:
		default:
			unreachable()
		}
	})
	testBlock(never, func() {
		ch := make(chan int, async)
		ch <- 7
		select {
		case <-ch:
		default:
			unreachable()
		}
	})

	// selects with closed channels behave like ordinary operations
	testBlock(never, func() {
		select {
		case <-closedch:
		}
	})
	testBlock(never, func() {
		select {
		case x := <-closedch:
			_ = x
		}
	})
	testBlock(never, func() {
		select {
		case x, ok := <-closedch:
			_, _ = x, ok
		}
	})
	testPanic(always, func() {
		select {
		case closedch <- 7:
		}
	})

	// select should not get confused if it sees itself
	testBlock(always, func() {
		c := make(chan int)
		select {
		case c <- 1:
		case <-c:
		}
	})
}
<commit_msg>test: test that x := <-c accepts a general expression<commit_after>// $G $D/$F.go && $L $F.$A && ./$A.out

// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Tests verifying the semantics of the select statement
// for basic empty/non-empty cases.

package main

import "time"

const always = "function did not"
const never = "function did"

func unreachable() {
	panic("control flow shouldn't reach here")
}

// Calls f and verifies that f always/never panics depending on signal.
func testPanic(signal string, f func()) {
	defer func() {
		s := never
		if recover() != nil {
			s = always // f panicked
		}
		if s != signal {
			panic(signal + " panic")
		}
	}()
	f()
}

// Calls f and empirically verifies that f always/never blocks depending on signal.
func testBlock(signal string, f func()) {
	c := make(chan string)
	go func() {
		f()
		c <- never // f didn't block
	}()
	go func() {
		time.Sleep(1e8) // 0.1s seems plenty long
		c <- always     // f blocked always
	}()
	if <-c != signal {
		panic(signal + " block")
	}
}

func main() {
	const async = 1 // asynchronous channels
	var nilch chan int
	closedch := make(chan int)
	close(closedch)

	// sending/receiving from a nil channel blocks
	testBlock(always, func() {
		nilch <- 7
	})
	testBlock(always, func() {
		<-nilch
	})

	// sending/receiving from a nil channel inside a select is never selected
	testPanic(never, func() {
		select {
		case nilch <- 7:
			unreachable()
		default:
		}
	})
	testPanic(never, func() {
		select {
		case <-nilch:
			unreachable()
		default:
		}
	})

	// sending to an async channel with free buffer space never blocks
	testBlock(never, func() {
		ch := make(chan int, async)
		ch <- 7
	})

	// receiving from a closed channel never blocks
	testBlock(never, func() {
		for i := 0; i < 10; i++ {
			if <-closedch != 0 {
				panic("expected zero value when reading from closed channel")
			}
			if x, ok := <-closedch; x != 0 || ok {
				println("closedch:", x, ok)
				panic("expected 0, false from closed channel")
			}
		}
	})

	// sending to a closed channel panics.
	testPanic(always, func() {
		closedch <- 7
	})

	// receiving from a non-ready channel always blocks
	testBlock(always, func() {
		ch := make(chan int)
		<-ch
	})

	// empty selects always block
	testBlock(always, func() {
		select {
		}
	})

	// selects with only nil channels always block
	testBlock(always, func() {
		select {
		case <-nilch:
			unreachable()
		}
	})
	testBlock(always, func() {
		select {
		case nilch <- 7:
			unreachable()
		}
	})
	testBlock(always, func() {
		select {
		case <-nilch:
			unreachable()
		case nilch <- 7:
			unreachable()
		}
	})

	// selects with non-ready non-nil channels always block
	testBlock(always, func() {
		ch := make(chan int)
		select {
		case <-ch:
			unreachable()
		}
	})

	// selects with default cases don't block
	testBlock(never, func() {
		select {
		default:
		}
	})
	testBlock(never, func() {
		select {
		case <-nilch:
			unreachable()
		default:
		}
	})
	testBlock(never, func() {
		select {
		case nilch <- 7:
			unreachable()
		default:
		}
	})

	// selects with ready channels don't block
	testBlock(never, func() {
		ch := make(chan int, async)
		select {
		case ch <- 7:
		default:
			unreachable()
		}
	})
	testBlock(never, func() {
		ch := make(chan int, async)
		ch <- 7
		select {
		case <-ch:
		default:
			unreachable()
		}
	})

	// selects with closed channels behave like ordinary operations
	testBlock(never, func() {
		select {
		case <-closedch:
		}
	})
	testBlock(never, func() {
		select {
		case x := (<-closedch):
			_ = x
		}
	})
	testBlock(never, func() {
		select {
		case x, ok := (<-closedch):
			_, _ = x, ok
		}
	})
	testPanic(always, func() {
		select {
		case closedch <- 7:
		}
	})

	// select should not get confused if it sees itself
	testBlock(always, func() {
		c := make(chan int)
		select {
		case c <- 1:
		case <-c:
		}
	})
}
<|endoftext|>


<commit_before>package utils

import (
	"os"
	"fmt"
)

//AreWeInAppRootDir returns nil if we're in the app root directory; otherwise it retuns the error.
func AreWeInAppRootDir() (string, error) {
  currentDir, err := os.Getwd();
  if err != nil {
    return "", err
  }
  //we are in the app root dir if we have both a ./app and
  //a ./config in the current working dir
  checkDirs := [...]string {
    fmt.Sprintf("%v/app", currentDir),
    fmt.Sprintf("%v/config", currentDir), //expand if needed
  }
  for _, checkDir := range checkDirs {
    if _, err := os.Stat(checkDir); err != nil {
      return "", err
    }
  }
  //if we made it here, all the checkDirs exist, which means we should be good
  return currentDir, nil
}
<commit_msg>better error message<commit_after>package utils

import (
	"fmt"
	"os"
)

func AppErrorMessage() error {
	currentDir, _ := os.Getwd()
	return fmt.Errorf(`
	Hi there!  You likely wanted to execute this command in a polka project directory.
	For example, if you ran:

	$ cd ~/code
	$ polka new todo

	Then ~/code/todo is your polka project dir.  It'll have a polka/app polka/config,
	and the rest of the polka generated files.

	This time you ran polka in %v
	`, currentDir)
}

//AreWeInAppRootDir returns nil if we're in the app root directory; otherwise it retuns the error.
func AreWeInAppRootDir() (string, error) {
	currentDir, err := os.Getwd()
	if err != nil {
		return "", err
	}
	//we are in the app root dir if we have both a ./app and
	//a ./config in the current working dir
	checkDirs := [...]string{
		fmt.Sprintf("%v/app", currentDir),
		fmt.Sprintf("%v/config", currentDir), //expand if needed
	}
	for _, checkDir := range checkDirs {
		if _, err := os.Stat(checkDir); err != nil {
			return "", AppErrorMessage()
		}
	}
	//if we made it here, all the checkDirs exist, which means we should be good
	return currentDir, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package retrieval

import (
	"fmt"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/extraction"
	"github.com/prometheus/client_golang/prometheus"

	clientmodel "github.com/prometheus/client_golang/model"

	"github.com/prometheus/prometheus/utility"
)

const (
	InstanceLabel clientmodel.LabelName = "instance"
	// The metric name for the synthetic health variable.
	ScrapeHealthMetricName clientmodel.LabelValue = "up"

	// Constants for instrumentation.
	address = "instance"
	alive   = "alive"
	failure = "failure"
	outcome = "outcome"
	state   = "state"
	success = "success"
)

var (
	localhostRepresentations = []string{"http://127.0.0.1", "http://localhost"}

	targetOperationLatencies = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Name:       "prometheus_target_operation_latency_ms",
			Help:       "The latencies for various target operations.",
			Objectives: []float64{0.01, 0.05, 0.5, 0.90, 0.99},
		},
		[]string{address, outcome},
	)
)

func init() {
	prometheus.MustRegister(targetOperationLatencies)
}

// The state of the given Target.
type TargetState int

func (t TargetState) String() string {
	switch t {
	case UNKNOWN:
		return "UNKNOWN"
	case ALIVE:
		return "ALIVE"
	case UNREACHABLE:
		return "UNREACHABLE"
	}

	panic("unknown state")
}

const (
	// The Target has not been seen; we know nothing about it, except that it is
	// on our docket for examination.
	UNKNOWN TargetState = iota
	// The Target has been found and successfully queried.
	ALIVE
	// The Target was either historically found or not found and then determined
	// to be unhealthy by either not responding or disappearing.
	UNREACHABLE
)

// A healthReporter is a type that can provide insight into its health state.
//
// It mainly exists for testability reasons to decouple the scheduler behaviors
// from fully-fledged Target and other types.
type healthReporter interface {
	// Report the last-known health state for this target.
	State() TargetState
}

// A Target represents an endpoint that should be interrogated for metrics.
//
// The protocol described by this type will likely change in future iterations,
// as it offers no good support for aggregated targets and fan out.  Thusly,
// it is likely that the current Target and target uses will be
// wrapped with some resolver type.
//
// For the future, the Target protocol will abstract away the exact means that
// metrics are retrieved and deserialized from the given instance to which it
// refers.
type Target interface {
	// Retrieve values from this target.
	//
	// earliest refers to the soonest available opportunity to reschedule the
	// target for a future retrieval.  It is up to the underlying scheduler type,
	// alluded to in the scheduledFor function, to use this as it wants to.  The
	// current use case is to create a common batching time for scraping multiple
	// Targets in the future through the TargetPool.
	Scrape(earliest time.Time, ingester extraction.Ingester) error
	// Fulfill the healthReporter interface.
	State() TargetState
	// Report the soonest time at which this Target may be scheduled for
	// retrieval.  This value needn't convey that the operation occurs at this
	// time, but it should occur no sooner than it.
	//
	// Right now, this is used as the sorting key in TargetPool.
	ScheduledFor() time.Time
	// EstimatedTimeToExecute emits the amount of time until the next prospective
	// scheduling opportunity for this target.
	EstimatedTimeToExecute() time.Duration
	// Return the last encountered scrape error, if any.
	LastError() error
	// The address to which the Target corresponds.  Out of all of the available
	// points in this interface, this one is the best candidate to change given
	// the ways to express the endpoint.
	Address() string
	// The address as seen from other hosts. References to localhost are resolved
	// to the address of the prometheus server.
	GlobalAddress() string
	// Return the target's base labels.
	BaseLabels() clientmodel.LabelSet
	// Merge a new externally supplied target definition (e.g. with changed base
	// labels) into an old target definition for the same endpoint. Preserve
	// remaining information - like health state - from the old target.
	Merge(newTarget Target)
}

// target is a Target that refers to a singular HTTP or HTTPS endpoint.
type target struct {
	// scheduler provides the scheduling strategy that is used to formulate what
	// is returned in Target.scheduledFor.
	scheduler scheduler
	// The current health state of the target.
	state TargetState
	// The last encountered scrape error, if any.
	lastError error

	address string
	// What is the deadline for the HTTP or HTTPS against this endpoint.
	Deadline time.Duration
	// Any base labels that are added to this target and its metrics.
	baseLabels clientmodel.LabelSet
	// The HTTP client used to scrape the target's endpoint.
	httpClient *http.Client
}

// Furnish a reasonably configured target for querying.
func NewTarget(address string, deadline time.Duration, baseLabels clientmodel.LabelSet) Target {
	target := &target{
		address:    address,
		Deadline:   deadline,
		baseLabels: baseLabels,
		httpClient: utility.NewDeadlineClient(deadline),
	}

	scheduler := &healthScheduler{
		target: target,
	}
	target.scheduler = scheduler

	return target
}

func (t *target) recordScrapeHealth(ingester extraction.Ingester, timestamp clientmodel.Timestamp, healthy bool) {
	metric := clientmodel.Metric{}
	for label, value := range t.baseLabels {
		metric[label] = value
	}
	metric[clientmodel.MetricNameLabel] = clientmodel.LabelValue(ScrapeHealthMetricName)
	metric[InstanceLabel] = clientmodel.LabelValue(t.Address())

	healthValue := clientmodel.SampleValue(0)
	if healthy {
		healthValue = clientmodel.SampleValue(1)
	}

	sample := &clientmodel.Sample{
		Metric:    metric,
		Timestamp: timestamp,
		Value:     healthValue,
	}

	ingester.Ingest(&extraction.Result{
		Err:     nil,
		Samples: clientmodel.Samples{sample},
	})
}

func (t *target) Scrape(earliest time.Time, ingester extraction.Ingester) error {
	now := clientmodel.Now()
	futureState := t.state
	err := t.scrape(now, ingester)
	if err != nil {
		t.recordScrapeHealth(ingester, now, false)
		futureState = UNREACHABLE
	} else {
		t.recordScrapeHealth(ingester, now, true)
		futureState = ALIVE
	}

	t.scheduler.Reschedule(earliest, futureState)
	t.state = futureState
	t.lastError = err

	return err
}

const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3,application/json;schema=prometheus/telemetry;version=0.0.2;q=0.2,*/*;q=0.1`

func (t *target) scrape(timestamp clientmodel.Timestamp, ingester extraction.Ingester) (err error) {
	defer func(start time.Time) {
		ms := float64(time.Since(start)) / float64(time.Millisecond)
		labels := prometheus.Labels{address: t.Address(), outcome: success}
		if err != nil {
			labels[outcome] = failure
		}

		targetOperationLatencies.With(labels).Observe(ms)
	}(time.Now())

	req, err := http.NewRequest("GET", t.Address(), nil)
	if err != nil {
		panic(err)
	}
	req.Header.Add("Accept", acceptHeader)

	resp, err := t.httpClient.Do(req)
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("server returned HTTP status %s", resp.Status)
	}
	defer resp.Body.Close()

	processor, err := extraction.ProcessorForRequestHeader(resp.Header)
	if err != nil {
		return err
	}

	// XXX: This is a wart; we need to handle this more gracefully down the
	//      road, especially once we have service discovery support.
	baseLabels := clientmodel.LabelSet{InstanceLabel: clientmodel.LabelValue(t.Address())}
	for baseLabel, baseValue := range t.baseLabels {
		baseLabels[baseLabel] = baseValue
	}

	i := &MergeLabelsIngester{
		Labels:          baseLabels,
		CollisionPrefix: clientmodel.ExporterLabelPrefix,

		Ingester: ingester,
	}
	processOptions := &extraction.ProcessOptions{
		Timestamp: timestamp,
	}
	return processor.ProcessSingle(resp.Body, i, processOptions)
}

func (t *target) State() TargetState {
	return t.state
}

func (t *target) ScheduledFor() time.Time {
	return t.scheduler.ScheduledFor()
}

func (t *target) EstimatedTimeToExecute() time.Duration {
	return time.Now().Sub(t.scheduler.ScheduledFor())
}

func (t *target) LastError() error {
	return t.lastError
}

func (t *target) Address() string {
	return t.address
}

func (t *target) GlobalAddress() string {
	address := t.address
	hostname, err := os.Hostname()
	if err != nil {
		glog.Warningf("Couldn't get hostname: %s, returning target.Address()", err)
		return address
	}
	for _, localhostRepresentation := range localhostRepresentations {
		address = strings.Replace(address, localhostRepresentation, fmt.Sprintf("http://%s", hostname), -1)
	}
	return address
}

func (t *target) BaseLabels() clientmodel.LabelSet {
	return t.baseLabels
}

// Merge a new externally supplied target definition (e.g. with changed base
// labels) into an old target definition for the same endpoint. Preserve
// remaining information - like health state - from the old target.
func (t *target) Merge(newTarget Target) {
	if t.Address() != newTarget.Address() {
		panic("targets don't refer to the same endpoint")
	}
	t.baseLabels = newTarget.BaseLabels()
}

type targets []Target

func (t targets) Len() int {
	return len(t)
}

func (t targets) Less(i, j int) bool {
	return t[i].ScheduledFor().Before(t[j].ScheduledFor())
}

func (t targets) Swap(i, j int) {
	t[i], t[j] = t[j], t[i]
}
<commit_msg>Fix HTTP connection leak upon non-OK status.<commit_after>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package retrieval

import (
	"fmt"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/extraction"
	"github.com/prometheus/client_golang/prometheus"

	clientmodel "github.com/prometheus/client_golang/model"

	"github.com/prometheus/prometheus/utility"
)

const (
	InstanceLabel clientmodel.LabelName = "instance"
	// The metric name for the synthetic health variable.
	ScrapeHealthMetricName clientmodel.LabelValue = "up"

	// Constants for instrumentation.
	address = "instance"
	alive   = "alive"
	failure = "failure"
	outcome = "outcome"
	state   = "state"
	success = "success"
)

var (
	localhostRepresentations = []string{"http://127.0.0.1", "http://localhost"}

	targetOperationLatencies = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Name:       "prometheus_target_operation_latency_ms",
			Help:       "The latencies for various target operations.",
			Objectives: []float64{0.01, 0.05, 0.5, 0.90, 0.99},
		},
		[]string{address, outcome},
	)
)

func init() {
	prometheus.MustRegister(targetOperationLatencies)
}

// The state of the given Target.
type TargetState int

func (t TargetState) String() string {
	switch t {
	case UNKNOWN:
		return "UNKNOWN"
	case ALIVE:
		return "ALIVE"
	case UNREACHABLE:
		return "UNREACHABLE"
	}

	panic("unknown state")
}

const (
	// The Target has not been seen; we know nothing about it, except that it is
	// on our docket for examination.
	UNKNOWN TargetState = iota
	// The Target has been found and successfully queried.
	ALIVE
	// The Target was either historically found or not found and then determined
	// to be unhealthy by either not responding or disappearing.
	UNREACHABLE
)

// A healthReporter is a type that can provide insight into its health state.
//
// It mainly exists for testability reasons to decouple the scheduler behaviors
// from fully-fledged Target and other types.
type healthReporter interface {
	// Report the last-known health state for this target.
	State() TargetState
}

// A Target represents an endpoint that should be interrogated for metrics.
//
// The protocol described by this type will likely change in future iterations,
// as it offers no good support for aggregated targets and fan out.  Thusly,
// it is likely that the current Target and target uses will be
// wrapped with some resolver type.
//
// For the future, the Target protocol will abstract away the exact means that
// metrics are retrieved and deserialized from the given instance to which it
// refers.
type Target interface {
	// Retrieve values from this target.
	//
	// earliest refers to the soonest available opportunity to reschedule the
	// target for a future retrieval.  It is up to the underlying scheduler type,
	// alluded to in the scheduledFor function, to use this as it wants to.  The
	// current use case is to create a common batching time for scraping multiple
	// Targets in the future through the TargetPool.
	Scrape(earliest time.Time, ingester extraction.Ingester) error
	// Fulfill the healthReporter interface.
	State() TargetState
	// Report the soonest time at which this Target may be scheduled for
	// retrieval.  This value needn't convey that the operation occurs at this
	// time, but it should occur no sooner than it.
	//
	// Right now, this is used as the sorting key in TargetPool.
	ScheduledFor() time.Time
	// EstimatedTimeToExecute emits the amount of time until the next prospective
	// scheduling opportunity for this target.
	EstimatedTimeToExecute() time.Duration
	// Return the last encountered scrape error, if any.
	LastError() error
	// The address to which the Target corresponds.  Out of all of the available
	// points in this interface, this one is the best candidate to change given
	// the ways to express the endpoint.
	Address() string
	// The address as seen from other hosts. References to localhost are resolved
	// to the address of the prometheus server.
	GlobalAddress() string
	// Return the target's base labels.
	BaseLabels() clientmodel.LabelSet
	// Merge a new externally supplied target definition (e.g. with changed base
	// labels) into an old target definition for the same endpoint. Preserve
	// remaining information - like health state - from the old target.
	Merge(newTarget Target)
}

// target is a Target that refers to a singular HTTP or HTTPS endpoint.
type target struct {
	// scheduler provides the scheduling strategy that is used to formulate what
	// is returned in Target.scheduledFor.
	scheduler scheduler
	// The current health state of the target.
	state TargetState
	// The last encountered scrape error, if any.
	lastError error

	address string
	// What is the deadline for the HTTP or HTTPS against this endpoint.
	Deadline time.Duration
	// Any base labels that are added to this target and its metrics.
	baseLabels clientmodel.LabelSet
	// The HTTP client used to scrape the target's endpoint.
	httpClient *http.Client
}

// Furnish a reasonably configured target for querying.
func NewTarget(address string, deadline time.Duration, baseLabels clientmodel.LabelSet) Target {
	target := &target{
		address:    address,
		Deadline:   deadline,
		baseLabels: baseLabels,
		httpClient: utility.NewDeadlineClient(deadline),
	}

	scheduler := &healthScheduler{
		target: target,
	}
	target.scheduler = scheduler

	return target
}

func (t *target) recordScrapeHealth(ingester extraction.Ingester, timestamp clientmodel.Timestamp, healthy bool) {
	metric := clientmodel.Metric{}
	for label, value := range t.baseLabels {
		metric[label] = value
	}
	metric[clientmodel.MetricNameLabel] = clientmodel.LabelValue(ScrapeHealthMetricName)
	metric[InstanceLabel] = clientmodel.LabelValue(t.Address())

	healthValue := clientmodel.SampleValue(0)
	if healthy {
		healthValue = clientmodel.SampleValue(1)
	}

	sample := &clientmodel.Sample{
		Metric:    metric,
		Timestamp: timestamp,
		Value:     healthValue,
	}

	ingester.Ingest(&extraction.Result{
		Err:     nil,
		Samples: clientmodel.Samples{sample},
	})
}

func (t *target) Scrape(earliest time.Time, ingester extraction.Ingester) error {
	now := clientmodel.Now()
	futureState := t.state
	err := t.scrape(now, ingester)
	if err != nil {
		t.recordScrapeHealth(ingester, now, false)
		futureState = UNREACHABLE
	} else {
		t.recordScrapeHealth(ingester, now, true)
		futureState = ALIVE
	}

	t.scheduler.Reschedule(earliest, futureState)
	t.state = futureState
	t.lastError = err

	return err
}

const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3,application/json;schema=prometheus/telemetry;version=0.0.2;q=0.2,*/*;q=0.1`

func (t *target) scrape(timestamp clientmodel.Timestamp, ingester extraction.Ingester) (err error) {
	defer func(start time.Time) {
		ms := float64(time.Since(start)) / float64(time.Millisecond)
		labels := prometheus.Labels{address: t.Address(), outcome: success}
		if err != nil {
			labels[outcome] = failure
		}

		targetOperationLatencies.With(labels).Observe(ms)
	}(time.Now())

	req, err := http.NewRequest("GET", t.Address(), nil)
	if err != nil {
		panic(err)
	}
	req.Header.Add("Accept", acceptHeader)

	resp, err := t.httpClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("server returned HTTP status %s", resp.Status)
	}

	processor, err := extraction.ProcessorForRequestHeader(resp.Header)
	if err != nil {
		return err
	}

	// XXX: This is a wart; we need to handle this more gracefully down the
	//      road, especially once we have service discovery support.
	baseLabels := clientmodel.LabelSet{InstanceLabel: clientmodel.LabelValue(t.Address())}
	for baseLabel, baseValue := range t.baseLabels {
		baseLabels[baseLabel] = baseValue
	}

	i := &MergeLabelsIngester{
		Labels:          baseLabels,
		CollisionPrefix: clientmodel.ExporterLabelPrefix,

		Ingester: ingester,
	}
	processOptions := &extraction.ProcessOptions{
		Timestamp: timestamp,
	}
	return processor.ProcessSingle(resp.Body, i, processOptions)
}

func (t *target) State() TargetState {
	return t.state
}

func (t *target) ScheduledFor() time.Time {
	return t.scheduler.ScheduledFor()
}

func (t *target) EstimatedTimeToExecute() time.Duration {
	return time.Now().Sub(t.scheduler.ScheduledFor())
}

func (t *target) LastError() error {
	return t.lastError
}

func (t *target) Address() string {
	return t.address
}

func (t *target) GlobalAddress() string {
	address := t.address
	hostname, err := os.Hostname()
	if err != nil {
		glog.Warningf("Couldn't get hostname: %s, returning target.Address()", err)
		return address
	}
	for _, localhostRepresentation := range localhostRepresentations {
		address = strings.Replace(address, localhostRepresentation, fmt.Sprintf("http://%s", hostname), -1)
	}
	return address
}

func (t *target) BaseLabels() clientmodel.LabelSet {
	return t.baseLabels
}

// Merge a new externally supplied target definition (e.g. with changed base
// labels) into an old target definition for the same endpoint. Preserve
// remaining information - like health state - from the old target.
func (t *target) Merge(newTarget Target) {
	if t.Address() != newTarget.Address() {
		panic("targets don't refer to the same endpoint")
	}
	t.baseLabels = newTarget.BaseLabels()
}

type targets []Target

func (t targets) Len() int {
	return len(t)
}

func (t targets) Less(i, j int) bool {
	return t[i].ScheduledFor().Before(t[j].ScheduledFor())
}

func (t targets) Swap(i, j int) {
	t[i], t[j] = t[j], t[i]
}
<|endoftext|>