<commit_before>package main

import (
	"github.com/prometheus/client_golang/prometheus"
)

type (
	executor struct {
		ID          string      `json:"executor_id"`
		Name        string      `json:"executor_name"`
		FrameworkID string      `json:"framework_id"`
		Source      string      `json:"source"`
		Statistics  *statistics `json:"statistics"`
		Tasks       []task      `json:"tasks"`
	}

	statistics struct {
		CpusLimit             float64 `json:"cpus_limit"`
		CpusSystemTimeSecs    float64 `json:"cpus_system_time_secs"`
		CpusUserTimeSecs      float64 `json:"cpus_user_time_secs"`
		CpusThrottledTimeSecs float64 `json:"cpus_throttled_time_secs"`

		MemLimitBytes float64 `json:"mem_limit_bytes"`
		MemRssBytes   float64 `json:"mem_rss_bytes"`

		NetRxBytes   float64 `json:"net_rx_bytes"`
		NetRxDropped float64 `json:"net_rx_dropped"`
		NetRxErrors  float64 `json:"net_rx_errors"`
		NetRxPackets float64 `json:"net_rx_packets"`
		NetTxBytes   float64 `json:"net_tx_bytes"`
		NetTxDropped float64 `json:"net_tx_dropped"`
		NetTxErrors  float64 `json:"net_tx_errors"`
		NetTxPackets float64 `json:"net_tx_packets"`
	}

	slaveCollector struct {
		*httpClient
		metrics map[*prometheus.Desc]metric
	}

	metric struct {
		valueType prometheus.ValueType
		get       func(*statistics) float64
	}
)

func newSlaveMonitorCollector(httpClient *httpClient) prometheus.Collector {
	labels := []string{"id", "framework_id", "source"}

	return &slaveCollector{
		httpClient: httpClient,
		metrics: map[*prometheus.Desc]metric{
			// CPU
			prometheus.NewDesc(
				"cpus_limit",
				"Current limit of CPUs for task",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.CpusLimit }},
			prometheus.NewDesc(
				"cpu_system_seconds_total",
				"Total system CPU seconds",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.CpusSystemTimeSecs }},
			prometheus.NewDesc(
				"cpu_user_seconds_total",
				"Total user CPU seconds",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.CpusUserTimeSecs }},
			prometheus.NewDesc(
				"cpu_throttled_seconds_total",
				"Total time CPU was throttled",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.CpusThrottledTimeSecs }},

			// Memory
			prometheus.NewDesc(
				"mem_limit_bytes",
				"Current memory limit in bytes",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.MemLimitBytes }},
			prometheus.NewDesc(
				"mem_rss_bytes",
				"Current rss memory usage",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.MemRssBytes }},

			// Network
			// - RX
			prometheus.NewDesc(
				"network_receive_bytes_total",
				"Total bytes received",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxBytes }},
			prometheus.NewDesc(
				"network_receive_dropped_total",
				"Total packets dropped while receiving",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxDropped }},
			prometheus.NewDesc(
				"network_receive_errors_total",
				"Total errors while receiving",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxBytes }},
			prometheus.NewDesc(
				"network_receive_packets_total",
				"Total packets received",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxBytes }},
			// - TX
			prometheus.NewDesc(
				"network_transmit_bytes_total",
				"Total bytes transmitted",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxBytes }},
			prometheus.NewDesc(
				"network_transmit_dropped_total",
				"Total packets dropped while transmitting",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxDropped }},
			prometheus.NewDesc(
				"network_transmit_errors_total",
				"Total errors while transmitting",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxBytes }},
			prometheus.NewDesc(
				"network_transmit_packets_total",
				"Total packets transmitted",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxBytes }},
		},
	}
}

func (c *slaveCollector) Collect(ch chan<- prometheus.Metric) {
	stats := []executor{}
	c.fetchAndDecode("/monitor/statistics", &stats)

	for _, exec := range stats {
		for desc, m := range c.metrics {
			ch <- prometheus.MustNewConstMetric(desc, m.valueType, m.get(exec.Statistics), exec.ID, exec.FrameworkID, exec.Source)
		}
	}
}

func (c *slaveCollector) Describe(ch chan<- *prometheus.Desc) {
	for metric := range c.metrics {
		ch <- metric
	}
}
<commit_msg>Fix network error and network packet metrics<commit_after>package main

import (
	"github.com/prometheus/client_golang/prometheus"
)

type (
	executor struct {
		ID          string      `json:"executor_id"`
		Name        string      `json:"executor_name"`
		FrameworkID string      `json:"framework_id"`
		Source      string      `json:"source"`
		Statistics  *statistics `json:"statistics"`
		Tasks       []task      `json:"tasks"`
	}

	statistics struct {
		CpusLimit             float64 `json:"cpus_limit"`
		CpusSystemTimeSecs    float64 `json:"cpus_system_time_secs"`
		CpusUserTimeSecs      float64 `json:"cpus_user_time_secs"`
		CpusThrottledTimeSecs float64 `json:"cpus_throttled_time_secs"`

		MemLimitBytes float64 `json:"mem_limit_bytes"`
		MemRssBytes   float64 `json:"mem_rss_bytes"`

		NetRxBytes   float64 `json:"net_rx_bytes"`
		NetRxDropped float64 `json:"net_rx_dropped"`
		NetRxErrors  float64 `json:"net_rx_errors"`
		NetRxPackets float64 `json:"net_rx_packets"`
		NetTxBytes   float64 `json:"net_tx_bytes"`
		NetTxDropped float64 `json:"net_tx_dropped"`
		NetTxErrors  float64 `json:"net_tx_errors"`
		NetTxPackets float64 `json:"net_tx_packets"`
	}

	slaveCollector struct {
		*httpClient
		metrics map[*prometheus.Desc]metric
	}

	metric struct {
		valueType prometheus.ValueType
		get       func(*statistics) float64
	}
)

func newSlaveMonitorCollector(httpClient *httpClient) prometheus.Collector {
	labels := []string{"id", "framework_id", "source"}

	return &slaveCollector{
		httpClient: httpClient,
		metrics: map[*prometheus.Desc]metric{
			// CPU
			prometheus.NewDesc(
				"cpus_limit",
				"Current limit of CPUs for task",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.CpusLimit }},
			prometheus.NewDesc(
				"cpu_system_seconds_total",
				"Total system CPU seconds",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.CpusSystemTimeSecs }},
			prometheus.NewDesc(
				"cpu_user_seconds_total",
				"Total user CPU seconds",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.CpusUserTimeSecs }},
			prometheus.NewDesc(
				"cpu_throttled_seconds_total",
				"Total time CPU was throttled",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.CpusThrottledTimeSecs }},

			// Memory
			prometheus.NewDesc(
				"mem_limit_bytes",
				"Current memory limit in bytes",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.MemLimitBytes }},
			prometheus.NewDesc(
				"mem_rss_bytes",
				"Current rss memory usage",
				labels, nil,
			): metric{prometheus.GaugeValue, func(s *statistics) float64 { return s.MemRssBytes }},

			// Network
			// - RX
			prometheus.NewDesc(
				"network_receive_bytes_total",
				"Total bytes received",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxBytes }},
			prometheus.NewDesc(
				"network_receive_dropped_total",
				"Total packets dropped while receiving",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxDropped }},
			prometheus.NewDesc(
				"network_receive_errors_total",
				"Total errors while receiving",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxErrors }},
			prometheus.NewDesc(
				"network_receive_packets_total",
				"Total packets received",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetRxPackets }},
			// - TX
			prometheus.NewDesc(
				"network_transmit_bytes_total",
				"Total bytes transmitted",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxBytes }},
			prometheus.NewDesc(
				"network_transmit_dropped_total",
				"Total packets dropped while transmitting",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxDropped }},
			prometheus.NewDesc(
				"network_transmit_errors_total",
				"Total errors while transmitting",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxErrors }},
			prometheus.NewDesc(
				"network_transmit_packets_total",
				"Total packets transmitted",
				labels, nil,
			): metric{prometheus.CounterValue, func(s *statistics) float64 { return s.NetTxPackets }},
		},
	}
}

func (c *slaveCollector) Collect(ch chan<- prometheus.Metric) {
	stats := []executor{}
	c.fetchAndDecode("/monitor/statistics", &stats)

	for _, exec := range stats {
		for desc, m := range c.metrics {
			ch <- prometheus.MustNewConstMetric(desc, m.valueType, m.get(exec.Statistics), exec.ID, exec.FrameworkID, exec.Source)
		}
	}
}

func (c *slaveCollector) Describe(ch chan<- *prometheus.Desc) {
	for metric := range c.metrics {
		ch <- metric
	}
}
<|endoftext|>


<commit_before>package units

type Unit struct {
	dim  Dimensionality
	base baseUnits
}

var unitByName map[string]*Unit = map[string]*Unit{
	// Mass Units
	"kg": &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: kilogram}},
	"g":  &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: gram}},
	"lb": &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: pound}},
	"st": &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: stone}},

	// Length Units
	"m":   &Unit{Dimensionality{Length: 1}, baseUnits{Length: meter}},
	"mm":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: millimeter}},
	"cm":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: centimeter}},
	"km":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: kilometer}},
	"mil": &Unit{Dimensionality{Length: 1}, baseUnits{Length: mil}},
	"in":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: inch}},
	"ft":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: foot}},
	"yd":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: yard}},

	// Angle Units
	"deg":  &Unit{Dimensionality{Angle: 1}, baseUnits{Angle: degree}},
	"rad":  &Unit{Dimensionality{Angle: 1}, baseUnits{Angle: radian}},
	"turn": &Unit{Dimensionality{Angle: 1}, baseUnits{Angle: turn}},

	// Time Units
	"s":  &Unit{Dimensionality{Time: 1}, baseUnits{Time: second}}, // There is no secs in physics.
	"ms": &Unit{Dimensionality{Time: 1}, baseUnits{Time: millisecond}},
	"us": &Unit{Dimensionality{Time: 1}, baseUnits{Time: microsecond}},

	// Electric Current Units
	"A": &Unit{Dimensionality{ElectricCurrent: 1}, baseUnits{ElectricCurrent: ampere}},

	// Luminous Intensity Units
	"cd": &Unit{Dimensionality{LuminousIntensity: 1}, baseUnits{LuminousIntensity: candela}},

	// Electic Resistance Units
	"ohm": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
	},
	// TODO: represent kohm, Mohm?

	// Electric Voltage Units
	"V": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -1},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
	},
	// TODO: represent mV, kV

	// Frequency Units
	"Hz": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
	},
	// TODO: represent kHz, mHz

	// Force Units
	"N": &Unit{
		Dimensionality{Mass: 1, Length: 1, Time: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second},
	},

	// Power Units
	"W": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3},
		baseUnits{Mass: kilogram, Length: meter, Time: second},
	},
	// TODO: represent mW, kW

	// Electrical Capacitance Units
	"F": &Unit{
		Dimensionality{Mass: -1, Length: -2, Time: 4, ElectricCurrent: 2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
	},
	// TODO: represent uF, mF

	// Electrical Inductance Units
	"H": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -2, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
	},
	// TODO: represent uH

	// Illuminance units
	"lx": &Unit{
		Dimensionality{Length: -2, LuminousIntensity: 1},
		baseUnits{Length: meter, LuminousIntensity: candela},
	},
}

var unitName map[Unit]string

func init() {
	unitName = make(map[Unit]string, len(unitByName))
	for name, unit := range unitByName {
		unitName[*unit] = name
	}
}
<commit_msg>units: support unit scaling<commit_after>package units

type Unit struct {
	dim  Dimensionality
	base baseUnits

	// For scaled units (millis, kilos, megas, etc) this stores the scale
	// factor. If positive then a value must be divided by it to recover
	// the unscaled unit, while if it's negative a value must be multiplied
	// by its absolute value. 0 means "unscaled"
	//
	// Scaling is only used for derived units. Units of base dimensions are
	// just represented directly.
	scale int
}

var unitByName map[string]*Unit = map[string]*Unit{
	// Mass Units
	"kg": &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: kilogram}, 0},
	"g":  &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: gram}, 0},
	"lb": &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: pound}, 0},
	"st": &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: stone}, 0},

	// Length Units
	"m":   &Unit{Dimensionality{Length: 1}, baseUnits{Length: meter}, 0},
	"mm":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: millimeter}, 0},
	"cm":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: centimeter}, 0},
	"km":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: kilometer}, 0},
	"mil": &Unit{Dimensionality{Length: 1}, baseUnits{Length: mil}, 0},
	"in":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: inch}, 0},
	"ft":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: foot}, 0},
	"yd":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: yard}, 0},

	// Angle Units
	"deg":  &Unit{Dimensionality{Angle: 1}, baseUnits{Angle: degree}, 0},
	"rad":  &Unit{Dimensionality{Angle: 1}, baseUnits{Angle: radian}, 0},
	"turn": &Unit{Dimensionality{Angle: 1}, baseUnits{Angle: turn}, 0},

	// Time Units
	"s":  &Unit{Dimensionality{Time: 1}, baseUnits{Time: second}, 0}, // There is no secs in physics.
	"ms": &Unit{Dimensionality{Time: 1}, baseUnits{Time: millisecond}, 0},
	"us": &Unit{Dimensionality{Time: 1}, baseUnits{Time: microsecond}, 0},

	// Electric Current Units
	"A": &Unit{Dimensionality{ElectricCurrent: 1}, baseUnits{ElectricCurrent: ampere}, 0},

	// Luminous Intensity Units
	"cd": &Unit{Dimensionality{LuminousIntensity: 1}, baseUnits{LuminousIntensity: candela}, 0},

	// Electic Resistance Units
	"ohm": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		0,
	},
	"kohm": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		1000,
	},
	"Mohm": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		1000000,
	},

	// Electric Voltage Units
	"V": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -1},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		0,
	},
	"mV": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -1},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		-1000,
	},
	"kV": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -1},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		1000,
	},

	// Frequency Units
	"Hz": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		0,
	},
	"kHz": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		1000,
	},
	"MHz": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		1000000,
	},

	// Force Units
	"N": &Unit{
		Dimensionality{Mass: 1, Length: 1, Time: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second},
		0,
	},

	// Power Units
	"W": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3},
		baseUnits{Mass: kilogram, Length: meter, Time: second},
		0,
	},
	"mW": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		-1000,
	},
	"kW": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		1000,
	},
	"MW": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		1000000,
	},
	"GW": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		1000000000,
	},

	// Electrical Capacitance Units
	"F": &Unit{
		Dimensionality{Mass: -1, Length: -2, Time: 4, ElectricCurrent: 2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		0,
	},
	"mF": &Unit{
		Dimensionality{Mass: -1, Length: -2, Time: 4, ElectricCurrent: 2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		-1000,
	},
	"uF": &Unit{
		Dimensionality{Mass: -1, Length: -2, Time: 4, ElectricCurrent: 2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		-1000000,
	},

	// Electrical Inductance Units
	"H": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -2, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		0,
	},
	"uH": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -2, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		-1000000,
	},

	// Illuminance units
	"lx": &Unit{
		Dimensionality{Length: -2, LuminousIntensity: 1},
		baseUnits{Length: meter, LuminousIntensity: candela},
		0,
	},
}

var unitName map[Unit]string

func init() {
	unitName = make(map[Unit]string, len(unitByName))
	for name, unit := range unitByName {
		unitName[*unit] = name
	}
}
<|endoftext|>


<commit_before>// Socket package provides a concise, powerful and high-performance TCP socket.
//
// Copyright 2017 HenryLee. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package socket

import (
	"encoding/json"
	"sync"

	"github.com/henrylee2cn/goutil"
)

var packetStack = new(struct {
	freePacket *Packet
	mu         sync.Mutex
})

// GetPacket gets a *Packet form packet stack.
// Note:
//  getBodyFunc is only for writing packet;
//  getBodyFunc should be nil when reading packet.
func GetPacket(getBodyFunc func(*Header) interface{}) *Packet {
	packetStack.mu.Lock()
	p := packetStack.freePacket
	if p == nil {
		p = newPacket(getBodyFunc)
	} else {
		packetStack.freePacket = p.next
		p.reset(getBodyFunc)
	}
	packetStack.mu.Unlock()
	return p
}

// PutPacket puts a *Packet to packet stack.
func PutPacket(p *Packet) {
	packetStack.mu.Lock()
	p.Body = nil
	p.next = packetStack.freePacket
	packetStack.freePacket = p
	packetStack.mu.Unlock()
}

// Packet provides header and body's containers for receiving and sending packet.
type Packet struct {
	// header content
	Header *Header `json:"header"`
	// body content
	Body interface{} `json:"body"`
	// header length
	HeaderLength int64 `json:"header_length"`
	// body length
	BodyLength int64 `json:"body_length"`
	// HeaderLength + BodyLength
	Length int64 `json:"length"`
	// get body by header
	// Note:
	//  only for writing packet;
	//  should be nil when reading packet.
	getBodyByHeader func(*Header) interface{} `json:"-"`
	next            *Packet                   `json:"-"`
}

func newPacket(getBodyFunc func(*Header) interface{}) *Packet {
	var p = &Packet{
		Header:          new(Header),
		getBodyByHeader: getBodyFunc,
	}
	return p
}

func (p *Packet) reset(getBodyFunc func(*Header) interface{}) {
	p.Header.Reset()
	p.Body = nil
	p.getBodyByHeader = getBodyFunc
	p.HeaderLength = 0
	p.BodyLength = 0
	p.Length = 0
}

func (p *Packet) loadBody() interface{} {
	p.Body = p.getBodyByHeader(p.Header)
	return p.Body
}

func (p *Packet) String() string {
	b, _ := json.MarshalIndent(p, "", "  ")
	return goutil.BytesToString(b)
}
<commit_msg>Insert a blank line in the bottom of the package comment<commit_after>// Socket package provides a concise, powerful and high-performance TCP socket.
//
// Copyright 2017 HenryLee. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package socket

import (
	"encoding/json"
	"sync"

	"github.com/henrylee2cn/goutil"
)

var packetStack = new(struct {
	freePacket *Packet
	mu         sync.Mutex
})

// GetPacket gets a *Packet form packet stack.
// Note:
//  getBodyFunc is only for writing packet;
//  getBodyFunc should be nil when reading packet.
func GetPacket(getBodyFunc func(*Header) interface{}) *Packet {
	packetStack.mu.Lock()
	p := packetStack.freePacket
	if p == nil {
		p = newPacket(getBodyFunc)
	} else {
		packetStack.freePacket = p.next
		p.reset(getBodyFunc)
	}
	packetStack.mu.Unlock()
	return p
}

// PutPacket puts a *Packet to packet stack.
func PutPacket(p *Packet) {
	packetStack.mu.Lock()
	p.Body = nil
	p.next = packetStack.freePacket
	packetStack.freePacket = p
	packetStack.mu.Unlock()
}

// Packet provides header and body's containers for receiving and sending packet.
type Packet struct {
	// header content
	Header *Header `json:"header"`
	// body content
	Body interface{} `json:"body"`
	// header length
	HeaderLength int64 `json:"header_length"`
	// body length
	BodyLength int64 `json:"body_length"`
	// HeaderLength + BodyLength
	Length int64 `json:"length"`
	// get body by header
	// Note:
	//  only for writing packet;
	//  should be nil when reading packet.
	getBodyByHeader func(*Header) interface{} `json:"-"`
	next            *Packet                   `json:"-"`
}

func newPacket(getBodyFunc func(*Header) interface{}) *Packet {
	var p = &Packet{
		Header:          new(Header),
		getBodyByHeader: getBodyFunc,
	}
	return p
}

func (p *Packet) reset(getBodyFunc func(*Header) interface{}) {
	p.Header.Reset()
	p.Body = nil
	p.getBodyByHeader = getBodyFunc
	p.HeaderLength = 0
	p.BodyLength = 0
	p.Length = 0
}

func (p *Packet) loadBody() interface{} {
	p.Body = p.getBodyByHeader(p.Header)
	return p.Body
}

func (p *Packet) String() string {
	b, _ := json.MarshalIndent(p, "", "  ")
	return goutil.BytesToString(b)
}
<|endoftext|>


<commit_before>package cache

import (
	"errors"
	"fmt"
	"testing"

	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fstest/mockfs"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var (
	called      = 0
	errSentinel = errors.New("an error")
)

func mockNewFs(t *testing.T) (func(), func(path string) (fs.Fs, error)) {
	called = 0
	create := func(path string) (fs.Fs, error) {
		assert.Equal(t, 0, called)
		called++
		switch path {
		case "/":
			return mockfs.NewFs("mock", "mock"), nil
		case "/file.txt":
			return mockfs.NewFs("mock", "mock"), fs.ErrorIsFile
		case "/error":
			return nil, errSentinel
		}
		panic(fmt.Sprintf("Unknown path %q", path))
	}
	cleanup := func() {
		c.Clear()
	}
	return cleanup, create
}

func TestGet(t *testing.T) {
	cleanup, create := mockNewFs(t)
	defer cleanup()

	assert.Equal(t, 0, c.Entries())

	f, err := GetFn("/", create)
	require.NoError(t, err)

	assert.Equal(t, 1, c.Entries())

	f2, err := GetFn("/", create)
	require.NoError(t, err)

	assert.Equal(t, f, f2)
}

func TestGetFile(t *testing.T) {
	cleanup, create := mockNewFs(t)
	defer cleanup()

	assert.Equal(t, 0, c.Entries())

	f, err := GetFn("/file.txt", create)
	require.Equal(t, fs.ErrorIsFile, err)

	assert.Equal(t, 1, c.Entries())

	f2, err := GetFn("/file.txt", create)
	require.Equal(t, fs.ErrorIsFile, err)

	assert.Equal(t, f, f2)
}

func TestGetError(t *testing.T) {
	cleanup, create := mockNewFs(t)
	defer cleanup()

	assert.Equal(t, 0, c.Entries())

	f, err := GetFn("/error", create)
	require.Equal(t, errSentinel, err)
	require.Equal(t, nil, f)

	assert.Equal(t, 0, c.Entries())
}

func TestPut(t *testing.T) {
	cleanup, create := mockNewFs(t)
	defer cleanup()

	f := mockfs.NewFs("mock", "mock")

	assert.Equal(t, 0, c.Entries())

	Put("/alien", f)

	assert.Equal(t, 1, c.Entries())

	fNew, err := GetFn("/alien", create)
	require.NoError(t, err)
	require.Equal(t, f, fNew)

	assert.Equal(t, 1, c.Entries())
}
<commit_msg>fs/cache: test for fix cached values pointing to files #3424<commit_after>package cache

import (
	"errors"
	"fmt"
	"testing"

	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fstest/mockfs"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var (
	called      = 0
	errSentinel = errors.New("an error")
)

func mockNewFs(t *testing.T) (func(), func(path string) (fs.Fs, error)) {
	called = 0
	create := func(path string) (fs.Fs, error) {
		assert.Equal(t, 0, called)
		called++
		switch path {
		case "/":
			return mockfs.NewFs("mock", "mock"), nil
		case "/file.txt":
			return mockfs.NewFs("mock", "mock"), fs.ErrorIsFile
		case "/error":
			return nil, errSentinel
		}
		panic(fmt.Sprintf("Unknown path %q", path))
	}
	cleanup := func() {
		c.Clear()
	}
	return cleanup, create
}

func TestGet(t *testing.T) {
	cleanup, create := mockNewFs(t)
	defer cleanup()

	assert.Equal(t, 0, c.Entries())

	f, err := GetFn("/", create)
	require.NoError(t, err)

	assert.Equal(t, 1, c.Entries())

	f2, err := GetFn("/", create)
	require.NoError(t, err)

	assert.Equal(t, f, f2)
}

func TestGetFile(t *testing.T) {
	cleanup, create := mockNewFs(t)
	defer cleanup()

	assert.Equal(t, 0, c.Entries())

	f, err := GetFn("/file.txt", create)
	require.Equal(t, fs.ErrorIsFile, err)
	require.NotNil(t, f)

	assert.Equal(t, 1, c.Entries())

	f2, err := GetFn("/file.txt", create)
	require.Equal(t, fs.ErrorIsFile, err)
	require.NotNil(t, f2)

	assert.Equal(t, f, f2)
}

func TestGetError(t *testing.T) {
	cleanup, create := mockNewFs(t)
	defer cleanup()

	assert.Equal(t, 0, c.Entries())

	f, err := GetFn("/error", create)
	require.Equal(t, errSentinel, err)
	require.Equal(t, nil, f)

	assert.Equal(t, 0, c.Entries())
}

func TestPut(t *testing.T) {
	cleanup, create := mockNewFs(t)
	defer cleanup()

	f := mockfs.NewFs("mock", "mock")

	assert.Equal(t, 0, c.Entries())

	Put("/alien", f)

	assert.Equal(t, 1, c.Entries())

	fNew, err := GetFn("/alien", create)
	require.NoError(t, err)
	require.Equal(t, f, fNew)

	assert.Equal(t, 1, c.Entries())
}
<|endoftext|>


<commit_before>package sokudoku

import "testing"

func TestBuildPhrase(t *testing.T) {
	p, _ := phraseInit()

	got, _ := p.Parse("ぼくはくま。フワフワした赤い大きな熊。「やあ、こんにちは。」「やあ」と挨拶")

	expects := []string{
		"ぼくは",
		"くま。",
		"フワフワした",
		"赤い",
		"大きな",
		"熊。",
		"「やあ、",
		"こんにちは。」",
		"「やあ」と",
		"挨拶",
	}
	for i, expect := range expects {
		if expect != got[i] {
			t.Error(
				"\n   got:", got[i],
				"\nexpect:", expect,
			)
			break
		}
	}
}
<commit_msg>refactored test code<commit_after>package sokudoku

import "testing"

type testCase struct {
	sample string
	expect []string
}

func TestBuildPhrase(t *testing.T) {
	p, _ := phraseInit()

	for _, test_case := range []testCase{
		{
			"ぼくはくま。フワフワした赤い大きな熊。「やあ、こんにちは。」「やあ」と挨拶",
			[]string{"ぼくは", "くま。", "フワフワした", "赤い", "大きな", "熊。", "「やあ、", "こんにちは。」", "「やあ」と", "挨拶"},
		},
		{
			"荒波に負けない心と、切なさと、優しさと。",
			[]string{"荒波に", "負けない", "心と、", "切なさと、", "優しさと。"},
		},
	} {
		got, _ := p.Parse(test_case.sample)
		for i, expect := range test_case.expect {
			if expect != got[i] {
				t.Error(
					"\n   got:", got[i],
					"\nexpect:", expect,
				)
				break
			}
		}
	}

}
<|endoftext|>


<commit_before>package utils

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"time"
)

type RestClient struct {
	client           *http.Client
	Headers          map[string]string
	lastResponse     http.Response
	lastResponseBody []byte
}

func NewRestClient() *RestClient {
	timeout := time.Duration(5 * time.Second)
	return &RestClient{&http.Client{Timeout: timeout}, nil, http.Response{}, nil}
}

func (c *RestClient) ResponseBody() []byte {
	return c.lastResponseBody
}

func (c *RestClient) ResponseStatus() int {
	return c.lastResponse.StatusCode
}

func doRequest(c *RestClient, req *http.Request) error {
	for name, value := range c.Headers {
		req.Header.Add(name, value)
	}
	response, err := c.client.Do(req)
	if err != nil {
		return err
	}
	defer response.Body.Close()
	c.lastResponse = *response
	c.lastResponseBody, err = ioutil.ReadAll(response.Body)
	return err
}

func (c *RestClient) GetWithBytes(url string, byteData []byte) error {
	req, err := http.NewRequest("GET", url, bytes.NewBuffer(byteData))
	if err != nil {
		return err
	}

	return doRequest(c, req)
}

func (c *RestClient) Get(url string) error {
	return c.GetWithBytes(url, nil)
}

func (c *RestClient) Post(url string, data io.Reader) error {

	req, err := http.NewRequest("POST", url, data)
	if err != nil {
		return err
	}

	return doRequest(c, req)
}

func (c *RestClient) PostString(url string, data string) error {
	return c.Post(url, bytes.NewBufferString(data))
}

func (c *RestClient) PostJson(url string, data interface{}) error {

	byteData, err := json.Marshal(data)
	if err != nil {
		return err
	}
	return c.Post(url, bytes.NewBuffer(byteData))

}
<commit_msg>Added accessor for http.client<commit_after>package utils

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"time"
)

type RestClient struct {
	Headers map[string]string

	client           *http.Client
	lastResponse     http.Response
	lastResponseBody []byte
}

func NewRestClient() *RestClient {
	timeout := time.Duration(5 * time.Second)
	return &RestClient{nil, &http.Client{Timeout: timeout}, http.Response{}, nil}
}

func (c *RestClient) Client() *http.Client {
	return c.Client()
}

func (c *RestClient) LastResponseBody() []byte {
	return c.lastResponseBody
}

func (c *RestClient) LastResponseStatus() int {
	return c.lastResponse.StatusCode
}

func doRequest(c *RestClient, req *http.Request) error {
	for name, value := range c.Headers {
		req.Header.Add(name, value)
	}
	response, err := c.client.Do(req)
	if err != nil {
		return err
	}
	defer response.Body.Close()
	c.lastResponse = *response
	c.lastResponseBody, err = ioutil.ReadAll(response.Body)
	return err
}

func (c *RestClient) GetWithBytes(url string, byteData []byte) error {
	req, err := http.NewRequest("GET", url, bytes.NewBuffer(byteData))
	if err != nil {
		return err
	}

	return doRequest(c, req)
}

func (c *RestClient) Get(url string) error {
	return c.GetWithBytes(url, nil)
}

func (c *RestClient) Post(url string, data io.Reader) error {

	req, err := http.NewRequest("POST", url, data)
	if err != nil {
		return err
	}

	return doRequest(c, req)
}

func (c *RestClient) PostString(url string, data string) error {
	return c.Post(url, bytes.NewBufferString(data))
}

func (c *RestClient) PostJson(url string, data interface{}) error {

	byteData, err := json.Marshal(data)
	if err != nil {
		return err
	}
	return c.Post(url, bytes.NewBuffer(byteData))

}
<|endoftext|>


<commit_before>// Copyright 2016-2021 The Libsacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package libsacloud

// Version バージョン
const Version = "2.26.0"
<commit_msg>Bump to v2.27.0<commit_after>// Copyright 2016-2021 The Libsacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package libsacloud

// Version バージョン
const Version = "2.27.0"
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"sort"
	"testing"

	"github.com/bazelbuild/bazelisk/core"
	"github.com/bazelbuild/bazelisk/runfiles"
)

func TestScanIssuesForIncompatibleFlags(t *testing.T) {
	samplesJSON, err := runfiles.ReadFile("sample-issues-migration.json")
	if err != nil {
		t.Errorf("Can not load sample github issues")
	}
	flags, err := core.ScanIssuesForIncompatibleFlags(samplesJSON)
	if flags == nil {
		t.Errorf("Could not parse sample issues")
	}
	expectedFlagnames := []string{
		"--//some/path:incompatible_user_defined_flag",
		"--incompatible_always_check_depset_elements",
		"--incompatible_no_implicit_file_export",
		"--incompatible_remove_enabled_toolchain_types",
		"--incompatible_remove_ram_utilization_factor",
		"--incompatible_validate_top_level_header_inclusions",
	}
	var gotFlags []string
	for _, flag := range flags {
		fmt.Printf("%s\n", flag.String())
		gotFlags = append(gotFlags, flag.Name)
	}
	sort.Strings(gotFlags)
	mismatch := false
	for i, got := range gotFlags {
		if expectedFlagnames[i] != got {
			mismatch = true
			break
		}
	}
	if mismatch || len(expectedFlagnames) != len(gotFlags) {
		t.Errorf("Expected %s, got %s", expectedFlagnames, gotFlags)
	}
}
<commit_msg>Check error in TestScanIssuesForIncompatibleFlags (#183)<commit_after>package main

import (
	"fmt"
	"sort"
	"testing"

	"github.com/bazelbuild/bazelisk/core"
	"github.com/bazelbuild/bazelisk/runfiles"
)

func TestScanIssuesForIncompatibleFlags(t *testing.T) {
	samplesJSON, err := runfiles.ReadFile("sample-issues-migration.json")
	if err != nil {
		t.Errorf("Can not load sample github issues")
	}
	flags, err := core.ScanIssuesForIncompatibleFlags(samplesJSON)
	if err != nil || flags == nil {
		t.Errorf("Could not parse sample issues")
	}
	expectedFlagnames := []string{
		"--//some/path:incompatible_user_defined_flag",
		"--incompatible_always_check_depset_elements",
		"--incompatible_no_implicit_file_export",
		"--incompatible_remove_enabled_toolchain_types",
		"--incompatible_remove_ram_utilization_factor",
		"--incompatible_validate_top_level_header_inclusions",
	}
	var gotFlags []string
	for _, flag := range flags {
		fmt.Printf("%s\n", flag.String())
		gotFlags = append(gotFlags, flag.Name)
	}
	sort.Strings(gotFlags)
	mismatch := false
	for i, got := range gotFlags {
		if expectedFlagnames[i] != got {
			mismatch = true
			break
		}
	}
	if mismatch || len(expectedFlagnames) != len(gotFlags) {
		t.Errorf("Expected %s, got %s", expectedFlagnames, gotFlags)
	}
}
<|endoftext|>


<commit_before>package downloader

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"sync"
	"path/filepath"

	"github.com/ibmjstart/cf-download/cmd_exec"
	"github.com/ibmjstart/cf-download/dir_parser"
	"github.com/ibmjstart/cf-download/filter"
	"github.com/mgutz/ansi"
)

type Downloader interface {
	Download(files, dirs []string, readPath, writePath string, filterList []string) error
	DownloadFile(readPath, writePath string, fileDownloadGroup *sync.WaitGroup) error
	WriteFile(readPath, writePath string, output []byte, err error) error
	CheckDownload(readPath string, file []string, err error) error
	GetFilesDownloadedCount() int
	GetFailedDownloads() []string
}

type downloader struct {
	cmdExec                    cmd_exec.CmdExec
	rootWorkingDirectoryServer string
	appName                    string
	instance                   string
	verbose                    bool
	onWindows                  bool
	failedDownloads            []string
	filesDownloaded            int
	parser                     dir_parser.Parser
	wg                         *sync.WaitGroup
}

func NewDownloader(cmdExec cmd_exec.CmdExec, WG *sync.WaitGroup, appName, instance, rootWorkingDirectoryServer string, verbose, onWindows bool) *downloader {

	return &downloader{
		cmdExec:                    cmdExec,
		rootWorkingDirectoryServer: rootWorkingDirectoryServer,
		appName:                    appName,
		instance:                   instance,
		verbose:                    verbose,
		onWindows:                  onWindows,
		parser:                     dir_parser.NewParser(cmdExec, appName, instance, onWindows, verbose),
		wg:                         WG,
	}
}

// error struct that allows appending error messages
type cliError struct {
	err    error
	errMsg string
}

/*
*	given file and directory names, download() will download the files from
* 	'readPath' and write them to disk on the 'writepath'.
* 	the function calls it's self recursively for each directory as it travels down the tree.
* 	Each call runs on a seperate go routine and and calls a go routine for every
* 	file download.
 */
func (d *downloader) Download(files, dirs []string, readPath, writePath string, filterList []string) error {
	defer d.wg.Done()

	//create dir if does not exist
	err := os.MkdirAll(writePath, 0755)
	check(err, "Error D1: failed to create directory.")

	// download each file
	for _, val := range files {
		fileWPath := writePath + val
		fileRPath := readPath + val

		filePath := strings.TrimPrefix(strings.TrimSuffix(fileRPath, "/"), d.rootWorkingDirectoryServer)

		if filter.CheckToFilter(filePath, filterList) {
			continue
		}

		d.wg.Add(1)
		go d.DownloadFile(fileRPath, fileWPath, d.wg)
	}

	// call download on every sub directory
	for _, val := range dirs {
		dirWPath := writePath + filepath.FromSlash(val)
		dirRPath := readPath + val
		dirPath := strings.TrimPrefix(strings.TrimSuffix(dirRPath, "/"), d.rootWorkingDirectoryServer)

		if filter.CheckToFilter(dirPath, filterList) {
			continue
		}

		err := os.MkdirAll(dirWPath, 0755)
		check(err, "Error D2: failed to create directory.")

		files, dirs = d.parser.ExecParseDir(dirRPath)

		d.wg.Add(1)
		go d.Download(files, dirs, dirRPath, dirWPath, filterList)
	}
	return nil
}

/*
*	downloadFile() takes a 'readPath' which corresponds to a file in the cf app. The file is
*	downloaded using the cmd_exec package which uses the os/exec library to call cf files with the given readPath. The output is
*	written to a file at writePath.
 */
func (d *downloader) DownloadFile(readPath, writePath string, fileDownloadGroup *sync.WaitGroup) error {
	defer fileDownloadGroup.Done()

	output, err := d.cmdExec.GetFile(d.appName, readPath, d.instance)

	d.WriteFile(readPath, writePath, output, err)

	return nil
}

func (d *downloader) WriteFile(readPath, writePath string, output []byte, err error) error {
	file := strings.SplitAfterN(string(output), "\n", 3)

	// check for invalid files or download issues
	downloadErr := d.CheckDownload(readPath, file, err)

	if d.verbose {
		fmt.Printf("Writing file: %s\n", readPath)
	} else {
		// increment download counter for commandline display
		// see consoleWriter() in main.go
		d.filesDownloaded++
	}

	if downloadErr == nil {
		fileAsString := file[2]
		// write downloaded file to writePath
		err = ioutil.WriteFile(writePath, []byte(fileAsString), 0644)
	}

	return err
}

func (d *downloader) CheckDownload(readPath string, file []string, err error) error {
	errMsg := createMessage(" Server Error: '"+readPath+"' not downloaded", "yellow", d.onWindows)
	if len(file) < 2 {
		// Sometimes files are inaccessible, but the api does not alert us by using a FAILED response.
		// alert users if it occurs (verbose). It usually happens in vendor files.
		d.failedDownloads = append(d.failedDownloads, errMsg)
		if d.verbose {
			fmt.Println(errMsg)
		}
		return errors.New("download failed (empty)")
	} else if strings.Contains(file[1], "502") || strings.Contains(file[1], "500") {
		// Caused by cf cli api connection issues
		PrintSlice(file)
		d.failedDownloads = append(d.failedDownloads, errMsg)
		return errors.New("502")
		// TODO: add these files to a retry queue and retry downloading them at the end. (see feature branch)
	} else if strings.Contains(file[1], "status code: 400") {
		// Caused by cf cli api connection issues
		PrintSlice(file)
		d.failedDownloads = append(d.failedDownloads, errMsg)
		return errors.New("400")
		// TODO: add these files to a retry queue and retry downloading them at the end. (see feature branch)
	} else if strings.Contains(file[1], "FAILED") {
		// some files are inaccesible from the cf files (permission issues) this is rare but we need to
		// alert users if it occurs. It usually happens in vendor files.
		d.failedDownloads = append(d.failedDownloads, errMsg)
		if d.verbose {
			fmt.Println(errMsg)
		}
		return errors.New("download failed")
	} else {
		// check for other errors
		if err != nil {
			PrintSlice(file)
		}
		check(err, "Error CD1: [cf files "+d.appName+" "+readPath+"]")
	}
	return nil
}

func (d *downloader) GetFilesDownloadedCount() int {
	return d.filesDownloaded
}

func (d *downloader) GetFailedDownloads() []string {
	return d.failedDownloads
}

// error check function
func check(e error, errMsg string) {
	if e != nil {
		fmt.Println("\nError: ", e)
		if errMsg != "" {
			fmt.Println("Message: ", errMsg)
		}
		os.Exit(1)
	}
}

// prints slices in readable format
func PrintSlice(slice []string) error {
	for index, val := range slice {
		fmt.Println(index, ": ", val)
	}
	return nil
}

func createMessage(message, color string, onWindows bool) string {
	errmsg := ansi.Color(message, color)
	if onWindows == true {
		errmsg = message
	}

	return errmsg
}
<commit_msg>changed error handling to treat all failures the same in download.go<commit_after>package downloader

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"sync"
	"path/filepath"

	"github.com/ibmjstart/cf-download/cmd_exec"
	"github.com/ibmjstart/cf-download/dir_parser"
	"github.com/ibmjstart/cf-download/filter"
	"github.com/mgutz/ansi"
)

type Downloader interface {
	Download(files, dirs []string, readPath, writePath string, filterList []string) error
	DownloadFile(readPath, writePath string, fileDownloadGroup *sync.WaitGroup) error
	WriteFile(readPath, writePath string, output []byte, err error) error
	CheckDownload(readPath string, file []string, err error) error
	GetFilesDownloadedCount() int
	GetFailedDownloads() []string
}

type downloader struct {
	cmdExec                    cmd_exec.CmdExec
	rootWorkingDirectoryServer string
	appName                    string
	instance                   string
	verbose                    bool
	onWindows                  bool
	failedDownloads            []string
	filesDownloaded            int
	parser                     dir_parser.Parser
	wg                         *sync.WaitGroup
}

func NewDownloader(cmdExec cmd_exec.CmdExec, WG *sync.WaitGroup, appName, instance, rootWorkingDirectoryServer string, verbose, onWindows bool) *downloader {

	return &downloader{
		cmdExec:                    cmdExec,
		rootWorkingDirectoryServer: rootWorkingDirectoryServer,
		appName:                    appName,
		instance:                   instance,
		verbose:                    verbose,
		onWindows:                  onWindows,
		parser:                     dir_parser.NewParser(cmdExec, appName, instance, onWindows, verbose),
		wg:                         WG,
	}
}

// error struct that allows appending error messages
type cliError struct {
	err    error
	errMsg string
}

/*
*	given file and directory names, download() will download the files from
* 	'readPath' and write them to disk on the 'writepath'.
* 	the function calls it's self recursively for each directory as it travels down the tree.
* 	Each call runs on a seperate go routine and and calls a go routine for every
* 	file download.
 */
func (d *downloader) Download(files, dirs []string, readPath, writePath string, filterList []string) error {
	defer d.wg.Done()

	//create dir if does not exist
	err := os.MkdirAll(writePath, 0755)
	check(err, "Error D1: failed to create directory.")

	// download each file
	for _, val := range files {
		fileWPath := writePath + val
		fileRPath := readPath + val

		filePath := strings.TrimPrefix(strings.TrimSuffix(fileRPath, "/"), d.rootWorkingDirectoryServer)

		if filter.CheckToFilter(filePath, filterList) {
			continue
		}

		d.wg.Add(1)
		go d.DownloadFile(fileRPath, fileWPath, d.wg)
	}

	// call download on every sub directory
	for _, val := range dirs {
		dirWPath := writePath + filepath.FromSlash(val)
		dirRPath := readPath + val
		dirPath := strings.TrimPrefix(strings.TrimSuffix(dirRPath, "/"), d.rootWorkingDirectoryServer)

		if filter.CheckToFilter(dirPath, filterList) {
			continue
		}

		err := os.MkdirAll(dirWPath, 0755)
		check(err, "Error D2: failed to create directory.")

		files, dirs = d.parser.ExecParseDir(dirRPath)

		d.wg.Add(1)
		go d.Download(files, dirs, dirRPath, dirWPath, filterList)
	}
	return nil
}

/*
*	downloadFile() takes a 'readPath' which corresponds to a file in the cf app. The file is
*	downloaded using the cmd_exec package which uses the os/exec library to call cf files with the given readPath. The output is
*	written to a file at writePath.
 */
func (d *downloader) DownloadFile(readPath, writePath string, fileDownloadGroup *sync.WaitGroup) error {
	defer fileDownloadGroup.Done()

	output, err := d.cmdExec.GetFile(d.appName, readPath, d.instance)

	d.WriteFile(readPath, writePath, output, err)

	return nil
}

func (d *downloader) WriteFile(readPath, writePath string, output []byte, err error) error {
	file := strings.SplitAfterN(string(output), "\n", 3)

	// check for invalid files or download issues
	downloadErr := d.CheckDownload(readPath, file, err)

	if d.verbose {
		if downloadErr == nil {
			fmt.Printf("Writing file: %s\n", readPath)
		}
	} else {
		// increment download counter for commandline display
		// see consoleWriter() in main.go
		d.filesDownloaded++
	}

	if downloadErr == nil {
		fileAsString := file[2]

		// there is currently an issue open to change the behavior for empty files
		// https://github.com/cloudfoundry/cli/issues/869
		if strings.Compare(fileAsString, "\nNo files found\n") == 0 {
			fileAsString = ""
		}

		// write downloaded file to writePath
		err = ioutil.WriteFile(writePath, []byte(fileAsString), 0644)
	}

	return err
}

func (d *downloader) CheckDownload(readPath string, file []string, err error) error {
	if len(file) >= 2 && strings.Contains(file[1], "OK"){
		return nil
	} else {
		errMsg := createMessage(" Server Error: '"+readPath+"' not downloaded", "yellow", d.onWindows)

		d.failedDownloads = append(d.failedDownloads, errMsg)

		if d.verbose {
			fmt.Println(errMsg)
			// check for other errors
			if err != nil && len(file) >= 2 {
				PrintSlice(file)
			}
		}
		return errors.New("download failed")
	}
}

func (d *downloader) GetFilesDownloadedCount() int {
	return d.filesDownloaded
}

func (d *downloader) GetFailedDownloads() []string {
	return d.failedDownloads
}

// error check function
func check(e error, errMsg string) {
	if e != nil {
		fmt.Println("\nError: ", e)
		if errMsg != "" {
			fmt.Println("Message: ", errMsg)
		}
		os.Exit(1)
	}
}

// prints slices in readable format
func PrintSlice(slice []string) error {
	for index, val := range slice {
		fmt.Println(index, ": ", val)
	}
	return nil
}

func createMessage(message, color string, onWindows bool) string {
	errmsg := ansi.Color(message, color)
	if onWindows == true {
		errmsg = message
	}

	return errmsg
}
<|endoftext|>


<commit_before>package beater

import (
	"bytes"
	"fmt"
	"regexp"
	"time"

	"github.com/e-travel/cloudwatchlogsbeat/config"
	"github.com/elastic/beats/libbeat/logp"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
)

type Stream struct {
	// the stream's name
	Name string
	// the group to which the stream belongs
	Group *Group
	// our AWS client
	Client cloudwatchlogsiface.CloudWatchLogsAPI
	// parameters for stream event query
	Params *cloudwatchlogs.GetLogEventsInput
	// the stream registry storage
	Registry Registry
	// This is used for multi line mode. We store all text needed until we find
	// the end of message
	Buffer     bytes.Buffer
	Multiline  *config.Multiline
	MultiRegex *regexp.Regexp
	// the publisher for our events
	Publisher EventPublisher
	// the last event that we've processed (in milliseconds since 1970)
	LastEventTimestamp int64
	// channel for the stream to signal that its processing is over
	finished chan<- bool
	// number of published events
	publishedEvents int64
}

func NewStream(name string, group *Group, client cloudwatchlogsiface.CloudWatchLogsAPI,
	registry Registry, finished chan<- bool) *Stream {

	startTime := time.Now().UTC().Add(-group.Beat.Config.StreamEventHorizon)

	params := &cloudwatchlogs.GetLogEventsInput{
		LogGroupName:  aws.String(group.Name),
		LogStreamName: aws.String(name),
		StartFromHead: aws.Bool(true),
		Limit:         aws.Int64(100),
		StartTime:     aws.Int64(startTime.UnixNano() / 1e6),
	}

	stream := &Stream{
		Name:               name,
		Group:              group,
		Client:             client,
		Params:             params,
		Registry:           registry,
		Multiline:          group.Prospector.Multiline,
		Publisher:          Publisher{},
		finished:           finished,
		LastEventTimestamp: 1000 * time.Now().Unix(),
	}

	// Construct regular expression if multiline mode
	var regx *regexp.Regexp
	var err error
	if stream.Multiline != nil {
		regx, err = regexp.Compile(stream.Multiline.Pattern)
		Fatal(err)
	}
	stream.MultiRegex = regx

	return stream
}

// Fetches the next batch of events from the cloudwatchlogs stream
// returns the error (if any) otherwise nil
func (stream *Stream) Next() error {
	var err error

	output, err := stream.Client.GetLogEvents(stream.Params)
	if err != nil {
		return err
	}

	// have we got anything new?
	if len(output.Events) == 0 {
		return nil
	}
	// process the events
	for _, streamEvent := range output.Events {
		stream.digest(streamEvent)
		stream.LastEventTimestamp = aws.Int64Value(streamEvent.Timestamp)
	}
	stream.Params.NextToken = output.NextForwardToken
	err = stream.Registry.WriteStreamInfo(stream)
	return err
}

// Coninuously monitors the stream for new events. If an error is
// encountered, monitoring will stop and the stream will send an event
// to the finished channel for the group to cleanup
func (stream *Stream) Monitor() {
	logp.Info("[stream] %s started", stream.FullName())

	defer func() {
		stream.finished <- true
	}()

	defer logp.Info("[stream] %s stopped", stream.FullName())

	// first of all, read the stream's info from our registry storage
	err := stream.Registry.ReadStreamInfo(stream)
	if err != nil {
		return
	}

	reportTicker := time.NewTicker(reportFrequency)
	defer reportTicker.Stop()

	var eventRefreshFrequency = stream.Group.Beat.Config.StreamEventRefreshFrequency

	for {
		err := stream.Next()
		if err != nil {
			logp.Err("%s %s", stream.FullName(), err.Error())
			return
		}
		// is the stream expired?
		if IsBefore(stream.Group.Beat.Config.StreamEventHorizon, stream.LastEventTimestamp) {
			return
		}
		// is the stream "hot"?
		if stream.IsHot(stream.LastEventTimestamp) {
			eventRefreshFrequency = stream.Group.Beat.Config.HotStreamEventRefreshFrequency
		} else {
			eventRefreshFrequency = stream.Group.Beat.Config.StreamEventRefreshFrequency
		}
		select {
		case <-reportTicker.C:
			stream.report()
		default:
			time.Sleep(eventRefreshFrequency)
		}
	}
}

func (stream *Stream) IsHot(lastEventTimestamp int64) bool {
	return !IsBefore(stream.Group.Beat.Config.HotStreamEventHorizon, lastEventTimestamp)
}

func (stream *Stream) report() {
	logp.Info("report[stream] %d %s %s",
		stream.publishedEvents, stream.FullName(), reportFrequency)
	stream.publishedEvents = 0
}

func (stream *Stream) FullName() string {
	return fmt.Sprintf("%s/%s", stream.Group.Name, stream.Name)
}

// fills the buffer's contents into the event,
// publishes the message and empties the buffer
func (stream *Stream) publish(event *Event) {
	if stream.Buffer.Len() == 0 {
		return
	}
	event.Message = stream.Buffer.String()
	stream.Publisher.Publish(event)
	stream.Buffer.Reset()
	stream.publishedEvents++
}

func (stream *Stream) digest(streamEvent *cloudwatchlogs.OutputLogEvent) {
	event := &Event{
		Stream:    stream,
		Timestamp: aws.Int64Value(streamEvent.Timestamp),
	}
	if stream.Multiline == nil {
		stream.Buffer.WriteString(*streamEvent.Message)
		stream.publish(event)
	} else {
		switch stream.Multiline.Match {
		case "after":
			if stream.MultiRegex.MatchString(*streamEvent.Message) == stream.Multiline.Negate {
				stream.publish(event)
			}
			stream.Buffer.WriteString(*streamEvent.Message)
		case "before":
			stream.Buffer.WriteString(*streamEvent.Message)
			if stream.MultiRegex.MatchString(*streamEvent.Message) == stream.Multiline.Negate {
				stream.publish(event)
			}
		}
	}
}
<commit_msg>Consolidates defer statements in stream's Monitor()<commit_after>package beater

import (
	"bytes"
	"fmt"
	"regexp"
	"time"

	"github.com/e-travel/cloudwatchlogsbeat/config"
	"github.com/elastic/beats/libbeat/logp"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
)

type Stream struct {
	// the stream's name
	Name string
	// the group to which the stream belongs
	Group *Group
	// our AWS client
	Client cloudwatchlogsiface.CloudWatchLogsAPI
	// parameters for stream event query
	Params *cloudwatchlogs.GetLogEventsInput
	// the stream registry storage
	Registry Registry
	// This is used for multi line mode. We store all text needed until we find
	// the end of message
	Buffer     bytes.Buffer
	Multiline  *config.Multiline
	MultiRegex *regexp.Regexp
	// the publisher for our events
	Publisher EventPublisher
	// the last event that we've processed (in milliseconds since 1970)
	LastEventTimestamp int64
	// channel for the stream to signal that its processing is over
	finished chan<- bool
	// number of published events
	publishedEvents int64
}

func NewStream(name string, group *Group, client cloudwatchlogsiface.CloudWatchLogsAPI,
	registry Registry, finished chan<- bool) *Stream {

	startTime := time.Now().UTC().Add(-group.Beat.Config.StreamEventHorizon)

	params := &cloudwatchlogs.GetLogEventsInput{
		LogGroupName:  aws.String(group.Name),
		LogStreamName: aws.String(name),
		StartFromHead: aws.Bool(true),
		Limit:         aws.Int64(100),
		StartTime:     aws.Int64(startTime.UnixNano() / 1e6),
	}

	stream := &Stream{
		Name:               name,
		Group:              group,
		Client:             client,
		Params:             params,
		Registry:           registry,
		Multiline:          group.Prospector.Multiline,
		Publisher:          Publisher{},
		finished:           finished,
		LastEventTimestamp: 1000 * time.Now().Unix(),
	}

	// Construct regular expression if multiline mode
	var regx *regexp.Regexp
	var err error
	if stream.Multiline != nil {
		regx, err = regexp.Compile(stream.Multiline.Pattern)
		Fatal(err)
	}
	stream.MultiRegex = regx

	return stream
}

// Fetches the next batch of events from the cloudwatchlogs stream
// returns the error (if any) otherwise nil
func (stream *Stream) Next() error {
	var err error

	output, err := stream.Client.GetLogEvents(stream.Params)
	if err != nil {
		return err
	}

	// have we got anything new?
	if len(output.Events) == 0 {
		return nil
	}
	// process the events
	for _, streamEvent := range output.Events {
		stream.digest(streamEvent)
		stream.LastEventTimestamp = aws.Int64Value(streamEvent.Timestamp)
	}
	stream.Params.NextToken = output.NextForwardToken
	err = stream.Registry.WriteStreamInfo(stream)
	return err
}

// Coninuously monitors the stream for new events. If an error is
// encountered, monitoring will stop and the stream will send an event
// to the finished channel for the group to cleanup
func (stream *Stream) Monitor() {
	logp.Info("[stream] %s started", stream.FullName())

	defer func() {
		logp.Info("[stream] %s stopped", stream.FullName())
		stream.finished <- true
	}()

	// first of all, read the stream's info from our registry storage
	err := stream.Registry.ReadStreamInfo(stream)
	if err != nil {
		return
	}

	reportTicker := time.NewTicker(reportFrequency)
	defer reportTicker.Stop()

	var eventRefreshFrequency = stream.Group.Beat.Config.StreamEventRefreshFrequency

	for {
		err := stream.Next()
		if err != nil {
			logp.Err("%s %s", stream.FullName(), err.Error())
			return
		}
		// is the stream expired?
		if IsBefore(stream.Group.Beat.Config.StreamEventHorizon, stream.LastEventTimestamp) {
			return
		}
		// is the stream "hot"?
		if stream.IsHot(stream.LastEventTimestamp) {
			eventRefreshFrequency = stream.Group.Beat.Config.HotStreamEventRefreshFrequency
		} else {
			eventRefreshFrequency = stream.Group.Beat.Config.StreamEventRefreshFrequency
		}
		select {
		case <-reportTicker.C:
			stream.report()
		default:
			time.Sleep(eventRefreshFrequency)
		}
	}
}

func (stream *Stream) IsHot(lastEventTimestamp int64) bool {
	return !IsBefore(stream.Group.Beat.Config.HotStreamEventHorizon, lastEventTimestamp)
}

func (stream *Stream) report() {
	logp.Info("report[stream] %d %s %s",
		stream.publishedEvents, stream.FullName(), reportFrequency)
	stream.publishedEvents = 0
}

func (stream *Stream) FullName() string {
	return fmt.Sprintf("%s/%s", stream.Group.Name, stream.Name)
}

// fills the buffer's contents into the event,
// publishes the message and empties the buffer
func (stream *Stream) publish(event *Event) {
	if stream.Buffer.Len() == 0 {
		return
	}
	event.Message = stream.Buffer.String()
	stream.Publisher.Publish(event)
	stream.Buffer.Reset()
	stream.publishedEvents++
}

func (stream *Stream) digest(streamEvent *cloudwatchlogs.OutputLogEvent) {
	event := &Event{
		Stream:    stream,
		Timestamp: aws.Int64Value(streamEvent.Timestamp),
	}
	if stream.Multiline == nil {
		stream.Buffer.WriteString(*streamEvent.Message)
		stream.publish(event)
	} else {
		switch stream.Multiline.Match {
		case "after":
			if stream.MultiRegex.MatchString(*streamEvent.Message) == stream.Multiline.Negate {
				stream.publish(event)
			}
			stream.Buffer.WriteString(*streamEvent.Message)
		case "before":
			stream.Buffer.WriteString(*streamEvent.Message)
			if stream.MultiRegex.MatchString(*streamEvent.Message) == stream.Multiline.Negate {
				stream.publish(event)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

// Interface used by the UDP Listener
type RemoteEventCollector interface {
	NewRemoteEvent(stateChange)
}

// Called by Server Notifications
func (s collector) NewRemoteEvent(c stateChange) {
	s.collect <- c
}

// TODO move to tests
var _ RemoteEventCollector = collector{}
<commit_msg>[cmd/gobox-client] simple watcher implemented for an io.Reader<commit_after>package main

import (
	"bufio"
	"io"
)

// Interface used by the UDP Listener
type RemoteEventCollector interface {
	NewRemoteEvent(stateChange)
}

// Called by Server Notifications
func (s collector) NewRemoteEvent(c stateChange) {
	s.collect <- c
}

// TODO move to tests
var _ RemoteEventCollector = collector{}

type readerWatcher struct {
	// TODO There should probly be some communication if
	//      watcher encounters an error while it's running

	// Stores the error that caused watcher to exit
	err error
}

func (w *readerWatcher) watchFrom(r io.Reader, sink RemoteEventCollector) {
	go func() {
		scanner := bufio.NewScanner(r)
		scanner.Split(bufio.ScanLines)

		for scanner.Scan() {
			sink.NewRemoteEvent(stateChange(scanner.Text()))
		}

		w.err = scanner.Err()
	}()
}
<|endoftext|>


<commit_before>// Package vagrantutil is a high level wrapper around Vagrant which provides an
// idiomatic go API.
package vagrantutil

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/koding/logging"
)

//go:generate stringer -type=Status  -output=stringer.go

type Status int

const (
	// Some possible states:
	// https://github.com/mitchellh/vagrant/blob/master/templates/locales/en.yml#L1504
	Unknown Status = iota
	NotCreated
	Running
	Saved
	PowerOff
	Aborted
	Preparing
)

// Box represents a single line of `vagrant box list` output.
type Box struct {
	Name     string
	Provider string
	Version  string
}

// CommandOutput is the streaming output of a command
type CommandOutput struct {
	Line  string
	Error error
}

type Vagrant struct {
	// VagrantfilePath is the directory with specifies the directory where
	// Vagrantfile is being stored.
	VagrantfilePath string

	// ProviderName overwrites the default provider used for the Vagrantfile.
	ProviderName string

	// ID is the unique ID of the given box.
	ID string

	// State is populated/updated if the Status() or List() method is called.
	State string

	// Log is used for logging output of vagrant commands in debug mode.
	Log logging.Logger
}

// NewVagrant returns a new Vagrant instance for the given path. The path
// should be unique. If the path already exists in the system it'll be used, if
// not a new setup will be createad.
func NewVagrant(path string) (*Vagrant, error) {
	if path == "" {
		return nil, errors.New("vagrant: path is empty")
	}

	if err := os.MkdirAll(path, 0755); err != nil {
		return nil, err
	}

	return &Vagrant{
		VagrantfilePath: path,
	}, nil
}

// Create creates the vagrantFile in the pre initialized vagrant path.
func (v *Vagrant) Create(vagrantFile string) error {
	// Recreate the directory in case it was removed between
	// call to NewVagrant and Create.
	if err := os.MkdirAll(v.VagrantfilePath, 0755); err != nil {
		return v.error(err)
	}

	v.debugf("create:\n%s", vagrantFile)

	return v.error(ioutil.WriteFile(v.vagrantfile(), []byte(vagrantFile), 0644))
}

// Version returns the current installed vagrant version
func (v *Vagrant) Version() (string, error) {
	out, err := v.vagrantCommand().run("version", "--machine-readable")
	if err != nil {
		return "", err
	}

	records, err := parseRecords(out)
	if err != nil {
		return "", v.error(err)
	}

	versionInstalled, err := parseData(records, "version-installed")
	if err != nil {
		return "", v.error(err)
	}

	return versionInstalled, nil
}

// Status returns the state of the box, such as "Running", "NotCreated", etc...
func (v *Vagrant) Status() (s Status, err error) {
	defer func() {
		v.State = s.String()
	}()

	var notCreated bool
	cmd := v.vagrantCommand()
	cmd.ignoreErr = func(err error) bool {
		if isNotCreated(err) {
			notCreated = true
			return true
		}

		return false
	}

	out, err := cmd.run("status", "--machine-readable")
	if err != nil {
		return Unknown, err
	}

	if notCreated {
		return NotCreated, nil
	}

	records, err := parseRecords(out)
	if err != nil {
		return Unknown, v.error(err)
	}

	status, err := parseData(records, "state")
	if err != nil {
		return Unknown, v.error(err)
	}

	s, err = toStatus(status)
	if err != nil {
		return Unknown, err
	}

	return s, nil
}

func (v *Vagrant) Provider() (string, error) {
	out, err := v.vagrantCommand().run("status", "--machine-readable")
	if err != nil {
		return "", err
	}

	records, err := parseRecords(out)
	if err != nil {
		return "", v.error(err)
	}

	provider, err := parseData(records, "provider-name")
	if err != nil {
		return "", v.error(err)
	}

	return provider, nil
}

// List returns all available boxes on the system. Under the hood it calls
// "global-status" and parses the output.
func (v *Vagrant) List() ([]*Vagrant, error) {
	// Refresh box status cache. So it does not report that aborted
	// box is running etc.
	_, err := v.vagrantCommand().run("global-status", "--prune")
	if err != nil {
		return nil, err
	}
	out, err := v.vagrantCommand().run("global-status")
	if err != nil {
		return nil, err
	}

	output := make([][]string, 0)

	scanner := bufio.NewScanner(strings.NewReader(out))
	collectStarted := false

	for scanner.Scan() {
		if strings.HasPrefix(scanner.Text(), "--") {
			scanner.Scan() // advance to next line
			collectStarted = true
		}

		if !collectStarted {
			continue
		}

		trimmedLine := strings.TrimSpace(scanner.Text())
		if trimmedLine == "" {
			break // we are finished with collecting the boxes
		}

		output = append(output, strings.Fields(trimmedLine))
	}
	if err := scanner.Err(); err != nil {
		return nil, v.error(err)
	}

	boxes := make([]*Vagrant, len(output))

	for i, box := range output {
		// example box: [0c269f6 default virtualbox aborted /Users/fatih/path]
		boxes[i] = &Vagrant{
			ID:              box[0],
			VagrantfilePath: box[len(box)-1],
			State:           box[3],
		}
	}

	return boxes, nil
}

// Up executes "vagrant up" for the given vagrantfile. The returned channel
// contains the output stream. At the end of the output, the error is put into
// the Error field if there is any.
func (v *Vagrant) Up() (<-chan *CommandOutput, error) {
	if v.Provider != "" {
		return v.vagrantCommand().start("up", "--provider", v.Provider)
	}

	return v.vagrantCommand().start("up")
}

// Halt executes "vagrant halt". The returned reader contains the output
// stream. The client is responsible of calling the Close method of the
// returned reader.
func (v *Vagrant) Halt() (<-chan *CommandOutput, error) {
	return v.vagrantCommand().start("halt")
}

// Destroy executes "vagrant destroy". The returned reader contains the output
// stream. The client is responsible of calling the Close method of the
// returned reader.
func (v *Vagrant) Destroy() (<-chan *CommandOutput, error) {
	if _, err := os.Stat(v.VagrantfilePath); os.IsNotExist(err) {
		// Makes Destroy idempotent if called consecutively multiple times on
		// the same path.
		//
		// Returning closed channel to not make existing like the one
		// below hang:
		//
		//   ch, err := vg.Destroy()
		//   if err != nil {
		//     ...
		//   }
		//   for line := range ch {
		//     ...
		//   }
		//
		ch := make(chan *CommandOutput)
		close(ch)

		return ch, nil
	}

	cmd := v.vagrantCommand()
	cmd.onSuccess = func() {
		// cleanup vagrant directory on success, as it's no longer needed;
		// after destroy it should be not possible to call vagrant up
		// again, call to Create is required first
		if err := os.RemoveAll(v.VagrantfilePath); err != nil {
			v.debugf("failed to cleanup %q after destroy: %s", v.VagrantfilePath, err)
		}

		// We leave empty directory to not make other commands fail
		// due to missing cwd.
		//
		// TODO(rjeczalik): rework lookup to use box id instead
		if err := os.MkdirAll(v.VagrantfilePath, 0755); err != nil {
			v.debugf("failed to create empty dir %q after destroy: %s", v.VagrantfilePath, err)
		}
	}
	// if vagrant box is not created, return success - the destroy
	// should be effectively a nop
	cmd.ignoreErr = isNotCreated

	return cmd.start("destroy", "--force")
}

var stripFmt = strings.NewReplacer("(", "", ",", "", ")", "")

// BoxList executes "vagrant box list", parses the output and returns all
// available base boxes.
func (v *Vagrant) BoxList() ([]*Box, error) {
	out, err := v.vagrantCommand().run("box", "list")
	if err != nil {
		return nil, err
	}

	var boxes []*Box
	scanner := bufio.NewScanner(strings.NewReader(out))

	for scanner.Scan() {
		line := strings.TrimSpace(stripFmt.Replace(scanner.Text()))
		if line == "" {
			continue
		}

		var box Box
		n, err := fmt.Sscanf(line, "%s %s %s", &box.Name, &box.Provider, &box.Version)
		if err != nil {
			return nil, v.errorf("%s for line: %s", err, line)
		}
		if n != 3 {
			return nil, v.errorf("unable to parse output line: %s", line)
		}

		boxes = append(boxes, &box)
	}
	if err := scanner.Err(); err != nil {
		return nil, v.error(err)
	}

	return boxes, nil
}

// BoxAdd executes "vagrant box add" for the given box. The returned channel
// contains the output stream. At the end of the output, the error is put into
// the Error field if there is any.
//
// TODO(rjeczalik): BoxAdd does not support currently adding boxes directly
// from files.
func (v *Vagrant) BoxAdd(box *Box) (<-chan *CommandOutput, error) {
	args := append([]string{"box", "add"}, toArgs(box)...)
	return v.vagrantCommand().start(args...)
}

// BoxRemove executes "vagrant box remove" for the given box. The returned channel
// contains the output stream. At the end of the output, the error is put into
// the Error field if there is any.
func (v *Vagrant) BoxRemove(box *Box) (<-chan *CommandOutput, error) {
	args := append([]string{"box", "remove"}, toArgs(box)...)
	return v.vagrantCommand().start(args...)
}

// SSH executes "vagrant ssh" for the given vagrantfile. The returned channel
// contains the output stream. At the end of the output, the error is put into
// the Error field if there is any.
func (v *Vagrant) SSH(command string) (<-chan *CommandOutput, error) {
	args := []string{"ssh", "-c", command}
	return v.vagrantCommand().start(args...)
}

// vagrantfile returns the Vagrantfile path
func (v *Vagrant) vagrantfile() string {
	return filepath.Join(v.VagrantfilePath, "Vagrantfile")
}

// vagrantfileExists checks if a Vagrantfile exists in the given path. It
// returns a nil error if exists.
func (v *Vagrant) vagrantfileExists() error {
	if _, err := os.Stat(v.vagrantfile()); os.IsNotExist(err) {
		return err
	}
	return nil
}

// vagrantCommand creates a command which is setup to be run next to
// Vagrantfile
func (v *Vagrant) vagrantCommand() *command {
	return newCommand(v.VagrantfilePath, v.Log)
}

func (v *Vagrant) debugf(format string, args ...interface{}) {
	if v.Log != nil {
		v.Log.New(v.VagrantfilePath).Debug(format, args...)
	}
}

func (v *Vagrant) errorf(format string, args ...interface{}) error {
	err := fmt.Errorf(format, args...)
	v.debugf("%s", err)
	return err
}

func (v *Vagrant) error(err error) error {
	if err != nil {
		v.debugf("%s", err)
	}
	return err
}

// toArgs converts the given box to argument list for `vagrant box add/remove`
// commands
func toArgs(box *Box) (args []string) {
	if box.Provider != "" {
		args = append(args, "--provider", box.Provider)
	}
	if box.Version != "" {
		args = append(args, "--box-version", box.Version)
	}
	return append(args, box.Name)
}

// toStatus converts the given state string to Status type
func toStatus(state string) (Status, error) {
	switch state {
	case "running":
		return Running, nil
	case "not_created":
		return NotCreated, nil
	case "saved":
		return Saved, nil
	case "poweroff":
		return PowerOff, nil
	case "aborted":
		return Aborted, nil
	case "preparing":
		return Preparing, nil
	default:
		return Unknown, fmt.Errorf("Unknown state: %s", state)
	}

}
<commit_msg>vagrantutil: fix build<commit_after>// Package vagrantutil is a high level wrapper around Vagrant which provides an
// idiomatic go API.
package vagrantutil

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/koding/logging"
)

//go:generate stringer -type=Status  -output=stringer.go

type Status int

const (
	// Some possible states:
	// https://github.com/mitchellh/vagrant/blob/master/templates/locales/en.yml#L1504
	Unknown Status = iota
	NotCreated
	Running
	Saved
	PowerOff
	Aborted
	Preparing
)

// Box represents a single line of `vagrant box list` output.
type Box struct {
	Name     string
	Provider string
	Version  string
}

// CommandOutput is the streaming output of a command
type CommandOutput struct {
	Line  string
	Error error
}

type Vagrant struct {
	// VagrantfilePath is the directory with specifies the directory where
	// Vagrantfile is being stored.
	VagrantfilePath string

	// ProviderName overwrites the default provider used for the Vagrantfile.
	ProviderName string

	// ID is the unique ID of the given box.
	ID string

	// State is populated/updated if the Status() or List() method is called.
	State string

	// Log is used for logging output of vagrant commands in debug mode.
	Log logging.Logger
}

// NewVagrant returns a new Vagrant instance for the given path. The path
// should be unique. If the path already exists in the system it'll be used, if
// not a new setup will be createad.
func NewVagrant(path string) (*Vagrant, error) {
	if path == "" {
		return nil, errors.New("vagrant: path is empty")
	}

	if err := os.MkdirAll(path, 0755); err != nil {
		return nil, err
	}

	return &Vagrant{
		VagrantfilePath: path,
	}, nil
}

// Create creates the vagrantFile in the pre initialized vagrant path.
func (v *Vagrant) Create(vagrantFile string) error {
	// Recreate the directory in case it was removed between
	// call to NewVagrant and Create.
	if err := os.MkdirAll(v.VagrantfilePath, 0755); err != nil {
		return v.error(err)
	}

	v.debugf("create:\n%s", vagrantFile)

	return v.error(ioutil.WriteFile(v.vagrantfile(), []byte(vagrantFile), 0644))
}

// Version returns the current installed vagrant version
func (v *Vagrant) Version() (string, error) {
	out, err := v.vagrantCommand().run("version", "--machine-readable")
	if err != nil {
		return "", err
	}

	records, err := parseRecords(out)
	if err != nil {
		return "", v.error(err)
	}

	versionInstalled, err := parseData(records, "version-installed")
	if err != nil {
		return "", v.error(err)
	}

	return versionInstalled, nil
}

// Status returns the state of the box, such as "Running", "NotCreated", etc...
func (v *Vagrant) Status() (s Status, err error) {
	defer func() {
		v.State = s.String()
	}()

	var notCreated bool
	cmd := v.vagrantCommand()
	cmd.ignoreErr = func(err error) bool {
		if isNotCreated(err) {
			notCreated = true
			return true
		}

		return false
	}

	out, err := cmd.run("status", "--machine-readable")
	if err != nil {
		return Unknown, err
	}

	if notCreated {
		return NotCreated, nil
	}

	records, err := parseRecords(out)
	if err != nil {
		return Unknown, v.error(err)
	}

	status, err := parseData(records, "state")
	if err != nil {
		return Unknown, v.error(err)
	}

	s, err = toStatus(status)
	if err != nil {
		return Unknown, err
	}

	return s, nil
}

func (v *Vagrant) Provider() (string, error) {
	out, err := v.vagrantCommand().run("status", "--machine-readable")
	if err != nil {
		return "", err
	}

	records, err := parseRecords(out)
	if err != nil {
		return "", v.error(err)
	}

	provider, err := parseData(records, "provider-name")
	if err != nil {
		return "", v.error(err)
	}

	return provider, nil
}

// List returns all available boxes on the system. Under the hood it calls
// "global-status" and parses the output.
func (v *Vagrant) List() ([]*Vagrant, error) {
	// Refresh box status cache. So it does not report that aborted
	// box is running etc.
	_, err := v.vagrantCommand().run("global-status", "--prune")
	if err != nil {
		return nil, err
	}
	out, err := v.vagrantCommand().run("global-status")
	if err != nil {
		return nil, err
	}

	output := make([][]string, 0)

	scanner := bufio.NewScanner(strings.NewReader(out))
	collectStarted := false

	for scanner.Scan() {
		if strings.HasPrefix(scanner.Text(), "--") {
			scanner.Scan() // advance to next line
			collectStarted = true
		}

		if !collectStarted {
			continue
		}

		trimmedLine := strings.TrimSpace(scanner.Text())
		if trimmedLine == "" {
			break // we are finished with collecting the boxes
		}

		output = append(output, strings.Fields(trimmedLine))
	}
	if err := scanner.Err(); err != nil {
		return nil, v.error(err)
	}

	boxes := make([]*Vagrant, len(output))

	for i, box := range output {
		// example box: [0c269f6 default virtualbox aborted /Users/fatih/path]
		boxes[i] = &Vagrant{
			ID:              box[0],
			VagrantfilePath: box[len(box)-1],
			State:           box[3],
		}
	}

	return boxes, nil
}

// Up executes "vagrant up" for the given vagrantfile. The returned channel
// contains the output stream. At the end of the output, the error is put into
// the Error field if there is any.
func (v *Vagrant) Up() (<-chan *CommandOutput, error) {
	if v.ProviderName != "" {
		return v.vagrantCommand().start("up", "--provider", v.ProviderName)
	}

	return v.vagrantCommand().start("up")
}

// Halt executes "vagrant halt". The returned reader contains the output
// stream. The client is responsible of calling the Close method of the
// returned reader.
func (v *Vagrant) Halt() (<-chan *CommandOutput, error) {
	return v.vagrantCommand().start("halt")
}

// Destroy executes "vagrant destroy". The returned reader contains the output
// stream. The client is responsible of calling the Close method of the
// returned reader.
func (v *Vagrant) Destroy() (<-chan *CommandOutput, error) {
	if _, err := os.Stat(v.VagrantfilePath); os.IsNotExist(err) {
		// Makes Destroy idempotent if called consecutively multiple times on
		// the same path.
		//
		// Returning closed channel to not make existing like the one
		// below hang:
		//
		//   ch, err := vg.Destroy()
		//   if err != nil {
		//     ...
		//   }
		//   for line := range ch {
		//     ...
		//   }
		//
		ch := make(chan *CommandOutput)
		close(ch)

		return ch, nil
	}

	cmd := v.vagrantCommand()
	cmd.onSuccess = func() {
		// cleanup vagrant directory on success, as it's no longer needed;
		// after destroy it should be not possible to call vagrant up
		// again, call to Create is required first
		if err := os.RemoveAll(v.VagrantfilePath); err != nil {
			v.debugf("failed to cleanup %q after destroy: %s", v.VagrantfilePath, err)
		}

		// We leave empty directory to not make other commands fail
		// due to missing cwd.
		//
		// TODO(rjeczalik): rework lookup to use box id instead
		if err := os.MkdirAll(v.VagrantfilePath, 0755); err != nil {
			v.debugf("failed to create empty dir %q after destroy: %s", v.VagrantfilePath, err)
		}
	}
	// if vagrant box is not created, return success - the destroy
	// should be effectively a nop
	cmd.ignoreErr = isNotCreated

	return cmd.start("destroy", "--force")
}

var stripFmt = strings.NewReplacer("(", "", ",", "", ")", "")

// BoxList executes "vagrant box list", parses the output and returns all
// available base boxes.
func (v *Vagrant) BoxList() ([]*Box, error) {
	out, err := v.vagrantCommand().run("box", "list")
	if err != nil {
		return nil, err
	}

	var boxes []*Box
	scanner := bufio.NewScanner(strings.NewReader(out))

	for scanner.Scan() {
		line := strings.TrimSpace(stripFmt.Replace(scanner.Text()))
		if line == "" {
			continue
		}

		var box Box
		n, err := fmt.Sscanf(line, "%s %s %s", &box.Name, &box.Provider, &box.Version)
		if err != nil {
			return nil, v.errorf("%s for line: %s", err, line)
		}
		if n != 3 {
			return nil, v.errorf("unable to parse output line: %s", line)
		}

		boxes = append(boxes, &box)
	}
	if err := scanner.Err(); err != nil {
		return nil, v.error(err)
	}

	return boxes, nil
}

// BoxAdd executes "vagrant box add" for the given box. The returned channel
// contains the output stream. At the end of the output, the error is put into
// the Error field if there is any.
//
// TODO(rjeczalik): BoxAdd does not support currently adding boxes directly
// from files.
func (v *Vagrant) BoxAdd(box *Box) (<-chan *CommandOutput, error) {
	args := append([]string{"box", "add"}, toArgs(box)...)
	return v.vagrantCommand().start(args...)
}

// BoxRemove executes "vagrant box remove" for the given box. The returned channel
// contains the output stream. At the end of the output, the error is put into
// the Error field if there is any.
func (v *Vagrant) BoxRemove(box *Box) (<-chan *CommandOutput, error) {
	args := append([]string{"box", "remove"}, toArgs(box)...)
	return v.vagrantCommand().start(args...)
}

// SSH executes "vagrant ssh" for the given vagrantfile. The returned channel
// contains the output stream. At the end of the output, the error is put into
// the Error field if there is any.
func (v *Vagrant) SSH(command string) (<-chan *CommandOutput, error) {
	args := []string{"ssh", "-c", command}
	return v.vagrantCommand().start(args...)
}

// vagrantfile returns the Vagrantfile path
func (v *Vagrant) vagrantfile() string {
	return filepath.Join(v.VagrantfilePath, "Vagrantfile")
}

// vagrantfileExists checks if a Vagrantfile exists in the given path. It
// returns a nil error if exists.
func (v *Vagrant) vagrantfileExists() error {
	if _, err := os.Stat(v.vagrantfile()); os.IsNotExist(err) {
		return err
	}
	return nil
}

// vagrantCommand creates a command which is setup to be run next to
// Vagrantfile
func (v *Vagrant) vagrantCommand() *command {
	return newCommand(v.VagrantfilePath, v.Log)
}

func (v *Vagrant) debugf(format string, args ...interface{}) {
	if v.Log != nil {
		v.Log.New(v.VagrantfilePath).Debug(format, args...)
	}
}

func (v *Vagrant) errorf(format string, args ...interface{}) error {
	err := fmt.Errorf(format, args...)
	v.debugf("%s", err)
	return err
}

func (v *Vagrant) error(err error) error {
	if err != nil {
		v.debugf("%s", err)
	}
	return err
}

// toArgs converts the given box to argument list for `vagrant box add/remove`
// commands
func toArgs(box *Box) (args []string) {
	if box.Provider != "" {
		args = append(args, "--provider", box.Provider)
	}
	if box.Version != "" {
		args = append(args, "--box-version", box.Version)
	}
	return append(args, box.Name)
}

// toStatus converts the given state string to Status type
func toStatus(state string) (Status, error) {
	switch state {
	case "running":
		return Running, nil
	case "not_created":
		return NotCreated, nil
	case "saved":
		return Saved, nil
	case "poweroff":
		return PowerOff, nil
	case "aborted":
		return Aborted, nil
	case "preparing":
		return Preparing, nil
	default:
		return Unknown, fmt.Errorf("Unknown state: %s", state)
	}

}
<|endoftext|>