<commit_before>package service

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"text/template"
	"time"
)

type Service struct {
	URLTemplate string
	CmdTemplate []string
	Timeout     time.Duration
	url         string
	process     *os.Process
}

type addressInfo struct {
	Address string
	Host    string
	Port    string
}

func (s *Service) URL() (string, error) {
	if s.process == nil {
		return "", errors.New("not running")
	}

	return s.url, nil
}

func (s *Service) Start() error {
	if s.process != nil {
		return errors.New("already running")
	}

	address, err := freeAddress()
	if err != nil {
		return fmt.Errorf("failed to locate a free port: %s", err)
	}

	if s.url, err = buildURL(s.URLTemplate, address); err != nil {
		return fmt.Errorf("failed to parse URL: %s", err)
	}

	command, err := buildCommand(s.CmdTemplate, address)
	if err != nil {
		return fmt.Errorf("failed to parse command: %s", err)
	}

	if err := command.Start(); err != nil {
		return fmt.Errorf("failed to run command: %s", err)
	}

	s.process = command.Process

	return s.waitForServer()
}

func (s *Service) Stop() {
	if s.process == nil {
		return
	}
	s.process.Signal(syscall.SIGINT)
	s.process.Wait()
	s.process = nil
}

func buildURL(url string, address addressInfo) (string, error) {
	urlTemplate, err := template.New("URL").Parse(url)
	if err != nil {
		return "", err
	}
	urlBuffer := &bytes.Buffer{}
	if err := urlTemplate.Execute(urlBuffer, address); err != nil {
		return "", err
	}
	return urlBuffer.String(), nil
}

func buildCommand(arguments []string, address addressInfo) (*exec.Cmd, error) {
	if len(arguments) == 0 {
		return nil, errors.New("empty command")
	}

	command := []string{}
	for _, argument := range arguments {
		argTemplate, err := template.New("command").Parse(argument)
		if err != nil {
			return nil, err
		}

		argBuffer := &bytes.Buffer{}
		if err := argTemplate.Execute(argBuffer, address); err != nil {
			return nil, err
		}
		command = append(command, argBuffer.String())
	}

	return exec.Command(command[0], command[1:]...), nil
}

func freeAddress() (addressInfo, error) {
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return addressInfo{}, err
	}
	defer listener.Close()

	address := listener.Addr().String()
	addressParts := strings.SplitN(address, ":", 2)
	return addressInfo{address, addressParts[0], addressParts[1]}, nil
}

func (s *Service) waitForServer() error {
	timeoutChan := time.After(s.Timeout)
	failedChan := make(chan struct{}, 1)
	startedChan := make(chan struct{})

	go func() {
		up := s.checkStatus()
		for !up {
			select {
			case <-failedChan:
				return
			default:
				time.Sleep(500 * time.Millisecond)
				up = s.checkStatus()
			}
		}
		startedChan <- struct{}{}
	}()

	select {
	case <-timeoutChan:
		failedChan <- struct{}{}
		s.Stop()
		return errors.New("failed to start before timeout")
	case <-startedChan:
		return nil
	}
}

func (s *Service) checkStatus() bool {
	client := &http.Client{}
	request, _ := http.NewRequest("GET", fmt.Sprintf("%s/status", s.url), nil)
	response, err := client.Do(request)
	if err == nil && response.StatusCode == 200 {
		return true
	}
	return false
}
<commit_msg>Stop WebDriver using "SIGKILL" on Windows + minor service refactorings<commit_after>package service

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os/exec"
	"runtime"
	"strings"
	"syscall"
	"text/template"
	"time"
)

type Service struct {
	URLTemplate string
	CmdTemplate []string
	Timeout     time.Duration
	url         string
	command     *exec.Cmd
}

type addressInfo struct {
	Address string
	Host    string
	Port    string
}

func (s *Service) URL() (string, error) {
	if s.command == nil {
		return "", errors.New("not running")
	}

	return s.url, nil
}

func (s *Service) Start() error {
	if s.command != nil {
		return errors.New("already running")
	}

	address, err := freeAddress()
	if err != nil {
		return fmt.Errorf("failed to locate a free port: %s", err)
	}

	if s.url, err = buildURL(s.URLTemplate, address); err != nil {
		return fmt.Errorf("failed to parse URL: %s", err)
	}

	command, err := buildCommand(s.CmdTemplate, address)
	if err != nil {
		return fmt.Errorf("failed to parse command: %s", err)
	}

	if err := command.Start(); err != nil {
		return fmt.Errorf("failed to run command: %s", err)
	}

	s.command = command

	return s.waitForServer()
}

func (s *Service) Stop() {
	if s.command == nil {
		return
	}

	if runtime.GOOS == "windows" {
		s.command.Process.Kill()
	} else {
		s.command.Process.Signal(syscall.SIGINT)
	}

	s.command.Wait()
	s.command = nil
}

func buildURL(url string, address addressInfo) (string, error) {
	urlTemplate, err := template.New("URL").Parse(url)
	if err != nil {
		return "", err
	}
	urlBuffer := &bytes.Buffer{}
	if err := urlTemplate.Execute(urlBuffer, address); err != nil {
		return "", err
	}
	return urlBuffer.String(), nil
}

func buildCommand(arguments []string, address addressInfo) (*exec.Cmd, error) {
	if len(arguments) == 0 {
		return nil, errors.New("empty command")
	}

	command := []string{}
	for _, argument := range arguments {
		argTemplate, err := template.New("command").Parse(argument)
		if err != nil {
			return nil, err
		}

		argBuffer := &bytes.Buffer{}
		if err := argTemplate.Execute(argBuffer, address); err != nil {
			return nil, err
		}
		command = append(command, argBuffer.String())
	}

	return exec.Command(command[0], command[1:]...), nil
}

func freeAddress() (addressInfo, error) {
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return addressInfo{}, err
	}
	defer listener.Close()

	address := listener.Addr().String()
	addressParts := strings.SplitN(address, ":", 2)
	return addressInfo{address, addressParts[0], addressParts[1]}, nil
}

func (s *Service) waitForServer() error {
	timeoutChan := time.After(s.Timeout)
	failedChan := make(chan struct{}, 1)
	startedChan := make(chan struct{})

	go func() {
		up := s.checkStatus()
		for !up {
			select {
			case <-failedChan:
				return
			default:
				time.Sleep(500 * time.Millisecond)
				up = s.checkStatus()
			}
		}
		startedChan <- struct{}{}
	}()

	select {
	case <-timeoutChan:
		failedChan <- struct{}{}
		s.Stop()
		return errors.New("failed to start before timeout")
	case <-startedChan:
		return nil
	}
}

func (s *Service) checkStatus() bool {
	client := &http.Client{}
	request, _ := http.NewRequest("GET", fmt.Sprintf("%s/status", s.url), nil)
	response, err := client.Do(request)
	if err == nil && response.StatusCode == 200 {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gg

import (
	"fmt"
	"log"
	"os"
	"strconv"

	"github.com/aclements/go-gg/table"
)

// TODO: Split transforms, scalers, and layers into their own packages
// to clean up the name spaces and un-prefix their names?

// Warning is a logger for reporting conditions that don't prevent the
// production of a plot, but may lead to unexpected results.
var Warning = log.New(os.Stderr, "[gg] ", log.Lshortfile)

type Plot struct {
	//facet  facet

	env    *plotEnv
	scales map[string]scalerTree

	scaledData map[scaledDataKey]*scaledData
	scaleSet   map[scaleKey]bool
	marks      []plotMark

	axisLabels     map[string]string
	autoAxisLabels map[string][]string

	constNonce int
}

func NewPlot(data table.Grouping) *Plot {
	p := &Plot{
		env: &plotEnv{
			data: data,
		},
		scales:         make(map[string]scalerTree),
		scaledData:     make(map[scaledDataKey]*scaledData),
		scaleSet:       make(map[scaleKey]bool),
		axisLabels:     make(map[string]string),
		autoAxisLabels: make(map[string][]string),
	}
	return p
}

type plotEnv struct {
	parent *plotEnv
	data   table.Grouping
}

type scaleKey struct {
	gid   table.GroupID
	aes   string
	scale Scaler
}

// SetData sets p's current data table. The caller must not modify
// data in this table after this point.
func (p *Plot) SetData(data table.Grouping) *Plot {
	p.env.data = data
	return p
}

// Data returns p's current data table.
func (p *Plot) Data() table.Grouping {
	return p.env.data
}

// Const creates a new constant column bound to val in all groups and
// returns the generated column name. This is a convenient way to pass
// constant values to layers as columns.
//
// TODO: Typically this should be used with PreScaled or physical types.
func (p *Plot) Const(val interface{}) string {
	tab := p.Data()

retry:
	col := fmt.Sprintf("[gg-const-%d]", p.constNonce)
	p.constNonce++
	for _, col2 := range tab.Columns() {
		if col == col2 {
			goto retry
		}
	}

	p.SetData(table.MapTables(func(_ table.GroupID, t *table.Table) *table.Table {
		return t.AddConst(col, val)
	}, tab))

	return col
}

type scalerTree struct {
	scales map[table.GroupID]Scaler
}

func newScalerTree() scalerTree {
	return scalerTree{map[table.GroupID]Scaler{
		table.RootGroupID: &defaultScale{},
	}}
}

func (t scalerTree) bind(gid table.GroupID, s Scaler) {
	// Unbind scales under gid.
	for ogid := range t.scales {
		if gid == table.RootGroupID {
			// Optimize binding the root GID.
			delete(t.scales, ogid)
			continue
		}

		for p := ogid; ; p = p.Parent() {
			if p == gid {
				delete(t.scales, ogid)
				break
			}
			if p == table.RootGroupID {
				break
			}
		}
	}
	t.scales[gid] = s
}

func (t scalerTree) find(gid table.GroupID) Scaler {
	for {
		if s, ok := t.scales[gid]; ok {
			return s
		}
		if gid == table.RootGroupID {
			// This should never happen.
			panic("no scale for group " + gid.String())
		}
		gid = gid.Parent()
	}
}

func (p *Plot) getScales(aes string) scalerTree {
	st, ok := p.scales[aes]
	if !ok {
		st = newScalerTree()
		p.scales[aes] = st
	}
	return st
}

// SetScale binds a scale to the given visual aesthetic. SetScale is
// shorthand for SetScaleAt(aes, s, table.RootGroupID).
//
// SetScale returns p for ease of chaining.
func (p *Plot) SetScale(aes string, s Scaler) *Plot {
	return p.SetScaleAt(aes, s, table.RootGroupID)
}

// SetScaleAt binds a scale to the given visual aesthetic for all data
// in group gid or descendants of gid.
func (p *Plot) SetScaleAt(aes string, s Scaler, gid table.GroupID) *Plot {
	// TODO: Should aes be an enum so you can't mix up aesthetics
	// and column names?
	p.getScales(aes).bind(gid, s)
	return p
}

// GetScale returns the scale for the given visual aesthetic used for
// data in group gid.
func (p *Plot) GetScale(aes string, gid table.GroupID) Scaler {
	return p.getScales(aes).find(gid)
}

type scaledDataKey struct {
	aes  string
	data table.Grouping
	col  string
}

// use binds a column of data to an aesthetic. It expands the domain
// of the aesthetic's scale to include the data in col, and returns
// the scaled data.
//
// col may be "", in which case it simply returns nil.
//
// TODO: Should aes be an enum?
func (p *Plot) use(aes string, col string) *scaledData {
	if col == "" {
		return nil
	}

	if col[0] == '@' {
		// TODO: Document this.
		n, err := strconv.Atoi(col[1:])
		if err == nil {
			cols := p.Data().Columns()
			if n >= len(cols) {
				panic(fmt.Sprintf("column index %d out of range; table has %d columns", n, len(cols)))
			}
			col = cols[n]
		}
	}

	// TODO: This is wrong. If the scale tree for aes changes,
	// this may return a stale scaledData bound to the wrong
	// scalers. If I got rid of scale trees, I could just put the
	// scaler in the key. Or I could clean up the cache when the
	// scale tree changes.

	sd := p.scaledData[scaledDataKey{aes, p.Data(), col}]
	if sd == nil {
		// Construct the scaledData.
		sd = &scaledData{
			seqs: make(map[table.GroupID]scaledSeq),
		}

		// Get the scale tree.
		st := p.getScales(aes)

		for _, gid := range p.Data().Tables() {
			table := p.Data().Table(gid)

			// Get the data.
			seq := table.MustColumn(col)

			// Find the scale.
			scaler := st.find(gid)

			// Add the scale to the scale set.
			p.scaleSet[scaleKey{gid, aes, scaler}] = true

			// Train the scale.
			scaler.ExpandDomain(seq)

			// Add it to the scaledData.
			sd.seqs[gid] = scaledSeq{seq, scaler}
		}

		p.scaledData[scaledDataKey{aes, p.Data(), col}] = sd
	}

	// Update axis labels.
	if aes == "x" || aes == "y" {
		p.autoAxisLabels[aes] = append(p.autoAxisLabels[aes], col)
	}

	return sd
}

// Save saves the current data table of p to a stack.
func (p *Plot) Save() *Plot {
	p.env = &plotEnv{
		parent: p.env,
		data:   p.env.data,
	}
	return p
}

// Restore restores the data table of p from the save stack.
func (p *Plot) Restore() *Plot {
	if p.env.parent == nil {
		panic("unbalanced Save/Restore")
	}
	p.env = p.env.parent
	return p
}

type Plotter interface {
	Apply(*Plot)
}

func (p *Plot) Add(plotters ...Plotter) *Plot {
	for _, plotter := range plotters {
		plotter.Apply(p)
	}
	return p
}

// AxisLabel returns a Plotter that sets the label of an axis on a
// Plot. By default, Plot constructs automatic axis labels from column
// names, but AxisLabel lets callers override these.
//
// TODO: Should labels be attached to aesthetics, generally?
//
// TODO: Should this really be a Plotter or just a method of Plot?
func AxisLabel(axis, label string) Plotter {
	return axisLabel{axis, label}
}

type axisLabel struct {
	axis, label string
}

func (a axisLabel) Apply(p *Plot) {
	p.axisLabels[a.axis] = a.label
}
<commit_msg>gg: add Plot.Stat method<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gg

import (
	"fmt"
	"log"
	"os"
	"strconv"

	"github.com/aclements/go-gg/table"
)

// TODO: Split transforms, scalers, and layers into their own packages
// to clean up the name spaces and un-prefix their names?

// Warning is a logger for reporting conditions that don't prevent the
// production of a plot, but may lead to unexpected results.
var Warning = log.New(os.Stderr, "[gg] ", log.Lshortfile)

type Plot struct {
	//facet  facet

	env    *plotEnv
	scales map[string]scalerTree

	scaledData map[scaledDataKey]*scaledData
	scaleSet   map[scaleKey]bool
	marks      []plotMark

	axisLabels     map[string]string
	autoAxisLabels map[string][]string

	constNonce int
}

func NewPlot(data table.Grouping) *Plot {
	p := &Plot{
		env: &plotEnv{
			data: data,
		},
		scales:         make(map[string]scalerTree),
		scaledData:     make(map[scaledDataKey]*scaledData),
		scaleSet:       make(map[scaleKey]bool),
		axisLabels:     make(map[string]string),
		autoAxisLabels: make(map[string][]string),
	}
	return p
}

type plotEnv struct {
	parent *plotEnv
	data   table.Grouping
}

type scaleKey struct {
	gid   table.GroupID
	aes   string
	scale Scaler
}

// SetData sets p's current data table. The caller must not modify
// data in this table after this point.
func (p *Plot) SetData(data table.Grouping) *Plot {
	p.env.data = data
	return p
}

// Data returns p's current data table.
func (p *Plot) Data() table.Grouping {
	return p.env.data
}

// Const creates a new constant column bound to val in all groups and
// returns the generated column name. This is a convenient way to pass
// constant values to layers as columns.
//
// TODO: Typically this should be used with PreScaled or physical types.
func (p *Plot) Const(val interface{}) string {
	tab := p.Data()

retry:
	col := fmt.Sprintf("[gg-const-%d]", p.constNonce)
	p.constNonce++
	for _, col2 := range tab.Columns() {
		if col == col2 {
			goto retry
		}
	}

	p.SetData(table.MapTables(func(_ table.GroupID, t *table.Table) *table.Table {
		return t.AddConst(col, val)
	}, tab))

	return col
}

type scalerTree struct {
	scales map[table.GroupID]Scaler
}

func newScalerTree() scalerTree {
	return scalerTree{map[table.GroupID]Scaler{
		table.RootGroupID: &defaultScale{},
	}}
}

func (t scalerTree) bind(gid table.GroupID, s Scaler) {
	// Unbind scales under gid.
	for ogid := range t.scales {
		if gid == table.RootGroupID {
			// Optimize binding the root GID.
			delete(t.scales, ogid)
			continue
		}

		for p := ogid; ; p = p.Parent() {
			if p == gid {
				delete(t.scales, ogid)
				break
			}
			if p == table.RootGroupID {
				break
			}
		}
	}
	t.scales[gid] = s
}

func (t scalerTree) find(gid table.GroupID) Scaler {
	for {
		if s, ok := t.scales[gid]; ok {
			return s
		}
		if gid == table.RootGroupID {
			// This should never happen.
			panic("no scale for group " + gid.String())
		}
		gid = gid.Parent()
	}
}

func (p *Plot) getScales(aes string) scalerTree {
	st, ok := p.scales[aes]
	if !ok {
		st = newScalerTree()
		p.scales[aes] = st
	}
	return st
}

// SetScale binds a scale to the given visual aesthetic. SetScale is
// shorthand for SetScaleAt(aes, s, table.RootGroupID).
//
// SetScale returns p for ease of chaining.
func (p *Plot) SetScale(aes string, s Scaler) *Plot {
	return p.SetScaleAt(aes, s, table.RootGroupID)
}

// SetScaleAt binds a scale to the given visual aesthetic for all data
// in group gid or descendants of gid.
func (p *Plot) SetScaleAt(aes string, s Scaler, gid table.GroupID) *Plot {
	// TODO: Should aes be an enum so you can't mix up aesthetics
	// and column names?
	p.getScales(aes).bind(gid, s)
	return p
}

// GetScale returns the scale for the given visual aesthetic used for
// data in group gid.
func (p *Plot) GetScale(aes string, gid table.GroupID) Scaler {
	return p.getScales(aes).find(gid)
}

type scaledDataKey struct {
	aes  string
	data table.Grouping
	col  string
}

// use binds a column of data to an aesthetic. It expands the domain
// of the aesthetic's scale to include the data in col, and returns
// the scaled data.
//
// col may be "", in which case it simply returns nil.
//
// TODO: Should aes be an enum?
func (p *Plot) use(aes string, col string) *scaledData {
	if col == "" {
		return nil
	}

	if col[0] == '@' {
		// TODO: Document this.
		n, err := strconv.Atoi(col[1:])
		if err == nil {
			cols := p.Data().Columns()
			if n >= len(cols) {
				panic(fmt.Sprintf("column index %d out of range; table has %d columns", n, len(cols)))
			}
			col = cols[n]
		}
	}

	// TODO: This is wrong. If the scale tree for aes changes,
	// this may return a stale scaledData bound to the wrong
	// scalers. If I got rid of scale trees, I could just put the
	// scaler in the key. Or I could clean up the cache when the
	// scale tree changes.

	sd := p.scaledData[scaledDataKey{aes, p.Data(), col}]
	if sd == nil {
		// Construct the scaledData.
		sd = &scaledData{
			seqs: make(map[table.GroupID]scaledSeq),
		}

		// Get the scale tree.
		st := p.getScales(aes)

		for _, gid := range p.Data().Tables() {
			table := p.Data().Table(gid)

			// Get the data.
			seq := table.MustColumn(col)

			// Find the scale.
			scaler := st.find(gid)

			// Add the scale to the scale set.
			p.scaleSet[scaleKey{gid, aes, scaler}] = true

			// Train the scale.
			scaler.ExpandDomain(seq)

			// Add it to the scaledData.
			sd.seqs[gid] = scaledSeq{seq, scaler}
		}

		p.scaledData[scaledDataKey{aes, p.Data(), col}] = sd
	}

	// Update axis labels.
	if aes == "x" || aes == "y" {
		p.autoAxisLabels[aes] = append(p.autoAxisLabels[aes], col)
	}

	return sd
}

// Save saves the current data table of p to a stack.
func (p *Plot) Save() *Plot {
	p.env = &plotEnv{
		parent: p.env,
		data:   p.env.data,
	}
	return p
}

// Restore restores the data table of p from the save stack.
func (p *Plot) Restore() *Plot {
	if p.env.parent == nil {
		panic("unbalanced Save/Restore")
	}
	p.env = p.env.parent
	return p
}

type Plotter interface {
	Apply(*Plot)
}

func (p *Plot) Add(plotters ...Plotter) *Plot {
	for _, plotter := range plotters {
		plotter.Apply(p)
	}
	return p
}

// AxisLabel returns a Plotter that sets the label of an axis on a
// Plot. By default, Plot constructs automatic axis labels from column
// names, but AxisLabel lets callers override these.
//
// TODO: Should labels be attached to aesthetics, generally?
//
// TODO: Should this really be a Plotter or just a method of Plot?
func AxisLabel(axis, label string) Plotter {
	return axisLabel{axis, label}
}

type axisLabel struct {
	axis, label string
}

func (a axisLabel) Apply(p *Plot) {
	p.axisLabels[a.axis] = a.label
}

// A Stat transforms a table.Grouping.
type Stat interface {
	F(table.Grouping) table.Grouping
}

// Stat applies each of stats in order to p's data.
//
// TODO: Perform scale transforms before applying stats.
func (p *Plot) Stat(stats ...Stat) *Plot {
	data := p.Data()
	for _, stat := range stats {
		data = stat.F(data)
	}
	return p.SetData(data)
}
<|endoftext|>


<commit_before>/**
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package session

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"reflect"
	"strconv"
	"strings"

	"github.com/softlayer/softlayer-go/datatypes"
	"github.com/softlayer/softlayer-go/sl"
)

type RestTransport struct{}

// DoRequest - Implementation of the TransportHandler interface for handling
// calls to the REST endpoint.
func (r *RestTransport) DoRequest(sess *Session, service string, method string, args []interface{}, options *sl.Options, pResult interface{}) error {
	restMethod := httpMethod(method, args)

	// Parse any method parameters and determine the HTTP method
	var parameters []byte
	if len(args) > 0 {
		// parse the parameters
		parameters, _ = json.Marshal(
			map[string]interface{}{
				"parameters": args,
			})
	}

	path := buildPath(service, method, options)

	resp, code, err := makeHTTPRequest(
		sess,
		path,
		restMethod,
		bytes.NewBuffer(parameters),
		options)

	if err != nil {
		return sl.Error{Wrapped: err}
	}

	if code < 200 || code > 299 {
		e := sl.Error{StatusCode: code}

		err = json.Unmarshal(resp, &e)

		// If unparseable, wrap the json error
		if err != nil {
			e.Wrapped = err
			e.Message = err.Error()
		}

		return e
	}

	// Some APIs that normally return a collection, omit the []'s when the API returns a single value
	returnType := reflect.TypeOf(pResult).String()
	if strings.Index(returnType, "[]") == 1 && strings.Index(string(resp), "[") != 0 {
		resp = []byte("[" + string(resp) + "]")
	}

	// At this point, all that's left to do is parse the return value to the appropriate type, and return
	// any parse errors (or nil if successful)

	err = nil
	switch pResult.(type) {
	case *[]uint8:
		// exclude quotes
		*pResult.(*[]uint8) = resp[1 : len(resp)-1]
	case *datatypes.Void:
	case *uint:
		var val uint64
		val, err = strconv.ParseUint(string(resp), 0, 64)
		if err == nil {
			*pResult.(*uint) = uint(val)
		}
	case *bool:
		*pResult.(*bool), err = strconv.ParseBool(string(resp))
	case *string:
		str := string(resp)
		strIdx := len(str) - 1
		if str == "null" {
			str = ""
		} else if str[0] == '"' && str[strIdx] == '"' {
			rawStr := rawString{str}
			err = json.Unmarshal([]byte(`{"val":`+str+`}`), &rawStr)
			if err == nil {
				str = rawStr.Val
			}
		}
		*pResult.(*string) = str
	default:
		// Must be a json representation of one of the many softlayer datatypes
		err = json.Unmarshal(resp, pResult)
	}

	if err != nil {
		err = sl.Error{Message: err.Error(), Wrapped: err}
	}

	return err
}

type rawString struct {
	Val string
}

func buildPath(service string, method string, options *sl.Options) string {
	path := service

	if options.Id != nil {
		path = path + "/" + strconv.Itoa(*options.Id)
	}

	// omit the API method name if the method represents one of the basic REST methods
	if method != "getObject" && method != "deleteObject" && method != "createObject" &&
		method != "createObjects" && method != "editObject" && method != "editObjects" {
		path = path + "/" + method
	}

	return path + ".json"
}

func encodeQuery(opts *sl.Options) string {
	query := new(url.URL).Query()

	if opts.Mask != "" {
		query.Add("objectMask", opts.Mask)
	}

	if opts.Filter != "" {
		query.Add("objectFilter", opts.Filter)
	}

	// resultLimit=<offset>,<limit>
	// If offset unspecified, default to 0
	if opts.Limit != nil {
		startOffset := 0
		if opts.Offset != nil {
			startOffset = *opts.Offset
		}

		query.Add("resultLimit", fmt.Sprintf("%d,%d", startOffset, *opts.Limit))
	}

	return query.Encode()
}

func makeHTTPRequest(session *Session, path string, requestType string, requestBody *bytes.Buffer, options *sl.Options) ([]byte, int, error) {
	client := http.DefaultClient
	client.Timeout = DefaultTimeout
	if session.Timeout != 0 {
		client.Timeout = session.Timeout
	}

	var url string
	if session.Endpoint == "" {
		url = url + DefaultEndpoint
	} else {
		url = url + session.Endpoint
	}
	url = fmt.Sprintf("%s/%s", strings.TrimRight(url, "/"), path)
	req, err := http.NewRequest(requestType, url, requestBody)
	if err != nil {
		return nil, 0, err
	}

	if session.APIKey != "" {
		req.SetBasicAuth(session.UserName, session.APIKey)
	} else {
		req.SetBasicAuth(fmt.Sprintf("%d", session.UserId), session.AuthToken)
	}

	req.URL.RawQuery = encodeQuery(options)

	if session.Debug {
		log.Println("[DEBUG] Request URL: ", requestType, req.URL)
		log.Println("[DEBUG] Parameters: ", requestBody.String())
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, 520, err
	}

	defer resp.Body.Close()

	responseBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, resp.StatusCode, err
	}

	if session.Debug {
		log.Println("[DEBUG] Response: ", string(responseBody))
	}
	return responseBody, resp.StatusCode, nil
}

func httpMethod(name string, args []interface{}) string {
	if name == "deleteObject" {
		return "DELETE"
	} else if name == "editObject" || name == "editObjects" {
		return "PUT"
	} else if name == "createObject" || name == "createObjects" || len(args) > 0 {
		return "POST"
	}

	return "GET"
}
<commit_msg>Check for non-empty auth token before adding to basic auth header<commit_after>/**
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package session

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"reflect"
	"strconv"
	"strings"

	"github.com/softlayer/softlayer-go/datatypes"
	"github.com/softlayer/softlayer-go/sl"
)

type RestTransport struct{}

// DoRequest - Implementation of the TransportHandler interface for handling
// calls to the REST endpoint.
func (r *RestTransport) DoRequest(sess *Session, service string, method string, args []interface{}, options *sl.Options, pResult interface{}) error {
	restMethod := httpMethod(method, args)

	// Parse any method parameters and determine the HTTP method
	var parameters []byte
	if len(args) > 0 {
		// parse the parameters
		parameters, _ = json.Marshal(
			map[string]interface{}{
				"parameters": args,
			})
	}

	path := buildPath(service, method, options)

	resp, code, err := makeHTTPRequest(
		sess,
		path,
		restMethod,
		bytes.NewBuffer(parameters),
		options)

	if err != nil {
		return sl.Error{Wrapped: err}
	}

	if code < 200 || code > 299 {
		e := sl.Error{StatusCode: code}

		err = json.Unmarshal(resp, &e)

		// If unparseable, wrap the json error
		if err != nil {
			e.Wrapped = err
			e.Message = err.Error()
		}

		return e
	}

	// Some APIs that normally return a collection, omit the []'s when the API returns a single value
	returnType := reflect.TypeOf(pResult).String()
	if strings.Index(returnType, "[]") == 1 && strings.Index(string(resp), "[") != 0 {
		resp = []byte("[" + string(resp) + "]")
	}

	// At this point, all that's left to do is parse the return value to the appropriate type, and return
	// any parse errors (or nil if successful)

	err = nil
	switch pResult.(type) {
	case *[]uint8:
		// exclude quotes
		*pResult.(*[]uint8) = resp[1 : len(resp)-1]
	case *datatypes.Void:
	case *uint:
		var val uint64
		val, err = strconv.ParseUint(string(resp), 0, 64)
		if err == nil {
			*pResult.(*uint) = uint(val)
		}
	case *bool:
		*pResult.(*bool), err = strconv.ParseBool(string(resp))
	case *string:
		str := string(resp)
		strIdx := len(str) - 1
		if str == "null" {
			str = ""
		} else if str[0] == '"' && str[strIdx] == '"' {
			rawStr := rawString{str}
			err = json.Unmarshal([]byte(`{"val":`+str+`}`), &rawStr)
			if err == nil {
				str = rawStr.Val
			}
		}
		*pResult.(*string) = str
	default:
		// Must be a json representation of one of the many softlayer datatypes
		err = json.Unmarshal(resp, pResult)
	}

	if err != nil {
		err = sl.Error{Message: err.Error(), Wrapped: err}
	}

	return err
}

type rawString struct {
	Val string
}

func buildPath(service string, method string, options *sl.Options) string {
	path := service

	if options.Id != nil {
		path = path + "/" + strconv.Itoa(*options.Id)
	}

	// omit the API method name if the method represents one of the basic REST methods
	if method != "getObject" && method != "deleteObject" && method != "createObject" &&
		method != "createObjects" && method != "editObject" && method != "editObjects" {
		path = path + "/" + method
	}

	return path + ".json"
}

func encodeQuery(opts *sl.Options) string {
	query := new(url.URL).Query()

	if opts.Mask != "" {
		query.Add("objectMask", opts.Mask)
	}

	if opts.Filter != "" {
		query.Add("objectFilter", opts.Filter)
	}

	// resultLimit=<offset>,<limit>
	// If offset unspecified, default to 0
	if opts.Limit != nil {
		startOffset := 0
		if opts.Offset != nil {
			startOffset = *opts.Offset
		}

		query.Add("resultLimit", fmt.Sprintf("%d,%d", startOffset, *opts.Limit))
	}

	return query.Encode()
}

func makeHTTPRequest(session *Session, path string, requestType string, requestBody *bytes.Buffer, options *sl.Options) ([]byte, int, error) {
	client := http.DefaultClient
	client.Timeout = DefaultTimeout
	if session.Timeout != 0 {
		client.Timeout = session.Timeout
	}

	var url string
	if session.Endpoint == "" {
		url = url + DefaultEndpoint
	} else {
		url = url + session.Endpoint
	}
	url = fmt.Sprintf("%s/%s", strings.TrimRight(url, "/"), path)
	req, err := http.NewRequest(requestType, url, requestBody)
	if err != nil {
		return nil, 0, err
	}

	if session.APIKey != "" {
		req.SetBasicAuth(session.UserName, session.APIKey)
	} else if session.AuthToken != "" {
		req.SetBasicAuth(fmt.Sprintf("%d", session.UserId), session.AuthToken)
	}

	req.URL.RawQuery = encodeQuery(options)

	if session.Debug {
		log.Println("[DEBUG] Request URL: ", requestType, req.URL)
		log.Println("[DEBUG] Parameters: ", requestBody.String())
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, 520, err
	}

	defer resp.Body.Close()

	responseBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, resp.StatusCode, err
	}

	if session.Debug {
		log.Println("[DEBUG] Response: ", string(responseBody))
	}
	return responseBody, resp.StatusCode, nil
}

func httpMethod(name string, args []interface{}) string {
	if name == "deleteObject" {
		return "DELETE"
	} else if name == "editObject" || name == "editObjects" {
		return "PUT"
	} else if name == "createObject" || name == "createObjects" || len(args) > 0 {
		return "POST"
	}

	return "GET"
}
<|endoftext|>


<commit_before>package cliutils

const (
	// General CLI constants
	CliVersion = "1.16.2-dev"
	ClientAgent = "jfrog-cli-go"
	OnErrorPanic OnError = "panic"

	// CLI base commands constants:
	CmdArtifactory = "rt"
	CmdBintray = "bt"
	CmdMissionControl = "mc"
	CmdXray = "xr"

	// Download
	DownloadMinSplitKb    = 5120
	DownloadSplitCount    = 3
	DownloadRetries       = 3
	DownloadMaxSplitCount = 15
)
<commit_msg>Version 1.16.2.<commit_after>package cliutils

const (
	// General CLI constants
	CliVersion = "1.16.2"
	ClientAgent = "jfrog-cli-go"
	OnErrorPanic OnError = "panic"

	// CLI base commands constants:
	CmdArtifactory = "rt"
	CmdBintray = "bt"
	CmdMissionControl = "mc"
	CmdXray = "xr"

	// Download
	DownloadMinSplitKb    = 5120
	DownloadSplitCount    = 3
	DownloadRetries       = 3
	DownloadMaxSplitCount = 15
)
<|endoftext|>


<commit_before>// +build !windows

package execdriver

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/docker/docker/daemon/execdriver/native/template"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/pkg/mount"
	"github.com/docker/go-units"
	"github.com/opencontainers/runc/libcontainer"
	"github.com/opencontainers/runc/libcontainer/cgroups/fs"
	"github.com/opencontainers/runc/libcontainer/configs"
	blkiodev "github.com/opencontainers/runc/libcontainer/configs"
)

// Mount contains information for a mount operation.
type Mount struct {
	Source      string `json:"source"`
	Destination string `json:"destination"`
	Writable    bool   `json:"writable"`
	Data        string `json:"data"`
	Propagation string `json:"mountpropagation"`
}

// Resources contains all resource configs for a driver.
// Currently these are all for cgroup configs.
type Resources struct {
	CommonResources

	// Fields below here are platform specific

	BlkioWeightDevice            []*blkiodev.WeightDevice   `json:"blkio_weight_device"`
	BlkioThrottleReadBpsDevice   []*blkiodev.ThrottleDevice `json:"blkio_throttle_read_bps_device"`
	BlkioThrottleWriteBpsDevice  []*blkiodev.ThrottleDevice `json:"blkio_throttle_write_bps_device"`
	BlkioThrottleReadIOpsDevice  []*blkiodev.ThrottleDevice `json:"blkio_throttle_read_iops_device"`
	BlkioThrottleWriteIOpsDevice []*blkiodev.ThrottleDevice `json:"blkio_throttle_write_iops_device"`
	MemorySwap                   int64                      `json:"memory_swap"`
	KernelMemory                 int64                      `json:"kernel_memory"`
	CPUQuota                     int64                      `json:"cpu_quota"`
	CpusetCpus                   string                     `json:"cpuset_cpus"`
	CpusetMems                   string                     `json:"cpuset_mems"`
	CPUPeriod                    int64                      `json:"cpu_period"`
	Rlimits                      []*units.Rlimit            `json:"rlimits"`
	OomKillDisable               bool                       `json:"oom_kill_disable"`
	MemorySwappiness             int64                      `json:"memory_swappiness"`
}

// ProcessConfig is the platform specific structure that describes a process
// that will be run inside a container.
type ProcessConfig struct {
	CommonProcessConfig

	// Fields below here are platform specific
	Privileged bool   `json:"privileged"`
	User       string `json:"user"`
	Console    string `json:"-"` // dev/console path
}

// Ipc settings of the container
// It is for IPC namespace setting. Usually different containers
// have their own IPC namespace, however this specifies to use
// an existing IPC namespace.
// You can join the host's or a container's IPC namespace.
type Ipc struct {
	ContainerID string `json:"container_id"` // id of the container to join ipc.
	HostIpc     bool   `json:"host_ipc"`
}

// Pid settings of the container
// It is for PID namespace setting. Usually different containers
// have their own PID namespace, however this specifies to use
// an existing PID namespace.
// Joining the host's PID namespace is currently the only supported
// option.
type Pid struct {
	HostPid bool `json:"host_pid"`
}

// UTS settings of the container
// It is for UTS namespace setting. Usually different containers
// have their own UTS namespace, however this specifies to use
// an existing UTS namespace.
// Joining the host's UTS namespace is currently the only supported
// option.
type UTS struct {
	HostUTS bool `json:"host_uts"`
}

// Network settings of the container
type Network struct {
	Mtu            int    `json:"mtu"`
	ContainerID    string `json:"container_id"` // id of the container to join network.
	NamespacePath  string `json:"namespace_path"`
	HostNetworking bool   `json:"host_networking"`
}

// Command wraps an os/exec.Cmd to add more metadata
type Command struct {
	CommonCommand

	// Fields below here are platform specific

	AllowedDevices     []*configs.Device `json:"allowed_devices"`
	AppArmorProfile    string            `json:"apparmor_profile"`
	AutoCreatedDevices []*configs.Device `json:"autocreated_devices"`
	CapAdd             []string          `json:"cap_add"`
	CapDrop            []string          `json:"cap_drop"`
	CgroupParent       string            `json:"cgroup_parent"` // The parent cgroup for this command.
	GIDMapping         []idtools.IDMap   `json:"gidmapping"`
	GroupAdd           []string          `json:"group_add"`
	Ipc                *Ipc              `json:"ipc"`
	OomScoreAdj        int               `json:"oom_score_adj"`
	Pid                *Pid              `json:"pid"`
	ReadonlyRootfs     bool              `json:"readonly_rootfs"`
	RemappedRoot       *User             `json:"remap_root"`
	SeccompProfile     string            `json:"seccomp_profile"`
	UIDMapping         []idtools.IDMap   `json:"uidmapping"`
	UTS                *UTS              `json:"uts"`
}

// SetRootPropagation sets the root mount propagation mode.
func SetRootPropagation(config *configs.Config, propagation int) {
	config.RootPropagation = propagation
}

// InitContainer is the initialization of a container config.
// It returns the initial configs for a container. It's mostly
// defined by the default template.
func InitContainer(c *Command) *configs.Config {
	container := template.New()

	container.Hostname = getEnv("HOSTNAME", c.ProcessConfig.Env)
	container.Cgroups.Name = c.ID
	container.Cgroups.Resources.AllowedDevices = c.AllowedDevices
	container.Devices = c.AutoCreatedDevices
	container.Rootfs = c.Rootfs
	container.Readonlyfs = c.ReadonlyRootfs
	// This can be overridden later by driver during mount setup based
	// on volume options
	SetRootPropagation(container, mount.RPRIVATE)
	container.Cgroups.Parent = c.CgroupParent

	// check to see if we are running in ramdisk to disable pivot root
	container.NoPivotRoot = os.Getenv("DOCKER_RAMDISK") != ""

	return container
}

func getEnv(key string, env []string) string {
	for _, pair := range env {
		parts := strings.SplitN(pair, "=", 2)
		if parts[0] == key {
			return parts[1]
		}
	}
	return ""
}

// SetupCgroups setups cgroup resources for a container.
func SetupCgroups(container *configs.Config, c *Command) error {
	if c.Resources != nil {
		container.Cgroups.Resources.CpuShares = c.Resources.CPUShares
		container.Cgroups.Resources.Memory = c.Resources.Memory
		container.Cgroups.Resources.MemoryReservation = c.Resources.MemoryReservation
		container.Cgroups.Resources.MemorySwap = c.Resources.MemorySwap
		container.Cgroups.Resources.KernelMemory = c.Resources.KernelMemory
		container.Cgroups.Resources.CpusetCpus = c.Resources.CpusetCpus
		container.Cgroups.Resources.CpusetMems = c.Resources.CpusetMems
		container.Cgroups.Resources.CpuPeriod = c.Resources.CPUPeriod
		container.Cgroups.Resources.CpuQuota = c.Resources.CPUQuota
		container.Cgroups.Resources.BlkioWeight = c.Resources.BlkioWeight
		container.Cgroups.Resources.BlkioWeightDevice = c.Resources.BlkioWeightDevice
		container.Cgroups.Resources.BlkioThrottleReadBpsDevice = c.Resources.BlkioThrottleReadBpsDevice
		container.Cgroups.Resources.BlkioThrottleWriteBpsDevice = c.Resources.BlkioThrottleWriteBpsDevice
		container.Cgroups.Resources.BlkioThrottleReadIOPSDevice = c.Resources.BlkioThrottleReadIOpsDevice
		container.Cgroups.Resources.BlkioThrottleWriteIOPSDevice = c.Resources.BlkioThrottleWriteIOpsDevice
		container.Cgroups.Resources.OomKillDisable = c.Resources.OomKillDisable
		container.Cgroups.Resources.MemorySwappiness = c.Resources.MemorySwappiness
	}

	return nil
}

// Returns the network statistics for the network interfaces represented by the NetworkRuntimeInfo.
func getNetworkInterfaceStats(interfaceName string) (*libcontainer.NetworkInterface, error) {
	out := &libcontainer.NetworkInterface{Name: interfaceName}
	// This can happen if the network runtime information is missing - possible if the
	// container was created by an old version of libcontainer.
	if interfaceName == "" {
		return out, nil
	}
	type netStatsPair struct {
		// Where to write the output.
		Out *uint64
		// The network stats file to read.
		File string
	}
	// Ingress for host veth is from the container. Hence tx_bytes stat on the host veth is actually number of bytes received by the container.
	netStats := []netStatsPair{
		{Out: &out.RxBytes, File: "tx_bytes"},
		{Out: &out.RxPackets, File: "tx_packets"},
		{Out: &out.RxErrors, File: "tx_errors"},
		{Out: &out.RxDropped, File: "tx_dropped"},

		{Out: &out.TxBytes, File: "rx_bytes"},
		{Out: &out.TxPackets, File: "rx_packets"},
		{Out: &out.TxErrors, File: "rx_errors"},
		{Out: &out.TxDropped, File: "rx_dropped"},
	}
	for _, netStat := range netStats {
		data, err := readSysfsNetworkStats(interfaceName, netStat.File)
		if err != nil {
			return nil, err
		}
		*(netStat.Out) = data
	}
	return out, nil
}

// Reads the specified statistics available under /sys/class/net/<EthInterface>/statistics
func readSysfsNetworkStats(ethInterface, statsFile string) (uint64, error) {
	data, err := ioutil.ReadFile(filepath.Join("/sys/class/net", ethInterface, "statistics", statsFile))
	if err != nil {
		return 0, err
	}
	return strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64)
}

// Stats collects all the resource usage information from a container.
func Stats(containerDir string, containerMemoryLimit int64, machineMemory int64) (*ResourceStats, error) {
	f, err := os.Open(filepath.Join(containerDir, "state.json"))
	if err != nil {
		return nil, err
	}
	defer f.Close()

	type network struct {
		Type              string
		HostInterfaceName string
	}

	state := struct {
		CgroupPaths map[string]string `json:"cgroup_paths"`
		Networks    []network
	}{}

	if err := json.NewDecoder(f).Decode(&state); err != nil {
		return nil, err
	}
	now := time.Now()

	mgr := fs.Manager{Paths: state.CgroupPaths}
	cstats, err := mgr.GetStats()
	if err != nil {
		return nil, err
	}
	stats := &libcontainer.Stats{CgroupStats: cstats}
	// if the container does not have any memory limit specified set the
	// limit to the machines memory
	memoryLimit := containerMemoryLimit
	if memoryLimit == 0 {
		memoryLimit = machineMemory
	}
	for _, iface := range state.Networks {
		switch iface.Type {
		case "veth":
			istats, err := getNetworkInterfaceStats(iface.HostInterfaceName)
			if err != nil {
				return nil, err
			}
			stats.Interfaces = append(stats.Interfaces, istats)
		}
	}
	return &ResourceStats{
		Stats:       stats,
		Read:        now,
		MemoryLimit: memoryLimit,
	}, nil
}

// User contains the uid and gid representing a Unix user
type User struct {
	UID int `json:"root_uid"`
	GID int `json:"root_gid"`
}

// ExitStatus provides exit reasons for a container.
type ExitStatus struct {
	// The exit code with which the container exited.
	ExitCode int

	// Whether the container encountered an OOM.
	OOMKilled bool
}
<commit_msg>Filter auto-created device list if user namespaces enabled<commit_after>// +build !windows

package execdriver

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/docker/docker/daemon/execdriver/native/template"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/pkg/mount"
	"github.com/docker/go-units"
	"github.com/opencontainers/runc/libcontainer"
	"github.com/opencontainers/runc/libcontainer/cgroups/fs"
	"github.com/opencontainers/runc/libcontainer/configs"
	blkiodev "github.com/opencontainers/runc/libcontainer/configs"
)

// Mount contains information for a mount operation.
type Mount struct {
	Source      string `json:"source"`
	Destination string `json:"destination"`
	Writable    bool   `json:"writable"`
	Data        string `json:"data"`
	Propagation string `json:"mountpropagation"`
}

// Resources contains all resource configs for a driver.
// Currently these are all for cgroup configs.
type Resources struct {
	CommonResources

	// Fields below here are platform specific

	BlkioWeightDevice            []*blkiodev.WeightDevice   `json:"blkio_weight_device"`
	BlkioThrottleReadBpsDevice   []*blkiodev.ThrottleDevice `json:"blkio_throttle_read_bps_device"`
	BlkioThrottleWriteBpsDevice  []*blkiodev.ThrottleDevice `json:"blkio_throttle_write_bps_device"`
	BlkioThrottleReadIOpsDevice  []*blkiodev.ThrottleDevice `json:"blkio_throttle_read_iops_device"`
	BlkioThrottleWriteIOpsDevice []*blkiodev.ThrottleDevice `json:"blkio_throttle_write_iops_device"`
	MemorySwap                   int64                      `json:"memory_swap"`
	KernelMemory                 int64                      `json:"kernel_memory"`
	CPUQuota                     int64                      `json:"cpu_quota"`
	CpusetCpus                   string                     `json:"cpuset_cpus"`
	CpusetMems                   string                     `json:"cpuset_mems"`
	CPUPeriod                    int64                      `json:"cpu_period"`
	Rlimits                      []*units.Rlimit            `json:"rlimits"`
	OomKillDisable               bool                       `json:"oom_kill_disable"`
	MemorySwappiness             int64                      `json:"memory_swappiness"`
}

// ProcessConfig is the platform specific structure that describes a process
// that will be run inside a container.
type ProcessConfig struct {
	CommonProcessConfig

	// Fields below here are platform specific
	Privileged bool   `json:"privileged"`
	User       string `json:"user"`
	Console    string `json:"-"` // dev/console path
}

// Ipc settings of the container
// It is for IPC namespace setting. Usually different containers
// have their own IPC namespace, however this specifies to use
// an existing IPC namespace.
// You can join the host's or a container's IPC namespace.
type Ipc struct {
	ContainerID string `json:"container_id"` // id of the container to join ipc.
	HostIpc     bool   `json:"host_ipc"`
}

// Pid settings of the container
// It is for PID namespace setting. Usually different containers
// have their own PID namespace, however this specifies to use
// an existing PID namespace.
// Joining the host's PID namespace is currently the only supported
// option.
type Pid struct {
	HostPid bool `json:"host_pid"`
}

// UTS settings of the container
// It is for UTS namespace setting. Usually different containers
// have their own UTS namespace, however this specifies to use
// an existing UTS namespace.
// Joining the host's UTS namespace is currently the only supported
// option.
type UTS struct {
	HostUTS bool `json:"host_uts"`
}

// Network settings of the container
type Network struct {
	Mtu            int    `json:"mtu"`
	ContainerID    string `json:"container_id"` // id of the container to join network.
	NamespacePath  string `json:"namespace_path"`
	HostNetworking bool   `json:"host_networking"`
}

// Command wraps an os/exec.Cmd to add more metadata
type Command struct {
	CommonCommand

	// Fields below here are platform specific

	AllowedDevices     []*configs.Device `json:"allowed_devices"`
	AppArmorProfile    string            `json:"apparmor_profile"`
	AutoCreatedDevices []*configs.Device `json:"autocreated_devices"`
	CapAdd             []string          `json:"cap_add"`
	CapDrop            []string          `json:"cap_drop"`
	CgroupParent       string            `json:"cgroup_parent"` // The parent cgroup for this command.
	GIDMapping         []idtools.IDMap   `json:"gidmapping"`
	GroupAdd           []string          `json:"group_add"`
	Ipc                *Ipc              `json:"ipc"`
	OomScoreAdj        int               `json:"oom_score_adj"`
	Pid                *Pid              `json:"pid"`
	ReadonlyRootfs     bool              `json:"readonly_rootfs"`
	RemappedRoot       *User             `json:"remap_root"`
	SeccompProfile     string            `json:"seccomp_profile"`
	UIDMapping         []idtools.IDMap   `json:"uidmapping"`
	UTS                *UTS              `json:"uts"`
}

// SetRootPropagation sets the root mount propagation mode.
func SetRootPropagation(config *configs.Config, propagation int) {
	config.RootPropagation = propagation
}

// InitContainer is the initialization of a container config.
// It returns the initial configs for a container. It's mostly
// defined by the default template.
func InitContainer(c *Command) *configs.Config {
	container := template.New()

	container.Hostname = getEnv("HOSTNAME", c.ProcessConfig.Env)
	container.Cgroups.Name = c.ID
	container.Cgroups.Resources.AllowedDevices = c.AllowedDevices
	container.Devices = filterDevices(c.AutoCreatedDevices, (c.RemappedRoot.UID != 0))
	container.Rootfs = c.Rootfs
	container.Readonlyfs = c.ReadonlyRootfs
	// This can be overridden later by driver during mount setup based
	// on volume options
	SetRootPropagation(container, mount.RPRIVATE)
	container.Cgroups.Parent = c.CgroupParent

	// check to see if we are running in ramdisk to disable pivot root
	container.NoPivotRoot = os.Getenv("DOCKER_RAMDISK") != ""

	return container
}

func filterDevices(devices []*configs.Device, userNamespacesEnabled bool) []*configs.Device {
	if !userNamespacesEnabled {
		return devices
	}

	filtered := []*configs.Device{}
	// if we have user namespaces enabled, these devices will not be created
	// because of the mknod limitation in the kernel for an unprivileged process.
	// Rather, they will be bind-mounted, which will only work if they exist;
	// check for existence and remove non-existent entries from the list
	for _, device := range devices {
		if _, err := os.Stat(device.Path); err == nil {
			filtered = append(filtered, device)
		}
	}
	return filtered
}

func getEnv(key string, env []string) string {
	for _, pair := range env {
		parts := strings.SplitN(pair, "=", 2)
		if parts[0] == key {
			return parts[1]
		}
	}
	return ""
}

// SetupCgroups setups cgroup resources for a container.
func SetupCgroups(container *configs.Config, c *Command) error {
	if c.Resources != nil {
		container.Cgroups.Resources.CpuShares = c.Resources.CPUShares
		container.Cgroups.Resources.Memory = c.Resources.Memory
		container.Cgroups.Resources.MemoryReservation = c.Resources.MemoryReservation
		container.Cgroups.Resources.MemorySwap = c.Resources.MemorySwap
		container.Cgroups.Resources.KernelMemory = c.Resources.KernelMemory
		container.Cgroups.Resources.CpusetCpus = c.Resources.CpusetCpus
		container.Cgroups.Resources.CpusetMems = c.Resources.CpusetMems
		container.Cgroups.Resources.CpuPeriod = c.Resources.CPUPeriod
		container.Cgroups.Resources.CpuQuota = c.Resources.CPUQuota
		container.Cgroups.Resources.BlkioWeight = c.Resources.BlkioWeight
		container.Cgroups.Resources.BlkioWeightDevice = c.Resources.BlkioWeightDevice
		container.Cgroups.Resources.BlkioThrottleReadBpsDevice = c.Resources.BlkioThrottleReadBpsDevice
		container.Cgroups.Resources.BlkioThrottleWriteBpsDevice = c.Resources.BlkioThrottleWriteBpsDevice
		container.Cgroups.Resources.BlkioThrottleReadIOPSDevice = c.Resources.BlkioThrottleReadIOpsDevice
		container.Cgroups.Resources.BlkioThrottleWriteIOPSDevice = c.Resources.BlkioThrottleWriteIOpsDevice
		container.Cgroups.Resources.OomKillDisable = c.Resources.OomKillDisable
		container.Cgroups.Resources.MemorySwappiness = c.Resources.MemorySwappiness
	}

	return nil
}

// Returns the network statistics for the network interfaces represented by the NetworkRuntimeInfo.
func getNetworkInterfaceStats(interfaceName string) (*libcontainer.NetworkInterface, error) {
	out := &libcontainer.NetworkInterface{Name: interfaceName}
	// This can happen if the network runtime information is missing - possible if the
	// container was created by an old version of libcontainer.
	if interfaceName == "" {
		return out, nil
	}
	type netStatsPair struct {
		// Where to write the output.
		Out *uint64
		// The network stats file to read.
		File string
	}
	// Ingress for host veth is from the container. Hence tx_bytes stat on the host veth is actually number of bytes received by the container.
	netStats := []netStatsPair{
		{Out: &out.RxBytes, File: "tx_bytes"},
		{Out: &out.RxPackets, File: "tx_packets"},
		{Out: &out.RxErrors, File: "tx_errors"},
		{Out: &out.RxDropped, File: "tx_dropped"},

		{Out: &out.TxBytes, File: "rx_bytes"},
		{Out: &out.TxPackets, File: "rx_packets"},
		{Out: &out.TxErrors, File: "rx_errors"},
		{Out: &out.TxDropped, File: "rx_dropped"},
	}
	for _, netStat := range netStats {
		data, err := readSysfsNetworkStats(interfaceName, netStat.File)
		if err != nil {
			return nil, err
		}
		*(netStat.Out) = data
	}
	return out, nil
}

// Reads the specified statistics available under /sys/class/net/<EthInterface>/statistics
func readSysfsNetworkStats(ethInterface, statsFile string) (uint64, error) {
	data, err := ioutil.ReadFile(filepath.Join("/sys/class/net", ethInterface, "statistics", statsFile))
	if err != nil {
		return 0, err
	}
	return strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64)
}

// Stats collects all the resource usage information from a container.
func Stats(containerDir string, containerMemoryLimit int64, machineMemory int64) (*ResourceStats, error) {
	f, err := os.Open(filepath.Join(containerDir, "state.json"))
	if err != nil {
		return nil, err
	}
	defer f.Close()

	type network struct {
		Type              string
		HostInterfaceName string
	}

	state := struct {
		CgroupPaths map[string]string `json:"cgroup_paths"`
		Networks    []network
	}{}

	if err := json.NewDecoder(f).Decode(&state); err != nil {
		return nil, err
	}
	now := time.Now()

	mgr := fs.Manager{Paths: state.CgroupPaths}
	cstats, err := mgr.GetStats()
	if err != nil {
		return nil, err
	}
	stats := &libcontainer.Stats{CgroupStats: cstats}
	// if the container does not have any memory limit specified set the
	// limit to the machines memory
	memoryLimit := containerMemoryLimit
	if memoryLimit == 0 {
		memoryLimit = machineMemory
	}
	for _, iface := range state.Networks {
		switch iface.Type {
		case "veth":
			istats, err := getNetworkInterfaceStats(iface.HostInterfaceName)
			if err != nil {
				return nil, err
			}
			stats.Interfaces = append(stats.Interfaces, istats)
		}
	}
	return &ResourceStats{
		Stats:       stats,
		Read:        now,
		MemoryLimit: memoryLimit,
	}, nil
}

// User contains the uid and gid representing a Unix user
type User struct {
	UID int `json:"root_uid"`
	GID int `json:"root_gid"`
}

// ExitStatus provides exit reasons for a container.
type ExitStatus struct {
	// The exit code with which the container exited.
	ExitCode int

	// Whether the container encountered an OOM.
	OOMKilled bool
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2006-2011 Philipp Meinen <philipp@bind.ch>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the Software
 * is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH
 * THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

package properties

import (
	"errors"
	"fmt"
	"io/ioutil"
	"strings"

	"github.com/PhiCode/l10n_check/validate"
)

type Property struct {
	Key   string
	Value string
	Line  int
}

type Properties struct {
	file  string
	props []*Property
	ByKey map[string]*Property
}

func ReadAndParse(filename string) (*Properties, *validate.Results, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, nil, errors.New(fmt.Sprintf("could not open/read file '%s': %s", filename, err.Error()))
	}
	var v validate.Results
	v.Resource = filename
	var p Properties
	p.file = filename
	parse(data, &p, &v)
	return &p, &v, nil
}

func (props *Properties) String() string {
	parts := make([]string, len(props.props))
	for i, prop := range props.props {
		parts[i] = fmt.Sprintf("line %d '%s' = '%s'", prop.Line, prop.Key, prop.Value)
	}
	return strings.Join(parts, "\n")
}

func (props *Properties) PrintAll(indent string) {
	for _, prop := range props.props {
		fmt.Printf("%sline %d '%s' = '%s'\n", indent, prop.Line, prop.Key, prop.Value)
	}
}
<commit_msg>properties: simplify error return value generation<commit_after>/*
 * Copyright (c) 2006-2011 Philipp Meinen <philipp@bind.ch>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the Software
 * is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH
 * THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

package properties

import (
	"fmt"
	"io/ioutil"
	"strings"

	"github.com/PhiCode/l10n_check/validate"
)

type Property struct {
	Key   string
	Value string
	Line  int
}

type Properties struct {
	file  string
	props []*Property
	ByKey map[string]*Property
}

func ReadAndParse(filename string) (*Properties, *validate.Results, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, nil, fmt.Errorf("could not open/read file '%s': %s", filename, err.Error())
	}
	var v validate.Results
	v.Resource = filename
	var p Properties
	p.file = filename
	parse(data, &p, &v)
	return &p, &v, nil
}

func (props *Properties) String() string {
	parts := make([]string, len(props.props))
	for i, prop := range props.props {
		parts[i] = fmt.Sprintf("line %d '%s' = '%s'", prop.Line, prop.Key, prop.Value)
	}
	return strings.Join(parts, "\n")
}

func (props *Properties) PrintAll(indent string) {
	for _, prop := range props.props {
		fmt.Printf("%sline %d '%s' = '%s'\n", indent, prop.Line, prop.Key, prop.Value)
	}
}
<|endoftext|>


<commit_before>// Generated by reader_util_test.go.
// Only edit to bootstrap new entries or change existing entries.

package fit_test

var decodeTestFiles = [...]struct {
	folder      string
	name        string
	wantErr     bool
	fingerprint uint64
	compress    bool
	dopts       testingDecodeOpts
	skipEncode  bool
	encodeNote  string
}{
	{
		"me",
		"activity-small-fenix2-run.fit",
		false,
		16272604713108132935,
		true,
		tdoAllWithDiscardLogger,
		true,
		"Decode mismatch due to unknown fields",
	},
	{
		"fitsdk",
		"Activity.fit",
		false,
		5393063379197673570,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"fitsdk",
		"MonitoringFile.fit",
		false,
		11936585269915402423,
		true,
		tdoNone,
		true,
		"Fails because first message has different valid fields (#36)",
	},
	{
		"fitsdk",
		"Settings.fit",
		false,
		18422634047426156243,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"fitsdk",
		"WeightScaleMultiUser.fit",
		false,
		15668939108214135507,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"fitsdk",
		"WorkoutCustomTargetValues.fit",
		false,
		14786462533817802322,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"fitsdk",
		"WorkoutIndividualSteps.fit",
		false,
		5594723163894392697,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"fitsdk",
		"WorkoutRepeatGreaterThanStep.fit",
		false,
		17460594330644784595,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"fitsdk",
		"WorkoutRepeatSteps.fit",
		false,
		4256650519608265147,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"fitsdk",
		"WeightScaleSingleUser.fit",
		false,
		16394851171432919279,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"python-fitparse",
		"garmin-edge-500-activitiy.fit",
		false,
		17514013668470756651,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"python-fitparse",
		"sample-activity-indoor-trainer.fit",
		false,
		945649812206588852,
		true,
		tdoNone,
		true,
		"Fails because first message has different valid fields (#36)",
	},
	{
		"python-fitparse",
		"compressed-speed-distance.fit",
		false,
		0,
		false,
		tdoNone,
		false,
		"",
	},
	{
		"python-fitparse",
		"antfs-dump.63.fit",
		false,
		6282273622209975218,
		true,
		tdoNone,
		true,
		"Fails because first message has different valid fields (#36)",
	},
	{
		"sram",
		"Settings.fit",
		false,
		5866657363356029809,
		true,
		tdoNone,
		true,
		"Fails due to encoder using profile length for arrays (#37)",
	},
	{
		"sram",
		"Settings2.fit",
		false,
		15709312684722569429,
		true,
		tdoNone,
		true,
		"Fails due to encoder using profile length for arrays (#37)",
	},
	{
		"dcrainmaker",
		"Edge810-Vector-2013-08-16-15-35-10.fit",
		false,
		12420128971150793206,
		true,
		tdoNone,
		true,
		"Fails because first message has different valid fields (#36)",
	},
	{
		"misc",
		"2013-02-06-12-11-14.fit",
		false,
		11959686082894445424,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"misc",
		"2015-10-13-08-43-15.fit",
		false,
		16776362073923423348,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"bpg",
		"garmin.fit",
		false,
		11468599866908951097,
		true,
		tdoNone,
		false,
		"Has a definition message with number of fields >85",
	},
	{
		"corrupt",
		"activity-filecrc.fit",
		true,
		13015127050946751954,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"corrupt",
		"activity-unexpected-eof.fit",
		true,
		5112456444205297678,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"misc",
		"0134902991.fit",
		false,
		1269717752691992296,
		true,
		tdoNone,
		true,
		"Contains developer data fields",
	},
	{
		"misc",
		"mornindew-broken.fit",
		false,
		864390381975294256,
		true,
		tdoNone,
		true,
		"Contains developer data fields",
	},
	{
		"fitsdk",
		"DeveloperData.fit",
		false,
		7735802126653373100,
		true,
		tdoNone,
		true,
		"Contains developer data fields",
	},
}
<commit_msg>reader_files_test: add 'soruce' field for test cases<commit_after>// Generated by reader_util_test.go.
// Only edit to bootstrap new entries or change existing entries.

package fit_test

var decodeTestFiles = [...]struct {
	folder      string
	name        string
	source      string
	wantErr     bool
	fingerprint uint64
	compress    bool
	dopts       testingDecodeOpts
	skipEncode  bool
	encodeNote  string
}{
	{
		"me",
		"activity-small-fenix2-run.fit",
		"",
		false,
		16272604713108132935,
		true,
		tdoAllWithDiscardLogger,
		true,
		"Decode mismatch due to unknown fields",
	},
	{
		"fitsdk",
		"Activity.fit",
		"",
		false,
		5393063379197673570,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"fitsdk",
		"MonitoringFile.fit",
		"",
		false,
		11936585269915402423,
		true,
		tdoNone,
		true,
		"Fails because first message has different valid fields (#36)",
	},
	{
		"fitsdk",
		"Settings.fit",
		"",
		false,
		18422634047426156243,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"fitsdk",
		"WeightScaleMultiUser.fit",
		"",
		false,
		15668939108214135507,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"fitsdk",
		"WorkoutCustomTargetValues.fit",
		"",
		false,
		14786462533817802322,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"fitsdk",
		"WorkoutIndividualSteps.fit",
		"",
		false,
		5594723163894392697,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"fitsdk",
		"WorkoutRepeatGreaterThanStep.fit",
		"",
		false,
		17460594330644784595,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"fitsdk",
		"WorkoutRepeatSteps.fit",
		"",
		false,
		4256650519608265147,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"fitsdk",
		"WeightScaleSingleUser.fit",
		"",
		false,
		16394851171432919279,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"python-fitparse",
		"garmin-edge-500-activitiy.fit",
		"",
		false,
		17514013668470756651,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"python-fitparse",
		"sample-activity-indoor-trainer.fit",
		"",
		false,
		945649812206588852,
		true,
		tdoNone,
		true,
		"Fails because first message has different valid fields (#36)",
	},
	{
		"python-fitparse",
		"compressed-speed-distance.fit",
		"",
		false,
		0,
		false,
		tdoNone,
		false,
		"",
	},
	{
		"python-fitparse",
		"antfs-dump.63.fit",
		"",
		false,
		6282273622209975218,
		true,
		tdoNone,
		true,
		"Fails because first message has different valid fields (#36)",
	},
	{
		"sram",
		"Settings.fit",
		"",
		false,
		5866657363356029809,
		true,
		tdoNone,
		true,
		"Fails due to encoder using profile length for arrays (#37)",
	},
	{
		"sram",
		"Settings2.fit",
		"",
		false,
		15709312684722569429,
		true,
		tdoNone,
		true,
		"Fails due to encoder using profile length for arrays (#37)",
	},
	{
		"dcrainmaker",
		"Edge810-Vector-2013-08-16-15-35-10.fit",
		"",
		false,
		12420128971150793206,
		true,
		tdoNone,
		true,
		"Fails because first message has different valid fields (#36)",
	},
	{
		"misc",
		"2013-02-06-12-11-14.fit",
		"",
		false,
		11959686082894445424,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"misc",
		"2015-10-13-08-43-15.fit",
		"",
		false,
		16776362073923423348,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"bpg",
		"garmin.fit",
		"https://github.com/tormoder/fit/pull/54",
		false,
		11468599866908951097,
		true,
		tdoNone,
		false,
		"Has a definition message with number of fields >85",
	},
	{
		"corrupt",
		"activity-filecrc.fit",
		"",
		true,
		13015127050946751954,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"corrupt",
		"activity-unexpected-eof.fit",
		"",
		true,
		5112456444205297678,
		true,
		tdoNone,
		false,
		"",
	},
	{
		"misc",
		"0134902991.fit",
		"https://github.com/tormoder/fit/pull/59",
		false,
		1269717752691992296,
		true,
		tdoNone,
		true,
		"Contains developer data fields",
	},
	{
		"misc",
		"mornindew-broken.fit",
		"https://github.com/tormoder/fit/issues/41",
		false,
		864390381975294256,
		true,
		tdoNone,
		true,
		"Contains developer data fields",
	},
	{
		"fitsdk",
		"DeveloperData.fit",
		"",
		false,
		7735802126653373100,
		true,
		tdoNone,
		true,
		"Contains developer data fields",
	},
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test mutator availability and end-to-end latency of an RPC-like system.
package main

// TODO: This benchmark doesn't target a particular aspect of the GC
// like other gcbench benchmarks do. It would be nice if we just had
// composable sets of metrics that a benchmark could opt in to without
// needing a whole "framework", but I'm not sure how to do that with
// the gctrace on stderr thing. Maybe we could dup our own FD over
// stderr to read our own gctrace? That's dangerous if something goes
// wrong since you'll lose the panic. We could pipe it by way of an
// external program that could report if something goes wrong (e.g.,
// tee if you just want to see the whole log). This might also make it
// easier to ignore initialization and warm-up in the GC log.

import (
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"net"
	"os"
	"os/exec"
	"runtime"
	"sync"
	"sync/atomic"
	"time"

	"github.com/aclements/go-gcbench/gcbench"
	"github.com/aclements/go-gcbench/gcbench/heapgen"
)

var (
	flagDuration = flag.Duration("benchtime", 10*time.Second, "steady state duration")
	flagBallast  = gcbench.FlagBytes("ballast", 64*gcbench.MB, "retain `x` bytes of global data")
	// XXX Make this reqs-per-sec-per-p so it scales properly.
	flagReqsPerSec = flag.Float64("reqs-per-sec", 8000, "send `rate` requests per second")
	flagClient     = flag.String("client", "", "internal flag to act as RPC client")
)

func main() {
	flag.Parse()
	if flag.NArg() > 0 {
		flag.Usage()
		os.Exit(2)
	}

	if *flagClient != "" {
		doClient(*flagClient)
		return
	}

	name := "RPC"
	gcbench.NewBenchmark(name, benchMain).Config("reqs-per-sec", *flagReqsPerSec).Config("ballast", *flagBallast).Run()
}

var sink1 interface{}
var requestCount int64
var serverLatency gcbench.LatencyDist

func benchMain() {
	// Create the ballast.
	m := heapgen.Measure(heapgen.MakeAST)
	sink1 = heapgen.Generate(m.Gen, m.BytesRetained, int(*flagBallast))

	// Divide GOMAXPROCS by two so it's split between client and
	// server.
	gomaxprocs := runtime.GOMAXPROCS(-1)
	runtime.GOMAXPROCS(gomaxprocs / 2)

	// Start the server.
	l := startServer()

	// Start the client.
	client := exec.Command(os.Args[0], "-client", l.Addr().String(), "-reqs-per-sec", fmt.Sprint(*flagReqsPerSec))
	client.Env = []string{"GODEBUG=gctrace=1", "GOGC=off",
		fmt.Sprintf("GOMAXPROCS=%d", gomaxprocs/2)}
	cin, err := client.StdinPipe()
	if err != nil {
		log.Fatal("creating client stdin pipe: ", err)
	}
	defer cin.Close()
	cout, err := client.StdoutPipe()
	if err != nil {
		log.Fatal("creating client stdout pipe: ", err)
	}
	defer cout.Close()
	client.Stderr = os.Stderr
	err = client.Start()
	if err != nil {
		log.Fatal("starting client: ", err)
	}

	// Run the server.
	done := make(chan bool)
	startTime := time.Now()
	time.AfterFunc(*flagDuration, func() {
		gcbench.ReportExtra("reqs/sec", float64(atomic.LoadInt64(&requestCount))/float64(time.Since(startTime))*float64(time.Second))

		// Shut down client.
		cin.Close()
		err := client.Wait()
		if err != nil {
			log.Fatal("client failed: ", err)
		}
		close(done)
	})
	go func() {
		for {
			c, err := l.Accept()
			if err != nil {
				log.Fatal("accept: ", err)
			}
			go handler(c)
		}
	}()
	<-done
}

func startServer() net.Listener {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		if l, err = net.Listen("tcp6", "[::1]:0"); err != nil {
			log.Fatalf("failed to listen: %v", err)
		}
	}
	return l
}

func handler(c net.Conn) {
	var header [1]byte
	r, w := bufio.NewReader(c), bufio.NewWriter(c)
	d := json.NewDecoder(r)
	for {
		// Read the request.
		n, err := r.Read(header[:])
		if err == io.EOF {
			c.Close()
			return
		}
		if n != 1 || header[0] != 'x' {
			log.Fatal("bad header: ", n, err, header)
		}

		var data interface{}
		err = d.Decode(&data)
		if err != nil {
			log.Fatal("decoding request: ", err)
		}

		atomic.AddInt64(&requestCount, 1)

		// Send the response.
		json.NewEncoder(w).Encode(data)
		w.Flush()
	}
}

func doClient(addr string) {
	var wg sync.WaitGroup
	stop := make(chan bool)

	var lat gcbench.LatencyDist

	// Wait for stdin to close, then shut down cleanly.
	go func() {
		var buf [1024]byte
		for {
			n, err := os.Stdin.Read(buf[:])
			if n <= 0 || err != nil {
				break
			}
		}
		log.Print("cleaning up client")
		close(stop)
		wg.Wait()
		log.Print("closing ", len(conns.q), " connections")
		for _, c := range conns.q {
			c.Close()
		}
		fmt.Fprintln(os.Stderr, "client-measured request latency:")
		lat.FprintHist(os.Stderr, 70, 5)
		gcbench.ReportExtra("P99-latency-ns", float64(lat.Quantile(0.99)))
		gcbench.ReportExtra("P99.9-latency-ns", float64(lat.Quantile(0.999)))
		gcbench.ReportExtra("max-latency-ns", float64(lat.Max))
		//log.Print("client exiting")
		os.Exit(0)
	}()

	// Generate a Poisson process by spacing events out by an
	// exponential distribution and using open-loop control. To
	// minimize the bottleneck in the coordinator, we shard this
	// process across several goroutines.
	const shards = 16
	reqsPerNSPerShard := *flagReqsPerSec / 1e9 / shards
	for i := 0; i < shards; i++ {
		wg.Add(1)
		go func() {
		loop:
			for {
				wait := rand.ExpFloat64() / reqsPerNSPerShard
				select {
				case <-stop:
					break loop
				case <-time.After(time.Duration(wait)):
				}
				wg.Add(1)
				go sendOneRequest(addr, &wg, &lat)
			}
			wg.Done()
		}()
	}
	select {}
}

var reqBytes = []byte(`x{
	"Dir": "src/runtime",
	"ImportPath": "runtime",
	"Name": "runtime",
	"Doc": "Package runtime contains operations that interact with Go's runtime system, such as functions to control goroutines.",
	"Target": "pkg/linux_amd64/runtime.a",
	"Goroot": true,
	"Standard": true,
	"Stale": true,
	"StaleReason": "stale dependency",
	"Root": "go.dev"
}`)

var conns struct {
	q []net.Conn
	sync.Mutex
}

func sendOneRequest(addr string, wg *sync.WaitGroup, lat *gcbench.LatencyDist) {
	defer wg.Done()

	var c net.Conn
	conns.Lock()
	if len(conns.q) > 0 {
		c = conns.q[len(conns.q)-1]
		conns.q = conns.q[:len(conns.q)-1]
	}
	conns.Unlock()
	if c == nil {
		var err error
		c, err = net.Dial("tcp", addr)
		if err != nil {
			// This could just mean the server is
			// overloaded. (TODO: Maybe it's this single
			// Accept that gets overloaded?)
			log.Print("dialing ", addr, ": ", err)
			return
		}
	}

	// Send the request.
	//log.Print("sending request", c)
	lt := lat.Start()
	n, err := c.Write(reqBytes)
	if err != nil {
		log.Fatal("writing request: ", err)
	}
	if n != len(reqBytes) {
		log.Fatal("short write")
	}
	//log.Print("sent request; reading response", c)

	// Read the response.
	var buf [1024]byte
	var done bool
	for !done {
		n, err := c.Read(buf[:])
		if bytes.IndexRune(buf[:n], '\n') >= 0 {
			done = true
		}
		if err == io.EOF {
			c.Close()
			return
		}
		if err != nil {
			log.Fatal("reading response: ", err)
		}
	}
	lt.Tick()
	//log.Print("read response", c)

	conns.Lock()
	conns.q = append(conns.q, c)
	conns.Unlock()
}
<commit_msg>gcbench/progs/rpc: report server-measured latency<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test mutator availability and end-to-end latency of an RPC-like system.
package main

// TODO: This benchmark doesn't target a particular aspect of the GC
// like other gcbench benchmarks do. It would be nice if we just had
// composable sets of metrics that a benchmark could opt in to without
// needing a whole "framework", but I'm not sure how to do that with
// the gctrace on stderr thing. Maybe we could dup our own FD over
// stderr to read our own gctrace? That's dangerous if something goes
// wrong since you'll lose the panic. We could pipe it by way of an
// external program that could report if something goes wrong (e.g.,
// tee if you just want to see the whole log). This might also make it
// easier to ignore initialization and warm-up in the GC log.

import (
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"net"
	"os"
	"os/exec"
	"runtime"
	"sync"
	"sync/atomic"
	"time"

	"github.com/aclements/go-gcbench/gcbench"
	"github.com/aclements/go-gcbench/gcbench/heapgen"
)

var (
	flagDuration = flag.Duration("benchtime", 10*time.Second, "steady state duration")
	flagBallast  = gcbench.FlagBytes("ballast", 64*gcbench.MB, "retain `x` bytes of global data")
	// XXX Make this reqs-per-sec-per-p so it scales properly.
	flagReqsPerSec = flag.Float64("reqs-per-sec", 8000, "send `rate` requests per second")
	flagClient     = flag.String("client", "", "internal flag to act as RPC client")
)

func main() {
	flag.Parse()
	if flag.NArg() > 0 {
		flag.Usage()
		os.Exit(2)
	}

	if *flagClient != "" {
		doClient(*flagClient)
		return
	}

	name := "RPC"
	gcbench.NewBenchmark(name, benchMain).Config("reqs-per-sec", *flagReqsPerSec).Config("ballast", *flagBallast).Run()
}

var sink1 interface{}
var requestCount int64
var serverLatency gcbench.LatencyDist

func benchMain() {
	// Create the ballast.
	m := heapgen.Measure(heapgen.MakeAST)
	sink1 = heapgen.Generate(m.Gen, m.BytesRetained, int(*flagBallast))

	// Divide GOMAXPROCS by two so it's split between client and
	// server.
	gomaxprocs := runtime.GOMAXPROCS(-1)
	runtime.GOMAXPROCS(gomaxprocs / 2)

	// Start the server.
	l := startServer()

	// Start the client.
	client := exec.Command(os.Args[0], "-client", l.Addr().String(), "-reqs-per-sec", fmt.Sprint(*flagReqsPerSec))
	client.Env = []string{"GODEBUG=gctrace=1", "GOGC=off",
		fmt.Sprintf("GOMAXPROCS=%d", gomaxprocs/2)}
	cin, err := client.StdinPipe()
	if err != nil {
		log.Fatal("creating client stdin pipe: ", err)
	}
	defer cin.Close()
	cout, err := client.StdoutPipe()
	if err != nil {
		log.Fatal("creating client stdout pipe: ", err)
	}
	defer cout.Close()
	var clientOut bytes.Buffer
	client.Stderr = &clientOut
	err = client.Start()
	if err != nil {
		log.Fatal("starting client: ", err)
	}

	// Run the server.
	done := make(chan bool)
	startTime := time.Now()
	time.AfterFunc(*flagDuration, func() {
		gcbench.ReportExtra("reqs/sec", float64(atomic.LoadInt64(&requestCount))/float64(time.Since(startTime))*float64(time.Second))

		// Server-reported latency.
		fmt.Fprintf(os.Stderr, "server-measured request latency:\n")
		serverLatency.FprintHist(os.Stderr, 70, 5)

		// Shut down client.
		cin.Close()
		err := client.Wait()
		fmt.Fprintf(os.Stderr, "client stderr:\n%s", clientOut.String())
		if err != nil {
			log.Fatal("client failed: ", err)
		}
		close(done)
	})
	go func() {
		for {
			c, err := l.Accept()
			if err != nil {
				log.Fatal("accept: ", err)
			}
			go handler(c)
		}
	}()
	<-done
}

func startServer() net.Listener {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		if l, err = net.Listen("tcp6", "[::1]:0"); err != nil {
			log.Fatalf("failed to listen: %v", err)
		}
	}
	return l
}

func handler(c net.Conn) {
	var header [1]byte
	r, w := bufio.NewReader(c), bufio.NewWriter(c)
	d := json.NewDecoder(r)
	for {
		// Read the request.
		n, err := r.Read(header[:])
		if err == io.EOF {
			c.Close()
			return
		}
		lt := serverLatency.Start()
		if n != 1 || header[0] != 'x' {
			log.Fatal("bad header: ", n, err, header)
		}

		var data interface{}
		err = d.Decode(&data)
		if err != nil {
			log.Fatal("decoding request: ", err)
		}

		atomic.AddInt64(&requestCount, 1)

		// Send the response.
		json.NewEncoder(w).Encode(data)
		w.Flush()
		lt.Tick()
	}
}

func doClient(addr string) {
	var wg sync.WaitGroup
	stop := make(chan bool)

	var lat gcbench.LatencyDist

	// Wait for stdin to close, then shut down cleanly.
	go func() {
		var buf [1024]byte
		for {
			n, err := os.Stdin.Read(buf[:])
			if n <= 0 || err != nil {
				break
			}
		}
		log.Print("cleaning up client")
		close(stop)
		wg.Wait()
		log.Print("closing ", len(conns.q), " connections")
		for _, c := range conns.q {
			c.Close()
		}
		fmt.Fprintln(os.Stderr, "client-measured request latency:")
		lat.FprintHist(os.Stderr, 70, 5)
		gcbench.ReportExtra("P99-latency-ns", float64(lat.Quantile(0.99)))
		gcbench.ReportExtra("P99.9-latency-ns", float64(lat.Quantile(0.999)))
		gcbench.ReportExtra("max-latency-ns", float64(lat.Max))
		//log.Print("client exiting")
		os.Exit(0)
	}()

	// Generate a Poisson process by spacing events out by an
	// exponential distribution and using open-loop control. To
	// minimize the bottleneck in the coordinator, we shard this
	// process across several goroutines.
	const shards = 16
	reqsPerNSPerShard := *flagReqsPerSec / 1e9 / shards
	for i := 0; i < shards; i++ {
		wg.Add(1)
		go func() {
		loop:
			for {
				wait := rand.ExpFloat64() / reqsPerNSPerShard
				select {
				case <-stop:
					break loop
				case <-time.After(time.Duration(wait)):
				}
				wg.Add(1)
				go sendOneRequest(addr, &wg, &lat)
			}
			wg.Done()
		}()
	}
	select {}
}

var reqBytes = []byte(`x{
	"Dir": "src/runtime",
	"ImportPath": "runtime",
	"Name": "runtime",
	"Doc": "Package runtime contains operations that interact with Go's runtime system, such as functions to control goroutines.",
	"Target": "pkg/linux_amd64/runtime.a",
	"Goroot": true,
	"Standard": true,
	"Stale": true,
	"StaleReason": "stale dependency",
	"Root": "go.dev"
}`)

var conns struct {
	q []net.Conn
	sync.Mutex
}

func sendOneRequest(addr string, wg *sync.WaitGroup, lat *gcbench.LatencyDist) {
	defer wg.Done()

	var c net.Conn
	conns.Lock()
	if len(conns.q) > 0 {
		c = conns.q[len(conns.q)-1]
		conns.q = conns.q[:len(conns.q)-1]
	}
	conns.Unlock()
	if c == nil {
		var err error
		c, err = net.Dial("tcp", addr)
		if err != nil {
			// This could just mean the server is
			// overloaded. (TODO: Maybe it's this single
			// Accept that gets overloaded?)
			log.Print("dialing ", addr, ": ", err)
			return
		}
	}

	// Send the request.
	//log.Print("sending request", c)
	lt := lat.Start()
	n, err := c.Write(reqBytes)
	if err != nil {
		log.Fatal("writing request: ", err)
	}
	if n != len(reqBytes) {
		log.Fatal("short write")
	}
	//log.Print("sent request; reading response", c)

	// Read the response.
	var buf [1024]byte
	var done bool
	for !done {
		n, err := c.Read(buf[:])
		if bytes.IndexRune(buf[:n], '\n') >= 0 {
			done = true
		}
		if err == io.EOF {
			c.Close()
			return
		}
		if err != nil {
			log.Fatal("reading response: ", err)
		}
	}
	lt.Tick()
	//log.Print("read response", c)

	conns.Lock()
	conns.q = append(conns.q, c)
	conns.Unlock()
}
<|endoftext|>