<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel

import (
	"encoding/json"
	"runtime"
	"sort"

	"golang.org/x/net/context"
)

// IntrospectionOptions are the options used when introspecting the Channel.
type IntrospectionOptions struct {
	// IncludeExchanges will include all the IDs in the message exchanges.
	IncludeExchanges bool `json:"includeExchanges"`

	// IncludeEmptyPeers will include peers, even if they have no connections.
	IncludeEmptyPeers bool `json:"includeEmptyPeers"`
}

// RuntimeState is a snapshot of the runtime state for a channel.
type RuntimeState struct {
	// CreatedStack is the stack for how this channel was created.
	CreatedStack string `json:"createdStack"`

	// LocalPeer is the local peer information (service name, host-port, etc).
	LocalPeer LocalPeerInfo `json:"localPeer"`

	// SubChannels contains information about any subchannels.
	SubChannels map[string]SubChannelRuntimeState `json:"subChannels"`

	// RootPeers contains information about all the peers on this channel and their connections.
	RootPeers map[string]PeerRuntimeState `json:"rootPeers"`

	// Peers is the list of shared peers for this channel.
	Peers []SubPeerScore `json:"peers"`

	// NumConnections is the number of connections stored in the channel.
	NumConnections int `json:"numConnections"`

	// OtherChannels is information about any other channels running in this process.
	OtherChannels map[string][]ChannelInfo `json:"otherChannels,omitEmpty"`
}

// GoRuntimeStateOptions are the options used when getting Go runtime state.
type GoRuntimeStateOptions struct {
	// IncludeGoStacks will include all goroutine stacks.
	IncludeGoStacks bool `json:"includeGoStacks"`
}

// ChannelInfo is the state of other channels in the same process.
type ChannelInfo struct {
	CreatedStack string        `json:"createdStack"`
	LocalPeer    LocalPeerInfo `json:"localPeer"`
}

// GoRuntimeState is a snapshot of runtime stats from the runtime.
type GoRuntimeState struct {
	MemStats      runtime.MemStats `json:"memStats"`
	NumGoroutines int              `json:"numGoRoutines"`
	NumCPU        int              `json:"numCPU"`
	NumCGo        int64            `json:"numCGo"`
	GoStacks      []byte           `json:"goStacks,omitempty"`
}

// SubChannelRuntimeState is the runtime state for a subchannel.
type SubChannelRuntimeState struct {
	Service  string `json:"service"`
	Isolated bool   `json:"isolated"`
	// IsolatedPeers is the list of all isolated peers for this channel.
	IsolatedPeers []SubPeerScore      `json:"isolatedPeers,omitempty"`
	Handler       HandlerRuntimeState `json:"handler"`
}

// HandlerRuntimeState TODO
type HandlerRuntimeState struct {
	Type    handlerType `json:"type"`
	Methods []string    `json:"methods,omitempty"`
}

type handlerType string

func (h handlerType) String() string { return string(h) }

const (
	methodHandler   handlerType = "methods"
	overrideHandler             = "overriden"
)

// SubPeerScore show the runtime state of a peer with score.
type SubPeerScore struct {
	HostPort string `json:"hostPort"`
	Score    uint64 `json:"score"`
}

// ConnectionRuntimeState is the runtime state for a single connection.
type ConnectionRuntimeState struct {
	ID               uint32                  `json:"id"`
	ConnectionState  string                  `json:"connectionState"`
	LocalHostPort    string                  `json:"localHostPort"`
	RemoteHostPort   string                  `json:"remoteHostPort"`
	IsEphemeral      bool                    `json:"isEphemeral"`
	InboundExchange  ExchangeSetRuntimeState `json:"inboundExchange"`
	OutboundExchange ExchangeSetRuntimeState `json:"outboundExchange"`
}

// ExchangeSetRuntimeState is the runtime state for a message exchange set.
type ExchangeSetRuntimeState struct {
	Name      string                 `json:"name"`
	Count     int                    `json:"count"`
	Exchanges []ExchangeRuntimeState `json:"exchanges,omitempty"`
}

// ExchangeRuntimeState is the runtime state for a single message exchange.
type ExchangeRuntimeState struct {
	ID          uint32      `json:"id"`
	MessageType messageType `json:"messageType"`
}

// PeerRuntimeState is the runtime state for a single peer.
type PeerRuntimeState struct {
	HostPort            string                   `json:"hostPort"`
	OutboundConnections []ConnectionRuntimeState `json:"outboundConnections"`
	InboundConnections  []ConnectionRuntimeState `json:"inboundConnections"`
	ChosenCount         uint64                   `json:"chosenCount"`
	SCCount             uint32                   `json:"scCount"`
}

// IntrospectState returns the RuntimeState for this channel.
// Note: this is purely for debugging and monitoring, and may slow down your Channel.
func (ch *Channel) IntrospectState(opts *IntrospectionOptions) *RuntimeState {
	if opts == nil {
		opts = &IntrospectionOptions{}
	}

	ch.mutable.RLock()
	conns := len(ch.mutable.conns)
	ch.mutable.RUnlock()

	return &RuntimeState{
		CreatedStack:   ch.createdStack,
		LocalPeer:      ch.PeerInfo(),
		SubChannels:    ch.subChannels.IntrospectState(opts),
		RootPeers:      ch.rootPeers().IntrospectState(opts),
		Peers:          ch.Peers().IntrospectList(opts),
		NumConnections: conns,
		OtherChannels:  ch.IntrospectOthers(opts),
	}
}

// IntrospectOthers returns the ChannelInfo for all other channels in this process.
func (ch *Channel) IntrospectOthers(opts *IntrospectionOptions) map[string][]ChannelInfo {
	channelMap.Lock()
	defer channelMap.Unlock()

	states := make(map[string][]ChannelInfo)
	for svc, channels := range channelMap.existing {
		channelInfos := make([]ChannelInfo, 0, len(channels))
		for _, otherChan := range channels {
			if ch == otherChan {
				continue
			}
			channelInfos = append(channelInfos, otherChan.ReportInfo(opts))
		}
		states[svc] = channelInfos
	}

	return states
}

// ReportInfo returns ChannelInfo for a channel.
func (ch *Channel) ReportInfo(opts *IntrospectionOptions) ChannelInfo {
	return ChannelInfo{
		CreatedStack: ch.createdStack,
		LocalPeer:    ch.PeerInfo(),
	}
}

type containsPeerList interface {
	Copy() map[string]*Peer
}

func fromPeerList(peers containsPeerList, opts *IntrospectionOptions) map[string]PeerRuntimeState {
	m := make(map[string]PeerRuntimeState)
	for _, peer := range peers.Copy() {
		peerState := peer.IntrospectState(opts)
		if len(peerState.InboundConnections)+len(peerState.OutboundConnections) > 0 || opts.IncludeEmptyPeers {
			m[peer.HostPort()] = peerState
		}
	}
	return m
}

// IntrospectState returns the runtime state of the peer list.
func (l *PeerList) IntrospectState(opts *IntrospectionOptions) map[string]PeerRuntimeState {
	return fromPeerList(l, opts)
}

// IntrospectState returns the runtime state of the
func (l *RootPeerList) IntrospectState(opts *IntrospectionOptions) map[string]PeerRuntimeState {
	return fromPeerList(l, opts)
}

// IntrospectState returns the runtime state of the subchannels.
func (subChMap *subChannelMap) IntrospectState(opts *IntrospectionOptions) map[string]SubChannelRuntimeState {
	m := make(map[string]SubChannelRuntimeState)
	subChMap.RLock()
	for k, sc := range subChMap.subchannels {
		state := SubChannelRuntimeState{
			Service:  k,
			Isolated: sc.Isolated(),
		}
		if state.Isolated {
			state.IsolatedPeers = sc.Peers().IntrospectList(opts)
		}
		if hmap, ok := sc.handler.(*handlerMap); ok {
			state.Handler.Type = methodHandler
			methods := make([]string, 0, len(hmap.handlers))
			for k := range hmap.handlers {
				methods = append(methods, k)
			}
			sort.Strings(methods)
			state.Handler.Methods = methods
		} else {
			state.Handler.Type = overrideHandler
		}
		m[k] = state
	}
	subChMap.RUnlock()
	return m
}

func getConnectionRuntimeState(conns []*Connection, opts *IntrospectionOptions) []ConnectionRuntimeState {
	connStates := make([]ConnectionRuntimeState, len(conns))

	for i, conn := range conns {
		connStates[i] = conn.IntrospectState(opts)
	}

	return connStates
}

// IntrospectState returns the runtime state for this peer.
func (p *Peer) IntrospectState(opts *IntrospectionOptions) PeerRuntimeState {
	p.RLock()
	defer p.RUnlock()

	return PeerRuntimeState{
		HostPort:            p.hostPort,
		InboundConnections:  getConnectionRuntimeState(p.inboundConnections, opts),
		OutboundConnections: getConnectionRuntimeState(p.outboundConnections, opts),
		ChosenCount:         p.chosenCount.Load(),
		SCCount:             p.scCount,
	}
}

// IntrospectState returns the runtime state for this connection.
func (c *Connection) IntrospectState(opts *IntrospectionOptions) ConnectionRuntimeState {
	c.stateMut.RLock()
	defer c.stateMut.RUnlock()

	return ConnectionRuntimeState{
		ID:               c.connID,
		ConnectionState:  c.state.String(),
		LocalHostPort:    c.conn.LocalAddr().String(),
		RemoteHostPort:   c.conn.RemoteAddr().String(),
		IsEphemeral:      c.remotePeerInfo.IsEphemeral,
		InboundExchange:  c.inbound.IntrospectState(opts),
		OutboundExchange: c.outbound.IntrospectState(opts),
	}
}

// IntrospectState returns the runtime state for this messsage exchange set.
func (mexset *messageExchangeSet) IntrospectState(opts *IntrospectionOptions) ExchangeSetRuntimeState {
	mexset.RLock()
	setState := ExchangeSetRuntimeState{
		Name:  mexset.name,
		Count: len(mexset.exchanges),
	}

	if opts.IncludeExchanges {
		setState.Exchanges = make([]ExchangeRuntimeState, 0, len(mexset.exchanges))
		for k, v := range mexset.exchanges {
			state := ExchangeRuntimeState{
				ID:          k,
				MessageType: v.msgType,
			}
			setState.Exchanges = append(setState.Exchanges, state)
		}
	}

	mexset.RUnlock()
	return setState
}

func getStacks(all bool) []byte {
	var buf []byte
	for n := 4096; n < 10*1024*1024; n *= 2 {
		buf = make([]byte, n)
		stackLen := runtime.Stack(buf, all)
		if stackLen < n {
			return buf
		}
	}

	// return the first 10MB of stacks if we have more than 10MB.
	return buf
}
func (ch *Channel) handleIntrospection(arg3 []byte) interface{} {
	var opts IntrospectionOptions
	json.Unmarshal(arg3, &opts)
	return ch.IntrospectState(&opts)
}

// IntrospectList returns the list of peers (hostport, score) in this peer list.
func (l *PeerList) IntrospectList(opts *IntrospectionOptions) []SubPeerScore {
	var peers []SubPeerScore
	l.RLock()
	for _, ps := range l.peerHeap.peerScores {
		peers = append(peers, SubPeerScore{
			HostPort: ps.Peer.hostPort,
			Score:    ps.score,
		})
	}
	l.RUnlock()

	return peers
}

func handleInternalRuntime(arg3 []byte) interface{} {
	var opts GoRuntimeStateOptions
	json.Unmarshal(arg3, &opts)

	state := GoRuntimeState{
		NumGoroutines: runtime.NumGoroutine(),
		NumCPU:        runtime.NumCPU(),
		NumCGo:        runtime.NumCgoCall(),
	}
	runtime.ReadMemStats(&state.MemStats)
	if opts.IncludeGoStacks {
		state.GoStacks = getStacks(true /* all */)
	}

	return state
}

// registerInternal registers the following internal handlers which return runtime state:
//  _gometa_introspect: TChannel internal state.
//  _gometa_runtime: Golang runtime stats.
func (ch *Channel) registerInternal() {
	endpoints := []struct {
		name    string
		handler func([]byte) interface{}
	}{
		{"_gometa_introspect", ch.handleIntrospection},
		{"_gometa_runtime", handleInternalRuntime},
	}

	for _, ep := range endpoints {
		// We need ep in our closure.
		ep := ep
		handler := func(ctx context.Context, call *InboundCall) {
			var arg2, arg3 []byte
			if err := NewArgReader(call.Arg2Reader()).Read(&arg2); err != nil {
				return
			}
			if err := NewArgReader(call.Arg3Reader()).Read(&arg3); err != nil {
				return
			}
			if err := NewArgWriter(call.Response().Arg2Writer()).Write(nil); err != nil {
				return
			}
			NewArgWriter(call.Response().Arg3Writer()).WriteJSON(ep.handler(arg3))
		}
		ch.Register(HandlerFunc(handler), ep.name)
	}
}
<commit_msg>Fix for stacks ending with 0 bytes<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel

import (
	"encoding/json"
	"runtime"
	"sort"

	"golang.org/x/net/context"
)

// IntrospectionOptions are the options used when introspecting the Channel.
type IntrospectionOptions struct {
	// IncludeExchanges will include all the IDs in the message exchanges.
	IncludeExchanges bool `json:"includeExchanges"`

	// IncludeEmptyPeers will include peers, even if they have no connections.
	IncludeEmptyPeers bool `json:"includeEmptyPeers"`
}

// RuntimeState is a snapshot of the runtime state for a channel.
type RuntimeState struct {
	// CreatedStack is the stack for how this channel was created.
	CreatedStack string `json:"createdStack"`

	// LocalPeer is the local peer information (service name, host-port, etc).
	LocalPeer LocalPeerInfo `json:"localPeer"`

	// SubChannels contains information about any subchannels.
	SubChannels map[string]SubChannelRuntimeState `json:"subChannels"`

	// RootPeers contains information about all the peers on this channel and their connections.
	RootPeers map[string]PeerRuntimeState `json:"rootPeers"`

	// Peers is the list of shared peers for this channel.
	Peers []SubPeerScore `json:"peers"`

	// NumConnections is the number of connections stored in the channel.
	NumConnections int `json:"numConnections"`

	// OtherChannels is information about any other channels running in this process.
	OtherChannels map[string][]ChannelInfo `json:"otherChannels,omitEmpty"`
}

// GoRuntimeStateOptions are the options used when getting Go runtime state.
type GoRuntimeStateOptions struct {
	// IncludeGoStacks will include all goroutine stacks.
	IncludeGoStacks bool `json:"includeGoStacks"`
}

// ChannelInfo is the state of other channels in the same process.
type ChannelInfo struct {
	CreatedStack string        `json:"createdStack"`
	LocalPeer    LocalPeerInfo `json:"localPeer"`
}

// GoRuntimeState is a snapshot of runtime stats from the runtime.
type GoRuntimeState struct {
	MemStats      runtime.MemStats `json:"memStats"`
	NumGoroutines int              `json:"numGoRoutines"`
	NumCPU        int              `json:"numCPU"`
	NumCGo        int64            `json:"numCGo"`
	GoStacks      []byte           `json:"goStacks,omitempty"`
}

// SubChannelRuntimeState is the runtime state for a subchannel.
type SubChannelRuntimeState struct {
	Service  string `json:"service"`
	Isolated bool   `json:"isolated"`
	// IsolatedPeers is the list of all isolated peers for this channel.
	IsolatedPeers []SubPeerScore      `json:"isolatedPeers,omitempty"`
	Handler       HandlerRuntimeState `json:"handler"`
}

// HandlerRuntimeState TODO
type HandlerRuntimeState struct {
	Type    handlerType `json:"type"`
	Methods []string    `json:"methods,omitempty"`
}

type handlerType string

func (h handlerType) String() string { return string(h) }

const (
	methodHandler   handlerType = "methods"
	overrideHandler             = "overriden"
)

// SubPeerScore show the runtime state of a peer with score.
type SubPeerScore struct {
	HostPort string `json:"hostPort"`
	Score    uint64 `json:"score"`
}

// ConnectionRuntimeState is the runtime state for a single connection.
type ConnectionRuntimeState struct {
	ID               uint32                  `json:"id"`
	ConnectionState  string                  `json:"connectionState"`
	LocalHostPort    string                  `json:"localHostPort"`
	RemoteHostPort   string                  `json:"remoteHostPort"`
	IsEphemeral      bool                    `json:"isEphemeral"`
	InboundExchange  ExchangeSetRuntimeState `json:"inboundExchange"`
	OutboundExchange ExchangeSetRuntimeState `json:"outboundExchange"`
}

// ExchangeSetRuntimeState is the runtime state for a message exchange set.
type ExchangeSetRuntimeState struct {
	Name      string                 `json:"name"`
	Count     int                    `json:"count"`
	Exchanges []ExchangeRuntimeState `json:"exchanges,omitempty"`
}

// ExchangeRuntimeState is the runtime state for a single message exchange.
type ExchangeRuntimeState struct {
	ID          uint32      `json:"id"`
	MessageType messageType `json:"messageType"`
}

// PeerRuntimeState is the runtime state for a single peer.
type PeerRuntimeState struct {
	HostPort            string                   `json:"hostPort"`
	OutboundConnections []ConnectionRuntimeState `json:"outboundConnections"`
	InboundConnections  []ConnectionRuntimeState `json:"inboundConnections"`
	ChosenCount         uint64                   `json:"chosenCount"`
	SCCount             uint32                   `json:"scCount"`
}

// IntrospectState returns the RuntimeState for this channel.
// Note: this is purely for debugging and monitoring, and may slow down your Channel.
func (ch *Channel) IntrospectState(opts *IntrospectionOptions) *RuntimeState {
	if opts == nil {
		opts = &IntrospectionOptions{}
	}

	ch.mutable.RLock()
	conns := len(ch.mutable.conns)
	ch.mutable.RUnlock()

	return &RuntimeState{
		CreatedStack:   ch.createdStack,
		LocalPeer:      ch.PeerInfo(),
		SubChannels:    ch.subChannels.IntrospectState(opts),
		RootPeers:      ch.rootPeers().IntrospectState(opts),
		Peers:          ch.Peers().IntrospectList(opts),
		NumConnections: conns,
		OtherChannels:  ch.IntrospectOthers(opts),
	}
}

// IntrospectOthers returns the ChannelInfo for all other channels in this process.
func (ch *Channel) IntrospectOthers(opts *IntrospectionOptions) map[string][]ChannelInfo {
	channelMap.Lock()
	defer channelMap.Unlock()

	states := make(map[string][]ChannelInfo)
	for svc, channels := range channelMap.existing {
		channelInfos := make([]ChannelInfo, 0, len(channels))
		for _, otherChan := range channels {
			if ch == otherChan {
				continue
			}
			channelInfos = append(channelInfos, otherChan.ReportInfo(opts))
		}
		states[svc] = channelInfos
	}

	return states
}

// ReportInfo returns ChannelInfo for a channel.
func (ch *Channel) ReportInfo(opts *IntrospectionOptions) ChannelInfo {
	return ChannelInfo{
		CreatedStack: ch.createdStack,
		LocalPeer:    ch.PeerInfo(),
	}
}

type containsPeerList interface {
	Copy() map[string]*Peer
}

func fromPeerList(peers containsPeerList, opts *IntrospectionOptions) map[string]PeerRuntimeState {
	m := make(map[string]PeerRuntimeState)
	for _, peer := range peers.Copy() {
		peerState := peer.IntrospectState(opts)
		if len(peerState.InboundConnections)+len(peerState.OutboundConnections) > 0 || opts.IncludeEmptyPeers {
			m[peer.HostPort()] = peerState
		}
	}
	return m
}

// IntrospectState returns the runtime state of the peer list.
func (l *PeerList) IntrospectState(opts *IntrospectionOptions) map[string]PeerRuntimeState {
	return fromPeerList(l, opts)
}

// IntrospectState returns the runtime state of the
func (l *RootPeerList) IntrospectState(opts *IntrospectionOptions) map[string]PeerRuntimeState {
	return fromPeerList(l, opts)
}

// IntrospectState returns the runtime state of the subchannels.
func (subChMap *subChannelMap) IntrospectState(opts *IntrospectionOptions) map[string]SubChannelRuntimeState {
	m := make(map[string]SubChannelRuntimeState)
	subChMap.RLock()
	for k, sc := range subChMap.subchannels {
		state := SubChannelRuntimeState{
			Service:  k,
			Isolated: sc.Isolated(),
		}
		if state.Isolated {
			state.IsolatedPeers = sc.Peers().IntrospectList(opts)
		}
		if hmap, ok := sc.handler.(*handlerMap); ok {
			state.Handler.Type = methodHandler
			methods := make([]string, 0, len(hmap.handlers))
			for k := range hmap.handlers {
				methods = append(methods, k)
			}
			sort.Strings(methods)
			state.Handler.Methods = methods
		} else {
			state.Handler.Type = overrideHandler
		}
		m[k] = state
	}
	subChMap.RUnlock()
	return m
}

func getConnectionRuntimeState(conns []*Connection, opts *IntrospectionOptions) []ConnectionRuntimeState {
	connStates := make([]ConnectionRuntimeState, len(conns))

	for i, conn := range conns {
		connStates[i] = conn.IntrospectState(opts)
	}

	return connStates
}

// IntrospectState returns the runtime state for this peer.
func (p *Peer) IntrospectState(opts *IntrospectionOptions) PeerRuntimeState {
	p.RLock()
	defer p.RUnlock()

	return PeerRuntimeState{
		HostPort:            p.hostPort,
		InboundConnections:  getConnectionRuntimeState(p.inboundConnections, opts),
		OutboundConnections: getConnectionRuntimeState(p.outboundConnections, opts),
		ChosenCount:         p.chosenCount.Load(),
		SCCount:             p.scCount,
	}
}

// IntrospectState returns the runtime state for this connection.
func (c *Connection) IntrospectState(opts *IntrospectionOptions) ConnectionRuntimeState {
	c.stateMut.RLock()
	defer c.stateMut.RUnlock()

	return ConnectionRuntimeState{
		ID:               c.connID,
		ConnectionState:  c.state.String(),
		LocalHostPort:    c.conn.LocalAddr().String(),
		RemoteHostPort:   c.conn.RemoteAddr().String(),
		IsEphemeral:      c.remotePeerInfo.IsEphemeral,
		InboundExchange:  c.inbound.IntrospectState(opts),
		OutboundExchange: c.outbound.IntrospectState(opts),
	}
}

// IntrospectState returns the runtime state for this messsage exchange set.
func (mexset *messageExchangeSet) IntrospectState(opts *IntrospectionOptions) ExchangeSetRuntimeState {
	mexset.RLock()
	setState := ExchangeSetRuntimeState{
		Name:  mexset.name,
		Count: len(mexset.exchanges),
	}

	if opts.IncludeExchanges {
		setState.Exchanges = make([]ExchangeRuntimeState, 0, len(mexset.exchanges))
		for k, v := range mexset.exchanges {
			state := ExchangeRuntimeState{
				ID:          k,
				MessageType: v.msgType,
			}
			setState.Exchanges = append(setState.Exchanges, state)
		}
	}

	mexset.RUnlock()
	return setState
}

func getStacks(all bool) []byte {
	var buf []byte
	for n := 4096; n < 10*1024*1024; n *= 2 {
		buf = make([]byte, n)
		stackLen := runtime.Stack(buf, all)
		if stackLen < n {
			return buf[:stackLen]
		}
	}

	// return the first 10MB of stacks if we have more than 10MB.
	return buf
}
func (ch *Channel) handleIntrospection(arg3 []byte) interface{} {
	var opts IntrospectionOptions
	json.Unmarshal(arg3, &opts)
	return ch.IntrospectState(&opts)
}

// IntrospectList returns the list of peers (hostport, score) in this peer list.
func (l *PeerList) IntrospectList(opts *IntrospectionOptions) []SubPeerScore {
	var peers []SubPeerScore
	l.RLock()
	for _, ps := range l.peerHeap.peerScores {
		peers = append(peers, SubPeerScore{
			HostPort: ps.Peer.hostPort,
			Score:    ps.score,
		})
	}
	l.RUnlock()

	return peers
}

func handleInternalRuntime(arg3 []byte) interface{} {
	var opts GoRuntimeStateOptions
	json.Unmarshal(arg3, &opts)

	state := GoRuntimeState{
		NumGoroutines: runtime.NumGoroutine(),
		NumCPU:        runtime.NumCPU(),
		NumCGo:        runtime.NumCgoCall(),
	}
	runtime.ReadMemStats(&state.MemStats)
	if opts.IncludeGoStacks {
		state.GoStacks = getStacks(true /* all */)
	}

	return state
}

// registerInternal registers the following internal handlers which return runtime state:
//  _gometa_introspect: TChannel internal state.
//  _gometa_runtime: Golang runtime stats.
func (ch *Channel) registerInternal() {
	endpoints := []struct {
		name    string
		handler func([]byte) interface{}
	}{
		{"_gometa_introspect", ch.handleIntrospection},
		{"_gometa_runtime", handleInternalRuntime},
	}

	for _, ep := range endpoints {
		// We need ep in our closure.
		ep := ep
		handler := func(ctx context.Context, call *InboundCall) {
			var arg2, arg3 []byte
			if err := NewArgReader(call.Arg2Reader()).Read(&arg2); err != nil {
				return
			}
			if err := NewArgReader(call.Arg3Reader()).Read(&arg3); err != nil {
				return
			}
			if err := NewArgWriter(call.Response().Arg2Writer()).Write(nil); err != nil {
				return
			}
			NewArgWriter(call.Response().Arg3Writer()).WriteJSON(ep.handler(arg3))
		}
		ch.Register(HandlerFunc(handler), ep.name)
	}
}
<|endoftext|>


<commit_before>package mapreduce

import (
	"fmt"
	"sync"
)

// schedule starts and waits for all tasks in the given phase (Map or Reduce).
func (mr *Master) schedule(phase jobPhase) {
	var ntasks int
	var nios int // number of inputs (for reduce) or outputs (for map)
	switch phase {
	case mapPhase:
		ntasks = len(mr.files)
		nios = mr.nReduce
	case reducePhase:
		ntasks = mr.nReduce
		nios = len(mr.files)
	}

	fmt.Printf("Schedule: %v %v tasks (%d I/Os)\n", ntasks, phase, nios)

	// All ntasks tasks have to be scheduled on workers, and only once all of
	// them have been completed successfully should the function return.
	// Remember that workers may fail, and that any given worker may finish
	// multiple tasks.
	//
	// TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO
	//

	// Use WaitGroup to wait for all tasks to finish
	var wg sync.WaitGroup
	for i := 0; i < ntasks; i++ {
		wg.Add(1)
		go func(taskNum int) {
			defer wg.Done()
			dtarg := DoTaskArgs{}
			dtarg.JobName = mr.jobName
			dtarg.File = mr.files[taskNum]
			dtarg.Phase = phase
			dtarg.TaskNumber = taskNum
			dtarg.NumOtherPhase = nios
			for {
				availWorker := <- mr.registerChannel

				// RPC call worker to run synchronously
				ok := call(availWorker, "Worker.DoTask", &dtarg, new(struct{}))
				if ok {
					go func() {
						mr.registerChannel <- availWorker
					}()
					break;
				}
			}
		}(i)
	}

	// Wait for all tasks to complete
	wg.Wait()
	fmt.Printf("Schedule: %v phase done\n", phase)
}
<commit_msg>Use Higher level channels to synchronize<commit_after>package mapreduce

import (
	"fmt"
)

// schedule starts and waits for all tasks in the given phase (Map or Reduce).
func (mr *Master) schedule(phase jobPhase) {
	var ntasks int
	var nios int // number of inputs (for reduce) or outputs (for map)
	switch phase {
	case mapPhase:
		ntasks = len(mr.files)
		nios = mr.nReduce
	case reducePhase:
		ntasks = mr.nReduce
		nios = len(mr.files)
	}

	fmt.Printf("Schedule: %v %v tasks (%d I/Os)\n", ntasks, phase, nios)

	// All ntasks tasks have to be scheduled on workers, and only once all of
	// them have been completed successfully should the function return.
	// Remember that workers may fail, and that any given worker may finish
	// multiple tasks.
	//
	// TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO TODO
	//

	// Use WaitGroup to wait for all tasks to finish
	// var wg sync.WaitGroup

	wt := make(chan int, ntasks)
	for i := 0; i < ntasks; i++ {
		go func(taskNum int, c chan int) {
			dtarg := DoTaskArgs{}
			dtarg.JobName = mr.jobName
			dtarg.File = mr.files[taskNum]
			dtarg.Phase = phase
			dtarg.TaskNumber = taskNum
			dtarg.NumOtherPhase = nios
			for {
				availWorker := <- mr.registerChannel
				ok := call(availWorker, "Worker.DoTask", &dtarg, new(struct{}))
				if ok {
					go func() {
						mr.registerChannel <- availWorker
					}()
					break;
				}
			}
			c <- 1
		}(i, wt)
	}

	for j := 0; j < ntasks; j++ {
		<- wt
	}
	fmt.Printf("Schedule: %v phase done\n", phase)
}
<|endoftext|>


<commit_before>package matrix

func searchMatrix(matrix [][]int, target int) bool {
	return false
}
<commit_msg>solve 74 use binary search<commit_after>package matrix

func searchMatrix(matrix [][]int, target int) bool {
	return useBS(matrix, target)
}

// useBS time complexity O(log(m*n)), space complexity O(1)
func useBS(matrix [][]int, target int) bool {
	m := len(matrix)
	if m < 1 {
		return false
	}
	n := len(matrix[0])
	if n < 1 {
		return false
	}
	l, r := 0, m*n-1
	for l <= r {
		mid := l + (r-l)>>1
		nr, nc := mid/n, mid%n
		if matrix[nr][nc] == target {
			return true
		} else if matrix[nr][nc] < target {
			l = mid + 1
		} else {
			r = mid - 1
		}
	}
	return false
}
<|endoftext|>


<commit_before>package lib

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/nightlyone/lockfile"
	"github.com/pivotal-golang/archiver/extractor"
	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/clearsign"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/lib/gpg"
	"github.com/subutai-io/base/agent/lib/template"
	"github.com/subutai-io/base/agent/log"
)

var (
	lock lockfile.Lockfile
)

type templ struct {
	name    string
	file    string
	version string
	branch  string
	id      string
	owner   []string
	signa   signature
}

type metainfo struct {
	Name   string    `json:"name"`
	ID     string    `json:"id"`
	Owner  []string  `json:"owner"`
	Md5Sum string    `json:"md5Sum"`
	Signs  signature `json:"signature"`
}

type signature []struct {
	Author string
	Sign   string
}

func templId(t *templ, kurjun *http.Client) {
	var meta metainfo

	url := config.Cdn.Kurjun + "/template/info?name=" + t.name
	if t.name == "management" && len(t.branch) != 0 {
		url = config.Cdn.Kurjun + "/template/info?name=" + t.name + "&version=" + t.version + "-" + t.branch
	} else if t.name == "management" {
		url = config.Cdn.Kurjun + "/template/info?name=" + t.name + "&version=" + t.version
	}

	response, err := kurjun.Get(url)
	defer response.Body.Close()
	log.Debug("Retrieving id, get: " + url)

	if err == nil && response.StatusCode == 204 && t.name == "management" {
		log.Warn("Cannot get management with specified version, trying without version")
		response, err = kurjun.Get(config.Cdn.Kurjun + "/template/info?name=" + t.name)
	}
	if log.Check(log.WarnLevel, "Getting kurjun response", err) || response.StatusCode != 200 {
		return
	}

	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)

	if log.Check(log.WarnLevel, "Parsing response body", json.Unmarshal(body, &meta)) {
		return
	}

	t.id = meta.ID
	t.owner = meta.Owner
	t.signa = meta.Signs
}

func md5sum(filePath string) string {
	file, err := os.Open(filePath)
	if err != nil {
		return ""
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		return ""
	}
	return fmt.Sprintf("%x", hash.Sum(nil))
}

func checkLocal(t templ) bool {
	var response string
	files, _ := ioutil.ReadDir(config.Agent.LxcPrefix + "tmpdir")
	for _, f := range files {
		if t.file == f.Name() {
			if len(t.id) == 0 {
				fmt.Print("Cannot verify local template. Trust anyway? (y/n)")
				_, err := fmt.Scanln(&response)
				log.Check(log.FatalLevel, "Reading input", err)
				if response == "y" {
					return true
				}
				return false
			}
			if id := strings.Split(t.id, "."); len(id) > 0 && id[len(id)-1] == md5sum(config.Agent.LxcPrefix+"tmpdir/"+f.Name()) {
				return true
			}
		}
	}
	return false
}

func download(t templ, kurjun *http.Client) bool {
	if len(t.id) == 0 {
		return false
	}
	out, err := os.Create(config.Agent.LxcPrefix + "tmpdir/" + t.file)
	log.Check(log.FatalLevel, "Creating file "+t.file, err)
	defer out.Close()
	log.Info("Downloading " + t.name)

	response, err := kurjun.Get(config.Cdn.Kurjun + "/template/download?id=" + t.id)
	log.Check(log.FatalLevel, "Getting "+config.Cdn.Kurjun+"/template/download?id="+t.id, err)
	defer response.Body.Close()
	bar := pb.New(int(response.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()
	rd := bar.NewProxyReader(response.Body)

	_, err = io.Copy(out, rd)
	for c := 0; err != nil && c < 5; _, err = io.Copy(out, rd) {
		log.Info("Download interrupted, retrying")
		time.Sleep(3 * time.Second)
		c++

		//Repeating GET request to CDN, while need to continue interrupted download
		out, err = os.Create(config.Agent.LxcPrefix + "tmpdir/" + t.file)
		log.Check(log.FatalLevel, "Creating file "+t.file, err)
		defer out.Close()
		response, err = kurjun.Get(config.Cdn.Kurjun + "/template/download?id=" + t.id)
		log.Check(log.FatalLevel, "Getting "+config.Cdn.Kurjun+"/template/download?id="+t.id, err)
		defer response.Body.Close()
		bar = pb.New(int(response.ContentLength)).SetUnits(pb.U_BYTES)
		bar.Start()
		rd = bar.NewProxyReader(response.Body)
	}

	log.Check(log.FatalLevel, "Writing response body to file", err)

	if id := strings.Split(t.id, "."); len(id) > 0 && id[len(id)-1] == md5sum(config.Agent.LxcPrefix+"tmpdir/"+t.file) {
		return true
	}
	log.Error("Failed to check MD5 after download. Please check your connection and try again.")
	return false
}

func getOwnerKey(owner string) string {
	response, err := http.Get(config.Cdn.Kurjun + "/auth/key?user=" + owner)
	log.Check(log.FatalLevel, "Getting owner public key", err)
	defer response.Body.Close()
	key, err := ioutil.ReadAll(response.Body)
	log.Check(log.FatalLevel, "Reading key body", err)
	return string(key)
}

func verifySignature(key, signature string) string {
	entity, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(key))
	log.Check(log.WarnLevel, "Reading user public key", err)

	if block, _ := clearsign.Decode([]byte(signature)); block != nil {
		_, err = openpgp.CheckDetachedSignature(entity, bytes.NewBuffer(block.Bytes), block.ArmoredSignature.Body)
		if log.Check(log.ErrorLevel, "Checking signature", err) {
			return ""
		}
		return string(block.Bytes)
	}
	return ""
}

func idToName(id string, kurjun *http.Client) string {
	var meta metainfo

	//Since only kurjun knows template's ID, we cannot define if we have template already installed in system by ID as we do it by name, so unreachable kurjun in this case is a deadend for us
	//To omit this issue we should add ID into template config and use this ID as a "primary key" to any request
	response, err := kurjun.Get(config.Cdn.Kurjun + "/template/info?id=" + id)
	log.Check(log.ErrorLevel, "Getting kurjun response", err)
	defer response.Body.Close()

	body, err := ioutil.ReadAll(response.Body)

	if string(body) == "Not found" {
		log.Error("Template with id \"" + id + "\" not found")
	}
	log.Check(log.ErrorLevel, "Parsing response body", json.Unmarshal(body, &meta))

	return meta.Name
}

func lockSubutai(file string) bool {
	lock, err := lockfile.New("/var/run/lock/subutai." + file)
	if log.Check(log.DebugLevel, "Init lock "+file, err) {
		return false
	}

	err = lock.TryLock()
	if log.Check(log.DebugLevel, "Locking file "+file, err) {
		return false
	}

	return true
}

func unlockSubutai() {
	lock.Unlock()
}

func LxcImport(name, version, token string) {
	var kurjun *http.Client

	if container.IsContainer(name) && name == "management" && len(token) > 1 {
		gpg.ExchageAndEncrypt("management", token)
		return
	}

	if id := strings.Split(name, "id:"); len(id) > 1 {
		kurjun, _ = config.CheckKurjun()
		name = idToName(id[1], kurjun)
	}

	log.Info("Importing " + name)
	for !lockSubutai(name + ".import") {
		time.Sleep(time.Second * 1)
	}
	defer unlockSubutai()

	if container.IsContainer(name) {
		log.Info(name + " instance exist")
		return
	}

	var t templ

	t.name = name
	t.version = config.Template.Version
	t.branch = config.Template.Branch
	if len(version) != 0 {
		if strings.Contains(version, "-") {
			verstr := strings.Split(version, "-")
			if len(verstr) == 2 {
				t.version = verstr[0]
				t.branch = verstr[1]
			} else {
				log.Error("Invalid version")
			}
		} else {
			if strings.Contains(version, ".") {
				t.version = version
				t.branch = config.Template.Branch
			} else {
				t.version = config.Template.Version
				t.branch = version
			}
		}
	}

	log.Info("Version: " + t.version + ", branch: " + t.branch)

	if t.branch == "" || t.name != "management" {
		t.file = t.name + "-subutai-template_" + t.version + "_" + config.Template.Arch + ".tar.gz"
	} else {
		t.file = t.name + "-subutai-template_" + t.version + "-" + t.branch + "_" + config.Template.Arch + ".tar.gz"
	}

	if kurjun == nil {
		kurjun, _ = config.CheckKurjun()
	}
	if kurjun != nil {
		templId(&t, kurjun)
	} else {
		log.Info("Trying to import from local storage")
	}

	if len(t.id) != 0 && len(t.signa) == 0 {
		log.Warn("Template is not signed")
	}
	for _, v := range t.signa {
		// if v.Author == "public" || v.Author == "subutai" || v.Author == "jenkins" {
		signedhash := verifySignature(getOwnerKey(v.Author), v.Sign)
		if t.id != signedhash {
			log.Error("Signature does not match with template hash")
		}
		log.Info("Digital signature verification succeeded, owner and template integrity are valid")
		break
		// }
	}

	if !checkLocal(t) && !download(t, kurjun) {
		log.Error(t.name + " template not found")
	}

	log.Info("Unpacking template " + t.name)
	tgz := extractor.NewTgz()
	templdir := config.Agent.LxcPrefix + "tmpdir/" + t.name
	tgz.Extract(config.Agent.LxcPrefix+"tmpdir/"+t.file, templdir)
	parent := container.GetConfigItem(templdir+"/config", "subutai.parent")
	if parent != "" && parent != t.name && !container.IsTemplate(parent) {
		log.Info("Parent template required: " + parent)
		LxcImport(parent, "", token)
	}

	log.Info("Installing template " + t.name)
	template.Install(parent, t.name)
	// TODO following lines kept for back compatibility with old templates, should be deleted when all templates will be replaced.
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-home", config.Agent.LxcPrefix+t.name+"/home")
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-var", config.Agent.LxcPrefix+t.name+"/var")
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-opt", config.Agent.LxcPrefix+t.name+"/opt")
	log.Check(log.FatalLevel, "Removing temp dir "+templdir, os.RemoveAll(templdir))

	if t.name == "management" {
		template.MngInit()
		return
	}

	container.SetContainerConf(t.name, [][]string{
		{"lxc.rootfs", config.Agent.LxcPrefix + t.name + "/rootfs"},
		{"lxc.mount", config.Agent.LxcPrefix + t.name + "/fstab"},
		{"lxc.hook.pre-start", ""},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.common.conf"},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.userns.conf"},
		{"subutai.config.path", config.Agent.AppPrefix + "etc"},
		{"lxc.network.script.up", config.Agent.AppPrefix + "bin/create_ovs_interface"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/home home none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/opt opt none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/var var none bind,rw 0 0"},
	})
}
<commit_msg>Import with name autocompletion<commit_after>package lib

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/nightlyone/lockfile"
	"github.com/pivotal-golang/archiver/extractor"
	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/clearsign"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/lib/gpg"
	"github.com/subutai-io/base/agent/lib/template"
	"github.com/subutai-io/base/agent/log"
)

var (
	lock lockfile.Lockfile
)

type templ struct {
	name    string
	file    string
	version string
	branch  string
	id      string
	owner   []string
	signa   signature
}

type metainfo struct {
	Name   string    `json:"name"`
	ID     string    `json:"id"`
	Owner  []string  `json:"owner"`
	Md5Sum string    `json:"md5Sum"`
	Signs  signature `json:"signature"`
}

type signature []struct {
	Author string
	Sign   string
}

func templId(t *templ, kurjun *http.Client) {
	var meta metainfo

	url := config.Cdn.Kurjun + "/template/info?name=" + t.name
	if t.name == "management" && len(t.branch) != 0 {
		url = config.Cdn.Kurjun + "/template/info?name=" + t.name + "&version=" + t.version + "-" + t.branch
	} else if t.name == "management" {
		url = config.Cdn.Kurjun + "/template/info?name=" + t.name + "&version=" + t.version
	}

	response, err := kurjun.Get(url)
	defer response.Body.Close()
	log.Debug("Retrieving id, get: " + url)

	if err == nil && response.StatusCode == 204 && t.name == "management" {
		log.Warn("Cannot get management with specified version, trying without version")
		response, err = kurjun.Get(config.Cdn.Kurjun + "/template/info?name=" + t.name)
	}
	if log.Check(log.WarnLevel, "Getting kurjun response", err) || response.StatusCode != 200 {
		return
	}

	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)

	if log.Check(log.WarnLevel, "Parsing response body", json.Unmarshal(body, &meta)) {
		return
	}

	if t.name != meta.Name {
		log.Info("Found: " + t.name + " -> " + meta.Name)
		t.name = meta.Name
	}
	t.id = meta.ID
	t.owner = meta.Owner
	t.signa = meta.Signs
}

func md5sum(filePath string) string {
	file, err := os.Open(filePath)
	if err != nil {
		return ""
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		return ""
	}
	return fmt.Sprintf("%x", hash.Sum(nil))
}

func checkLocal(t templ) bool {
	var response string
	files, _ := ioutil.ReadDir(config.Agent.LxcPrefix + "tmpdir")
	for _, f := range files {
		if t.file == f.Name() {
			if len(t.id) == 0 {
				fmt.Print("Cannot verify local template. Trust anyway? (y/n)")
				_, err := fmt.Scanln(&response)
				log.Check(log.FatalLevel, "Reading input", err)
				if response == "y" {
					return true
				}
				return false
			}
			if id := strings.Split(t.id, "."); len(id) > 0 && id[len(id)-1] == md5sum(config.Agent.LxcPrefix+"tmpdir/"+f.Name()) {
				return true
			}
		}
	}
	return false
}

func download(t templ, kurjun *http.Client) bool {
	if len(t.id) == 0 {
		return false
	}
	out, err := os.Create(config.Agent.LxcPrefix + "tmpdir/" + t.file)
	log.Check(log.FatalLevel, "Creating file "+t.file, err)
	defer out.Close()
	log.Info("Downloading " + t.name)

	response, err := kurjun.Get(config.Cdn.Kurjun + "/template/download?id=" + t.id)
	log.Check(log.FatalLevel, "Getting "+config.Cdn.Kurjun+"/template/download?id="+t.id, err)
	defer response.Body.Close()
	bar := pb.New(int(response.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()
	rd := bar.NewProxyReader(response.Body)

	_, err = io.Copy(out, rd)
	for c := 0; err != nil && c < 5; _, err = io.Copy(out, rd) {
		log.Info("Download interrupted, retrying")
		time.Sleep(3 * time.Second)
		c++

		//Repeating GET request to CDN, while need to continue interrupted download
		out, err = os.Create(config.Agent.LxcPrefix + "tmpdir/" + t.file)
		log.Check(log.FatalLevel, "Creating file "+t.file, err)
		defer out.Close()
		response, err = kurjun.Get(config.Cdn.Kurjun + "/template/download?id=" + t.id)
		log.Check(log.FatalLevel, "Getting "+config.Cdn.Kurjun+"/template/download?id="+t.id, err)
		defer response.Body.Close()
		bar = pb.New(int(response.ContentLength)).SetUnits(pb.U_BYTES)
		bar.Start()
		rd = bar.NewProxyReader(response.Body)
	}

	log.Check(log.FatalLevel, "Writing response body to file", err)

	if id := strings.Split(t.id, "."); len(id) > 0 && id[len(id)-1] == md5sum(config.Agent.LxcPrefix+"tmpdir/"+t.file) {
		return true
	}
	log.Error("Failed to check MD5 after download. Please check your connection and try again.")
	return false
}

func getOwnerKey(owner string) string {
	response, err := http.Get(config.Cdn.Kurjun + "/auth/key?user=" + owner)
	log.Check(log.FatalLevel, "Getting owner public key", err)
	defer response.Body.Close()
	key, err := ioutil.ReadAll(response.Body)
	log.Check(log.FatalLevel, "Reading key body", err)
	return string(key)
}

func verifySignature(key, signature string) string {
	entity, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(key))
	log.Check(log.WarnLevel, "Reading user public key", err)

	if block, _ := clearsign.Decode([]byte(signature)); block != nil {
		_, err = openpgp.CheckDetachedSignature(entity, bytes.NewBuffer(block.Bytes), block.ArmoredSignature.Body)
		if log.Check(log.ErrorLevel, "Checking signature", err) {
			return ""
		}
		return string(block.Bytes)
	}
	return ""
}

func idToName(id string, kurjun *http.Client) string {
	var meta metainfo

	//Since only kurjun knows template's ID, we cannot define if we have template already installed in system by ID as we do it by name, so unreachable kurjun in this case is a deadend for us
	//To omit this issue we should add ID into template config and use this ID as a "primary key" to any request
	response, err := kurjun.Get(config.Cdn.Kurjun + "/template/info?id=" + id)
	log.Check(log.ErrorLevel, "Getting kurjun response", err)
	defer response.Body.Close()

	body, err := ioutil.ReadAll(response.Body)

	if string(body) == "Not found" {
		log.Error("Template with id \"" + id + "\" not found")
	}
	log.Check(log.ErrorLevel, "Parsing response body", json.Unmarshal(body, &meta))

	return meta.Name
}

func lockSubutai(file string) bool {
	lock, err := lockfile.New("/var/run/lock/subutai." + file)
	if log.Check(log.DebugLevel, "Init lock "+file, err) {
		return false
	}

	err = lock.TryLock()
	if log.Check(log.DebugLevel, "Locking file "+file, err) {
		return false
	}

	return true
}

func unlockSubutai() {
	lock.Unlock()
}

func LxcImport(name, version, token string) {
	var kurjun *http.Client

	if container.IsContainer(name) && name == "management" && len(token) > 1 {
		gpg.ExchageAndEncrypt("management", token)
		return
	}

	if id := strings.Split(name, "id:"); len(id) > 1 {
		kurjun, _ = config.CheckKurjun()
		name = idToName(id[1], kurjun)
	}

	log.Info("Importing " + name)
	for !lockSubutai(name + ".import") {
		time.Sleep(time.Second * 1)
	}
	defer unlockSubutai()

	if container.IsContainer(name) {
		log.Info(name + " instance exist")
		return
	}

	var t templ

	t.name = name
	t.version = config.Template.Version
	t.branch = config.Template.Branch
	if len(version) != 0 {
		if strings.Contains(version, "-") {
			verstr := strings.Split(version, "-")
			if len(verstr) == 2 {
				t.version = verstr[0]
				t.branch = verstr[1]
			} else {
				log.Error("Invalid version")
			}
		} else {
			if strings.Contains(version, ".") {
				t.version = version
				t.branch = config.Template.Branch
			} else {
				t.version = config.Template.Version
				t.branch = version
			}
		}
	}

	log.Info("Version: " + t.version + ", branch: " + t.branch)

	if t.branch == "" || t.name != "management" {
		t.file = t.name + "-subutai-template_" + t.version + "_" + config.Template.Arch + ".tar.gz"
	} else {
		t.file = t.name + "-subutai-template_" + t.version + "-" + t.branch + "_" + config.Template.Arch + ".tar.gz"
	}

	if kurjun == nil {
		kurjun, _ = config.CheckKurjun()
	}
	if kurjun != nil {
		templId(&t, kurjun)
	} else {
		log.Info("Trying to import from local storage")
	}

	if len(t.id) != 0 && len(t.signa) == 0 {
		log.Warn("Template is not signed")
	}
	for _, v := range t.signa {
		// if v.Author == "public" || v.Author == "subutai" || v.Author == "jenkins" {
		signedhash := verifySignature(getOwnerKey(v.Author), v.Sign)
		if t.id != signedhash {
			log.Error("Signature does not match with template hash")
		}
		log.Info("Digital signature verification succeeded, owner and template integrity are valid")
		break
		// }
	}

	if !checkLocal(t) && !download(t, kurjun) {
		log.Error(t.name + " template not found")
	}

	log.Info("Unpacking template " + t.name)
	tgz := extractor.NewTgz()
	templdir := config.Agent.LxcPrefix + "tmpdir/" + t.name
	tgz.Extract(config.Agent.LxcPrefix+"tmpdir/"+t.file, templdir)
	parent := container.GetConfigItem(templdir+"/config", "subutai.parent")
	if parent != "" && parent != t.name && !container.IsTemplate(parent) {
		log.Info("Parent template required: " + parent)
		LxcImport(parent, "", token)
	}

	log.Info("Installing template " + t.name)
	template.Install(parent, t.name)
	// TODO following lines kept for back compatibility with old templates, should be deleted when all templates will be replaced.
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-home", config.Agent.LxcPrefix+t.name+"/home")
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-var", config.Agent.LxcPrefix+t.name+"/var")
	os.Rename(config.Agent.LxcPrefix+t.name+"/"+t.name+"-opt", config.Agent.LxcPrefix+t.name+"/opt")
	log.Check(log.FatalLevel, "Removing temp dir "+templdir, os.RemoveAll(templdir))

	if t.name == "management" {
		template.MngInit()
		return
	}

	container.SetContainerConf(t.name, [][]string{
		{"lxc.rootfs", config.Agent.LxcPrefix + t.name + "/rootfs"},
		{"lxc.mount", config.Agent.LxcPrefix + t.name + "/fstab"},
		{"lxc.hook.pre-start", ""},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.common.conf"},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.userns.conf"},
		{"subutai.config.path", config.Agent.AppPrefix + "etc"},
		{"lxc.network.script.up", config.Agent.AppPrefix + "bin/create_ovs_interface"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/home home none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/opt opt none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + t.name + "/var var none bind,rw 0 0"},
	})
}
<|endoftext|>


<commit_before>package main

import "testing"

func TestHistogramAggregation(t *testing.T) {
	for i := 0.0; i < 101; i++ {
		histogram1 := metric{
			Name:      "latency",
			Timestamp: "2015-05-12T14:49:32",
			Type:      "histogram",
			Value:     i,
		}
		processMetric(histogram1)
	}

	aggregatedCount := buckets["latency"].Fields["count"]
	aggregatedAvg := buckets["latency"].Fields["avg"]
	aggregatedMax := buckets["latency"].Fields["max"]
	aggregatedMedian := buckets["latency"].Fields["median"]
	aggregated95Percentile := buckets["latency"].Fields["95percentile"]

	if aggregatedCount != 101.0 {
		t.Error("Expected count of 101, got ", aggregatedCount)
	}

	if aggregatedAvg != 50.0 {
		t.Error("Expected average of 50, got ", aggregatedAvg)
	}

	if aggregatedMax != 100.0 {
		t.Error("Expected max of 100, got ", aggregatedMax)
	}

	if aggregatedMedian != 50.0 {
		t.Error("Expected median of 50, got ", aggregatedMax)
	}

	if aggregated95Percentile != 95.0 {
		t.Error("Expected the 95th percentile to be 95, got ", aggregated95Percentile)
	}

}
func TestGaugeAggregation(t *testing.T) {
	for i := 0.0; i < 100; i++ {
		gauge1 := metric{
			Name:      "load",
			Timestamp: "2015-05-12T14:49:32",
			Type:      "gauge",
			Value:     i,
		}
		processMetric(gauge1)
	}

	gauge2 := metric{
		Name:      "load",
		Timestamp: "2015-05-12T14:49:31",
		Type:      "gauge",
		Value:     1,
	}

	processMetric(gauge2)

	aggregatedValue := buckets["load"].Fields["value"]

	if aggregatedValue != 1.0 {
		t.Error("Expected 1, got ", aggregatedValue)
	}
}

func TestCounterAggregation(t *testing.T) {
	counter1 := metric{
		Name:      "requests",
		Timestamp: "2015-05-12T14:49:32",
		Type:      "counter",
		Value:     10,
	}

	counter2 := metric{
		Name:      "requests",
		Timestamp: "2015-05-12T14:49:31",
		Type:      "counter",
		Value:     50,
	}

	counter3 := metric{
		Name:      "requests",
		Timestamp: "2015-05-12T14:49:31",
		Type:      "counter",
		Value:     -9,
	}
	processMetric(counter1)
	processMetric(counter2)
	processMetric(counter3)

	aggregatedValue := buckets["requests"].Fields["value"]

	if aggregatedValue != 51.0 {
		t.Error("Expected 51, got ", aggregatedValue)
	}
}
<commit_msg>Changed metric to input.Metric<commit_after>package main

import (
	"testing"

	"github.com/ccpgames/aggregateD/input"
)

func TestHistogramAggregation(t *testing.T) {
	for i := 0.0; i < 101; i++ {
		histogram1 := input.Metric{
			Name:      "latency",
			Timestamp: "2015-05-12T14:49:32",
			Type:      "histogram",
			Value:     i,
		}
		processMetric(histogram1)
	}

	aggregatedCount := buckets["latency"].Fields["count"]
	aggregatedAvg := buckets["latency"].Fields["avg"]
	aggregatedMax := buckets["latency"].Fields["max"]
	aggregatedMedian := buckets["latency"].Fields["median"]
	aggregated95Percentile := buckets["latency"].Fields["95percentile"]

	if aggregatedCount != 101.0 {
		t.Error("Expected count of 101, got ", aggregatedCount)
	}

	if aggregatedAvg != 50.0 {
		t.Error("Expected average of 50, got ", aggregatedAvg)
	}

	if aggregatedMax != 100.0 {
		t.Error("Expected max of 100, got ", aggregatedMax)
	}

	if aggregatedMedian != 50.0 {
		t.Error("Expected median of 50, got ", aggregatedMax)
	}

	if aggregated95Percentile != 95.0 {
		t.Error("Expected the 95th percentile to be 95, got ", aggregated95Percentile)
	}

}
func TestGaugeAggregation(t *testing.T) {
	for i := 0.0; i < 100; i++ {
		gauge1 := input.Metric{
			Name:      "load",
			Timestamp: "2015-05-12T14:49:32",
			Type:      "gauge",
			Value:     i,
		}
		processMetric(gauge1)
	}

	gauge2 := input.Metric{
		Name:      "load",
		Timestamp: "2015-05-12T14:49:31",
		Type:      "gauge",
		Value:     1,
	}

	processMetric(gauge2)

	aggregatedValue := buckets["load"].Fields["value"]

	if aggregatedValue != 1.0 {
		t.Error("Expected 1, got ", aggregatedValue)
	}
}

func TestCounterAggregation(t *testing.T) {
	counter1 := input.Metric{
		Name:      "requests",
		Timestamp: "2015-05-12T14:49:32",
		Type:      "counter",
		Value:     10,
	}

	counter2 := input.Metric{
		Name:      "requests",
		Timestamp: "2015-05-12T14:49:31",
		Type:      "counter",
		Value:     50,
	}

	counter3 := input.Metric{
		Name:      "requests",
		Timestamp: "2015-05-12T14:49:31",
		Type:      "counter",
		Value:     -9,
	}
	processMetric(counter1)
	processMetric(counter2)
	processMetric(counter3)

	aggregatedValue := buckets["requests"].Fields["value"]

	if aggregatedValue != 51.0 {
		t.Error("Expected 51, got ", aggregatedValue)
	}
}
<|endoftext|>


<commit_before>package bootstrap

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"time"

	homedir "github.com/mitchellh/go-homedir"
	"github.com/nightlyone/lockfile"
)

type knownHosts struct {
	*lockfile.Lockfile
	Path string
}

func findKnownHosts() (*knownHosts, error) {
	userHomePath, err := homedir.Dir()
	if err != nil {
		warningf("Could not find the current users home directory (%s)", err)
		return nil, err
	}

	// Construct paths to the known_hosts file
	sshDirectory := filepath.Join(userHomePath, ".ssh")
	knownHostPath := filepath.Join(sshDirectory, "known_hosts")

	// Ensure a directory exists and known_host file exist
	if !fileExists(knownHostPath) {
		os.MkdirAll(sshDirectory, 0755)
		ioutil.WriteFile(knownHostPath, []byte(""), 0644)
	}

	lockFile := filepath.Join(sshDirectory, "known_hosts.lock")

	// Create a lock on the known_host file so other agents don't try and
	// change it at the same time
	knownHostLock, err := acquireLockWithTimeout(lockFile, time.Second*30)
	if err != nil {
		warningf("Could not acquire a lock on `%s`: %v", lockFile, err)
		return nil, err
	}

	return &knownHosts{knownHostLock, knownHostPath}, nil
}

func (kh *knownHosts) Add(host string) error {
	// Try and open the existing hostfile in (append_only) mode
	f, err := os.OpenFile(kh.Path, os.O_APPEND|os.O_WRONLY, 0644)
	if err != nil {
		warningf("Could not open \"%s\" for reading (%s)", kh.Path, err)
		return err
	}
	defer f.Close()

	sshToolsDir, err := findSSHToolsDir()
	if err != nil {
		return err
	}

	// Grab the generated keys for the repo host
	keygenOutput, err := exec.Command(filepath.Join(sshToolsDir, "ssh-keygen"), "-f", kh.Path, "-F", host).Output()
	if err != nil {
		warningf("Could not perform `ssh-keygen` (%s)", err)
		return err
	}

	// If the keygen output already contains the host, we can skip!
	if bytes.Contains(keygenOutput, []byte(host)) {
		commentf("Host \"%s\" already in list of known hosts at \"%s\"", host, kh.Path)
		return nil
	}

	// Scan the key and then write it to the known_host file
	keyscanOutput, err := exec.Command(filepath.Join(sshToolsDir, "ssh-keyscan"), host).Output()
	if err != nil {
		warningf("Could not perform `ssh-keyscan` (%s)", err)
		return err
	}

	if _, err = fmt.Fprintf(f, "%s\n", keyscanOutput); err != nil {
		warningf("Could not write to \"%s\" (%s)", kh.Path, err)
		return err
	}

	commentf("Added \"%s\" to the list of known hosts at \"%s\"", host, kh.Path)
	return nil
}

func findSSHToolsDir() (string, error) {
	// On Windows, ssh-keygen isn't on the $PATH by default, but we know we can find it
	// relative to where git for windows is installed, so try that
	if runtime.GOOS == "windows" {
		gitExecPathOutput, _ := exec.Command("git", "--exec-path").Output()
		if len(gitExecPathOutput) > 0 {
			sshToolRelativePaths := [][]string{}
			sshToolRelativePaths = append(sshToolRelativePaths, []string{"..", "..", "..", "usr", "bin"})
			sshToolRelativePaths = append(sshToolRelativePaths, []string{"..", "..", "bin"})

			for _, segments := range sshToolRelativePaths {
				segments = append([]string{string(gitExecPathOutput)}, segments...)
				dir := filepath.Join(segments...)
				if _, err := os.Stat(filepath.Join(dir, "ssh-keygen.exe")); err == nil {
					return dir, nil
				}
			}
		}
	}

	keygen, err := exec.LookPath("ssh-keygen")
	if err != nil {
		return "", fmt.Errorf("Failed to find path for ssh-keygen: %v", err)
	}

	return filepath.Dir(keygen), nil
}
<commit_msg>exec.Command returns an ExitError, this is more backwards compat<commit_after>package bootstrap

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/buildkite/agent/shell"
	homedir "github.com/mitchellh/go-homedir"
	"github.com/nightlyone/lockfile"
)

type knownHosts struct {
	*lockfile.Lockfile
	Path string
}

func findKnownHosts() (*knownHosts, error) {
	userHomePath, err := homedir.Dir()
	if err != nil {
		warningf("Could not find the current users home directory (%s)", err)
		return nil, err
	}

	// Construct paths to the known_hosts file
	sshDirectory := filepath.Join(userHomePath, ".ssh")
	knownHostPath := filepath.Join(sshDirectory, "known_hosts")

	// Ensure a directory exists and known_host file exist
	if !fileExists(knownHostPath) {
		os.MkdirAll(sshDirectory, 0755)
		ioutil.WriteFile(knownHostPath, []byte(""), 0644)
	}

	lockFile := filepath.Join(sshDirectory, "known_hosts.lock")

	// Create a lock on the known_host file so other agents don't try and
	// change it at the same time
	knownHostLock, err := acquireLockWithTimeout(lockFile, time.Second*30)
	if err != nil {
		warningf("Could not acquire a lock on `%s`: %v", lockFile, err)
		return nil, err
	}

	return &knownHosts{knownHostLock, knownHostPath}, nil
}

func (kh *knownHosts) Add(host string) error {
	// Try and open the existing hostfile in (append_only) mode
	f, err := os.OpenFile(kh.Path, os.O_APPEND|os.O_WRONLY, 0644)
	if err != nil {
		warningf("Could not open \"%s\" for reading (%s)", kh.Path, err)
		return err
	}
	defer f.Close()

	sshToolsDir, err := findSSHToolsDir()
	if err != nil {
		return err
	}

	// Grab the generated keys for the repo host
	keygenOutput, err := runCommandSilentlyAndCaptureOutput(filepath.Join(sshToolsDir, "ssh-keygen"), "-f", kh.Path, "-F", host)
	if err != nil {
		warningf("Could not perform `ssh-keygen` (%s)", err)
		return err
	}

	// If the keygen output already contains the host, we can skip!
	if strings.Contains(keygenOutput, host) {
		commentf("Host \"%s\" already in list of known hosts at \"%s\"", host, kh.Path)
		return nil
	}

	// Scan the key and then write it to the known_host file
	keyscanOutput, err := runCommandSilentlyAndCaptureOutput(filepath.Join(sshToolsDir, "ssh-keyscan"), host)
	if err != nil {
		warningf("Could not perform `ssh-keyscan` (%s)", err)
		return err
	}

	if _, err = fmt.Fprintf(f, "%s\n", keyscanOutput); err != nil {
		warningf("Could not write to \"%s\" (%s)", kh.Path, err)
		return err
	}

	commentf("Added \"%s\" to the list of known hosts at \"%s\"", host, kh.Path)
	return nil
}

func findSSHToolsDir() (string, error) {
	// On Windows, ssh-keygen isn't on the $PATH by default, but we know we can find it
	// relative to where git for windows is installed, so try that
	if runtime.GOOS == "windows" {
		gitExecPathOutput, _ := runCommandSilentlyAndCaptureOutput("git", "--exec-path")
		if len(gitExecPathOutput) > 0 {
			sshToolRelativePaths := [][]string{}
			sshToolRelativePaths = append(sshToolRelativePaths, []string{"..", "..", "..", "usr", "bin"})
			sshToolRelativePaths = append(sshToolRelativePaths, []string{"..", "..", "bin"})

			for _, segments := range sshToolRelativePaths {
				segments = append([]string{gitExecPathOutput}, segments...)
				dir := filepath.Join(segments...)
				if _, err := os.Stat(filepath.Join(dir, "ssh-keygen.exe")); err == nil {
					return dir, nil
				}
			}
		}
	}

	keygen, err := exec.LookPath("ssh-keygen")
	if err != nil {
		return "", fmt.Errorf("Failed to find path for ssh-keygen: %v", err)
	}

	return filepath.Dir(keygen), nil
}

// This will be removed shortly
func runCommandSilentlyAndCaptureOutput(command string, args ...string) (string, error) {
	var buffer bytes.Buffer

	p := subprocess{
		Command: &shell.Command{Command: command, Args: args},
		Writer:  &buffer,
	}

	err := p.Run()
	return strings.TrimSpace(buffer.String()), err
}
<|endoftext|>


<commit_before><commit_msg>Create stopping game in delete game handler<commit_after><|endoftext|>


<commit_before>package main

import (
	"encoding/binary"
	"fmt"
	"io"
	"net"
	"os"
)

const defaultPort = "5000"

func main() {
	if len(os.Args) < 3 {
		fmt.Printf("Usage: %s <ip> <file>\n", os.Args[0])
		return
	}
	if isCiaFile(os.Args[2]) == false {
		fmt.Println("Error, not cia file")
		return
	}
	file, err := os.Open(os.Args[2])
	checkError(err)
	defer file.Close()
	fileInfo, err := file.Stat()
	checkError(err)
	fileSize := fileInfo.Size()
	ipPort := setDefaultPort(os.Args[1])
	raddr, err := net.ResolveTCPAddr("tcp", ipPort)
	checkError(err)
	out, err := net.DialTCP("tcp", nil, raddr)
	checkError(err)
	defer out.Close()
	buffer := make([]byte, 128*1024)

	//write filesize
	err = binary.Write(out, binary.BigEndian, &fileSize)
	checkError(err)
	for {
		n, err := file.Read(buffer)
		if err != nil {
			if err != io.EOF {
				fmt.Println(err)
			}
			break
		}
		nWrite, err := out.Write(buffer[:n])
		if err != nil {
			fmt.Println(err)
			break
		}
		if n != nWrite {
			fmt.Println("partial write...")
			break
		}
	}
}

func setDefaultPort(ip string) string {
	_, _, err := net.SplitHostPort(ip)
	if err != nil {
		return ip + ":" + defaultPort
	}
	return ip
}

func isCiaFile(path string) bool {
	if len(path) < 4 {
		return false
	}
	ext := path[len(path)-4:]
	if (ext[0] == '.') &&
		(ext[1] == 'c' || ext[1] == 'C') &&
		(ext[2] == 'i' || ext[2] == 'i') &&
		(ext[3] == 'a' || ext[3] == 'A') {
		return true
	}
	return false
}

func checkError(err error) {
	if err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}
<commit_msg>add progress bar<commit_after>package main

import (
	"encoding/binary"
	"fmt"
	"io"
	"net"
	"os"
	"time"

	"github.com/cheggaaa/pb"
)

const defaultPort = "5000"

func main() {
	if len(os.Args) < 3 {
		fmt.Printf("Usage: %s <ip> <file>\n", os.Args[0])
		return
	}
	if isCiaFile(os.Args[2]) == false {
		fmt.Println("Error, not cia file")
		return
	}
	file, err := os.Open(os.Args[2])
	checkError(err)
	defer file.Close()
	fileInfo, err := file.Stat()
	checkError(err)
	fileSize := fileInfo.Size()
	ipPort := setDefaultPort(os.Args[1])
	raddr, err := net.ResolveTCPAddr("tcp", ipPort)
	checkError(err)
	out, err := net.DialTCP("tcp", nil, raddr)
	checkError(err)
	defer out.Close()
	buffer := make([]byte, 128*1024)

	//write filesize
	err = binary.Write(out, binary.BigEndian, &fileSize)
	checkError(err)

	//progress bar
	bar := pb.New64(fileSize)
	bar.SetRefreshRate(time.Second)
	bar.ShowCounters = false
	bar.ShowTimeLeft = true
	bar.ShowSpeed = true
	bar.SetUnits(pb.U_BYTES)
	pbStartFlag := false
	ciaInstallSuccess := false

	for {
		n, err := file.Read(buffer)
		if err != nil {
			if err == io.EOF {
				bar.Set64(fileSize)
				ciaInstallSuccess = true
			} else {
				fmt.Println(err)
			}
			break
		}
		if pbStartFlag == false {
			pbStartFlag = true
			bar.Set64(0)
			bar.Start()
		}
		if n <= 0 {
			break
		}
		nWrite, err := out.Write(buffer[:n])
		if err != nil {
			fmt.Println(err)
			break
		}
		bar.Add(nWrite)
		if n != nWrite {
			fmt.Println("partial write...")
			break
		}
	}

	if ciaInstallSuccess {
		bar.FinishPrint("Install cia file success")
	} else {
		bar.FinishPrint("Install cia file failed")
	}
}

func setDefaultPort(ip string) string {
	_, _, err := net.SplitHostPort(ip)
	if err != nil {
		return ip + ":" + defaultPort
	}
	return ip
}

func isCiaFile(path string) bool {
	if len(path) < 4 {
		return false
	}
	ext := path[len(path)-4:]
	if (ext[0] == '.') &&
		(ext[1] == 'c' || ext[1] == 'C') &&
		(ext[2] == 'i' || ext[2] == 'i') &&
		(ext[3] == 'a' || ext[3] == 'A') {
		return true
	}
	return false
}

func checkError(err error) {
	if err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}
<|endoftext|>


<commit_before>package scanner

import (
	"bufio"
	"bytes"
	"github.com/github/git-media/pointer"
	// "github.com/rubyist/tracerx"
	"io"
	"os/exec"
	"strconv"
)

var blobSizeCutoff = 125

func Scan(ref string) ([]*pointer.Pointer, error) {
	revs, _ := revListStream(ref, ref == "")
	smallShas, _ := catFileBatchCheck(revs)
	pointerc, _ := catFileBatch(smallShas)

	pointers := make([]*pointer.Pointer, 0)
	for p := range pointerc {
		pointers = append(pointers, p)
	}

	return pointers, nil
}

func revListStream(ref string, all bool) (chan string, error) {
	refArgs := []string{"rev-list", "--objects"}
	if all {
		refArgs = append(refArgs, "--all")
	} else {
		refArgs = append(refArgs, ref)
	}

	cmd, err := startCommand("git", refArgs...)
	if err != nil {
		return nil, err
	}

	cmd.Stdin.Close()

	revs := make(chan string)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			revs <- scanner.Text()[0:40]
		}
		close(revs)
	}()

	return revs, nil
}

func catFileBatchCheck(revs chan string) (chan string, error) {
	cmd, err := startCommand("git", "cat-file", "--batch-check")
	if err != nil {
		return nil, err
	}

	smallRevs := make(chan string)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			line := scanner.Text()
			if line[41:45] == "blob" {
				size, err := strconv.Atoi(line[46:len(line)])
				if err != nil {
					continue
				}
				if size < blobSizeCutoff {
					smallRevs <- line[0:40]
				}
			}
		}
		close(smallRevs)
	}()

	go func() {
		for r := range revs {
			cmd.Stdin.Write([]byte(r + "\n"))
		}
		cmd.Stdin.Close()
	}()

	return smallRevs, nil
}

func catFileBatch(revs chan string) (chan *pointer.Pointer, error) {
	cmd, err := startCommand("git", "cat-file", "--batch")
	if err != nil {
		return nil, err
	}

	pointers := make(chan *pointer.Pointer)

	go func() {
		bstdout := bufio.NewReader(cmd.Stdout)
		for {
			l, err := bstdout.ReadBytes('\n')
			if err != nil { // Probably check for EOF
				break
			}

			// Line is formatted:
			// <sha1> <type> <size>
			fields := bytes.Fields(l)
			s, _ := strconv.Atoi(string(fields[2]))

			nbuf := make([]byte, s)
			_, err = io.ReadFull(bstdout, nbuf)
			if err != nil {
				break // Legit errors
			}

			p, err := pointer.Decode(bytes.NewBuffer(nbuf))
			if err == nil {
				pointers <- p
			}

			_, err = bstdout.ReadBytes('\n') // Extra \n inserted by cat-file
			if err != nil {                  // Probably check for EOF
				break
			}
		}
		close(pointers)
	}()

	// writes shas to cat-file stdin
	go func() {
		for r := range revs {
			cmd.Stdin.Write([]byte(r + "\n"))
		}
		cmd.Stdin.Close()
	}()

	return pointers, nil
}

type wrappedCmd struct {
	Stdin  io.WriteCloser
	Stdout io.ReadCloser
	*exec.Cmd
}

func startCommand(command string, args ...string) (*wrappedCmd, error) {
	cmd := exec.Command(command, args...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	if err := cmd.Start(); err != nil {
		return nil, err
	}

	return &wrappedCmd{stdin, stdout, cmd}, nil
}
<commit_msg>ラララララ ラー ウウウ フフフ<commit_after>package scanner

import (
	"bufio"
	"bytes"
	"github.com/github/git-media/pointer"
	// "github.com/rubyist/tracerx"
	"io"
	"os/exec"
	"strconv"
)

var (
	blobSizeCutoff = 125
	stdoutBufSize  = 16384
)

func Scan(ref string) ([]*pointer.Pointer, error) {
	revs, _ := revListStream(ref, ref == "")
	smallShas, _ := catFileBatchCheck(revs)
	pointerc, _ := catFileBatch(smallShas)

	pointers := make([]*pointer.Pointer, 0)
	for p := range pointerc {
		pointers = append(pointers, p)
	}

	return pointers, nil
}

func revListStream(ref string, all bool) (chan string, error) {
	refArgs := []string{"rev-list", "--objects"}
	if all {
		refArgs = append(refArgs, "--all")
	} else {
		refArgs = append(refArgs, ref)
	}

	cmd, err := startCommand("git", refArgs...)
	if err != nil {
		return nil, err
	}

	cmd.Stdin.Close()

	revs := make(chan string)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			revs <- scanner.Text()[0:40]
		}
		close(revs)
	}()

	return revs, nil
}

func catFileBatchCheck(revs chan string) (chan string, error) {
	cmd, err := startCommand("git", "cat-file", "--batch-check")
	if err != nil {
		return nil, err
	}

	smallRevs := make(chan string)

	go func() {
		scanner := bufio.NewScanner(cmd.Stdout)
		for scanner.Scan() {
			line := scanner.Text()
			if line[41:45] == "blob" {
				size, err := strconv.Atoi(line[46:len(line)])
				if err != nil {
					continue
				}
				if size < blobSizeCutoff {
					smallRevs <- line[0:40]
				}
			}
		}
		close(smallRevs)
	}()

	go func() {
		for r := range revs {
			cmd.Stdin.Write([]byte(r + "\n"))
		}
		cmd.Stdin.Close()
	}()

	return smallRevs, nil
}

func catFileBatch(revs chan string) (chan *pointer.Pointer, error) {
	cmd, err := startCommand("git", "cat-file", "--batch")
	if err != nil {
		return nil, err
	}

	pointers := make(chan *pointer.Pointer)

	go func() {
		for {
			l, err := cmd.Stdout.ReadBytes('\n')
			if err != nil { // Probably check for EOF
				break
			}

			// Line is formatted:
			// <sha1> <type> <size>
			fields := bytes.Fields(l)
			s, _ := strconv.Atoi(string(fields[2]))

			nbuf := make([]byte, s)
			_, err = io.ReadFull(cmd.Stdout, nbuf)
			if err != nil {
				break // Legit errors
			}

			p, err := pointer.Decode(bytes.NewBuffer(nbuf))
			if err == nil {
				pointers <- p
			}

			_, err = cmd.Stdout.ReadBytes('\n') // Extra \n inserted by cat-file
			if err != nil {                     // Probably check for EOF
				break
			}
		}
		close(pointers)
	}()

	// writes shas to cat-file stdin
	go func() {
		for r := range revs {
			cmd.Stdin.Write([]byte(r + "\n"))
		}
		cmd.Stdin.Close()
	}()

	return pointers, nil
}

type wrappedCmd struct {
	Stdin  io.WriteCloser
	Stdout *bufio.Reader
	*exec.Cmd
}

func startCommand(command string, args ...string) (*wrappedCmd, error) {
	cmd := exec.Command(command, args...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}

	if err := cmd.Start(); err != nil {
		return nil, err
	}

	return &wrappedCmd{stdin, bufio.NewReaderSize(stdout, stdoutBufSize), cmd}, nil
}
<|endoftext|>


<commit_before>package daemon // import "github.com/docker/docker/daemon"

import (
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/container"
	"github.com/docker/docker/errdefs"
	"github.com/docker/docker/pkg/archive"
	"github.com/docker/docker/pkg/chrootarchive"
	"github.com/docker/docker/pkg/ioutils"
	"github.com/docker/docker/pkg/system"
	"github.com/pkg/errors"
)

// ErrExtractPointNotDirectory is used to convey that the operation to extract
// a tar archive to a directory in a container has failed because the specified
// path does not refer to a directory.
var ErrExtractPointNotDirectory = errors.New("extraction point is not a directory")

// ContainerCopy performs a deprecated operation of archiving the resource at
// the specified path in the container identified by the given name.
func (daemon *Daemon) ContainerCopy(name string, res string) (io.ReadCloser, error) {
	ctr, err := daemon.GetContainer(name)
	if err != nil {
		return nil, err
	}

	// Make sure an online file-system operation is permitted.
	if err := daemon.isOnlineFSOperationPermitted(ctr); err != nil {
		return nil, errdefs.System(err)
	}

	data, err := daemon.containerCopy(ctr, res)
	if err == nil {
		return data, nil
	}

	if os.IsNotExist(err) {
		return nil, containerFileNotFound{res, name}
	}
	return nil, errdefs.System(err)
}

// ContainerStatPath stats the filesystem resource at the specified path in the
// container identified by the given name.
func (daemon *Daemon) ContainerStatPath(name string, path string) (stat *types.ContainerPathStat, err error) {
	ctr, err := daemon.GetContainer(name)
	if err != nil {
		return nil, err
	}

	// Make sure an online file-system operation is permitted.
	if err := daemon.isOnlineFSOperationPermitted(ctr); err != nil {
		return nil, errdefs.System(err)
	}

	stat, err = daemon.containerStatPath(ctr, path)
	if err == nil {
		return stat, nil
	}

	if os.IsNotExist(err) {
		return nil, containerFileNotFound{path, name}
	}
	return nil, errdefs.System(err)
}

// ContainerArchivePath creates an archive of the filesystem resource at the
// specified path in the container identified by the given name. Returns a
// tar archive of the resource and whether it was a directory or a single file.
func (daemon *Daemon) ContainerArchivePath(name string, path string) (content io.ReadCloser, stat *types.ContainerPathStat, err error) {
	ctr, err := daemon.GetContainer(name)
	if err != nil {
		return nil, nil, err
	}

	// Make sure an online file-system operation is permitted.
	if err := daemon.isOnlineFSOperationPermitted(ctr); err != nil {
		return nil, nil, errdefs.System(err)
	}

	content, stat, err = daemon.containerArchivePath(ctr, path)
	if err == nil {
		return content, stat, nil
	}

	if os.IsNotExist(err) {
		return nil, nil, containerFileNotFound{path, name}
	}
	return nil, nil, errdefs.System(err)
}

// ContainerExtractToDir extracts the given archive to the specified location
// in the filesystem of the container identified by the given name. The given
// path must be of a directory in the container. If it is not, the error will
// be ErrExtractPointNotDirectory. If noOverwriteDirNonDir is true then it will
// be an error if unpacking the given content would cause an existing directory
// to be replaced with a non-directory and vice versa.
func (daemon *Daemon) ContainerExtractToDir(name, path string, copyUIDGID, noOverwriteDirNonDir bool, content io.Reader) error {
	ctr, err := daemon.GetContainer(name)
	if err != nil {
		return err
	}

	// Make sure an online file-system operation is permitted.
	if err := daemon.isOnlineFSOperationPermitted(ctr); err != nil {
		return errdefs.System(err)
	}

	err = daemon.containerExtractToDir(ctr, path, copyUIDGID, noOverwriteDirNonDir, content)
	if err == nil {
		return nil
	}

	if os.IsNotExist(err) {
		return containerFileNotFound{path, name}
	}
	return errdefs.System(err)
}

// containerStatPath stats the filesystem resource at the specified path in this
// container. Returns stat info about the resource.
func (daemon *Daemon) containerStatPath(container *container.Container, path string) (stat *types.ContainerPathStat, err error) {
	container.Lock()
	defer container.Unlock()

	if err = daemon.Mount(container); err != nil {
		return nil, err
	}
	defer daemon.Unmount(container)

	err = daemon.mountVolumes(container)
	defer container.DetachAndUnmount(daemon.LogVolumeEvent)
	if err != nil {
		return nil, err
	}

	// Normalize path before sending to rootfs
	path = filepath.FromSlash(path)

	resolvedPath, absPath, err := container.ResolvePath(path)
	if err != nil {
		return nil, err
	}

	return container.StatPath(resolvedPath, absPath)
}

// containerArchivePath creates an archive of the filesystem resource at the specified
// path in this container. Returns a tar archive of the resource and stat info
// about the resource.
func (daemon *Daemon) containerArchivePath(container *container.Container, path string) (content io.ReadCloser, stat *types.ContainerPathStat, err error) {
	container.Lock()

	defer func() {
		if err != nil {
			// Wait to unlock the container until the archive is fully read
			// (see the ReadCloseWrapper func below) or if there is an error
			// before that occurs.
			container.Unlock()
		}
	}()

	if err = daemon.Mount(container); err != nil {
		return nil, nil, err
	}

	defer func() {
		if err != nil {
			// unmount any volumes
			container.DetachAndUnmount(daemon.LogVolumeEvent)
			// unmount the container's rootfs
			daemon.Unmount(container)
		}
	}()

	if err = daemon.mountVolumes(container); err != nil {
		return nil, nil, err
	}

	// Normalize path before sending to rootfs
	path = filepath.FromSlash(path)

	resolvedPath, absPath, err := container.ResolvePath(path)
	if err != nil {
		return nil, nil, err
	}

	stat, err = container.StatPath(resolvedPath, absPath)
	if err != nil {
		return nil, nil, err
	}

	// We need to rebase the archive entries if the last element of the
	// resolved path was a symlink that was evaluated and is now different
	// than the requested path. For example, if the given path was "/foo/bar/",
	// but it resolved to "/var/lib/docker/containers/{id}/foo/baz/", we want
	// to ensure that the archive entries start with "bar" and not "baz". This
	// also catches the case when the root directory of the container is
	// requested: we want the archive entries to start with "/" and not the
	// container ID.

	// Get the source and the base paths of the container resolved path in order
	// to get the proper tar options for the rebase tar.
	resolvedPath = filepath.Clean(resolvedPath)
	if filepath.Base(resolvedPath) == "." {
		resolvedPath += string(filepath.Separator) + "."
	}

	sourceDir := resolvedPath
	sourceBase := "."

	if stat.Mode&os.ModeDir == 0 { // not dir
		sourceDir, sourceBase = filepath.Split(resolvedPath)
	}
	opts := archive.TarResourceRebaseOpts(sourceBase, filepath.Base(absPath))

	data, err := chrootarchive.Tar(sourceDir, opts, container.BaseFS)
	if err != nil {
		return nil, nil, err
	}

	content = ioutils.NewReadCloserWrapper(data, func() error {
		err := data.Close()
		container.DetachAndUnmount(daemon.LogVolumeEvent)
		daemon.Unmount(container)
		container.Unlock()
		return err
	})

	daemon.LogContainerEvent(container, "archive-path")

	return content, stat, nil
}

// containerExtractToDir extracts the given tar archive to the specified location in the
// filesystem of this container. The given path must be of a directory in the
// container. If it is not, the error will be ErrExtractPointNotDirectory. If
// noOverwriteDirNonDir is true then it will be an error if unpacking the
// given content would cause an existing directory to be replaced with a non-
// directory and vice versa.
func (daemon *Daemon) containerExtractToDir(container *container.Container, path string, copyUIDGID, noOverwriteDirNonDir bool, content io.Reader) (err error) {
	container.Lock()
	defer container.Unlock()

	if err = daemon.Mount(container); err != nil {
		return err
	}
	defer daemon.Unmount(container)

	err = daemon.mountVolumes(container)
	defer container.DetachAndUnmount(daemon.LogVolumeEvent)
	if err != nil {
		return err
	}

	// Normalize path before sending to rootfs'
	path = filepath.FromSlash(path)

	// Check if a drive letter supplied, it must be the system drive. No-op except on Windows
	path, err = system.CheckSystemDriveAndRemoveDriveLetter(path)
	if err != nil {
		return err
	}

	// The destination path needs to be resolved to a host path, with all
	// symbolic links followed in the scope of the container's rootfs. Note
	// that we do not use `container.ResolvePath(path)` here because we need
	// to also evaluate the last path element if it is a symlink. This is so
	// that you can extract an archive to a symlink that points to a directory.

	// Consider the given path as an absolute path in the container.
	absPath := archive.PreserveTrailingDotOrSeparator(filepath.Join(string(filepath.Separator), path), path)

	// This will evaluate the last path element if it is a symlink.
	resolvedPath, err := container.GetResourcePath(absPath)
	if err != nil {
		return err
	}

	stat, err := os.Lstat(resolvedPath)
	if err != nil {
		return err
	}

	if !stat.IsDir() {
		return ErrExtractPointNotDirectory
	}

	// Need to check if the path is in a volume. If it is, it cannot be in a
	// read-only volume. If it is not in a volume, the container cannot be
	// configured with a read-only rootfs.

	// Use the resolved path relative to the container rootfs as the new
	// absPath. This way we fully follow any symlinks in a volume that may
	// lead back outside the volume.
	//
	// The Windows implementation of filepath.Rel in golang 1.4 does not
	// support volume style file path semantics. On Windows when using the
	// filter driver, we are guaranteed that the path will always be
	// a volume file path.
	var baseRel string
	if strings.HasPrefix(resolvedPath, `\\?\Volume{`) {
		if strings.HasPrefix(resolvedPath, container.BaseFS) {
			baseRel = resolvedPath[len(container.BaseFS):]
			if baseRel[:1] == `\` {
				baseRel = baseRel[1:]
			}
		}
	} else {
		baseRel, err = filepath.Rel(container.BaseFS, resolvedPath)
	}
	if err != nil {
		return err
	}
	// Make it an absolute path.
	absPath = filepath.Join(string(filepath.Separator), baseRel)

	toVolume, err := checkIfPathIsInAVolume(container, absPath)
	if err != nil {
		return err
	}

	if !toVolume && container.HostConfig.ReadonlyRootfs {
		return errdefs.InvalidParameter(errors.New("container rootfs is marked read-only"))
	}

	options := daemon.defaultTarCopyOptions(noOverwriteDirNonDir)

	if copyUIDGID {
		var err error
		// tarCopyOptions will appropriately pull in the right uid/gid for the
		// user/group and will set the options.
		options, err = daemon.tarCopyOptions(container, noOverwriteDirNonDir)
		if err != nil {
			return err
		}
	}

	if err := chrootarchive.UntarWithRoot(content, resolvedPath, options, container.BaseFS); err != nil {
		return err
	}

	daemon.LogContainerEvent(container, "extract-to-dir")

	return nil
}

func (daemon *Daemon) containerCopy(container *container.Container, resource string) (rc io.ReadCloser, err error) {
	if resource[0] == '/' || resource[0] == '\\' {
		resource = resource[1:]
	}
	container.Lock()

	defer func() {
		if err != nil {
			// Wait to unlock the container until the archive is fully read
			// (see the ReadCloseWrapper func below) or if there is an error
			// before that occurs.
			container.Unlock()
		}
	}()

	if err := daemon.Mount(container); err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			// unmount any volumes
			container.DetachAndUnmount(daemon.LogVolumeEvent)
			// unmount the container's rootfs
			daemon.Unmount(container)
		}
	}()

	if err := daemon.mountVolumes(container); err != nil {
		return nil, err
	}

	// Normalize path before sending to rootfs
	resource = filepath.FromSlash(resource)

	basePath, err := container.GetResourcePath(resource)
	if err != nil {
		return nil, err
	}
	stat, err := os.Stat(basePath)
	if err != nil {
		return nil, err
	}
	var filter []string
	if !stat.IsDir() {
		d, f := filepath.Split(basePath)
		basePath = d
		filter = []string{f}
	}
	archv, err := chrootarchive.Tar(basePath, &archive.TarOptions{
		Compression:  archive.Uncompressed,
		IncludeFiles: filter,
	}, container.BaseFS)
	if err != nil {
		return nil, err
	}

	reader := ioutils.NewReadCloserWrapper(archv, func() error {
		err := archv.Close()
		container.DetachAndUnmount(daemon.LogVolumeEvent)
		daemon.Unmount(container)
		container.Unlock()
		return err
	})
	daemon.LogContainerEvent(container, "copy")
	return reader, nil
}
<commit_msg>daemon: replace ErrExtractPointNotDirectory with errdefs<commit_after>package daemon // import "github.com/docker/docker/daemon"

import (
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/container"
	"github.com/docker/docker/errdefs"
	"github.com/docker/docker/pkg/archive"
	"github.com/docker/docker/pkg/chrootarchive"
	"github.com/docker/docker/pkg/ioutils"
	"github.com/docker/docker/pkg/system"
	"github.com/pkg/errors"
)

// ContainerCopy performs a deprecated operation of archiving the resource at
// the specified path in the container identified by the given name.
func (daemon *Daemon) ContainerCopy(name string, res string) (io.ReadCloser, error) {
	ctr, err := daemon.GetContainer(name)
	if err != nil {
		return nil, err
	}

	// Make sure an online file-system operation is permitted.
	if err := daemon.isOnlineFSOperationPermitted(ctr); err != nil {
		return nil, errdefs.System(err)
	}

	data, err := daemon.containerCopy(ctr, res)
	if err == nil {
		return data, nil
	}

	if os.IsNotExist(err) {
		return nil, containerFileNotFound{res, name}
	}
	return nil, errdefs.System(err)
}

// ContainerStatPath stats the filesystem resource at the specified path in the
// container identified by the given name.
func (daemon *Daemon) ContainerStatPath(name string, path string) (stat *types.ContainerPathStat, err error) {
	ctr, err := daemon.GetContainer(name)
	if err != nil {
		return nil, err
	}

	// Make sure an online file-system operation is permitted.
	if err := daemon.isOnlineFSOperationPermitted(ctr); err != nil {
		return nil, errdefs.System(err)
	}

	stat, err = daemon.containerStatPath(ctr, path)
	if err == nil {
		return stat, nil
	}

	if os.IsNotExist(err) {
		return nil, containerFileNotFound{path, name}
	}
	return nil, errdefs.System(err)
}

// ContainerArchivePath creates an archive of the filesystem resource at the
// specified path in the container identified by the given name. Returns a
// tar archive of the resource and whether it was a directory or a single file.
func (daemon *Daemon) ContainerArchivePath(name string, path string) (content io.ReadCloser, stat *types.ContainerPathStat, err error) {
	ctr, err := daemon.GetContainer(name)
	if err != nil {
		return nil, nil, err
	}

	// Make sure an online file-system operation is permitted.
	if err := daemon.isOnlineFSOperationPermitted(ctr); err != nil {
		return nil, nil, errdefs.System(err)
	}

	content, stat, err = daemon.containerArchivePath(ctr, path)
	if err == nil {
		return content, stat, nil
	}

	if os.IsNotExist(err) {
		return nil, nil, containerFileNotFound{path, name}
	}
	return nil, nil, errdefs.System(err)
}

// ContainerExtractToDir extracts the given archive to the specified location
// in the filesystem of the container identified by the given name. The given
// path must be of a directory in the container. If it is not, the error will
// be an errdefs.InvalidParameter. If noOverwriteDirNonDir is true then it will
// be an error if unpacking the given content would cause an existing directory
// to be replaced with a non-directory and vice versa.
func (daemon *Daemon) ContainerExtractToDir(name, path string, copyUIDGID, noOverwriteDirNonDir bool, content io.Reader) error {
	ctr, err := daemon.GetContainer(name)
	if err != nil {
		return err
	}

	// Make sure an online file-system operation is permitted.
	if err := daemon.isOnlineFSOperationPermitted(ctr); err != nil {
		return errdefs.System(err)
	}

	err = daemon.containerExtractToDir(ctr, path, copyUIDGID, noOverwriteDirNonDir, content)
	if err == nil {
		return nil
	}

	if os.IsNotExist(err) {
		return containerFileNotFound{path, name}
	}
	return errdefs.System(err)
}

// containerStatPath stats the filesystem resource at the specified path in this
// container. Returns stat info about the resource.
func (daemon *Daemon) containerStatPath(container *container.Container, path string) (stat *types.ContainerPathStat, err error) {
	container.Lock()
	defer container.Unlock()

	if err = daemon.Mount(container); err != nil {
		return nil, err
	}
	defer daemon.Unmount(container)

	err = daemon.mountVolumes(container)
	defer container.DetachAndUnmount(daemon.LogVolumeEvent)
	if err != nil {
		return nil, err
	}

	// Normalize path before sending to rootfs
	path = filepath.FromSlash(path)

	resolvedPath, absPath, err := container.ResolvePath(path)
	if err != nil {
		return nil, err
	}

	return container.StatPath(resolvedPath, absPath)
}

// containerArchivePath creates an archive of the filesystem resource at the specified
// path in this container. Returns a tar archive of the resource and stat info
// about the resource.
func (daemon *Daemon) containerArchivePath(container *container.Container, path string) (content io.ReadCloser, stat *types.ContainerPathStat, err error) {
	container.Lock()

	defer func() {
		if err != nil {
			// Wait to unlock the container until the archive is fully read
			// (see the ReadCloseWrapper func below) or if there is an error
			// before that occurs.
			container.Unlock()
		}
	}()

	if err = daemon.Mount(container); err != nil {
		return nil, nil, err
	}

	defer func() {
		if err != nil {
			// unmount any volumes
			container.DetachAndUnmount(daemon.LogVolumeEvent)
			// unmount the container's rootfs
			daemon.Unmount(container)
		}
	}()

	if err = daemon.mountVolumes(container); err != nil {
		return nil, nil, err
	}

	// Normalize path before sending to rootfs
	path = filepath.FromSlash(path)

	resolvedPath, absPath, err := container.ResolvePath(path)
	if err != nil {
		return nil, nil, err
	}

	stat, err = container.StatPath(resolvedPath, absPath)
	if err != nil {
		return nil, nil, err
	}

	// We need to rebase the archive entries if the last element of the
	// resolved path was a symlink that was evaluated and is now different
	// than the requested path. For example, if the given path was "/foo/bar/",
	// but it resolved to "/var/lib/docker/containers/{id}/foo/baz/", we want
	// to ensure that the archive entries start with "bar" and not "baz". This
	// also catches the case when the root directory of the container is
	// requested: we want the archive entries to start with "/" and not the
	// container ID.

	// Get the source and the base paths of the container resolved path in order
	// to get the proper tar options for the rebase tar.
	resolvedPath = filepath.Clean(resolvedPath)
	if filepath.Base(resolvedPath) == "." {
		resolvedPath += string(filepath.Separator) + "."
	}

	sourceDir := resolvedPath
	sourceBase := "."

	if stat.Mode&os.ModeDir == 0 { // not dir
		sourceDir, sourceBase = filepath.Split(resolvedPath)
	}
	opts := archive.TarResourceRebaseOpts(sourceBase, filepath.Base(absPath))

	data, err := chrootarchive.Tar(sourceDir, opts, container.BaseFS)
	if err != nil {
		return nil, nil, err
	}

	content = ioutils.NewReadCloserWrapper(data, func() error {
		err := data.Close()
		container.DetachAndUnmount(daemon.LogVolumeEvent)
		daemon.Unmount(container)
		container.Unlock()
		return err
	})

	daemon.LogContainerEvent(container, "archive-path")

	return content, stat, nil
}

// containerExtractToDir extracts the given tar archive to the specified location in the
// filesystem of this container. The given path must be of a directory in the
// container. If it is not, the error will be an errdefs.InvalidParameter. If
// noOverwriteDirNonDir is true then it will be an error if unpacking the
// given content would cause an existing directory to be replaced with a non-
// directory and vice versa.
func (daemon *Daemon) containerExtractToDir(container *container.Container, path string, copyUIDGID, noOverwriteDirNonDir bool, content io.Reader) (err error) {
	container.Lock()
	defer container.Unlock()

	if err = daemon.Mount(container); err != nil {
		return err
	}
	defer daemon.Unmount(container)

	err = daemon.mountVolumes(container)
	defer container.DetachAndUnmount(daemon.LogVolumeEvent)
	if err != nil {
		return err
	}

	// Normalize path before sending to rootfs'
	path = filepath.FromSlash(path)

	// Check if a drive letter supplied, it must be the system drive. No-op except on Windows
	path, err = system.CheckSystemDriveAndRemoveDriveLetter(path)
	if err != nil {
		return err
	}

	// The destination path needs to be resolved to a host path, with all
	// symbolic links followed in the scope of the container's rootfs. Note
	// that we do not use `container.ResolvePath(path)` here because we need
	// to also evaluate the last path element if it is a symlink. This is so
	// that you can extract an archive to a symlink that points to a directory.

	// Consider the given path as an absolute path in the container.
	absPath := archive.PreserveTrailingDotOrSeparator(filepath.Join(string(filepath.Separator), path), path)

	// This will evaluate the last path element if it is a symlink.
	resolvedPath, err := container.GetResourcePath(absPath)
	if err != nil {
		return err
	}

	stat, err := os.Lstat(resolvedPath)
	if err != nil {
		return err
	}

	if !stat.IsDir() {
		return errdefs.InvalidParameter(errors.New("extraction point is not a directory"))
	}

	// Need to check if the path is in a volume. If it is, it cannot be in a
	// read-only volume. If it is not in a volume, the container cannot be
	// configured with a read-only rootfs.

	// Use the resolved path relative to the container rootfs as the new
	// absPath. This way we fully follow any symlinks in a volume that may
	// lead back outside the volume.
	//
	// The Windows implementation of filepath.Rel in golang 1.4 does not
	// support volume style file path semantics. On Windows when using the
	// filter driver, we are guaranteed that the path will always be
	// a volume file path.
	var baseRel string
	if strings.HasPrefix(resolvedPath, `\\?\Volume{`) {
		if strings.HasPrefix(resolvedPath, container.BaseFS) {
			baseRel = resolvedPath[len(container.BaseFS):]
			if baseRel[:1] == `\` {
				baseRel = baseRel[1:]
			}
		}
	} else {
		baseRel, err = filepath.Rel(container.BaseFS, resolvedPath)
	}
	if err != nil {
		return err
	}
	// Make it an absolute path.
	absPath = filepath.Join(string(filepath.Separator), baseRel)

	toVolume, err := checkIfPathIsInAVolume(container, absPath)
	if err != nil {
		return err
	}

	if !toVolume && container.HostConfig.ReadonlyRootfs {
		return errdefs.InvalidParameter(errors.New("container rootfs is marked read-only"))
	}

	options := daemon.defaultTarCopyOptions(noOverwriteDirNonDir)

	if copyUIDGID {
		var err error
		// tarCopyOptions will appropriately pull in the right uid/gid for the
		// user/group and will set the options.
		options, err = daemon.tarCopyOptions(container, noOverwriteDirNonDir)
		if err != nil {
			return err
		}
	}

	if err := chrootarchive.UntarWithRoot(content, resolvedPath, options, container.BaseFS); err != nil {
		return err
	}

	daemon.LogContainerEvent(container, "extract-to-dir")

	return nil
}

func (daemon *Daemon) containerCopy(container *container.Container, resource string) (rc io.ReadCloser, err error) {
	if resource[0] == '/' || resource[0] == '\\' {
		resource = resource[1:]
	}
	container.Lock()

	defer func() {
		if err != nil {
			// Wait to unlock the container until the archive is fully read
			// (see the ReadCloseWrapper func below) or if there is an error
			// before that occurs.
			container.Unlock()
		}
	}()

	if err := daemon.Mount(container); err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			// unmount any volumes
			container.DetachAndUnmount(daemon.LogVolumeEvent)
			// unmount the container's rootfs
			daemon.Unmount(container)
		}
	}()

	if err := daemon.mountVolumes(container); err != nil {
		return nil, err
	}

	// Normalize path before sending to rootfs
	resource = filepath.FromSlash(resource)

	basePath, err := container.GetResourcePath(resource)
	if err != nil {
		return nil, err
	}
	stat, err := os.Stat(basePath)
	if err != nil {
		return nil, err
	}
	var filter []string
	if !stat.IsDir() {
		d, f := filepath.Split(basePath)
		basePath = d
		filter = []string{f}
	}
	archv, err := chrootarchive.Tar(basePath, &archive.TarOptions{
		Compression:  archive.Uncompressed,
		IncludeFiles: filter,
	}, container.BaseFS)
	if err != nil {
		return nil, err
	}

	reader := ioutils.NewReadCloserWrapper(archv, func() error {
		err := archv.Close()
		container.DetachAndUnmount(daemon.LogVolumeEvent)
		daemon.Unmount(container)
		container.Unlock()
		return err
	})
	daemon.LogContainerEvent(container, "copy")
	return reader, nil
}
<|endoftext|>