<commit_before>// Copyright 2019 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package plugin

import (
	"crypto/tls"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"strings"

	frpNet "github.com/fatedier/frp/pkg/util/net"
)

const PluginHTTP2HTTPS = "http2https"

func init() {
	Register(PluginHTTP2HTTPS, NewHTTP2HTTPSPlugin)
}

type HTTP2HTTPSPlugin struct {
	hostHeaderRewrite string
	localAddr         string
	headers           map[string]string

	l *Listener
	s *http.Server
}

func NewHTTP2HTTPSPlugin(params map[string]string) (Plugin, error) {
	localAddr := params["plugin_local_addr"]
	hostHeaderRewrite := params["plugin_host_header_rewrite"]
	headers := make(map[string]string)
	for k, v := range params {
		if !strings.HasPrefix(k, "plugin_header_") {
			continue
		}
		if k = strings.TrimPrefix(k, "plugin_header_"); k != "" {
			headers[k] = v
		}
	}

	if localAddr == "" {
		return nil, fmt.Errorf("plugin_local_addr is required")
	}

	listener := NewProxyListener()

	p := &HTTPS2HTTPPlugin{
		localAddr:         localAddr,
		hostHeaderRewrite: hostHeaderRewrite,
		headers:           headers,
		l:                 listener,
	}

	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	rp := &httputil.ReverseProxy{
		Director: func(req *http.Request) {
			req.URL.Scheme = "https"
			req.URL.Host = p.localAddr
			if p.hostHeaderRewrite != "" {
				req.Host = p.hostHeaderRewrite
			}
			for k, v := range p.headers {
				req.Header.Set(k, v)
			}
		},
		Transport: tr,
	}

	p.s = &http.Server{
		Handler: rp,
	}

	go p.s.Serve(listener)

	return p, nil
}

func (p *HTTP2HTTPSPlugin) Handle(conn io.ReadWriteCloser, realConn net.Conn, extraBufToLocal []byte) {
	wrapConn := frpNet.WrapReadWriteCloserToConn(conn, realConn)
	p.l.PutConn(wrapConn)
}

func (p *HTTP2HTTPSPlugin) Name() string {
	return PluginHTTP2HTTPS
}

func (p *HTTP2HTTPSPlugin) Close() error {
	if err := p.s.Close(); err != nil {
		return err
	}
	return nil
}
<commit_msg>fix: stuct name typo (#2458)<commit_after>// Copyright 2019 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package plugin

import (
	"crypto/tls"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"strings"

	frpNet "github.com/fatedier/frp/pkg/util/net"
)

const PluginHTTP2HTTPS = "http2https"

func init() {
	Register(PluginHTTP2HTTPS, NewHTTP2HTTPSPlugin)
}

type HTTP2HTTPSPlugin struct {
	hostHeaderRewrite string
	localAddr         string
	headers           map[string]string

	l *Listener
	s *http.Server
}

func NewHTTP2HTTPSPlugin(params map[string]string) (Plugin, error) {
	localAddr := params["plugin_local_addr"]
	hostHeaderRewrite := params["plugin_host_header_rewrite"]
	headers := make(map[string]string)
	for k, v := range params {
		if !strings.HasPrefix(k, "plugin_header_") {
			continue
		}
		if k = strings.TrimPrefix(k, "plugin_header_"); k != "" {
			headers[k] = v
		}
	}

	if localAddr == "" {
		return nil, fmt.Errorf("plugin_local_addr is required")
	}

	listener := NewProxyListener()

	p := &HTTP2HTTPSPlugin{
		localAddr:         localAddr,
		hostHeaderRewrite: hostHeaderRewrite,
		headers:           headers,
		l:                 listener,
	}

	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	rp := &httputil.ReverseProxy{
		Director: func(req *http.Request) {
			req.URL.Scheme = "https"
			req.URL.Host = p.localAddr
			if p.hostHeaderRewrite != "" {
				req.Host = p.hostHeaderRewrite
			}
			for k, v := range p.headers {
				req.Header.Set(k, v)
			}
		},
		Transport: tr,
	}

	p.s = &http.Server{
		Handler: rp,
	}

	go p.s.Serve(listener)

	return p, nil
}

func (p *HTTP2HTTPSPlugin) Handle(conn io.ReadWriteCloser, realConn net.Conn, extraBufToLocal []byte) {
	wrapConn := frpNet.WrapReadWriteCloserToConn(conn, realConn)
	p.l.PutConn(wrapConn)
}

func (p *HTTP2HTTPSPlugin) Name() string {
	return PluginHTTP2HTTPS
}

func (p *HTTP2HTTPSPlugin) Close() error {
	if err := p.s.Close(); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package plugin

import (
	"fmt"
	"sync"
	"time"

	"github.com/golang/glog"

	utilwait "k8s.io/apimachinery/pkg/util/wait"
	utildbus "k8s.io/kubernetes/pkg/util/dbus"
	kexec "k8s.io/kubernetes/pkg/util/exec"
	"k8s.io/kubernetes/pkg/util/iptables"
)

type FirewallRule struct {
	table string
	chain string
	args  []string
}

type NodeIPTables struct {
	ipt                iptables.Interface
	clusterNetworkCIDR string
	syncPeriod         time.Duration

	mu sync.Mutex // Protects concurrent access to syncIPTableRules()
}

const (
	OutputFilteringChain iptables.Chain = "OPENSHIFT-ADMIN-OUTPUT-RULES"
)

func newNodeIPTables(clusterNetworkCIDR string, syncPeriod time.Duration) *NodeIPTables {
	return &NodeIPTables{
		ipt:                iptables.New(kexec.New(), utildbus.New(), iptables.ProtocolIpv4),
		clusterNetworkCIDR: clusterNetworkCIDR,
		syncPeriod:         syncPeriod,
	}
}

func (n *NodeIPTables) Setup() error {
	if err := n.syncIPTableRules(); err != nil {
		return err
	}

	// If firewalld is running, reload will call this method
	n.ipt.AddReloadFunc(func() {
		if err := n.syncIPTableRules(); err != nil {
			glog.Errorf("Reloading openshift iptables failed: %v", err)
		}
	})

	go utilwait.Forever(n.syncLoop, 0)
	return nil
}

// syncLoop periodically calls syncIPTableRules().
// This is expected to run as a go routine or as the main loop. It does not return.
func (n *NodeIPTables) syncLoop() {
	t := time.NewTicker(n.syncPeriod)
	defer t.Stop()
	for {
		<-t.C
		glog.V(6).Infof("Periodic openshift iptables sync")
		err := n.syncIPTableRules()
		if err != nil {
			glog.Errorf("Syncing openshift iptables failed: %v", err)
		}
	}
}

// syncIPTableRules syncs the cluster network cidr iptables rules.
// Called from SyncLoop() or firwalld reload()
func (n *NodeIPTables) syncIPTableRules() error {
	n.mu.Lock()
	defer n.mu.Unlock()

	start := time.Now()
	defer func() {
		glog.V(4).Infof("syncIPTableRules took %v", time.Since(start))
	}()
	glog.V(3).Infof("Syncing openshift iptables rules")

	if _, err := n.ipt.EnsureChain(iptables.TableFilter, OutputFilteringChain); err != nil {
		return fmt.Errorf("failed to ensure chain %q exists: %v", OutputFilteringChain, err)
	}

	rules := n.getStaticNodeIPTablesRules()
	for _, rule := range rules {
		_, err := n.ipt.EnsureRule(iptables.Prepend, iptables.Table(rule.table), iptables.Chain(rule.chain), rule.args...)
		if err != nil {
			return fmt.Errorf("failed to ensure rule %v exists: %v", rule, err)
		}
	}
	return nil
}

const VXLAN_PORT = "4789"

// Get openshift iptables rules
func (n *NodeIPTables) getStaticNodeIPTablesRules() []FirewallRule {
	return []FirewallRule{
		{"nat", "POSTROUTING", []string{"-s", n.clusterNetworkCIDR, "-m", "comment", "--comment", "masquerade pod-to-service and pod-to-external traffic", "-j", "MASQUERADE"}},
		{"filter", "INPUT", []string{"-p", "udp", "--dport", VXLAN_PORT, "-m", "comment", "--comment", "VXLAN incoming", "-j", "ACCEPT"}},
		{"filter", "INPUT", []string{"-i", TUN, "-m", "comment", "--comment", "from SDN to localhost", "-j", "ACCEPT"}},
		{"filter", "INPUT", []string{"-i", "docker0", "-m", "comment", "--comment", "from docker to localhost", "-j", "ACCEPT"}},
		{"filter", "FORWARD", []string{"-s", n.clusterNetworkCIDR, "-m", "comment", "--comment", "attempted resend after connection close", "-m", "conntrack", "--ctstate", "INVALID", "-j", "DROP"}},
		{"filter", "FORWARD", []string{"-d", n.clusterNetworkCIDR, "-m", "comment", "--comment", "forward traffic from SDN", "-j", "ACCEPT"}},
		{"filter", "FORWARD", []string{"-s", n.clusterNetworkCIDR, "-m", "comment", "--comment", "forward traffic to SDN", "-j", "ACCEPT"}},
		{"filter", "FORWARD", []string{"-i", TUN, "!", "-o", TUN, "-m", "comment", "--comment", "administrator overrides", "-j", string(OutputFilteringChain)}},
	}
}
<commit_msg>Segregate OpenShift's iptables rules<commit_after>package plugin

import (
	"fmt"
	"sync"
	"time"

	"github.com/golang/glog"

	utilwait "k8s.io/apimachinery/pkg/util/wait"
	utildbus "k8s.io/kubernetes/pkg/util/dbus"
	kexec "k8s.io/kubernetes/pkg/util/exec"
	"k8s.io/kubernetes/pkg/util/iptables"
)

type NodeIPTables struct {
	ipt                iptables.Interface
	clusterNetworkCIDR string
	syncPeriod         time.Duration

	mu sync.Mutex // Protects concurrent access to syncIPTableRules()
}

func newNodeIPTables(clusterNetworkCIDR string, syncPeriod time.Duration) *NodeIPTables {
	return &NodeIPTables{
		ipt:                iptables.New(kexec.New(), utildbus.New(), iptables.ProtocolIpv4),
		clusterNetworkCIDR: clusterNetworkCIDR,
		syncPeriod:         syncPeriod,
	}
}

func (n *NodeIPTables) Setup() error {
	if err := n.syncIPTableRules(); err != nil {
		return err
	}

	// If firewalld is running, reload will call this method
	n.ipt.AddReloadFunc(func() {
		if err := n.syncIPTableRules(); err != nil {
			glog.Errorf("Reloading openshift iptables failed: %v", err)
		}
	})

	go utilwait.Forever(n.syncLoop, 0)
	return nil
}

// syncLoop periodically calls syncIPTableRules().
// This is expected to run as a go routine or as the main loop. It does not return.
func (n *NodeIPTables) syncLoop() {
	t := time.NewTicker(n.syncPeriod)
	defer t.Stop()
	for {
		<-t.C
		glog.V(6).Infof("Periodic openshift iptables sync")
		err := n.syncIPTableRules()
		if err != nil {
			glog.Errorf("Syncing openshift iptables failed: %v", err)
		}
	}
}

type Chain struct {
	table    string
	name     string
	srcChain string
	srcRule  []string
	rules    [][]string
}

// Adds all the rules in chain, returning true if they were all already present
func (n *NodeIPTables) addChainRules(chain Chain) (bool, error) {
	allExisted := true
	for _, rule := range chain.rules {
		existed, err := n.ipt.EnsureRule(iptables.Append, iptables.Table(chain.table), iptables.Chain(chain.name), rule...)
		if err != nil {
			return false, fmt.Errorf("failed to ensure rule %v exists: %v", rule, err)
		}
		if !existed {
			allExisted = false
		}
	}
	return allExisted, nil
}

// syncIPTableRules syncs the cluster network cidr iptables rules.
// Called from SyncLoop() or firewalld reload()
func (n *NodeIPTables) syncIPTableRules() error {
	n.mu.Lock()
	defer n.mu.Unlock()

	start := time.Now()
	defer func() {
		glog.V(4).Infof("syncIPTableRules took %v", time.Since(start))
	}()
	glog.V(3).Infof("Syncing openshift iptables rules")

	for _, chain := range n.getNodeIPTablesChains() {
		// Create chain if it does not already exist
		chainExisted, err := n.ipt.EnsureChain(iptables.Table(chain.table), iptables.Chain(chain.name))
		if err != nil {
			return fmt.Errorf("failed to ensure chain %s exists: %v", chain.name, err)
		}

		// Create the rule pointing to it from its parent chain. Note that since we
		// use iptables.Prepend each time, chains with the same table and srcChain
		// (ie, OPENSHIFT-FIREWALL-FORWARD and OPENSHIFT-ADMIN-OUTPUT-RULES) will
		// run in *reverse* order of how they are listed in getNodeIPTablesChains().
		_, err = n.ipt.EnsureRule(iptables.Prepend, iptables.Table(chain.table), iptables.Chain(chain.srcChain), append(chain.srcRule, "-j", chain.name)...)
		if err != nil {
			return fmt.Errorf("failed to ensure rule from %s to %s exists: %v", chain.srcChain, chain.name, err)
		}

		// Add/sync the rules
		rulesExisted, err := n.addChainRules(chain)
		if err != nil {
			return err
		}
		if chainExisted && !rulesExisted {
			// Chain existed but not with the expected rules; this probably means
			// it contained rules referring to a *different* subnet; flush them
			// and try again.
			if err = n.ipt.FlushChain(iptables.Table(chain.table), iptables.Chain(chain.name)); err != nil {
				return fmt.Errorf("failed to flush chain %s: %v", chain.name, err)
			}
			if _, err = n.addChainRules(chain); err != nil {
				return err
			}
		}
	}

	return nil
}

const VXLAN_PORT = "4789"

func (n *NodeIPTables) getNodeIPTablesChains() []Chain {
	return []Chain{
		{
			table:    "nat",
			name:     "OPENSHIFT-MASQUERADE",
			srcChain: "POSTROUTING",
			srcRule:  []string{"-m", "comment", "--comment", "rules for masquerading OpenShift traffic"},
			rules: [][]string{
				{"-s", n.clusterNetworkCIDR, "-m", "comment", "--comment", "masquerade pod-to-service and pod-to-external traffic", "-j", "MASQUERADE"},
			},
		},
		{
			table:    "filter",
			name:     "OPENSHIFT-FIREWALL-ALLOW",
			srcChain: "INPUT",
			srcRule:  []string{"-m", "comment", "--comment", "firewall overrides"},
			rules: [][]string{
				{"-p", "udp", "--dport", VXLAN_PORT, "-m", "comment", "--comment", "VXLAN incoming", "-j", "ACCEPT"},
				{"-i", TUN, "-m", "comment", "--comment", "from SDN to localhost", "-j", "ACCEPT"},
				{"-i", "docker0", "-m", "comment", "--comment", "from docker to localhost", "-j", "ACCEPT"},
			},
		},
		{
			table:    "filter",
			name:     "OPENSHIFT-FIREWALL-FORWARD",
			srcChain: "FORWARD",
			srcRule:  []string{"-m", "comment", "--comment", "firewall overrides"},
			rules: [][]string{
				{"-s", n.clusterNetworkCIDR, "-m", "comment", "--comment", "attempted resend after connection close", "-m", "conntrack", "--ctstate", "INVALID", "-j", "DROP"},
				{"-d", n.clusterNetworkCIDR, "-m", "comment", "--comment", "forward traffic from SDN", "-j", "ACCEPT"},
				{"-s", n.clusterNetworkCIDR, "-m", "comment", "--comment", "forward traffic to SDN", "-j", "ACCEPT"},
			},
		},
		{
			table:    "filter",
			name:     "OPENSHIFT-ADMIN-OUTPUT-RULES",
			srcChain: "FORWARD",
			srcRule:  []string{"-i", TUN, "!", "-o", TUN, "-m", "comment", "--comment", "administrator overrides"},
			rules:    nil,
		},
	}
}
<|endoftext|>


<commit_before>// Copyright 2021 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package control

import (
	"fmt"
	"strings"

	"gvisor.dev/gvisor/pkg/abi/linux"
	"gvisor.dev/gvisor/pkg/fd"
	"gvisor.dev/gvisor/pkg/log"
	"gvisor.dev/gvisor/pkg/sentry/fs/user"
	"gvisor.dev/gvisor/pkg/sentry/kernel"
	"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
	"gvisor.dev/gvisor/pkg/sentry/limits"
	"gvisor.dev/gvisor/pkg/sentry/vfs"
	"gvisor.dev/gvisor/pkg/sync"
	"gvisor.dev/gvisor/pkg/urpc"
)

// Lifecycle provides functions related to starting and stopping tasks.
type Lifecycle struct {
	// Kernel is the kernel where the tasks belong to.
	Kernel *kernel.Kernel

	// StartedCh is the channel used to send a message to the sentry that
	// all the containers in the sandbox have been started.
	StartedCh chan struct{}

	// mu protects the fields below.
	mu sync.RWMutex

	// containersStarted is the number of containers started in the sandbox.
	containersStarted int32

	// MountNamespacesMap is a map of container id/names and the mount
	// namespaces.
	MountNamespacesMap map[string]*vfs.MountNamespace
}

// StartContainerArgs is the set of arguments to start a container.
type StartContainerArgs struct {
	// Filename is the filename to load.
	//
	// If this is provided as "", then the file will be guessed via Argv[0].
	Filename string `json:"filename"`

	// Argv is a list of arguments.
	Argv []string `json:"argv"`

	// Envv is a list of environment variables.
	Envv []string `json:"envv"`

	// WorkingDirectory defines the working directory for the new process.
	WorkingDirectory string `json:"wd"`

	// KUID is the UID to run with in the root user namespace. Defaults to
	// root if not set explicitly.
	KUID auth.KUID `json:"KUID"`

	// KGID is the GID to run with in the root user namespace. Defaults to
	// the root group if not set explicitly.
	KGID auth.KGID `json:"KGID"`

	// ExtraKGIDs is the list of additional groups to which the user belongs.
	ExtraKGIDs []auth.KGID `json:"extraKGID"`

	// Capabilities is the list of capabilities to give to the process.
	Capabilities *auth.TaskCapabilities `json:"capabilities"`

	// FilePayload determines the files to give to the new process.
	urpc.FilePayload

	// ContainerID is the container for the process being executed.
	ContainerID string `json:"containerID"`

	// Limits is the limit set for the process being executed.
	Limits *limits.LimitSet `json:"limits"`
}

// String prints the StartContainerArgs.argv as a string.
func (args StartContainerArgs) String() string {
	if len(args.Argv) == 0 {
		return args.Filename
	}
	a := make([]string, len(args.Argv))
	copy(a, args.Argv)
	if args.Filename != "" {
		a[0] = args.Filename
	}
	return strings.Join(a, " ")
}

// StartContainer will start a new container in the sandbox.
func (l *Lifecycle) StartContainer(args *StartContainerArgs, _ *uint32) error {
	// Import file descriptors.
	fdTable := l.Kernel.NewFDTable()

	creds := auth.NewUserCredentials(
		args.KUID,
		args.KGID,
		args.ExtraKGIDs,
		args.Capabilities,
		l.Kernel.RootUserNamespace())

	limitSet := args.Limits
	if limitSet == nil {
		limitSet = limits.NewLimitSet()
	}
	initArgs := kernel.CreateProcessArgs{
		Filename:                args.Filename,
		Argv:                    args.Argv,
		Envv:                    args.Envv,
		WorkingDirectory:        args.WorkingDirectory,
		Credentials:             creds,
		FDTable:                 fdTable,
		Umask:                   0022,
		Limits:                  limitSet,
		MaxSymlinkTraversals:    linux.MaxSymlinkTraversals,
		UTSNamespace:            l.Kernel.RootUTSNamespace(),
		IPCNamespace:            l.Kernel.RootIPCNamespace(),
		AbstractSocketNamespace: l.Kernel.RootAbstractSocketNamespace(),
		ContainerID:             args.ContainerID,
		PIDNamespace:            l.Kernel.RootPIDNamespace(),
	}

	ctx := initArgs.NewContext(l.Kernel)
	defer fdTable.DecRef(ctx)

	// VFS2 is supported in multi-container mode by default.
	l.mu.RLock()
	mntns, ok := l.MountNamespacesMap[initArgs.ContainerID]
	if !ok {
		l.mu.RUnlock()
		return fmt.Errorf("mount namespace is nil for %s", initArgs.ContainerID)
	}
	initArgs.MountNamespaceVFS2 = mntns
	l.mu.RUnlock()
	initArgs.MountNamespaceVFS2.IncRef()

	resolved, err := user.ResolveExecutablePath(ctx, &initArgs)
	if err != nil {
		return err
	}
	initArgs.Filename = resolved

	fds, err := fd.NewFromFiles(args.Files)
	if err != nil {
		return fmt.Errorf("duplicating payload files: %w", err)
	}
	defer func() {
		for _, fd := range fds {
			_ = fd.Close()
		}
	}()

	tg, _, err := l.Kernel.CreateProcess(initArgs)
	if err != nil {
		return err
	}

	l.mu.Lock()
	numContainers := int32(len(l.MountNamespacesMap))

	// Start the newly created process.
	l.Kernel.StartProcess(tg)
	log.Infof("Started the new container %v ", l.containersStarted)
	l.containersStarted++
	if numContainers == l.containersStarted {
		l.StartedCh <- struct{}{}
	}
	l.mu.Unlock()
	return nil
}

// Pause pauses all tasks, blocking until they are stopped.
func (l *Lifecycle) Pause(_, _ *struct{}) error {
	l.Kernel.Pause()
	return nil
}

// Resume resumes all tasks.
func (l *Lifecycle) Resume(_, _ *struct{}) error {
	l.Kernel.Unpause()
	return nil
}
<commit_msg>Add a new control message for destroying sandbox in multi-container mode.<commit_after>// Copyright 2021 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package control

import (
	"fmt"
	"strings"

	"gvisor.dev/gvisor/pkg/abi/linux"
	"gvisor.dev/gvisor/pkg/fd"
	"gvisor.dev/gvisor/pkg/log"
	"gvisor.dev/gvisor/pkg/sentry/fs/user"
	"gvisor.dev/gvisor/pkg/sentry/kernel"
	"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
	"gvisor.dev/gvisor/pkg/sentry/limits"
	"gvisor.dev/gvisor/pkg/sentry/vfs"
	"gvisor.dev/gvisor/pkg/sync"
	"gvisor.dev/gvisor/pkg/urpc"
)

// Lifecycle provides functions related to starting and stopping tasks.
type Lifecycle struct {
	// Kernel is the kernel where the tasks belong to.
	Kernel *kernel.Kernel

	// ShutdownCh is the channel used to signal the sentry to shutdown
	// the sentry/sandbox.
	ShutdownCh chan struct{}

	// mu protects the fields below.
	mu sync.RWMutex

	// MountNamespacesMap is a map of container id/names and the mount
	// namespaces.
	MountNamespacesMap map[string]*vfs.MountNamespace
}

// StartContainerArgs is the set of arguments to start a container.
type StartContainerArgs struct {
	// Filename is the filename to load.
	//
	// If this is provided as "", then the file will be guessed via Argv[0].
	Filename string `json:"filename"`

	// Argv is a list of arguments.
	Argv []string `json:"argv"`

	// Envv is a list of environment variables.
	Envv []string `json:"envv"`

	// WorkingDirectory defines the working directory for the new process.
	WorkingDirectory string `json:"wd"`

	// KUID is the UID to run with in the root user namespace. Defaults to
	// root if not set explicitly.
	KUID auth.KUID `json:"KUID"`

	// KGID is the GID to run with in the root user namespace. Defaults to
	// the root group if not set explicitly.
	KGID auth.KGID `json:"KGID"`

	// ExtraKGIDs is the list of additional groups to which the user belongs.
	ExtraKGIDs []auth.KGID `json:"extraKGID"`

	// Capabilities is the list of capabilities to give to the process.
	Capabilities *auth.TaskCapabilities `json:"capabilities"`

	// FilePayload determines the files to give to the new process.
	urpc.FilePayload

	// ContainerID is the container for the process being executed.
	ContainerID string `json:"containerID"`

	// Limits is the limit set for the process being executed.
	Limits *limits.LimitSet `json:"limits"`
}

// String prints the StartContainerArgs.argv as a string.
func (args StartContainerArgs) String() string {
	if len(args.Argv) == 0 {
		return args.Filename
	}
	a := make([]string, len(args.Argv))
	copy(a, args.Argv)
	if args.Filename != "" {
		a[0] = args.Filename
	}
	return strings.Join(a, " ")
}

// StartContainer will start a new container in the sandbox.
func (l *Lifecycle) StartContainer(args *StartContainerArgs, _ *uint32) error {
	// Import file descriptors.
	fdTable := l.Kernel.NewFDTable()

	creds := auth.NewUserCredentials(
		args.KUID,
		args.KGID,
		args.ExtraKGIDs,
		args.Capabilities,
		l.Kernel.RootUserNamespace())

	limitSet := args.Limits
	if limitSet == nil {
		limitSet = limits.NewLimitSet()
	}
	initArgs := kernel.CreateProcessArgs{
		Filename:                args.Filename,
		Argv:                    args.Argv,
		Envv:                    args.Envv,
		WorkingDirectory:        args.WorkingDirectory,
		Credentials:             creds,
		FDTable:                 fdTable,
		Umask:                   0022,
		Limits:                  limitSet,
		MaxSymlinkTraversals:    linux.MaxSymlinkTraversals,
		UTSNamespace:            l.Kernel.RootUTSNamespace(),
		IPCNamespace:            l.Kernel.RootIPCNamespace(),
		AbstractSocketNamespace: l.Kernel.RootAbstractSocketNamespace(),
		ContainerID:             args.ContainerID,
		PIDNamespace:            l.Kernel.RootPIDNamespace(),
	}

	ctx := initArgs.NewContext(l.Kernel)
	defer fdTable.DecRef(ctx)

	// VFS2 is supported in multi-container mode by default.
	l.mu.RLock()
	mntns, ok := l.MountNamespacesMap[initArgs.ContainerID]
	if !ok {
		l.mu.RUnlock()
		return fmt.Errorf("mount namespace is nil for %s", initArgs.ContainerID)
	}
	initArgs.MountNamespaceVFS2 = mntns
	l.mu.RUnlock()
	initArgs.MountNamespaceVFS2.IncRef()

	resolved, err := user.ResolveExecutablePath(ctx, &initArgs)
	if err != nil {
		return err
	}
	initArgs.Filename = resolved

	fds, err := fd.NewFromFiles(args.Files)
	if err != nil {
		return fmt.Errorf("duplicating payload files: %w", err)
	}
	defer func() {
		for _, fd := range fds {
			_ = fd.Close()
		}
	}()

	tg, _, err := l.Kernel.CreateProcess(initArgs)
	if err != nil {
		return err
	}

	// Start the newly created process.
	l.Kernel.StartProcess(tg)
	log.Infof("Started the new container %v ", initArgs.ContainerID)
	return nil
}

// Pause pauses all tasks, blocking until they are stopped.
func (l *Lifecycle) Pause(_, _ *struct{}) error {
	l.Kernel.Pause()
	return nil
}

// Resume resumes all tasks.
func (l *Lifecycle) Resume(_, _ *struct{}) error {
	l.Kernel.Unpause()
	return nil
}

// Shutdown sends signal to destroy the sentry/sandbox.
func (l *Lifecycle) Shutdown(_, _ *struct{}) error {
	close(l.ShutdownCh)
	return nil
}
<|endoftext|>


<commit_before>// Access pass function propogation
//
// Alias accesses to function parameters passed by pointers, and record
// global variable accesses. Propogate accesses upwards through the blocks.
//
// func foo(ptr *DataA, val DataB) {
//   // Read ptr
//   // Write ptr
//   // Read val
//   // Write val
// }
//
// func bar(index int, val []DataB) int {
//   return val[index]
// }
//
// func main() {
//   a := &DataA{}
//   foo(a, DataB{}) // bubble up the reads and writes to "a" (ptr)
//   i := 0
//   aList := []DataA{a}
//   bar(i, aList) // bubble up val[index] -> aList[i] access
// }
package main

import (
	"fmt"
	"go/ast"
)

type AccessPassFuncPropogate struct {
	BasePass
}

type AccessPassFuncPropogateVisitor struct {
	pass      Pass
	p         *Package
	cur       *BasicBlock
	dataBlock *AccessPassData
	node      ast.Node
}

func (v AccessPassFuncPropogateVisitor) Done(block *BasicBlock) (modified bool, err error) {
	dataBlock := v.dataBlock

	block.Print("== Defines ==")
	for ident, expr := range dataBlock.defines {
		block.Printf("%s = %T %+v", ident, expr, expr)
	}
	block.Print("== Accesses ==")
	for _, access := range dataBlock.accesses {
		block.Printf(access.String())
	}

	return
}

func (v AccessPassFuncPropogateVisitor) Visit(node ast.Node) (w BasicBlockVisitor) {
	// Get the closest enclosing basic block for this node
	dataBlock := v.dataBlock
	b := v.cur
	pass := v.pass
	if node == nil {
		// post-order actions (all sub-nodes have been visited)
		node = v.node
		// Locate function calls
		switch t := node.(type) {
		case *ast.CallExpr:
			switch f := t.Fun.(type) {
			case *ast.FuncLit:
				// go down a FuncLit branch
			case *ast.Ident:
				fun := v.p.Lookup(f.Name)
				if fun == nil {
					// builtin function, or not found
					b.Print("Function not found", f.Name)
					return nil
				}
				funcDecl := fun.Decl.(*ast.FuncDecl)
				funcType := funcDecl.Type
				funcDataBlock := pass.GetCompiler().GetPassResult(BasicBlockPassType, funcDecl).(*BasicBlock).Get(AccessPassType).(*AccessPassData)

				child := v.cur
				parent := child.parent
				// Now fill in the accesses this call would have made, and propogate it
				// all the way to the top
				var funcAccesses []IdentifierGroup // only the accesses this function made

				// Fill in global accesses
				for _, access := range funcDataBlock.accesses {
					if _, ok := funcDataBlock.defines[access.group[0].id]; !ok {
						// if there is an array access that uses an identifier block defined in
						// this block, change the access from b[idx] to b
						var ig IdentifierGroup = access
						for idx, ident := range access.group {
							if _, ok := dataBlock.defines[ident.index]; ok && ident.isIndexed {
								ig.group = make([]Identifier, idx+1)
								copy(ig.group, access.group)
								parent.Printf("Leaving index scope [%s]", ig.group[idx].index)
								ig.group[idx].isIndexed = false
								ig.group[idx].index = ""
							}
							break
						}
						b.Print("Global access", ig.String())
						funcAccesses = append(funcAccesses, ig)
					}
				}

				// Fill in aliased arguments
				pos := 0 // argument position
				for _, arg := range funcType.Params.List {
					writeThrough := false
					// is the argument able to be modified?
					// builtin types (slice, map, chan), pointers
					switch arg.Type.(type) {
					case *ast.ArrayType, *ast.MapType, *ast.ChanType:
						b.Printf("Pass-by-reference %v %T", arg.Names, arg.Type)
						writeThrough = true
					case *ast.StarExpr:
						b.Printf("Pass-by-pointer %v %T", arg.Names, arg.Type)
						writeThrough = true
					}

					for _, argName := range arg.Names {
						if writeThrough {
							callArg := t.Args[pos]
							callIdent := &IdentifierGroup{}
							AccessIdentBuild(callIdent, callArg, nil)

							// Find all accesses to these variables
							for _, access := range funcDataBlock.accesses {
								// Replace the function arg name with the callIdent prefix
								if access.group[0].id == argName.Name {
									// check if an index variable is also a function argument and
									// remove it
									newAccess := make([]Identifier, len(access.group))

									copy(newAccess, access.group) // full copy
									for idx, ident := range newAccess {
										if _, ok := funcDataBlock.defines[ident.index]; ok && ident.isIndexed {
											newAccess = newAccess[0 : idx+1]
											newAccess[idx].isIndexed = false
											b.Printf("Stripping array index %s", ident.index)
											newAccess[idx].index = ""
										}
										break
									}

									// if the callsite is &a and the access is *a, make the access
									// a for this function
									var callIdentCopy []Identifier
									if callIdent.group[len(callIdent.group)-1].refType == AddressOf && newAccess[len(newAccess)-1].refType == Dereference {
										b.Print("Removing pointer alias & -> *")
										newAccess[len(newAccess)-1].refType = NoReference
										callIdentCopy = make([]Identifier, len(callIdent.group))
										copy(callIdentCopy, callIdent.group)
										callIdentCopy[len(callIdentCopy)-1].refType = NoReference
									} else {
										callIdentCopy = callIdent.group
									}
									// replace access[0] with callIdent
									var ig IdentifierGroup
									ig.t = access.t
									ig.group = append(ig.group, callIdentCopy...)
									ig.group = append(ig.group, newAccess[1:]...)
									b.Printf("%s -> %s", access.String(), ig.String())
									funcAccesses = append(funcAccesses, ig)
								}
							}
						}
						pos++
					}
				}

				// Propogate ONLY aliased argument accesses upwards
				// NOTE: doesn't work with recursive functions

				// Move upwards, replacing the placeholder access with the group of
				// accesses made by this function. Stop at variable define boundaries
				placeholderIdent := v.pass.GetCompiler().GetPassResult(AccessPassType, t).(*ast.Ident)
				b.Printf("\x1b[33m>> %s\x1b[0m filling in function effects: %+v, %+v", placeholderIdent.Name, t, funcDecl)

				// Walk up the parent blocks
				for ; child != nil; child = child.parent {
					// Find the placeholder
					dataBlock := child.Get(AccessPassType).(*AccessPassData)
					var placeholderIdx int
					var val IdentifierGroup
					for placeholderIdx, val = range dataBlock.accesses {
						if val.group[0].id == placeholderIdent.Name {
							break
						}
					}
					b.Printf("Replacing placeholder at %d", placeholderIdx)

					// Remove the placeholder
					dataBlock.accesses = append(dataBlock.accesses[0:placeholderIdx], dataBlock.accesses[placeholderIdx+1:]...)
					// Insert the function accesses
					funcAccessCopy := make([]IdentifierGroup, len(funcAccesses))
					copy(funcAccessCopy, funcAccesses)

					b.Print(" << Propogating up")
					for _, a := range funcAccessCopy {
						b.Print(a.String())
					}
					dataBlock.accesses = append(dataBlock.accesses[0:placeholderIdx], append(funcAccessCopy, dataBlock.accesses[placeholderIdx:]...)...)

					// Check if the identifier leaves scope
					for idx := 0; idx < len(funcAccesses); {
						access := funcAccesses[idx]
						if _, ok := dataBlock.defines[access.group[0].id]; ok {
							b.Print("Leaving scope", access.String())
							funcAccesses = append(funcAccesses[:idx], funcAccesses[idx+1:]...)
						} else {
							idx++
						}
					}

					// Check if an index variable leaves scope
					for accIdx, access := range funcAccesses {
						// check if an index variable is also a function argument and
						// remove it
						for idx, ident := range access.group {
							if _, ok := dataBlock.defines[ident.index]; ok && ident.isIndexed {
								// update the real value in funcAccesses
								access.group = access.group[0 : idx+1]
								access.group[idx].isIndexed = false
								b.Printf("Stripping array index %s", ident.index)
								access.group[idx].index = ""
								funcAccesses[accIdx] = access // write back to the original storage!
							}
							break
						}
					}
				}
				return nil
			}
		}
		return v
	}
	v.node = node

	return v
}

func NewAccessPassFuncPropogate() *AccessPassFuncPropogate {
	return &AccessPassFuncPropogate{
		BasePass: NewBasePass(),
	}
}

func (pass *AccessPassFuncPropogate) GetPassType() PassType {
	return AccessPassFuncPropogateType
}

func (pass *AccessPassFuncPropogate) GetPassMode() PassMode {
	return ModulePassMode
}

func (pass *AccessPassFuncPropogate) GetDependencies() []PassType {
	return []PassType{CallGraphPassType, AccessPassPropogateType}
}

// Declare two Run* functions

func (pass *AccessPassFuncPropogate) RunBasicBlockPass(block *BasicBlock, p *Package) BasicBlockVisitor {
	dataBlock := block.Get(AccessPassType).(*AccessPassData)
	return AccessPassFuncPropogateVisitor{pass: pass, cur: block, dataBlock: dataBlock, p: p}
}

func (pass *AccessPassFuncPropogate) RunModulePass(file *ast.File, p *Package) (modified bool, err error) {
	callGraph := pass.compiler.GetPassResult(CallGraphPassType, nil).(*CallGraphPassData)
	run := make(map[string]bool) // which functions have been propogated
	var orderGraph func(map[string][]string, string) []string
	orderGraph = func(graph map[string][]string, f string) (result []string) {
		for _, fn := range callGraph.graph[f] {
			result = append(result, orderGraph(graph, fn)...)
		}
		result = append(result, f)
		return
	}

	runOrder := orderGraph(callGraph.graph, "main")
	fmt.Println(runOrder)
	for _, fnName := range runOrder {
		fn := p.Lookup(fnName)
		if fn == nil || run[fnName] {
			continue
		}
		fnDecl := fn.Decl.(*ast.FuncDecl)
		block := pass.compiler.GetPassResult(BasicBlockPassType, fnDecl).(*BasicBlock)

		// Manually run the basic block pass in inverse call graph order
		var mod bool
		mod, err = RunBasicBlock(pass, block, p)
		modified = modified || mod
		if err != nil {
			return
		}
		run[fnName] = true
	}
	return
}
<commit_msg>fixed another deep copy bug in func prop, third parallel loop<commit_after>// Access pass function propogation
//
// Alias accesses to function parameters passed by pointers, and record
// global variable accesses. Propogate accesses upwards through the blocks.
//
// func foo(ptr *DataA, val DataB) {
//   // Read ptr
//   // Write ptr
//   // Read val
//   // Write val
// }
//
// func bar(index int, val []DataB) int {
//   return val[index]
// }
//
// func main() {
//   a := &DataA{}
//   foo(a, DataB{}) // bubble up the reads and writes to "a" (ptr)
//   i := 0
//   aList := []DataA{a}
//   bar(i, aList) // bubble up val[index] -> aList[i] access
// }
package main

import (
	"fmt"
	"go/ast"
)

type AccessPassFuncPropogate struct {
	BasePass
}

type AccessPassFuncPropogateVisitor struct {
	pass      Pass
	p         *Package
	cur       *BasicBlock
	dataBlock *AccessPassData
	node      ast.Node
}

func (v AccessPassFuncPropogateVisitor) Done(block *BasicBlock) (modified bool, err error) {
	dataBlock := v.dataBlock

	block.Print("== Defines ==")
	for ident, expr := range dataBlock.defines {
		block.Printf("%s = %T %+v", ident, expr, expr)
	}
	block.Print("== Accesses ==")
	for _, access := range dataBlock.accesses {
		block.Printf(access.String())
	}

	return
}

func (v AccessPassFuncPropogateVisitor) Visit(node ast.Node) (w BasicBlockVisitor) {
	// Get the closest enclosing basic block for this node
	dataBlock := v.dataBlock
	b := v.cur
	pass := v.pass
	if node == nil {
		// post-order actions (all sub-nodes have been visited)
		node = v.node
		// Locate function calls
		switch t := node.(type) {
		case *ast.CallExpr:
			switch f := t.Fun.(type) {
			case *ast.FuncLit:
				// go down a FuncLit branch
			case *ast.Ident:
				fun := v.p.Lookup(f.Name)
				if fun == nil {
					// builtin function, or not found
					b.Print("Function not found", f.Name)
					return nil
				}
				funcDecl := fun.Decl.(*ast.FuncDecl)
				funcType := funcDecl.Type
				funcDataBlock := pass.GetCompiler().GetPassResult(BasicBlockPassType, funcDecl).(*BasicBlock).Get(AccessPassType).(*AccessPassData)

				child := v.cur
				parent := child.parent
				// Now fill in the accesses this call would have made, and propogate it
				// all the way to the top
				var funcAccesses []IdentifierGroup // only the accesses this function made

				// Fill in global accesses
				for _, access := range funcDataBlock.accesses {
					if _, ok := funcDataBlock.defines[access.group[0].id]; !ok {
						// if there is an array access that uses an identifier block defined in
						// this block, change the access from b[idx] to b
						var ig IdentifierGroup = access
						for idx, ident := range access.group {
							if _, ok := dataBlock.defines[ident.index]; ok && ident.isIndexed {
								ig.group = make([]Identifier, idx+1)
								copy(ig.group, access.group)
								parent.Printf("Leaving index scope [%s]", ig.group[idx].index)
								ig.group[idx].isIndexed = false
								ig.group[idx].index = ""
							}
							break
						}
						b.Print("Global access", ig.String())
						funcAccesses = append(funcAccesses, ig)
					}
				}

				// Fill in aliased arguments
				pos := 0 // argument position
				for _, arg := range funcType.Params.List {
					writeThrough := false
					// is the argument able to be modified?
					// builtin types (slice, map, chan), pointers
					switch arg.Type.(type) {
					case *ast.ArrayType, *ast.MapType, *ast.ChanType:
						b.Printf("Pass-by-reference %v %T", arg.Names, arg.Type)
						writeThrough = true
					case *ast.StarExpr:
						b.Printf("Pass-by-pointer %v %T", arg.Names, arg.Type)
						writeThrough = true
					}

					for _, argName := range arg.Names {
						if writeThrough {
							callArg := t.Args[pos]
							callIdent := &IdentifierGroup{}
							AccessIdentBuild(callIdent, callArg, nil)

							// Find all accesses to these variables
							for _, access := range funcDataBlock.accesses {
								// Replace the function arg name with the callIdent prefix
								if access.group[0].id == argName.Name {
									// check if an index variable is also a function argument and
									// remove it
									newAccess := make([]Identifier, len(access.group))

									copy(newAccess, access.group) // full copy
									for idx, ident := range newAccess {
										if _, ok := funcDataBlock.defines[ident.index]; ok && ident.isIndexed {
											newAccess = newAccess[0 : idx+1]
											newAccess[idx].isIndexed = false
											b.Printf("Stripping array index %s", ident.index)
											newAccess[idx].index = ""
										}
										break
									}

									// if the callsite is &a and the access is *a, make the access
									// a for this function
									var callIdentCopy []Identifier
									if callIdent.group[len(callIdent.group)-1].refType == AddressOf && newAccess[len(newAccess)-1].refType == Dereference {
										b.Print("Removing pointer alias & -> *")
										newAccess[len(newAccess)-1].refType = NoReference
										callIdentCopy = make([]Identifier, len(callIdent.group))
										copy(callIdentCopy, callIdent.group)
										callIdentCopy[len(callIdentCopy)-1].refType = NoReference
									} else {
										callIdentCopy = callIdent.group
									}
									// replace access[0] with callIdent
									var ig IdentifierGroup
									ig.t = access.t
									ig.group = append(ig.group, callIdentCopy...)
									ig.group = append(ig.group, newAccess[1:]...)
									b.Printf("%s -> %s", access.String(), ig.String())
									funcAccesses = append(funcAccesses, ig)
								}
							}
						}
						pos++
					}
				}

				// Propogate ONLY aliased argument accesses upwards
				// NOTE: doesn't work with recursive functions??

				// Move upwards, replacing the placeholder access with the group of
				// accesses made by this function. Stop at variable define boundaries
				placeholderIdent := v.pass.GetCompiler().GetPassResult(AccessPassType, t).(*ast.Ident)
				b.Printf("\x1b[33m>> %s\x1b[0m filling in function effects: %+v, %+v", placeholderIdent.Name, t, funcDecl)

				// Walk up the parent blocks
				for ; child != nil; child = child.parent {
					// Find the placeholder
					dataBlock := child.Get(AccessPassType).(*AccessPassData)
					var placeholderIdx int
					var val IdentifierGroup
					for placeholderIdx, val = range dataBlock.accesses {
						if val.group[0].id == placeholderIdent.Name {
							break
						}
					}
					b.Printf("Replacing placeholder at %d", placeholderIdx)

					// Remove the placeholder
					dataBlock.accesses = append(dataBlock.accesses[0:placeholderIdx], dataBlock.accesses[placeholderIdx+1:]...)
					// Insert the function accesses, do a deep copy
					var funcAccessCopy []IdentifierGroup
					for _, v := range funcAccesses {
						// deep copy the identifers
						groupCopy := make([]Identifier, len(v.group))
						copy(groupCopy, v.group)
						v.group = groupCopy
						funcAccessCopy = append(funcAccessCopy, v)
					}

					b.Print(" << Propogating up")
					for _, a := range funcAccessCopy {
						b.Print(a.String())
					}
					dataBlock.accesses = append(dataBlock.accesses[0:placeholderIdx], append(funcAccessCopy, dataBlock.accesses[placeholderIdx:]...)...)

					// Check if the identifier leaves scope
					for idx := 0; idx < len(funcAccesses); {
						access := funcAccesses[idx]
						if _, ok := dataBlock.defines[access.group[0].id]; ok {
							b.Print("Leaving scope", access.String())
							funcAccesses = append(funcAccesses[:idx], funcAccesses[idx+1:]...)
						} else {
							idx++
						}
					}

					// Check if an index variable leaves scope
					for accIdx, access := range funcAccesses {
						// check if an index variable is also a function argument and
						// remove it
						for idx, ident := range access.group {
							if _, ok := dataBlock.defines[ident.index]; ok && ident.isIndexed {
								// update the real value in funcAccesses
								access.group = access.group[0 : idx+1]
								access.group[idx].isIndexed = false
								b.Printf("Stripping array index %s", ident.index)
								access.group[idx].index = ""
								funcAccesses[accIdx] = access // write back to the original storage!
							}
							break
						}
					}
				}
				return nil
			}
		}
		return v
	}
	v.node = node

	return v
}

func NewAccessPassFuncPropogate() *AccessPassFuncPropogate {
	return &AccessPassFuncPropogate{
		BasePass: NewBasePass(),
	}
}

func (pass *AccessPassFuncPropogate) GetPassType() PassType {
	return AccessPassFuncPropogateType
}

func (pass *AccessPassFuncPropogate) GetPassMode() PassMode {
	return ModulePassMode
}

func (pass *AccessPassFuncPropogate) GetDependencies() []PassType {
	return []PassType{CallGraphPassType, AccessPassPropogateType}
}

// Declare two Run* functions

func (pass *AccessPassFuncPropogate) RunBasicBlockPass(block *BasicBlock, p *Package) BasicBlockVisitor {
	dataBlock := block.Get(AccessPassType).(*AccessPassData)
	return AccessPassFuncPropogateVisitor{pass: pass, cur: block, dataBlock: dataBlock, p: p}
}

func (pass *AccessPassFuncPropogate) RunModulePass(file *ast.File, p *Package) (modified bool, err error) {
	callGraph := pass.compiler.GetPassResult(CallGraphPassType, nil).(*CallGraphPassData)
	run := make(map[string]bool) // which functions have been propogated
	var orderGraph func(map[string][]string, string) []string
	orderGraph = func(graph map[string][]string, f string) (result []string) {
		for _, fn := range callGraph.graph[f] {
			result = append(result, orderGraph(graph, fn)...)
		}
		result = append(result, f)
		return
	}

	runOrder := orderGraph(callGraph.graph, "main")
	fmt.Println(runOrder)
	for _, fnName := range runOrder {
		fn := p.Lookup(fnName)
		if fn == nil || run[fnName] {
			continue
		}
		fnDecl := fn.Decl.(*ast.FuncDecl)
		block := pass.compiler.GetPassResult(BasicBlockPassType, fnDecl).(*BasicBlock)

		// Manually run the basic block pass in inverse call graph order
		var mod bool
		mod, err = RunBasicBlock(pass, block, p)
		modified = modified || mod
		if err != nil {
			return
		}
		run[fnName] = true
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build dragonfly freebsd netbsd openbsd solaris

package runtime

import (
	"unsafe"
)

// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
//go:nosplit
func sysAlloc(n uintptr, sysStat *uint64) unsafe.Pointer {
	v, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
	if err != 0 {
		return nil
	}
	mSysStatInc(sysStat, n)
	return v
}

func sysUnused(v unsafe.Pointer, n uintptr) {
	madvise(v, n, _MADV_FREE)
}

func sysUsed(v unsafe.Pointer, n uintptr) {
}

func sysHugePage(v unsafe.Pointer, n uintptr) {
}

// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
//go:nosplit
func sysFree(v unsafe.Pointer, n uintptr, sysStat *uint64) {
	mSysStatDec(sysStat, n)
	munmap(v, n)
}

func sysFault(v unsafe.Pointer, n uintptr) {
	mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, -1, 0)
}

func sysReserve(v unsafe.Pointer, n uintptr) unsafe.Pointer {
	p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
	if err != 0 {
		return nil
	}
	return p
}

const _sunosEAGAIN = 11
const _ENOMEM = 12

func sysMap(v unsafe.Pointer, n uintptr, sysStat *uint64) {
	mSysStatInc(sysStat, n)

	p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
	if err == _ENOMEM || ((GOOS == "solaris" || GOOS == "illumos") && err == _sunosEAGAIN) {
		throw("runtime: out of memory")
	}
	if p != v || err != 0 {
		throw("runtime: cannot map pages in arena address space")
	}
}
<commit_msg>runtime: map reserved memory as NORESERVE on solaris<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build dragonfly freebsd netbsd openbsd solaris

package runtime

import (
	"unsafe"
)

// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
//go:nosplit
func sysAlloc(n uintptr, sysStat *uint64) unsafe.Pointer {
	v, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
	if err != 0 {
		return nil
	}
	mSysStatInc(sysStat, n)
	return v
}

func sysUnused(v unsafe.Pointer, n uintptr) {
	madvise(v, n, _MADV_FREE)
}

func sysUsed(v unsafe.Pointer, n uintptr) {
}

func sysHugePage(v unsafe.Pointer, n uintptr) {
}

// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
//go:nosplit
func sysFree(v unsafe.Pointer, n uintptr, sysStat *uint64) {
	mSysStatDec(sysStat, n)
	munmap(v, n)
}

func sysFault(v unsafe.Pointer, n uintptr) {
	mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, -1, 0)
}

// Indicates not to reserve swap space for the mapping.
const _sunosMAP_NORESERVE = 0x40

func sysReserve(v unsafe.Pointer, n uintptr) unsafe.Pointer {
	flags := int32(_MAP_ANON | _MAP_PRIVATE)
	if GOOS == "solaris" || GOOS == "illumos" {
		// Be explicit that we don't want to reserve swap space
		// for PROT_NONE anonymous mappings. This avoids an issue
		// wherein large mappings can cause fork to fail.
		flags |= _sunosMAP_NORESERVE
	}
	p, err := mmap(v, n, _PROT_NONE, flags, -1, 0)
	if err != 0 {
		return nil
	}
	return p
}

const _sunosEAGAIN = 11
const _ENOMEM = 12

func sysMap(v unsafe.Pointer, n uintptr, sysStat *uint64) {
	mSysStatInc(sysStat, n)

	p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
	if err == _ENOMEM || ((GOOS == "solaris" || GOOS == "illumos") && err == _sunosEAGAIN) {
		throw("runtime: out of memory")
	}
	if p != v || err != 0 {
		throw("runtime: cannot map pages in arena address space")
	}
}
<|endoftext|>


<commit_before>package main

import (

	"encoding/json"
	"errors"
	"fmt"
	"net"
	"os"
	"sync"
	"io/ioutil"
	"yaml"
	"time"
	"gopkg.in/yaml.v2"
	"github.com/joonakannisto/gocrypto/ssh"
)

type Config struct {
	HostKey string `yaml:"IdPHost"`
}
type sessionInfo struct {
	User string
	Keys []ssh.PublicKey
}

type Server struct {
	sshConfig    *ssh.ServerConfig
	mu          sync.RWMutex
	sessionInfo map[string]sessionInfo
}

func (s *Server) PublicKeyCallback(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
	s.mu.Lock()
	si := s.sessionInfo[string(conn.SessionID())]
	si.User = conn.User()
	configText, err := ioutil.ReadFile("config.yml")
	var C Config
	fatalIfErr(yaml.Unmarshal(configText, &C))

	sshConfig := &ssh.ClientConfig{
		User: si.User,
		Auth: []ssh.AuthMethod{ssh.WorkingKeys(key)},
	}
	_, err := ssh.ShakeThat("tcp", C.IdPHost, sshConfig)
        if err != nil {
             si.Keys = append(si.Keys, key)
	}

	s.sessionInfo[string(conn.SessionID())] = si
	s.mu.Unlock()

	// Never succeed a key, or we might not see the next. See KeyboardInteractiveCallback.
	return nil, errors.New("")

}

func (s *Server) KeyboardInteractiveCallback(ssh.ConnMetadata, ssh.KeyboardInteractiveChallenge) (*ssh.Permissions, error) {
	// keyboard-interactive is tried when all public keys failed, and
	// since it's server-driven we can just pass without user
	// interaction to let the user in once we got all the public keys
	return nil, nil
}

type logEntry struct {
	Timestamp     string
	Username      string
	ChannelTypes  []string
	RequestTypes  []string
	Error         string
	KeysOffered   []string
	ClientVersion string
}

func (s *Server) Handle(nConn net.Conn) {
	le := &logEntry{Timestamp: time.Now().Format(time.RFC3339)}
	defer json.NewEncoder(os.Stdout).Encode(le)

	conn, chans, reqs, err := ssh.NewServerConn(nConn, s.sshConfig)
	if err != nil {
		le.Error = "Handshake failed: " + err.Error()
		return
	}
	defer func() {
		s.mu.Lock()
		delete(s.sessionInfo, string(conn.SessionID()))
		s.mu.Unlock()
		conn.Close()
	}()
	go func(in <-chan *ssh.Request) {
		for req := range in {
			le.RequestTypes = append(le.RequestTypes, req.Type)
			if req.WantReply {
				req.Reply(false, nil)
			}
		}
	}(reqs)

	s.mu.RLock()
	si := s.sessionInfo[string(conn.SessionID())]
	s.mu.RUnlock()

	le.Username = conn.User()
	le.ClientVersion = fmt.Sprintf("%x", conn.ClientVersion())
	for _, key := range si.Keys {
		le.KeysOffered = append(le.KeysOffered, string(ssh.MarshalAuthorizedKey(key)))
	}

	for newChannel := range chans {
		le.ChannelTypes = append(le.ChannelTypes, newChannel.ChannelType())

		if newChannel.ChannelType() != "session" {
			newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}
		channel, requests, err := newChannel.Accept()
		if err != nil {
			le.Error = "Channel accept failed: " + err.Error()
			continue
		}

		agentFwd, x11 := false, false
		reqLock := &sync.Mutex{}
		reqLock.Lock()
		timeout := time.AfterFunc(30*time.Second, func() { reqLock.Unlock() })

		go func(in <-chan *ssh.Request) {
			for req := range in {
				le.RequestTypes = append(le.RequestTypes, req.Type)
				ok := false
				switch req.Type {
				case "shell":
					fallthrough
				case "pty-req":
					ok = true

					// "auth-agent-req@openssh.com" and "x11-req" always arrive
					// before the "pty-req", so we can go ahead now
					if timeout.Stop() {
						reqLock.Unlock()
					}

				case "auth-agent-req@openssh.com":
					agentFwd = true
				case "x11-req":
					x11 = true
				}

				if req.WantReply {
					req.Reply(ok, nil)
				}
			}
		}(requests)

		reqLock.Lock()

		if err != nil {
			le.Error = "findUser failed: " + err.Error()
			channel.Close()
			continue
		}

		channel.Close()
	}
}

<commit_msg>pubkey check<commit_after>package main

import (

	"encoding/json"
	"errors"
	"fmt"
	"net"
	"os"
	"sync"
	"io/ioutil"
	"time"
	"gopkg.in/yaml.v2"
	"github.com/joonakannisto/gocrypto/ssh"
)

type Config struct {
	HostKey string `yaml:"IdPHost"`
}
type sessionInfo struct {
	User string
	Keys []ssh.PublicKey
}

type Server struct {
	sshConfig    *ssh.ServerConfig
	mu          sync.RWMutex
	sessionInfo map[string]sessionInfo
}

func (s *Server) PublicKeyCallback(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
	s.mu.Lock()
	si := s.sessionInfo[string(conn.SessionID())]
	si.User = conn.User()
	configText, err := ioutil.ReadFile("config.yml")
	var C Config
	fatalIfErr(yaml.Unmarshal(configText, &C))

	sshConfig := &ssh.ClientConfig{
		User: si.User,
		Auth: []ssh.AuthMethod{ssh.WorkingKeys(key)},
	}
	_, err := ssh.ShakeThat("tcp", C.IdPHost, sshConfig)
        if err != nil {
             si.Keys = append(si.Keys, key)
	}

	s.sessionInfo[string(conn.SessionID())] = si
	s.mu.Unlock()

	// Never succeed a key, or we might not see the next. See KeyboardInteractiveCallback.
	return nil, errors.New("")

}

func (s *Server) KeyboardInteractiveCallback(ssh.ConnMetadata, ssh.KeyboardInteractiveChallenge) (*ssh.Permissions, error) {
	// keyboard-interactive is tried when all public keys failed, and
	// since it's server-driven we can just pass without user
	// interaction to let the user in once we got all the public keys
	return nil, nil
}

type logEntry struct {
	Timestamp     string
	Username      string
	ChannelTypes  []string
	RequestTypes  []string
	Error         string
	KeysOffered   []string
	ClientVersion string
}

func (s *Server) Handle(nConn net.Conn) {
	le := &logEntry{Timestamp: time.Now().Format(time.RFC3339)}
	defer json.NewEncoder(os.Stdout).Encode(le)

	conn, chans, reqs, err := ssh.NewServerConn(nConn, s.sshConfig)
	if err != nil {
		le.Error = "Handshake failed: " + err.Error()
		return
	}
	defer func() {
		s.mu.Lock()
		delete(s.sessionInfo, string(conn.SessionID()))
		s.mu.Unlock()
		conn.Close()
	}()
	go func(in <-chan *ssh.Request) {
		for req := range in {
			le.RequestTypes = append(le.RequestTypes, req.Type)
			if req.WantReply {
				req.Reply(false, nil)
			}
		}
	}(reqs)

	s.mu.RLock()
	si := s.sessionInfo[string(conn.SessionID())]
	s.mu.RUnlock()

	le.Username = conn.User()
	le.ClientVersion = fmt.Sprintf("%x", conn.ClientVersion())
	for _, key := range si.Keys {
		le.KeysOffered = append(le.KeysOffered, string(ssh.MarshalAuthorizedKey(key)))
	}

	for newChannel := range chans {
		le.ChannelTypes = append(le.ChannelTypes, newChannel.ChannelType())

		if newChannel.ChannelType() != "session" {
			newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}
		channel, requests, err := newChannel.Accept()
		if err != nil {
			le.Error = "Channel accept failed: " + err.Error()
			continue
		}

		agentFwd, x11 := false, false
		reqLock := &sync.Mutex{}
		reqLock.Lock()
		timeout := time.AfterFunc(30*time.Second, func() { reqLock.Unlock() })

		go func(in <-chan *ssh.Request) {
			for req := range in {
				le.RequestTypes = append(le.RequestTypes, req.Type)
				ok := false
				switch req.Type {
				case "shell":
					fallthrough
				case "pty-req":
					ok = true

					// "auth-agent-req@openssh.com" and "x11-req" always arrive
					// before the "pty-req", so we can go ahead now
					if timeout.Stop() {
						reqLock.Unlock()
					}

				case "auth-agent-req@openssh.com":
					agentFwd = true
				case "x11-req":
					x11 = true
				}

				if req.WantReply {
					req.Reply(ok, nil)
				}
			}
		}(requests)

		reqLock.Lock()

		if err != nil {
			le.Error = "findUser failed: " + err.Error()
			channel.Close()
			continue
		}

		channel.Close()
	}
}

<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e_node

import (
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/restclient"
	client "k8s.io/kubernetes/pkg/client/unversioned"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	retryTimeout = 4 * time.Minute
	pollInterval = time.Second * 5
)

var _ = Describe("Container Conformance Test", func() {
	var cl *client.Client

	BeforeEach(func() {
		// Setup the apiserver client
		cl = client.NewOrDie(&restclient.Config{Host: *apiServerAddress})
	})

	Describe("container conformance blackbox test", func() {
		Context("when testing images that exist", func() {
			var conformImages []ConformanceImage
			conformImageTags := []string{
				"gcr.io/google_containers/busybox:1.24",
				"gcr.io/google_containers/mounttest:0.2",
				"gcr.io/google_containers/nettest:1.7",
				"gcr.io/google_containers/nginx:1.7.9",
			}
			It("it should pull successfully [Conformance]", func() {
				for _, imageTag := range conformImageTags {
					image, _ := NewConformanceImage("docker", imageTag)
					conformImages = append(conformImages, image)
				}
				for _, image := range conformImages {
					if err := image.Pull(); err != nil {
						Expect(err).NotTo(HaveOccurred())
						break
					}
				}
			})
			It("it should list pulled images [Conformance]", func() {
				image, _ := NewConformanceImage("docker", "")
				tags, _ := image.List()
				for _, tag := range conformImageTags {
					Expect(tags).To(ContainElement(tag))
				}
			})
			It("it should remove successfully [Conformance]", func() {
				for _, image := range conformImages {
					if err := image.Remove(); err != nil {
						Expect(err).NotTo(HaveOccurred())
						break
					}
				}
			})
		})
		Context("when testing image that does not exist", func() {
			var invalidImage ConformanceImage
			var invalidImageTag string
			It("it should not pull successfully [Conformance]", func() {
				invalidImageTag = "foo.com/foo/foo"
				invalidImage, _ = NewConformanceImage("docker", invalidImageTag)
				err := invalidImage.Pull()
				Expect(err).To(HaveOccurred())
			})
			It("it should not list pulled images [Conformance]", func() {
				image, _ := NewConformanceImage("docker", "")
				tags, _ := image.List()
				Expect(tags).NotTo(ContainElement(invalidImageTag))
			})
			It("it should not remove successfully [Conformance]", func() {
				err := invalidImage.Remove()
				Expect(err).To(HaveOccurred())
			})
		})
		Context("when running a container that terminates", func() {
			var terminateCase ConformanceContainer
			It("it should run successfully to completion [Conformance]", func() {
				terminateCase = ConformanceContainer{
					Container: api.Container{
						Image:           "gcr.io/google_containers/busybox",
						Name:            "busybox",
						Command:         []string{"sh", "-c", "env"},
						ImagePullPolicy: api.PullIfNotPresent,
					},
					Client:   cl,
					Phase:    api.PodSucceeded,
					NodeName: *nodeName,
				}
				err := terminateCase.Create()
				Expect(err).NotTo(HaveOccurred())

				// TODO: Check that the container enters running state by sleeping in the container #23309
				Eventually(func() (api.PodPhase, error) {
					pod, err := terminateCase.Get()
					return pod.Phase, err
				}, retryTimeout, pollInterval).Should(Equal(terminateCase.Phase))
			})
			It("it should report its phase as 'succeeded' [Conformance]", func() {
				ccontainer, err := terminateCase.Get()
				Expect(err).NotTo(HaveOccurred())
				Expect(ccontainer).Should(CContainerEqual(terminateCase))
			})
			It("it should be possible to delete [Conformance]", func() {
				err := terminateCase.Delete()
				Expect(err).NotTo(HaveOccurred())
			})
		})
		Context("when running a container with invalid image", func() {
			var invalidImageCase ConformanceContainer
			It("it should not start successfully [Conformance]", func() {
				invalidImageCase = ConformanceContainer{
					Container: api.Container{
						Image:           "foo.com/foo/foo",
						Name:            "foo",
						Command:         []string{"foo", "'Should not work'"},
						ImagePullPolicy: api.PullIfNotPresent,
					},
					Client:   cl,
					Phase:    api.PodPending,
					NodeName: *nodeName,
				}
				err := invalidImageCase.Create()
				Expect(err).NotTo(HaveOccurred())
				Eventually(func() (api.PodPhase, error) {
					pod, err := invalidImageCase.Get()
					return pod.Phase, err
				}, retryTimeout, pollInterval).Should(Equal(invalidImageCase.Phase))
			})
			It("it should report its phase as 'pending' [Conformance]", func() {
				ccontainer, err := invalidImageCase.Get()
				Expect(err).NotTo(HaveOccurred())
				Expect(ccontainer).Should(CContainerEqual(invalidImageCase))
			})
			It("it should be possible to delete [Conformance]", func() {
				err := invalidImageCase.Delete()
				Expect(err).NotTo(HaveOccurred())
			})
		})
	})
})
<commit_msg>Retry failed image pulls.  Closes #23669.<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e_node

import (
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/restclient"
	client "k8s.io/kubernetes/pkg/client/unversioned"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	retryTimeout      = time.Minute * 4
	pollInterval      = time.Second * 5
	imageRetryTimeout = time.Minute * 2
	imagePullInterval = time.Second * 15
)

var _ = Describe("Container Conformance Test", func() {
	var cl *client.Client

	BeforeEach(func() {
		// Setup the apiserver client
		cl = client.NewOrDie(&restclient.Config{Host: *apiServerAddress})
	})

	Describe("container conformance blackbox test", func() {
		Context("when testing images that exist", func() {
			var conformImages []ConformanceImage
			conformImageTags := []string{
				"gcr.io/google_containers/busybox:1.24",
				"gcr.io/google_containers/mounttest:0.2",
				"gcr.io/google_containers/nettest:1.7",
				"gcr.io/google_containers/nginx:1.7.9",
			}
			It("it should pull successfully [Conformance]", func() {
				for _, imageTag := range conformImageTags {
					image, _ := NewConformanceImage("docker", imageTag)
					conformImages = append(conformImages, image)
				}
				for _, image := range conformImages {
					// Pulling images from gcr.io is flaky, so retry failures
					Eventually(func() error {
						return image.Pull()
					}, imageRetryTimeout, imagePullInterval).ShouldNot(HaveOccurred())
				}
			})
			It("it should list pulled images [Conformance]", func() {
				image, _ := NewConformanceImage("docker", "")
				tags, _ := image.List()
				for _, tag := range conformImageTags {
					Expect(tags).To(ContainElement(tag))
				}
			})
			It("it should remove successfully [Conformance]", func() {
				for _, image := range conformImages {
					if err := image.Remove(); err != nil {
						Expect(err).NotTo(HaveOccurred())
						break
					}
				}
			})
		})
		Context("when testing image that does not exist", func() {
			var invalidImage ConformanceImage
			var invalidImageTag string
			It("it should not pull successfully [Conformance]", func() {
				invalidImageTag = "foo.com/foo/foo"
				invalidImage, _ = NewConformanceImage("docker", invalidImageTag)
				err := invalidImage.Pull()
				Expect(err).To(HaveOccurred())
			})
			It("it should not list pulled images [Conformance]", func() {
				image, _ := NewConformanceImage("docker", "")
				tags, _ := image.List()
				Expect(tags).NotTo(ContainElement(invalidImageTag))
			})
			It("it should not remove successfully [Conformance]", func() {
				err := invalidImage.Remove()
				Expect(err).To(HaveOccurred())
			})
		})
		Context("when running a container that terminates", func() {
			var terminateCase ConformanceContainer
			It("it should run successfully to completion [Conformance]", func() {
				terminateCase = ConformanceContainer{
					Container: api.Container{
						Image:           "gcr.io/google_containers/busybox",
						Name:            "busybox",
						Command:         []string{"sh", "-c", "env"},
						ImagePullPolicy: api.PullIfNotPresent,
					},
					Client:   cl,
					Phase:    api.PodSucceeded,
					NodeName: *nodeName,
				}
				err := terminateCase.Create()
				Expect(err).NotTo(HaveOccurred())

				// TODO: Check that the container enters running state by sleeping in the container #23309
				Eventually(func() (api.PodPhase, error) {
					pod, err := terminateCase.Get()
					return pod.Phase, err
				}, retryTimeout, pollInterval).Should(Equal(terminateCase.Phase))
			})
			It("it should report its phase as 'succeeded' [Conformance]", func() {
				ccontainer, err := terminateCase.Get()
				Expect(err).NotTo(HaveOccurred())
				Expect(ccontainer).Should(CContainerEqual(terminateCase))
			})
			It("it should be possible to delete [Conformance]", func() {
				err := terminateCase.Delete()
				Expect(err).NotTo(HaveOccurred())
			})
		})
		Context("when running a container with invalid image", func() {
			var invalidImageCase ConformanceContainer
			It("it should not start successfully [Conformance]", func() {
				invalidImageCase = ConformanceContainer{
					Container: api.Container{
						Image:           "foo.com/foo/foo",
						Name:            "foo",
						Command:         []string{"foo", "'Should not work'"},
						ImagePullPolicy: api.PullIfNotPresent,
					},
					Client:   cl,
					Phase:    api.PodPending,
					NodeName: *nodeName,
				}
				err := invalidImageCase.Create()
				Expect(err).NotTo(HaveOccurred())
				Eventually(func() (api.PodPhase, error) {
					pod, err := invalidImageCase.Get()
					return pod.Phase, err
				}, retryTimeout, pollInterval).Should(Equal(invalidImageCase.Phase))
			})
			It("it should report its phase as 'pending' [Conformance]", func() {
				ccontainer, err := invalidImageCase.Get()
				Expect(err).NotTo(HaveOccurred())
				Expect(ccontainer).Should(CContainerEqual(invalidImageCase))
			})
			It("it should be possible to delete [Conformance]", func() {
				err := invalidImageCase.Delete()
				Expect(err).NotTo(HaveOccurred())
			})
		})
	})
})
<|endoftext|>


<commit_before>// Package bootstrap implements the bootstrapping logic: generation of a .go file to
// launch the actual generator and launching the generator itself.
//
// The package may be preferred to a command-line utility if generating the serializers
// from golang code is required.
package bootstrap

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
)

const genPackage = "github.com/mailru/easyjson/gen"
const pkgWriter = "github.com/mailru/easyjson/jwriter"
const pkgLexer = "github.com/mailru/easyjson/jlexer"

type Generator struct {
	PkgPath, PkgName string
	Types            []string

	NoStdMarshalers       bool
	SnakeCase             bool
	LowerCamelCase        bool
	OmitEmpty             bool
	DisallowUnknownFields bool

	OutName   string
	BuildTags string

	StubsOnly  bool
	LeaveTemps bool
	NoFormat   bool
}

// writeStub outputs an initial stubs for marshalers/unmarshalers so that the package
// using marshalers/unmarshales compiles correctly for boostrapping code.
func (g *Generator) writeStub() error {
	f, err := os.Create(g.OutName)
	if err != nil {
		return err
	}
	defer f.Close()

	if g.BuildTags != "" {
		fmt.Fprintln(f, "// +build ", g.BuildTags)
		fmt.Fprintln(f)
	}
	fmt.Fprintln(f, "// TEMPORARY AUTOGENERATED FILE: easyjson stub code to make the package")
	fmt.Fprintln(f, "// compilable during generation.")
	fmt.Fprintln(f)
	fmt.Fprintln(f, "package ", g.PkgName)

	if len(g.Types) > 0 {
		fmt.Fprintln(f)
		fmt.Fprintln(f, "import (")
		fmt.Fprintln(f, `  "`+pkgWriter+`"`)
		fmt.Fprintln(f, `  "`+pkgLexer+`"`)
		fmt.Fprintln(f, ")")
	}

	sort.Strings(g.Types)
	for _, t := range g.Types {
		fmt.Fprintln(f)
		if !g.NoStdMarshalers {
			fmt.Fprintln(f, "func (", t, ") MarshalJSON() ([]byte, error) { return nil, nil }")
			fmt.Fprintln(f, "func (*", t, ") UnmarshalJSON([]byte) error { return nil }")
		}

		fmt.Fprintln(f, "func (", t, ") MarshalEasyJSON(w *jwriter.Writer) {}")
		fmt.Fprintln(f, "func (*", t, ") UnmarshalEasyJSON(l *jlexer.Lexer) {}")
		fmt.Fprintln(f)
		fmt.Fprintln(f, "type EasyJSON_exporter_"+t+" *"+t)
	}
	return nil
}

// writeMain creates a .go file that launches the generator if 'go run'.
func (g *Generator) writeMain() (path string, err error) {
	f, err := ioutil.TempFile(filepath.Dir(g.OutName), "easyjson-bootstrap")
	if err != nil {
		return "", err
	}

	fmt.Fprintln(f, "// +build ignore")
	fmt.Fprintln(f)
	fmt.Fprintln(f, "// TEMPORARY AUTOGENERATED FILE: easyjson bootstapping code to launch")
	fmt.Fprintln(f, "// the actual generator.")
	fmt.Fprintln(f)
	fmt.Fprintln(f, "package main")
	fmt.Fprintln(f)
	fmt.Fprintln(f, "import (")
	fmt.Fprintln(f, `  "fmt"`)
	fmt.Fprintln(f, `  "os"`)
	fmt.Fprintln(f)
	fmt.Fprintf(f, "  %q\n", genPackage)
	if len(g.Types) > 0 {
		fmt.Fprintln(f)
		fmt.Fprintf(f, "  pkg %q\n", g.PkgPath)
	}
	fmt.Fprintln(f, ")")
	fmt.Fprintln(f)
	fmt.Fprintln(f, "func main() {")
	fmt.Fprintf(f, "  g := gen.NewGenerator(%q)\n", filepath.Base(g.OutName))
	fmt.Fprintf(f, "  g.SetPkg(%q, %q)\n", g.PkgName, g.PkgPath)
	if g.BuildTags != "" {
		fmt.Fprintf(f, "  g.SetBuildTags(%q)\n", g.BuildTags)
	}
	if g.SnakeCase {
		fmt.Fprintln(f, "  g.UseSnakeCase()")
	}
	if g.LowerCamelCase {
		fmt.Fprintln(f, "  g.UseLowerCamelCase()")
	}
	if g.OmitEmpty {
		fmt.Fprintln(f, "  g.OmitEmpty()")
	}
	if g.NoStdMarshalers {
		fmt.Fprintln(f, "  g.NoStdMarshalers()")
	}
	if g.DisallowUnknownFields {
		fmt.Fprintln(f, "  g.DisallowUnknownFields()")
	}

	sort.Strings(g.Types)
	for _, v := range g.Types {
		fmt.Fprintln(f, "  g.Add(pkg.EasyJSON_exporter_"+v+"(nil))")
	}

	fmt.Fprintln(f, "  if err := g.Run(os.Stdout); err != nil {")
	fmt.Fprintln(f, "    fmt.Fprintln(os.Stderr, err)")
	fmt.Fprintln(f, "    os.Exit(1)")
	fmt.Fprintln(f, "  }")
	fmt.Fprintln(f, "}")

	src := f.Name()
	if err := f.Close(); err != nil {
		return src, err
	}

	dest := src + ".go"
	return dest, os.Rename(src, dest)
}

func (g *Generator) Run() error {
	if err := g.writeStub(); err != nil {
		return err
	}
	if g.StubsOnly {
		return nil
	}

	path, err := g.writeMain()
	if err != nil {
		return err
	}
	if !g.LeaveTemps {
		defer os.Remove(path)
	}

	f, err := os.Create(g.OutName + ".tmp")
	if err != nil {
		return err
	}
	if !g.LeaveTemps {
		defer os.Remove(f.Name()) // will not remove after rename
	}

	cmd := exec.Command("go", "run", "-tags", g.BuildTags, filepath.Base(path))
	cmd.Stdout = f
	cmd.Stderr = os.Stderr
	cmd.Dir = filepath.Dir(path)
	if err = cmd.Run(); err != nil {
		return err
	}

	f.Close()

	if !g.NoFormat {
		cmd = exec.Command("gofmt", "-w", f.Name())
		cmd.Stderr = os.Stderr
		cmd.Stdout = os.Stdout

		if err = cmd.Run(); err != nil {
			return err
		}
	}

	return os.Rename(f.Name(), g.OutName)
}
<commit_msg>Fix a little typo<commit_after>// Package bootstrap implements the bootstrapping logic: generation of a .go file to
// launch the actual generator and launching the generator itself.
//
// The package may be preferred to a command-line utility if generating the serializers
// from golang code is required.
package bootstrap

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
)

const genPackage = "github.com/mailru/easyjson/gen"
const pkgWriter = "github.com/mailru/easyjson/jwriter"
const pkgLexer = "github.com/mailru/easyjson/jlexer"

type Generator struct {
	PkgPath, PkgName string
	Types            []string

	NoStdMarshalers       bool
	SnakeCase             bool
	LowerCamelCase        bool
	OmitEmpty             bool
	DisallowUnknownFields bool

	OutName   string
	BuildTags string

	StubsOnly  bool
	LeaveTemps bool
	NoFormat   bool
}

// writeStub outputs an initial stub for marshalers/unmarshalers so that the package
// using marshalers/unmarshales compiles correctly for boostrapping code.
func (g *Generator) writeStub() error {
	f, err := os.Create(g.OutName)
	if err != nil {
		return err
	}
	defer f.Close()

	if g.BuildTags != "" {
		fmt.Fprintln(f, "// +build ", g.BuildTags)
		fmt.Fprintln(f)
	}
	fmt.Fprintln(f, "// TEMPORARY AUTOGENERATED FILE: easyjson stub code to make the package")
	fmt.Fprintln(f, "// compilable during generation.")
	fmt.Fprintln(f)
	fmt.Fprintln(f, "package ", g.PkgName)

	if len(g.Types) > 0 {
		fmt.Fprintln(f)
		fmt.Fprintln(f, "import (")
		fmt.Fprintln(f, `  "`+pkgWriter+`"`)
		fmt.Fprintln(f, `  "`+pkgLexer+`"`)
		fmt.Fprintln(f, ")")
	}

	sort.Strings(g.Types)
	for _, t := range g.Types {
		fmt.Fprintln(f)
		if !g.NoStdMarshalers {
			fmt.Fprintln(f, "func (", t, ") MarshalJSON() ([]byte, error) { return nil, nil }")
			fmt.Fprintln(f, "func (*", t, ") UnmarshalJSON([]byte) error { return nil }")
		}

		fmt.Fprintln(f, "func (", t, ") MarshalEasyJSON(w *jwriter.Writer) {}")
		fmt.Fprintln(f, "func (*", t, ") UnmarshalEasyJSON(l *jlexer.Lexer) {}")
		fmt.Fprintln(f)
		fmt.Fprintln(f, "type EasyJSON_exporter_"+t+" *"+t)
	}
	return nil
}

// writeMain creates a .go file that launches the generator if 'go run'.
func (g *Generator) writeMain() (path string, err error) {
	f, err := ioutil.TempFile(filepath.Dir(g.OutName), "easyjson-bootstrap")
	if err != nil {
		return "", err
	}

	fmt.Fprintln(f, "// +build ignore")
	fmt.Fprintln(f)
	fmt.Fprintln(f, "// TEMPORARY AUTOGENERATED FILE: easyjson bootstapping code to launch")
	fmt.Fprintln(f, "// the actual generator.")
	fmt.Fprintln(f)
	fmt.Fprintln(f, "package main")
	fmt.Fprintln(f)
	fmt.Fprintln(f, "import (")
	fmt.Fprintln(f, `  "fmt"`)
	fmt.Fprintln(f, `  "os"`)
	fmt.Fprintln(f)
	fmt.Fprintf(f, "  %q\n", genPackage)
	if len(g.Types) > 0 {
		fmt.Fprintln(f)
		fmt.Fprintf(f, "  pkg %q\n", g.PkgPath)
	}
	fmt.Fprintln(f, ")")
	fmt.Fprintln(f)
	fmt.Fprintln(f, "func main() {")
	fmt.Fprintf(f, "  g := gen.NewGenerator(%q)\n", filepath.Base(g.OutName))
	fmt.Fprintf(f, "  g.SetPkg(%q, %q)\n", g.PkgName, g.PkgPath)
	if g.BuildTags != "" {
		fmt.Fprintf(f, "  g.SetBuildTags(%q)\n", g.BuildTags)
	}
	if g.SnakeCase {
		fmt.Fprintln(f, "  g.UseSnakeCase()")
	}
	if g.LowerCamelCase {
		fmt.Fprintln(f, "  g.UseLowerCamelCase()")
	}
	if g.OmitEmpty {
		fmt.Fprintln(f, "  g.OmitEmpty()")
	}
	if g.NoStdMarshalers {
		fmt.Fprintln(f, "  g.NoStdMarshalers()")
	}
	if g.DisallowUnknownFields {
		fmt.Fprintln(f, "  g.DisallowUnknownFields()")
	}

	sort.Strings(g.Types)
	for _, v := range g.Types {
		fmt.Fprintln(f, "  g.Add(pkg.EasyJSON_exporter_"+v+"(nil))")
	}

	fmt.Fprintln(f, "  if err := g.Run(os.Stdout); err != nil {")
	fmt.Fprintln(f, "    fmt.Fprintln(os.Stderr, err)")
	fmt.Fprintln(f, "    os.Exit(1)")
	fmt.Fprintln(f, "  }")
	fmt.Fprintln(f, "}")

	src := f.Name()
	if err := f.Close(); err != nil {
		return src, err
	}

	dest := src + ".go"
	return dest, os.Rename(src, dest)
}

func (g *Generator) Run() error {
	if err := g.writeStub(); err != nil {
		return err
	}
	if g.StubsOnly {
		return nil
	}

	path, err := g.writeMain()
	if err != nil {
		return err
	}
	if !g.LeaveTemps {
		defer os.Remove(path)
	}

	f, err := os.Create(g.OutName + ".tmp")
	if err != nil {
		return err
	}
	if !g.LeaveTemps {
		defer os.Remove(f.Name()) // will not remove after rename
	}

	cmd := exec.Command("go", "run", "-tags", g.BuildTags, filepath.Base(path))
	cmd.Stdout = f
	cmd.Stderr = os.Stderr
	cmd.Dir = filepath.Dir(path)
	if err = cmd.Run(); err != nil {
		return err
	}

	f.Close()

	if !g.NoFormat {
		cmd = exec.Command("gofmt", "-w", f.Name())
		cmd.Stderr = os.Stderr
		cmd.Stdout = os.Stdout

		if err = cmd.Run(); err != nil {
			return err
		}
	}

	return os.Rename(f.Name(), g.OutName)
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

var (
	// Major is the current major version of master branch.
	Major = 0
	// Minor is the current minor version of master branch.
	Minor = 2
	// Patch is the curernt patched version of the master branch.
	Patch = 1
	// Release is the current release level of the master branch. Valid values
	// are dev (developement unreleased), rcX (release candidate with current
	// iteration), stable (indicates a final released version).
	Release = "stable"
)
<commit_msg>Bumping ups to 0.2.2 dev<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

var (
	// Major is the current major version of master branch.
	Major = 0
	// Minor is the current minor version of master branch.
	Minor = 2
	// Patch is the curernt patched version of the master branch.
	Patch = 2
	// Release is the current release level of the master branch. Valid values
	// are dev (developement unreleased), rcX (release candidate with current
	// iteration), stable (indicates a final released version).
	Release = "dev"
)
<|endoftext|>


<commit_before>package operators

import (
	"context"
	"fmt"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"
	"github.com/stretchr/objx"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/dynamic"
	e2e "k8s.io/kubernetes/test/e2e/framework"
	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"

	exutil "github.com/openshift/origin/test/extended/util"
	"github.com/openshift/origin/test/extended/util/prometheus"
)

var _ = g.Describe("[sig-cluster-lifecycle][Feature:Machines][Early] Managed cluster should", func() {
	defer g.GinkgoRecover()

	g.It("have same number of Machines and Nodes", func() {
		cfg, err := e2e.LoadConfig()
		o.Expect(err).NotTo(o.HaveOccurred())
		c, err := e2e.LoadClientset()
		o.Expect(err).NotTo(o.HaveOccurred())
		dc, err := dynamic.NewForConfig(cfg)
		o.Expect(err).NotTo(o.HaveOccurred())

		g.By("getting MachineSet list")
		machineSetClient := dc.Resource(schema.GroupVersionResource{Group: "machine.openshift.io", Resource: "machinesets", Version: "v1beta1"})
		msList, err := machineSetClient.List(context.Background(), metav1.ListOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		machineSetList := objx.Map(msList.UnstructuredContent())
		machineSetItems := objects(machineSetList.Get("items"))

		if len(machineSetItems) == 0 {
			e2eskipper.Skipf("cluster does not have machineset resources")
		}

		g.By("getting Node list")
		nodeList, err := c.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		nodeItems := nodeList.Items

		g.By("getting Machine list")
		machineClient := dc.Resource(schema.GroupVersionResource{Group: "machine.openshift.io", Resource: "machines", Version: "v1beta1"})
		obj, err := machineClient.List(context.Background(), metav1.ListOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		machineList := objx.Map(obj.UnstructuredContent())
		machineItems := objects(machineList.Get("items"))

		g.By("ensure number of Machines and Nodes are equal")
		o.Expect(len(nodeItems)).To(o.Equal(len(machineItems)))
	})
})

var _ = g.Describe("[sig-node] Managed cluster", func() {
	defer g.GinkgoRecover()
	var (
		oc = exutil.NewCLIWithoutNamespace("managed-cluster-node")

		url, bearerToken string
	)
	g.BeforeEach(func() {
		var ok bool
		url, bearerToken, ok = prometheus.LocatePrometheus(oc)
		if !ok {
			e2e.Failf("Prometheus could not be located on this cluster, failing prometheus test")
		}
	})

	g.It("should report ready nodes the entire duration of the test run [Late]", func() {
		ns := oc.SetupNamespace()
		execPod := exutil.CreateExecPodOrFail(oc.AdminKubeClient(), ns, "execpod")
		defer func() {
			oc.AdminKubeClient().CoreV1().Pods(ns).Delete(context.Background(), execPod.Name, *metav1.NewDeleteOptions(1))
		}()

		// we only consider samples since the beginning of the test
		testDuration := exutil.DurationSinceStartInSeconds().String()

		tests := map[string]bool{
			// all nodes should be reporting ready throughout the entire run, as long as they are older than 6m
			fmt.Sprintf(`(min_over_time((max by (node) (kube_node_status_condition{condition="Ready",status="true"}) and (((max by (node) (kube_node_status_condition))) and (0*max by (node) (kube_node_status_condition offset 6m))))[%s:1s])) < 1`, testDuration): false,
		}
		err := prometheus.RunQueries(tests, oc, ns, execPod.Name, url, bearerToken)
		o.Expect(err).NotTo(o.HaveOccurred())
	})
})
<commit_msg>test: Nodes that are deleted should not fire the unready alert<commit_after>package operators

import (
	"context"
	"fmt"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"
	"github.com/stretchr/objx"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/dynamic"
	e2e "k8s.io/kubernetes/test/e2e/framework"
	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"

	exutil "github.com/openshift/origin/test/extended/util"
	"github.com/openshift/origin/test/extended/util/prometheus"
)

var _ = g.Describe("[sig-cluster-lifecycle][Feature:Machines][Early] Managed cluster should", func() {
	defer g.GinkgoRecover()

	g.It("have same number of Machines and Nodes", func() {
		cfg, err := e2e.LoadConfig()
		o.Expect(err).NotTo(o.HaveOccurred())
		c, err := e2e.LoadClientset()
		o.Expect(err).NotTo(o.HaveOccurred())
		dc, err := dynamic.NewForConfig(cfg)
		o.Expect(err).NotTo(o.HaveOccurred())

		g.By("getting MachineSet list")
		machineSetClient := dc.Resource(schema.GroupVersionResource{Group: "machine.openshift.io", Resource: "machinesets", Version: "v1beta1"})
		msList, err := machineSetClient.List(context.Background(), metav1.ListOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		machineSetList := objx.Map(msList.UnstructuredContent())
		machineSetItems := objects(machineSetList.Get("items"))

		if len(machineSetItems) == 0 {
			e2eskipper.Skipf("cluster does not have machineset resources")
		}

		g.By("getting Node list")
		nodeList, err := c.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		nodeItems := nodeList.Items

		g.By("getting Machine list")
		machineClient := dc.Resource(schema.GroupVersionResource{Group: "machine.openshift.io", Resource: "machines", Version: "v1beta1"})
		obj, err := machineClient.List(context.Background(), metav1.ListOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		machineList := objx.Map(obj.UnstructuredContent())
		machineItems := objects(machineList.Get("items"))

		g.By("ensure number of Machines and Nodes are equal")
		o.Expect(len(nodeItems)).To(o.Equal(len(machineItems)))
	})
})

var _ = g.Describe("[sig-node] Managed cluster", func() {
	defer g.GinkgoRecover()
	var (
		oc = exutil.NewCLIWithoutNamespace("managed-cluster-node")

		url, bearerToken string
	)
	g.BeforeEach(func() {
		var ok bool
		url, bearerToken, ok = prometheus.LocatePrometheus(oc)
		if !ok {
			e2e.Failf("Prometheus could not be located on this cluster, failing prometheus test")
		}
	})

	g.It("should report ready nodes the entire duration of the test run [Late]", func() {
		ns := oc.SetupNamespace()
		execPod := exutil.CreateExecPodOrFail(oc.AdminKubeClient(), ns, "execpod")
		defer func() {
			oc.AdminKubeClient().CoreV1().Pods(ns).Delete(context.Background(), execPod.Name, *metav1.NewDeleteOptions(1))
		}()

		// we only consider samples since the beginning of the test
		testDuration := exutil.DurationSinceStartInSeconds().String()

		tests := map[string]bool{
			// all nodes should be reporting ready throughout the entire run, as long as they are older than 6m, and they still
			// exist in 1m (because prometheus doesn't support negative offsets, we have to shift the entire query left). Since
			// the late test might not catch a node not ready at the very end of the run anyway, we don't do anything special
			// to shift the test execution later, we just note that there's a scrape_interval+wait_interval gap here of up to
			// 1m30s and we can live with ith
			fmt.Sprintf(`(min_over_time((max by (node) (kube_node_status_condition{condition="Ready",status="true"} offset 1m) and (((max by (node) (kube_node_status_condition offset 1m))) and (0*max by (node) (kube_node_status_condition offset 7m)) and (0*max by (node) (kube_node_status_condition))))[%s:1s])) < 1`, testDuration): false,
		}
		err := prometheus.RunQueries(tests, oc, ns, execPod.Name, url, bearerToken)
		o.Expect(err).NotTo(o.HaveOccurred())
	})
})
<|endoftext|>


<commit_before>package effio

// encoding/csv doesn't strip whitespace and does a fair bit of
// work to handle strings & quoting which are totally unnecessary
// overhead for these files so skip it

import (
	"bufio"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
)

// the input is ints but plotinum uses float64 so might as well
// start there and avoid the type conversions later
type LatRec struct {
	time float64 // time offset from beginning of fio run
	perf float64 // latency value
}

type LatRecs []LatRec

/*
time, perf, ??, block
3, 205274611861, 0, 4096
16, 205274624691, 0, 4096
*/
func LoadCSV(filename string) LatRecs {
	fd, err := os.Open(filename)
	if err != nil {
		log.Printf("Failed to open file '%s' for read: %s\n", filename, err)
		return LatRecs{}
	}
	defer fd.Close()

	records := make(LatRecs, 0)
	var time, perf float64
	bfd := bufio.NewReader(fd)
	var lno int = 0
	for {
		line, _, err := bfd.ReadLine()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatalf("Read from file '%s' failed: %s", filename, err)
		}
		lno++

		// fio always uses ", " instead of "," as far as I can tell
		r := strings.SplitN(string(line), ", ", 4)
		// probably an impartial record at the end of the file
		if len(r) < 4 || r[0] == "" || r[1] == "" {
			continue
		}

		time, err = strconv.ParseFloat(r[0], 64)
		if err != nil {
			log.Fatalf("Parsing time integer failed in file '%s' at line %d: %s", filename, lno, err)
		}
		perf, err = strconv.ParseFloat(r[1], 64)
		if err != nil {
			log.Fatalf("Parsing perf integer failed in file '%s' at line %d: %s", filename, lno, err)
		}
		// r[2:3] are unused, 2 is reserved, 3 is block size

		lr := LatRec{time, perf}
		records = append(records, lr)
	}
	log.Printf("Done parsing file '%s'.\n", filename)

	return records
}

// implement some plotinum interfaces
func (lrs LatRecs) Len() int {
	return len(lrs)
}

func (lrs LatRecs) XY(i int) (float64, float64) {
	return lrs[i].time, lrs[i].perf
}

func (lrs LatRecs) Value(i int) float64 {
	return lrs[i].perf
}

func (lrs LatRecs) Values(i int) (vals []float64) {
	for _, l := range lrs {
		vals = append(vals, l.perf)
	}
	return
}

// reduces the number of data points to sz by taking the mean across buckets
func (lrs LatRecs) Histogram(sz int) (out LatRecs) {
	if sz > len(lrs) {
		log.Fatalf("Error: Histogram(%d) is smaller than the dataset of length %d.", sz, len(lrs))
	}

	bktsz := len(lrs) / sz
	log.Printf("Bucket size for %d/%d is %d\n", len(lrs), sz, bktsz)

	var total, time float64
	var count int = 0
	for _, v := range lrs {
		if count == 0 {
			time = v.time
			total = 0.0
		}

		total += v.perf
		count++

		if count == bktsz {
			val := total / float64(count)
			out = append(out, LatRec{time, val})
			count = 0
			continue
		}
	}
	return
}
<commit_msg>add ..... progress to CVS loading (for now)<commit_after>package effio

// encoding/csv doesn't strip whitespace and does a fair bit of
// work to handle strings & quoting which are totally unnecessary
// overhead for these files so skip it

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
)

// the input is ints but plotinum uses float64 so might as well
// start there and avoid the type conversions later
type LatRec struct {
	time float64 // time offset from beginning of fio run
	perf float64 // latency value
}

type LatRecs []LatRec

/*
time, perf, ??, block
3, 205274611861, 0, 4096
16, 205274624691, 0, 4096
*/
func LoadCSV(filename string) LatRecs {
	fmt.Printf("Parsing file: '%s' ... ", filename)

	fd, err := os.Open(filename)
	if err != nil {
		fmt.Printf(" Failed.\nCould not open file '%s' for read: %s\n", filename, err)
		return LatRecs{}
	}
	defer fd.Close()

	records := make(LatRecs, 0)
	var time, perf float64
	bfd := bufio.NewReader(fd)
	var lno int = 0
	for {
		line, _, err := bfd.ReadLine()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatalf("\nRead from file '%s' failed: %s", filename, err)
		}
		lno++

		if lno%10000 == 0 {
			fmt.Printf(".")
		}

		// fio always uses ", " instead of "," as far as I can tell
		r := strings.SplitN(string(line), ", ", 4)
		// probably an impartial record at the end of the file
		if len(r) < 4 || r[0] == "" || r[1] == "" {
			continue
		}

		time, err = strconv.ParseFloat(r[0], 64)
		if err != nil {
			log.Fatalf("\nParsing time integer failed in file '%s' at line %d: %s", filename, lno, err)
		}
		perf, err = strconv.ParseFloat(r[1], 64)
		if err != nil {
			log.Fatalf("\nParsing perf integer failed in file '%s' at line %d: %s", filename, lno, err)
		}
		// r[2:3] are unused, 2 is reserved, 3 is block size

		lr := LatRec{time, perf}
		records = append(records, lr)
	}
	fmt.Println(" Done.")

	return records
}

// implement some plotinum interfaces
func (lrs LatRecs) Len() int {
	return len(lrs)
}

func (lrs LatRecs) XY(i int) (float64, float64) {
	return lrs[i].time, lrs[i].perf
}

func (lrs LatRecs) Value(i int) float64 {
	return lrs[i].perf
}

func (lrs LatRecs) Values(i int) (vals []float64) {
	for _, l := range lrs {
		vals = append(vals, l.perf)
	}
	return
}

// reduces the number of data points to sz by taking the mean across buckets
func (lrs LatRecs) Histogram(sz int) (out LatRecs) {
	if sz > len(lrs) {
		log.Fatalf("Error: Histogram(%d) is smaller than the dataset of length %d.", sz, len(lrs))
	}

	bktsz := len(lrs) / sz
	log.Printf("Bucket size for %d/%d is %d\n", len(lrs), sz, bktsz)

	var total, time float64
	var count int = 0
	for _, v := range lrs {
		if count == 0 {
			time = v.time
			total = 0.0
		}

		total += v.perf
		count++

		if count == bktsz {
			val := total / float64(count)
			out = append(out, LatRec{time, val})
			count = 0
			continue
		}
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2015-2016, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package net provides additional network helper functions.
package net
<commit_msg>net: Update doc<commit_after>// Copyright 2015-2016, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package net provides additional network helper functions.
//
// context Package: https://twitter.com/peterbourgon/status/752022730812317696
package net
<|endoftext|>