<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dwarf

// This file provides simple methods to access the symbol table by name and address.

import "fmt"

// lookupEntry returns the Entry for the name. If tag is non-zero, only entries
// with that tag are considered.
func (d *Data) lookupEntry(name string, tag Tag) (*Entry, error) {
	r := d.Reader()
	for {
		entry, err := r.Next()
		if err != nil {
			return nil, err
		}
		if entry == nil {
			// TODO: why don't we get an error here?
			break
		}
		if tag != 0 && tag != entry.Tag {
			continue
		}
		nameAttr := entry.Val(AttrName)
		if nameAttr == nil {
			continue
		}
		if nameAttr.(string) == name {
			return entry, nil
		}
	}
	return nil, fmt.Errorf("DWARF entry for %q not found", name)
}

// LookupEntry returns the Entry for the named symbol.
func (d *Data) LookupEntry(name string) (*Entry, error) {
	return d.lookupEntry(name, 0)
}

// LookupFunction returns the address of the named symbol, a function.
func (d *Data) LookupFunction(name string) (uint64, error) {
	entry, err := d.lookupEntry(name, TagSubprogram)
	if err != nil {
		return 0, err
	}
	addrAttr := entry.Val(AttrLowpc)
	if addrAttr == nil {
		return 0, fmt.Errorf("symbol %q has no LowPC attribute", name)
	}
	addr, ok := addrAttr.(uint64)
	if !ok {
		return 0, fmt.Errorf("symbol %q has non-uint64 LowPC attribute", name)
	}
	return addr, nil
}

// TODO: should LookupVariable handle both globals and locals? Locals don't
// necessarily have a fixed address. They may be in a register, or otherwise
// move around. Should this function be renamed LookupGlobal?

// LookupVariable returns the location of a named symbol, a variable.
func (d *Data) LookupVariable(name string) (uint64, error) {
	entry, err := d.lookupEntry(name, TagVariable)
	if err != nil {
		return 0, err
	}
	loc, _ := entry.Val(AttrLocation).([]byte)
	if len(loc) == 0 {
		return 0, fmt.Errorf("name %q has no Location attribute", name)
	}
	// TODO: implement the DWARF Location bytecode. What we have here only
	// recognizes a program with a single literal opAddr bytecode.
	// TODO: is d.unit[0] always the right unit to take?
	if asize := d.unit[0].asize; loc[0] == opAddr || len(loc) == 1+asize {
		switch asize {
		case 1:
			return uint64(loc[1]), nil
		case 2:
			return uint64(d.order.Uint16(loc[1:])), nil
		case 4:
			return uint64(d.order.Uint32(loc[1:])), nil
		case 8:
			return d.order.Uint64(loc[1:]), nil
		}
	}
	return 0, fmt.Errorf("DWARF: unimplemented Location op")
}

// LookupPC returns the name of a symbol at the specified PC.
func (d *Data) LookupPC(pc uint64) (string, error) {
	entry, _, err := d.EntryForPC(pc)
	if err != nil {
		return "", err
	}
	nameAttr := entry.Val(AttrName)
	if nameAttr == nil {
		// TODO: this shouldn't be possible.
		return "", fmt.Errorf("LookupPC: TODO")
	}
	name, ok := nameAttr.(string)
	if !ok {
		return "", fmt.Errorf("name for PC %#x is not a string", pc)
	}
	return name, nil
}

// EntryForPC returns the entry and address for a symbol at the specified PC.
func (d *Data) EntryForPC(pc uint64) (entry *Entry, lowpc uint64, err error) {
	// TODO: do something better than a linear scan?
	r := d.Reader()
	for {
		entry, err := r.Next()
		if err != nil {
			return nil, 0, err
		}
		if entry == nil {
			// TODO: why don't we get an error here.
			break
		}
		if entry.Tag != TagSubprogram {
			continue
		}
		lowpc, lok := entry.Val(AttrLowpc).(uint64)
		highpc, hok := entry.Val(AttrHighpc).(uint64)
		if !lok || !hok || pc < lowpc || highpc <= pc {
			continue
		}
		return entry, lowpc, nil
	}
	return nil, 0, fmt.Errorf("PC %#x not found", pc)
}
<commit_msg>ogle/debug/dwarf: fix a logic typo.<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dwarf

// This file provides simple methods to access the symbol table by name and address.

import "fmt"

// lookupEntry returns the Entry for the name. If tag is non-zero, only entries
// with that tag are considered.
func (d *Data) lookupEntry(name string, tag Tag) (*Entry, error) {
	r := d.Reader()
	for {
		entry, err := r.Next()
		if err != nil {
			return nil, err
		}
		if entry == nil {
			// TODO: why don't we get an error here?
			break
		}
		if tag != 0 && tag != entry.Tag {
			continue
		}
		nameAttr := entry.Val(AttrName)
		if nameAttr == nil {
			continue
		}
		if nameAttr.(string) == name {
			return entry, nil
		}
	}
	return nil, fmt.Errorf("DWARF entry for %q not found", name)
}

// LookupEntry returns the Entry for the named symbol.
func (d *Data) LookupEntry(name string) (*Entry, error) {
	return d.lookupEntry(name, 0)
}

// LookupFunction returns the address of the named symbol, a function.
func (d *Data) LookupFunction(name string) (uint64, error) {
	entry, err := d.lookupEntry(name, TagSubprogram)
	if err != nil {
		return 0, err
	}
	addrAttr := entry.Val(AttrLowpc)
	if addrAttr == nil {
		return 0, fmt.Errorf("symbol %q has no LowPC attribute", name)
	}
	addr, ok := addrAttr.(uint64)
	if !ok {
		return 0, fmt.Errorf("symbol %q has non-uint64 LowPC attribute", name)
	}
	return addr, nil
}

// TODO: should LookupVariable handle both globals and locals? Locals don't
// necessarily have a fixed address. They may be in a register, or otherwise
// move around.

// LookupVariable returns the location of a named symbol, a variable.
func (d *Data) LookupVariable(name string) (uint64, error) {
	entry, err := d.lookupEntry(name, TagVariable)
	if err != nil {
		return 0, err
	}
	loc, _ := entry.Val(AttrLocation).([]byte)
	if len(loc) == 0 {
		return 0, fmt.Errorf("name %q has no Location attribute", name)
	}
	// TODO: implement the DWARF Location bytecode. What we have here only
	// recognizes a program with a single literal opAddr bytecode.
	if asize := d.unit[0].asize; loc[0] == opAddr && len(loc) == 1+asize {
		switch asize {
		case 1:
			return uint64(loc[1]), nil
		case 2:
			return uint64(d.order.Uint16(loc[1:])), nil
		case 4:
			return uint64(d.order.Uint32(loc[1:])), nil
		case 8:
			return d.order.Uint64(loc[1:]), nil
		}
	}
	return 0, fmt.Errorf("DWARF: unimplemented Location op")
}

// LookupPC returns the name of a symbol at the specified PC.
func (d *Data) LookupPC(pc uint64) (string, error) {
	entry, _, err := d.EntryForPC(pc)
	if err != nil {
		return "", err
	}
	nameAttr := entry.Val(AttrName)
	if nameAttr == nil {
		// TODO: this shouldn't be possible.
		return "", fmt.Errorf("LookupPC: TODO")
	}
	name, ok := nameAttr.(string)
	if !ok {
		return "", fmt.Errorf("name for PC %#x is not a string", pc)
	}
	return name, nil
}

// EntryForPC returns the entry and address for a symbol at the specified PC.
func (d *Data) EntryForPC(pc uint64) (entry *Entry, lowpc uint64, err error) {
	// TODO: do something better than a linear scan?
	r := d.Reader()
	for {
		entry, err := r.Next()
		if err != nil {
			return nil, 0, err
		}
		if entry == nil {
			// TODO: why don't we get an error here.
			break
		}
		if entry.Tag != TagSubprogram {
			continue
		}
		lowpc, lok := entry.Val(AttrLowpc).(uint64)
		highpc, hok := entry.Val(AttrHighpc).(uint64)
		if !lok || !hok || pc < lowpc || highpc <= pc {
			continue
		}
		return entry, lowpc, nil
	}
	return nil, 0, fmt.Errorf("PC %#x not found", pc)
}
<|endoftext|>


<commit_before>// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package indexer

import (
	"code.gitea.io/gitea/models"
	code_indexer "code.gitea.io/gitea/modules/indexer/code"
	issue_indexer "code.gitea.io/gitea/modules/indexer/issues"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/notification/base"
	"code.gitea.io/gitea/modules/repository"
	"code.gitea.io/gitea/modules/setting"
)

type indexerNotifier struct {
	base.NullNotifier
}

var (
	_ base.Notifier = &indexerNotifier{}
)

// NewNotifier create a new indexerNotifier notifier
func NewNotifier() base.Notifier {
	return &indexerNotifier{}
}

func (r *indexerNotifier) NotifyCreateIssueComment(doer *models.User, repo *models.Repository,
	issue *models.Issue, comment *models.Comment) {
	if comment.Type == models.CommentTypeComment {
		if issue.Comments == nil {
			if err := issue.LoadDiscussComments(); err != nil {
				log.Error("LoadComments failed: %v", err)
				return
			}
		} else {
			issue.Comments = append(issue.Comments, comment)
		}

		issue_indexer.UpdateIssueIndexer(issue)
	}
}

func (r *indexerNotifier) NotifyNewIssue(issue *models.Issue) {
	issue_indexer.UpdateIssueIndexer(issue)
}

func (r *indexerNotifier) NotifyNewPullRequest(pr *models.PullRequest) {
	issue_indexer.UpdateIssueIndexer(pr.Issue)
}

func (r *indexerNotifier) NotifyUpdateComment(doer *models.User, c *models.Comment, oldContent string) {
	if c.Type == models.CommentTypeComment {
		var found bool
		if c.Issue.Comments != nil {
			for i := 0; i < len(c.Issue.Comments); i++ {
				if c.Issue.Comments[i].ID == c.ID {
					c.Issue.Comments[i] = c
					found = true
					break
				}
			}
		}

		if !found {
			if err := c.Issue.LoadDiscussComments(); err != nil {
				log.Error("LoadComments failed: %v", err)
				return
			}
		}

		issue_indexer.UpdateIssueIndexer(c.Issue)
	}
}

func (r *indexerNotifier) NotifyDeleteComment(doer *models.User, comment *models.Comment) {
	if comment.Type == models.CommentTypeComment {
		if err := comment.LoadIssue(); err != nil {
			log.Error("LoadIssue: %v", err)
			return
		}

		var found bool
		if comment.Issue.Comments != nil {
			for i := 0; i < len(comment.Issue.Comments); i++ {
				if comment.Issue.Comments[i].ID == comment.ID {
					comment.Issue.Comments = append(comment.Issue.Comments[:i], comment.Issue.Comments[i+1:]...)
					found = true
					break
				}
			}
		}

		if !found {
			if err := comment.Issue.LoadDiscussComments(); err != nil {
				log.Error("LoadComments failed: %v", err)
				return
			}
		}
		// reload comments to delete the old comment
		issue_indexer.UpdateIssueIndexer(comment.Issue)
	}
}

func (r *indexerNotifier) NotifyDeleteRepository(doer *models.User, repo *models.Repository) {
	issue_indexer.DeleteRepoIssueIndexer(repo)
	if setting.Indexer.RepoIndexerEnabled {
		code_indexer.DeleteRepoFromIndexer(repo)
	}
}

func (r *indexerNotifier) NotifyMigrateRepository(doer *models.User, u *models.User, repo *models.Repository) {
	issue_indexer.UpdateRepoIndexer(repo)
	if setting.Indexer.RepoIndexerEnabled && !repo.IsEmpty {
		code_indexer.UpdateRepoIndexer(repo)
	}
}

func (r *indexerNotifier) NotifyPushCommits(pusher *models.User, repo *models.Repository, refName, oldCommitID, newCommitID string, commits *repository.PushCommits) {
	if setting.Indexer.RepoIndexerEnabled && refName == repo.DefaultBranch {
		code_indexer.UpdateRepoIndexer(repo)
	}
}

func (r *indexerNotifier) NotifyIssueChangeContent(doer *models.User, issue *models.Issue, oldContent string) {
	issue_indexer.UpdateIssueIndexer(issue)
}

func (r *indexerNotifier) NotifyIssueChangeTitle(doer *models.User, issue *models.Issue, oldTitle string) {
	issue_indexer.UpdateIssueIndexer(issue)
}
<commit_msg>Fix repo indexer not updating upon push (#9957)<commit_after>// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package indexer

import (
	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/git"
	code_indexer "code.gitea.io/gitea/modules/indexer/code"
	issue_indexer "code.gitea.io/gitea/modules/indexer/issues"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/notification/base"
	"code.gitea.io/gitea/modules/repository"
	"code.gitea.io/gitea/modules/setting"
)

type indexerNotifier struct {
	base.NullNotifier
}

var (
	_ base.Notifier = &indexerNotifier{}
)

// NewNotifier create a new indexerNotifier notifier
func NewNotifier() base.Notifier {
	return &indexerNotifier{}
}

func (r *indexerNotifier) NotifyCreateIssueComment(doer *models.User, repo *models.Repository,
	issue *models.Issue, comment *models.Comment) {
	if comment.Type == models.CommentTypeComment {
		if issue.Comments == nil {
			if err := issue.LoadDiscussComments(); err != nil {
				log.Error("LoadComments failed: %v", err)
				return
			}
		} else {
			issue.Comments = append(issue.Comments, comment)
		}

		issue_indexer.UpdateIssueIndexer(issue)
	}
}

func (r *indexerNotifier) NotifyNewIssue(issue *models.Issue) {
	issue_indexer.UpdateIssueIndexer(issue)
}

func (r *indexerNotifier) NotifyNewPullRequest(pr *models.PullRequest) {
	issue_indexer.UpdateIssueIndexer(pr.Issue)
}

func (r *indexerNotifier) NotifyUpdateComment(doer *models.User, c *models.Comment, oldContent string) {
	if c.Type == models.CommentTypeComment {
		var found bool
		if c.Issue.Comments != nil {
			for i := 0; i < len(c.Issue.Comments); i++ {
				if c.Issue.Comments[i].ID == c.ID {
					c.Issue.Comments[i] = c
					found = true
					break
				}
			}
		}

		if !found {
			if err := c.Issue.LoadDiscussComments(); err != nil {
				log.Error("LoadComments failed: %v", err)
				return
			}
		}

		issue_indexer.UpdateIssueIndexer(c.Issue)
	}
}

func (r *indexerNotifier) NotifyDeleteComment(doer *models.User, comment *models.Comment) {
	if comment.Type == models.CommentTypeComment {
		if err := comment.LoadIssue(); err != nil {
			log.Error("LoadIssue: %v", err)
			return
		}

		var found bool
		if comment.Issue.Comments != nil {
			for i := 0; i < len(comment.Issue.Comments); i++ {
				if comment.Issue.Comments[i].ID == comment.ID {
					comment.Issue.Comments = append(comment.Issue.Comments[:i], comment.Issue.Comments[i+1:]...)
					found = true
					break
				}
			}
		}

		if !found {
			if err := comment.Issue.LoadDiscussComments(); err != nil {
				log.Error("LoadComments failed: %v", err)
				return
			}
		}
		// reload comments to delete the old comment
		issue_indexer.UpdateIssueIndexer(comment.Issue)
	}
}

func (r *indexerNotifier) NotifyDeleteRepository(doer *models.User, repo *models.Repository) {
	issue_indexer.DeleteRepoIssueIndexer(repo)
	if setting.Indexer.RepoIndexerEnabled {
		code_indexer.DeleteRepoFromIndexer(repo)
	}
}

func (r *indexerNotifier) NotifyMigrateRepository(doer *models.User, u *models.User, repo *models.Repository) {
	issue_indexer.UpdateRepoIndexer(repo)
	if setting.Indexer.RepoIndexerEnabled && !repo.IsEmpty {
		code_indexer.UpdateRepoIndexer(repo)
	}
}

func (r *indexerNotifier) NotifyPushCommits(pusher *models.User, repo *models.Repository, refName, oldCommitID, newCommitID string, commits *repository.PushCommits) {
	if setting.Indexer.RepoIndexerEnabled && refName == git.BranchPrefix+repo.DefaultBranch {
		code_indexer.UpdateRepoIndexer(repo)
	}
}

func (r *indexerNotifier) NotifyIssueChangeContent(doer *models.User, issue *models.Issue, oldContent string) {
	issue_indexer.UpdateIssueIndexer(issue)
}

func (r *indexerNotifier) NotifyIssueChangeTitle(doer *models.User, issue *models.Issue, oldTitle string) {
	issue_indexer.UpdateIssueIndexer(issue)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"sort"
	"time"
    "strings"
	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/controller/framework"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/util"
	utildbus "k8s.io/kubernetes/pkg/util/dbus"
	"k8s.io/kubernetes/pkg/util/exec"
	"k8s.io/kubernetes/pkg/util/intstr"
	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
)

const (
	reloadQPS    = 10.0
	resyncPeriod = 10 * time.Second
)

type service struct {
	IP   string
	Port int
}

type serviceByIPPort []service

func (c serviceByIPPort) Len() int      { return len(c) }
func (c serviceByIPPort) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
func (c serviceByIPPort) Less(i, j int) bool {
	iIP := c[i].IP
	jIP := c[j].IP
	if iIP != jIP {
		return iIP < jIP
	}

	iPort := c[i].Port
	jPort := c[j].Port
	return iPort < jPort
}

type vip struct {
	Name      string
	IP        string
	Port      int
	Protocol  string
	LVSMethod string
	Backends  []service
}

type vipByNameIPPort []vip

func (c vipByNameIPPort) Len() int      { return len(c) }
func (c vipByNameIPPort) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
func (c vipByNameIPPort) Less(i, j int) bool {
	iName := c[i].Name
	jName := c[j].Name
	if iName != jName {
		return iName < jName
	}

	iIP := c[i].IP
	jIP := c[j].IP
	if iIP != jIP {
		return iIP < jIP
	}

	iPort := c[i].Port
	jPort := c[j].Port
	return iPort < jPort
}

// ipvsControllerController watches the kubernetes api and adds/removes
// services from LVS throgh ipvsadmin.
type ipvsControllerController struct {
	client            *unversioned.Client
	epController      *framework.Controller
	svcController     *framework.Controller
	svcLister         cache.StoreToServiceLister
	epLister          cache.StoreToEndpointsLister
	reloadRateLimiter util.RateLimiter
	keepalived        *keepalived
	configMapName     string
	ruCfg             []vip
	ruMD5             string
}

// getEndpoints returns a list of <endpoint ip>:<port> for a given service/target port combination.
func (ipvsc *ipvsControllerController) getEndpoints(
	s *api.Service, servicePort *api.ServicePort) []service {
	ep, err := ipvsc.epLister.GetServiceEndpoints(s)
	if err != nil {
		glog.Warningf("unexpected error getting service endpoints: %v", err)
		return []service{}
	}

	var endpoints []service

	// The intent here is to create a union of all subsets that match a targetPort.
	// We know the endpoint already matches the service, so all pod ips that have
	// the target port are capable of service traffic for it.
	for _, ss := range ep.Subsets {
		for _, epPort := range ss.Ports {
			var targetPort int
			switch servicePort.TargetPort.Type {
			case intstr.Int:
				if epPort.Port == servicePort.TargetPort.IntValue() {
					targetPort = epPort.Port
				}
			case intstr.String:
				if epPort.Name == servicePort.TargetPort.StrVal {
					targetPort = epPort.Port
				}
			}
			if targetPort == 0 {
				continue
			}
			for _, epAddress := range ss.Addresses {
				endpoints = append(endpoints, service{IP: epAddress.IP, Port: targetPort})
			}
		}
	}

	return endpoints
}

// getServices returns a list of services and their endpoints.
func (ipvsc *ipvsControllerController) getServices() []vip {
	svcs := []vip{}

	ns, name, err := parseNsName(ipvsc.configMapName)
	if err != nil {
		glog.Warningf("%v", err)
		return []vip{}
	}
	cfgMap, err := ipvsc.getConfigMap(ns, name)
	if err != nil {
		glog.Warningf("%v", err)
		return []vip{}
	}

	// k -> IP to use
	// v -> <namespace>/<service name>:<lvs method>
	for externalIPIndex, nsSvcLvs := range cfgMap.Data {
        	var externalIP string
        	if colonIndex := strings.Index(externalIPIndex, "-"); colonIndex < 0 {
            		externalIP = externalIPIndex
        	} else {
            		externalIP = externalIPIndex[:colonIndex]
        	}
		ns, svc, lvsm, err := parseNsSvcLVS(nsSvcLvs)
		if err != nil {
			glog.Warningf("%v", err)
			continue
		}

		nsSvc := fmt.Sprintf("%v/%v", ns, svc)
		svcObj, svcExists, err := ipvsc.svcLister.Store.GetByKey(nsSvc)
		if err != nil {
			glog.Warningf("error getting service %v: %v", nsSvc, err)
			continue
		}

		if !svcExists {
			glog.Warningf("service %v not found", nsSvc)
			continue
		}

		s := svcObj.(*api.Service)
		for _, servicePort := range s.Spec.Ports {
			ep := ipvsc.getEndpoints(s, &servicePort)
			if len(ep) == 0 {
				glog.Warningf("no endpoints found for service %v, port %+v", s.Name, servicePort)
				continue
			}

			sort.Sort(serviceByIPPort(ep))

			svcs = append(svcs, vip{
				Name:      fmt.Sprintf("%v/%v", s.Namespace, s.Name),
				IP:        externalIP,
				Port:      servicePort.Port,
				LVSMethod: lvsm,
				Backends:  ep,
				Protocol:  fmt.Sprintf("%v", servicePort.Protocol),
			})
			glog.V(2).Infof("found service: %v:%v", s.Name, servicePort.Port)
		}
	}

	sort.Sort(vipByNameIPPort(svcs))

	return svcs
}

func (ipvsc *ipvsControllerController) getConfigMap(ns, name string) (*api.ConfigMap, error) {
	return ipvsc.client.ConfigMaps(ns).Get(name)
}

// sync all services with the
func (ipvsc *ipvsControllerController) sync() {
	ipvsc.reloadRateLimiter.Accept()

	if !ipvsc.epController.HasSynced() || !ipvsc.svcController.HasSynced() {
		time.Sleep(100 * time.Millisecond)
		return
	}

	svc := ipvsc.getServices()
	ipvsc.ruCfg = svc

	err := ipvsc.keepalived.WriteCfg(svc)
	if err != nil {
		return
	}
	glog.V(2).Infof("services: %v", svc)

	md5, err := checksum(keepalivedCfg)
	if err == nil && md5 == ipvsc.ruMD5 {
		return
	}

	ipvsc.ruMD5 = md5
	err = ipvsc.keepalived.Reload()
	if err != nil {
		glog.Errorf("error reloading keepalived: %v", err)
	}
}

// newIPVSController creates a new controller from the given config.
func newIPVSController(kubeClient *unversioned.Client, namespace string, useUnicast bool, configMapName string) *ipvsControllerController {
	ipvsc := ipvsControllerController{
		client:            kubeClient,
		reloadRateLimiter: util.NewTokenBucketRateLimiter(reloadQPS, int(reloadQPS)),
		ruCfg:             []vip{},
		configMapName:     configMapName,
	}

	podInfo, err := getPodDetails(kubeClient)
	if err != nil {
		glog.Fatalf("Error getting POD information: %v", err)
	}

	pod, err := kubeClient.Pods(podInfo.PodNamespace).Get(podInfo.PodName)
	if err != nil {
		glog.Fatalf("Error getting %v: %v", podInfo.PodName, err)
	}

	selector := parseNodeSelector(pod.Spec.NodeSelector)
	clusterNodes := getClusterNodesIP(kubeClient, selector)

	nodeInfo, err := getNetworkInfo(podInfo.NodeIP)
	if err != nil {
		glog.Fatalf("Error getting local IP from nodes in the cluster: %v", err)
	}

	neighbors := getNodeNeighbors(nodeInfo, clusterNodes)

	execer := exec.New()
	dbus := utildbus.New()
	iptInterface := utiliptables.New(execer, dbus, utiliptables.ProtocolIpv4)

	ipvsc.keepalived = &keepalived{
		iface:      nodeInfo.iface,
		ip:         nodeInfo.ip,
		netmask:    nodeInfo.netmask,
		nodes:      clusterNodes,
		neighbors:  neighbors,
		priority:   getNodePriority(nodeInfo.ip, clusterNodes),
		useUnicast: useUnicast,
		ipt:        iptInterface,
	}

	err = ipvsc.keepalived.loadTemplate()
	if err != nil {
		glog.Fatalf("Error loading keepalived template: %v", err)
	}

	eventHandlers := framework.ResourceEventHandlerFuncs{}

	ipvsc.svcLister.Store, ipvsc.svcController = framework.NewInformer(
		cache.NewListWatchFromClient(
			ipvsc.client, "services", namespace, fields.Everything()),
		&api.Service{}, resyncPeriod, eventHandlers)

	ipvsc.epLister.Store, ipvsc.epController = framework.NewInformer(
		cache.NewListWatchFromClient(
			ipvsc.client, "endpoints", namespace, fields.Everything()),
		&api.Endpoints{}, resyncPeriod, eventHandlers)

	return &ipvsc
}

func checksum(filename string) (string, error) {
	var result []byte
	file, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hash := md5.New()
	_, err = io.Copy(hash, file)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(hash.Sum(result)), nil
}
<commit_msg>select service NodePort as LVS port<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"sort"
	"time"
    "strings"
	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/controller/framework"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/util"
	utildbus "k8s.io/kubernetes/pkg/util/dbus"
	"k8s.io/kubernetes/pkg/util/exec"
	"k8s.io/kubernetes/pkg/util/intstr"
	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
)

const (
	reloadQPS    = 10.0
	resyncPeriod = 10 * time.Second
)

type service struct {
	IP   string
	Port int
}

type serviceByIPPort []service

func (c serviceByIPPort) Len() int      { return len(c) }
func (c serviceByIPPort) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
func (c serviceByIPPort) Less(i, j int) bool {
	iIP := c[i].IP
	jIP := c[j].IP
	if iIP != jIP {
		return iIP < jIP
	}

	iPort := c[i].Port
	jPort := c[j].Port
	return iPort < jPort
}

type vip struct {
	Name      string
	IP        string
	Port      int
	Protocol  string
	LVSMethod string
	Backends  []service
}

type vipByNameIPPort []vip

func (c vipByNameIPPort) Len() int      { return len(c) }
func (c vipByNameIPPort) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
func (c vipByNameIPPort) Less(i, j int) bool {
	iName := c[i].Name
	jName := c[j].Name
	if iName != jName {
		return iName < jName
	}

	iIP := c[i].IP
	jIP := c[j].IP
	if iIP != jIP {
		return iIP < jIP
	}

	iPort := c[i].Port
	jPort := c[j].Port
	return iPort < jPort
}

// ipvsControllerController watches the kubernetes api and adds/removes
// services from LVS throgh ipvsadmin.
type ipvsControllerController struct {
	client            *unversioned.Client
	epController      *framework.Controller
	svcController     *framework.Controller
	svcLister         cache.StoreToServiceLister
	epLister          cache.StoreToEndpointsLister
	reloadRateLimiter util.RateLimiter
	keepalived        *keepalived
	configMapName     string
	ruCfg             []vip
	ruMD5             string
}

// getEndpoints returns a list of <endpoint ip>:<port> for a given service/target port combination.
func (ipvsc *ipvsControllerController) getEndpoints(
	s *api.Service, servicePort *api.ServicePort) []service {
	ep, err := ipvsc.epLister.GetServiceEndpoints(s)
	if err != nil {
		glog.Warningf("unexpected error getting service endpoints: %v", err)
		return []service{}
	}

	var endpoints []service

	// The intent here is to create a union of all subsets that match a targetPort.
	// We know the endpoint already matches the service, so all pod ips that have
	// the target port are capable of service traffic for it.
	for _, ss := range ep.Subsets {
		for _, epPort := range ss.Ports {
			var targetPort int
			switch servicePort.TargetPort.Type {
			case intstr.Int:
				if epPort.Port == servicePort.TargetPort.IntValue() {
					targetPort = epPort.Port
				}
			case intstr.String:
				if epPort.Name == servicePort.TargetPort.StrVal {
					targetPort = epPort.Port
				}
			}
			if targetPort == 0 {
				continue
			}
			for _, epAddress := range ss.Addresses {
				endpoints = append(endpoints, service{IP: epAddress.IP, Port: targetPort})
			}
		}
	}

	return endpoints
}

// getServices returns a list of services and their endpoints.
func (ipvsc *ipvsControllerController) getServices() []vip {
	svcs := []vip{}

	ns, name, err := parseNsName(ipvsc.configMapName)
	if err != nil {
		glog.Warningf("%v", err)
		return []vip{}
	}
	cfgMap, err := ipvsc.getConfigMap(ns, name)
	if err != nil {
		glog.Warningf("%v", err)
		return []vip{}
	}

	// k -> IP to use
	// v -> <namespace>/<service name>:<lvs method>
	for externalIPIndex, nsSvcLvs := range cfgMap.Data {
		var externalIP string
		if colonIndex := strings.Index(externalIPIndex, "-"); colonIndex < 0 {
			externalIP = externalIPIndex
		} else {
			externalIP = externalIPIndex[:colonIndex]
		}
		ns, svc, lvsm, err := parseNsSvcLVS(nsSvcLvs)
		if err != nil {
			glog.Warningf("%v", err)
			continue
		}

		nsSvc := fmt.Sprintf("%v/%v", ns, svc)
		svcObj, svcExists, err := ipvsc.svcLister.Store.GetByKey(nsSvc)
		if err != nil {
			glog.Warningf("error getting service %v: %v", nsSvc, err)
			continue
		}

		if !svcExists {
			glog.Warningf("service %v not found", nsSvc)
			continue
		}

		s := svcObj.(*api.Service)
		for _, servicePort := range s.Spec.Ports {
            		np := servicePort.NodePort
           		if np == 0 {
           		    	glog.Infof("No nodePort found for service %v, port %+v", s.Name, servicePort)
                		continue
            		}

			ep := ipvsc.getEndpoints(s, &servicePort)
			if len(ep) == 0 {
				glog.Warningf("no endpoints found for service %v, port %+v", s.Name, servicePort)
				continue
			}

			sort.Sort(serviceByIPPort(ep))

			svcs = append(svcs, vip{
				Name:      fmt.Sprintf("%v/%v", s.Namespace, s.Name),
				IP:        externalIP,
				Port:      servicePort.NodePort,
				LVSMethod: lvsm,
				Backends:  ep,
				Protocol:  fmt.Sprintf("%v", servicePort.Protocol),
			})
			glog.V(2).Infof("Found service: %v %v %v:%v->%v:%v", s.Name, servicePort.Protocol, externalIP, servicePort.NodePort, s.Spec.ClusterIP, servicePort.Port)
		}
	}

	sort.Sort(vipByNameIPPort(svcs))

	return svcs
}

func (ipvsc *ipvsControllerController) getConfigMap(ns, name string) (*api.ConfigMap, error) {
	return ipvsc.client.ConfigMaps(ns).Get(name)
}

// sync all services with the
func (ipvsc *ipvsControllerController) sync() {
	ipvsc.reloadRateLimiter.Accept()

	if !ipvsc.epController.HasSynced() || !ipvsc.svcController.HasSynced() {
		time.Sleep(100 * time.Millisecond)
		return
	}

	svc := ipvsc.getServices()
	ipvsc.ruCfg = svc

	err := ipvsc.keepalived.WriteCfg(svc)
	if err != nil {
		return
	}
	glog.V(2).Infof("services: %v", svc)

	md5, err := checksum(keepalivedCfg)
	if err == nil && md5 == ipvsc.ruMD5 {
		return
	}

	ipvsc.ruMD5 = md5
	err = ipvsc.keepalived.Reload()
	if err != nil {
		glog.Errorf("error reloading keepalived: %v", err)
	}
}

// newIPVSController creates a new controller from the given config.
func newIPVSController(kubeClient *unversioned.Client, namespace string, useUnicast bool, configMapName string) *ipvsControllerController {
	ipvsc := ipvsControllerController{
		client:            kubeClient,
		reloadRateLimiter: util.NewTokenBucketRateLimiter(reloadQPS, int(reloadQPS)),
		ruCfg:             []vip{},
		configMapName:     configMapName,
	}

	podInfo, err := getPodDetails(kubeClient)
	if err != nil {
		glog.Fatalf("Error getting POD information: %v", err)
	}

	pod, err := kubeClient.Pods(podInfo.PodNamespace).Get(podInfo.PodName)
	if err != nil {
		glog.Fatalf("Error getting %v: %v", podInfo.PodName, err)
	}

	selector := parseNodeSelector(pod.Spec.NodeSelector)
	clusterNodes := getClusterNodesIP(kubeClient, selector)

	nodeInfo, err := getNetworkInfo(podInfo.NodeIP)
	if err != nil {
		glog.Fatalf("Error getting local IP from nodes in the cluster: %v", err)
	}

	neighbors := getNodeNeighbors(nodeInfo, clusterNodes)

	execer := exec.New()
	dbus := utildbus.New()
	iptInterface := utiliptables.New(execer, dbus, utiliptables.ProtocolIpv4)

	ipvsc.keepalived = &keepalived{
		iface:      nodeInfo.iface,
		ip:         nodeInfo.ip,
		netmask:    nodeInfo.netmask,
		nodes:      clusterNodes,
		neighbors:  neighbors,
		priority:   getNodePriority(nodeInfo.ip, clusterNodes),
		useUnicast: useUnicast,
		ipt:        iptInterface,
	}

	err = ipvsc.keepalived.loadTemplate()
	if err != nil {
		glog.Fatalf("Error loading keepalived template: %v", err)
	}

	eventHandlers := framework.ResourceEventHandlerFuncs{}

	ipvsc.svcLister.Store, ipvsc.svcController = framework.NewInformer(
		cache.NewListWatchFromClient(
			ipvsc.client, "services", namespace, fields.Everything()),
		&api.Service{}, resyncPeriod, eventHandlers)

	ipvsc.epLister.Store, ipvsc.epController = framework.NewInformer(
		cache.NewListWatchFromClient(
			ipvsc.client, "endpoints", namespace, fields.Everything()),
		&api.Endpoints{}, resyncPeriod, eventHandlers)

	return &ipvsc
}

func checksum(filename string) (string, error) {
	var result []byte
	file, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hash := md5.New()
	_, err = io.Copy(hash, file)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(hash.Sum(result)), nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package build

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"

	"k8s.io/test-infra/kind/pkg/build/kube"
	"k8s.io/test-infra/kind/pkg/build/sources"
	"k8s.io/test-infra/kind/pkg/exec"
)

// NodeImageBuildContext is used to build the kind node image, and contains
// build configuration
type NodeImageBuildContext struct {
	SourceDir string
	ImageTag  string
	Arch      string
	BaseImage string
	KubeRoot  string
	Bits      kube.Bits
}

// NewNodeImageBuildContext creates a new NodeImageBuildContext with
// default configuration
func NewNodeImageBuildContext(mode string) (ctx *NodeImageBuildContext, err error) {
	kubeRoot := ""
	// apt should not fail on finding kube root as it does not use it
	if mode != "apt" {
		kubeRoot, err = kube.FindSource()
		if err != nil {
			return nil, fmt.Errorf("error finding kuberoot: %v", err)
		}
	}
	bits, err := kube.NewNamedBits(mode, kubeRoot)
	if err != nil {
		return nil, err
	}
	return &NodeImageBuildContext{
		ImageTag:  "kind-node",
		Arch:      "amd64",
		BaseImage: "kind-base",
		KubeRoot:  kubeRoot,
		Bits:      bits,
	}, nil
}

// Build builds the cluster node image, the sourcedir must be set on
// the NodeImageBuildContext
func (c *NodeImageBuildContext) Build() (err error) {
	// ensure kubernetes build is up to date first
	glog.Infof("Starting to build Kubernetes")
	if err = c.Bits.Build(); err != nil {
		glog.Errorf("Failed to build Kubernetes: %v", err)
		return errors.Wrap(err, "failed to build kubernetes")
	}
	glog.Infof("Finished building Kubernetes")

	// create tempdir to build the image in
	tmpDir, err := TempDir("", "kind-node-image")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpDir)

	// populate with image sources
	// if SourceDir is unset, use the baked in sources
	buildDir := tmpDir
	if c.SourceDir == "" {
		// populate with image sources
		err = sources.RestoreAssets(buildDir, "images/node")
		if err != nil {
			return err
		}
		buildDir = filepath.Join(buildDir, "images", "node")

	} else {
		err = copyDir(c.SourceDir, buildDir)
		if err != nil {
			glog.Errorf("failed to copy sources to build dir %v", err)
			return err
		}
	}

	glog.Infof("Building node image in: %s", buildDir)

	// populate the kubernetes artifacts first
	if err := c.populateBits(buildDir); err != nil {
		return err
	}

	// then the perform the actual docker image build
	return c.buildImage(buildDir)
}

func (c *NodeImageBuildContext) populateBits(buildDir string) error {
	// always create bits dir
	bitsDir := path.Join(buildDir, "bits")
	if err := os.Mkdir(bitsDir, 0777); err != nil {
		return errors.Wrap(err, "failed to make bits dir")
	}
	// copy all bits from their source path to where we will COPY them into
	// the dockerfile, see images/node/Dockerfile
	bitPaths := c.Bits.Paths()
	for src, dest := range bitPaths {
		realDest := path.Join(bitsDir, dest)
		if err := copyFile(src, realDest); err != nil {
			return errors.Wrap(err, "failed to copy build artifact")
		}
	}
	return nil
}

// BuildContainerLabelKey is applied to each build container
const BuildContainerLabelKey = "io.k8s.test-infra.kind-build"

// private kube.InstallContext implementation, local to the image build
type installContext struct {
	basePath    string
	containerID string
}

var _ kube.InstallContext = &installContext{}

func (ic *installContext) BasePath() string {
	return ic.basePath
}

func (ic *installContext) Run(command string, args ...string) error {
	cmd := exec.Command("docker", "exec", ic.containerID, command)
	cmd.Args = append(cmd.Args, args...)
	cmd.Debug = true
	cmd.InheritOutput = true
	return cmd.Run()
}

func (ic *installContext) CombinedOutputLines(command string, args ...string) ([]string, error) {
	cmd := exec.Command("docker", "exec", ic.containerID, command)
	cmd.Args = append(cmd.Args, args...)
	cmd.Debug = true
	return cmd.CombinedOutputLines()
}

func (c *NodeImageBuildContext) buildImage(dir string) error {
	// build the image, tagged as tagImageAs, using the our tempdir as the context
	glog.Info("Starting image build ...")
	// create build container
	// NOTE: we are using docker run + docker commit so we can install
	// debians without permanently copying them into the image.
	// if docker gets proper squash support, we can rm them instead
	// This also allows the KubeBit implementations to perform programmatic
	// isntall in the image
	containerID, err := c.createBuildContainer(dir)
	if err != nil {
		glog.Errorf("Image build Failed! %v", err)
		return err
	}

	// ensure we will delete it
	defer func() {
		exec.Command("docker", "rm", "-f", containerID).Run()
	}()

	// helper we will use to run "build steps"
	execInBuild := func(command ...string) error {
		cmd := exec.Command("docker", "exec", containerID)
		cmd.Args = append(cmd.Args, command...)
		cmd.Debug = true
		cmd.InheritOutput = true
		return cmd.Run()
	}

	// make artifacts directory
	if err = execInBuild("mkdir", "/kind/"); err != nil {
		glog.Errorf("Image build Failed! %v", err)
		return err
	}

	// copy artifacts in
	if err = execInBuild("rsync", "-r", "/build/bits/", "/kind/"); err != nil {
		glog.Errorf("Image build Failed! %v", err)
		return err
	}

	// install the kube bits
	ic := &installContext{
		basePath:    "/kind/",
		containerID: containerID,
	}
	if err = c.Bits.Install(ic); err != nil {
		glog.Errorf("Image build Failed! %v", err)
		return err
	}

	// ensure we don't fail if swap is enabled on the host
	if err = execInBuild("/bin/sh", "-c",
		`echo "KUBELET_EXTRA_ARGS=--fail-swap-on=false" >> /etc/default/kubelet`,
	); err != nil {
		glog.Errorf("Image build Failed! %v", err)
		return err
	}

	// Save the image changes to a new image
	cmd := exec.Command("docker", "commit", containerID, c.ImageTag)
	cmd.Debug = true
	cmd.InheritOutput = true
	if err = cmd.Run(); err != nil {
		glog.Errorf("Image build Failed! %v", err)
		return err
	}

	glog.Info("Image build completed.")
	return nil
}

func (c *NodeImageBuildContext) createBuildContainer(buildDir string) (id string, err error) {
	cmd := exec.Command("docker", "run")
	cmd.Args = append(cmd.Args,
		"-d", // make the client exit while the container continues to run
		// label the container to make them easier to track
		"--label", fmt.Sprintf("%s=%s", BuildContainerLabelKey, time.Now().Format(time.RFC3339Nano)),
		"-v", fmt.Sprintf("%s:/build", buildDir),
		c.BaseImage,
	)
	cmd.Debug = true
	lines, err := cmd.CombinedOutputLines()
	if err != nil {
		return "", errors.Wrap(err, "failed to create build container")
	}
	if len(lines) != 1 {
		return "", fmt.Errorf("invalid container creation output: %v", lines)
	}
	return lines[0], nil
}
<commit_msg>don't populate non-existent assets when building node image<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package build

import (
	"fmt"
	"os"
	"path"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"

	"k8s.io/test-infra/kind/pkg/build/kube"
	"k8s.io/test-infra/kind/pkg/exec"
)

// NodeImageBuildContext is used to build the kind node image, and contains
// build configuration
type NodeImageBuildContext struct {
	ImageTag  string
	Arch      string
	BaseImage string
	KubeRoot  string
	Bits      kube.Bits
}

// NewNodeImageBuildContext creates a new NodeImageBuildContext with
// default configuration
func NewNodeImageBuildContext(mode string) (ctx *NodeImageBuildContext, err error) {
	kubeRoot := ""
	// apt should not fail on finding kube root as it does not use it
	if mode != "apt" {
		kubeRoot, err = kube.FindSource()
		if err != nil {
			return nil, fmt.Errorf("error finding kuberoot: %v", err)
		}
	}
	bits, err := kube.NewNamedBits(mode, kubeRoot)
	if err != nil {
		return nil, err
	}
	return &NodeImageBuildContext{
		ImageTag:  "kind-node",
		Arch:      "amd64",
		BaseImage: "kind-base",
		KubeRoot:  kubeRoot,
		Bits:      bits,
	}, nil
}

// Build builds the cluster node image, the sourcedir must be set on
// the NodeImageBuildContext
func (c *NodeImageBuildContext) Build() (err error) {
	// ensure kubernetes build is up to date first
	glog.Infof("Starting to build Kubernetes")
	if err = c.Bits.Build(); err != nil {
		glog.Errorf("Failed to build Kubernetes: %v", err)
		return errors.Wrap(err, "failed to build kubernetes")
	}
	glog.Infof("Finished building Kubernetes")

	// create tempdir to build the image in
	buildDir, err := TempDir("", "kind-node-image")
	if err != nil {
		return err
	}
	defer os.RemoveAll(buildDir)

	glog.Infof("Building node image in: %s", buildDir)

	// populate the kubernetes artifacts first
	if err := c.populateBits(buildDir); err != nil {
		return err
	}

	// then the perform the actual docker image build
	return c.buildImage(buildDir)
}

func (c *NodeImageBuildContext) populateBits(buildDir string) error {
	// always create bits dir
	bitsDir := path.Join(buildDir, "bits")
	if err := os.Mkdir(bitsDir, 0777); err != nil {
		return errors.Wrap(err, "failed to make bits dir")
	}
	// copy all bits from their source path to where we will COPY them into
	// the dockerfile, see images/node/Dockerfile
	bitPaths := c.Bits.Paths()
	for src, dest := range bitPaths {
		realDest := path.Join(bitsDir, dest)
		if err := copyFile(src, realDest); err != nil {
			return errors.Wrap(err, "failed to copy build artifact")
		}
	}
	return nil
}

// BuildContainerLabelKey is applied to each build container
const BuildContainerLabelKey = "io.k8s.test-infra.kind-build"

// private kube.InstallContext implementation, local to the image build
type installContext struct {
	basePath    string
	containerID string
}

var _ kube.InstallContext = &installContext{}

func (ic *installContext) BasePath() string {
	return ic.basePath
}

func (ic *installContext) Run(command string, args ...string) error {
	cmd := exec.Command("docker", "exec", ic.containerID, command)
	cmd.Args = append(cmd.Args, args...)
	cmd.Debug = true
	cmd.InheritOutput = true
	return cmd.Run()
}

func (ic *installContext) CombinedOutputLines(command string, args ...string) ([]string, error) {
	cmd := exec.Command("docker", "exec", ic.containerID, command)
	cmd.Args = append(cmd.Args, args...)
	cmd.Debug = true
	return cmd.CombinedOutputLines()
}

func (c *NodeImageBuildContext) buildImage(dir string) error {
	// build the image, tagged as tagImageAs, using the our tempdir as the context
	glog.Info("Starting image build ...")
	// create build container
	// NOTE: we are using docker run + docker commit so we can install
	// debians without permanently copying them into the image.
	// if docker gets proper squash support, we can rm them instead
	// This also allows the KubeBit implementations to perform programmatic
	// isntall in the image
	containerID, err := c.createBuildContainer(dir)
	if err != nil {
		glog.Errorf("Image build Failed! %v", err)
		return err
	}

	// ensure we will delete it
	defer func() {
		exec.Command("docker", "rm", "-f", containerID).Run()
	}()

	// helper we will use to run "build steps"
	execInBuild := func(command ...string) error {
		cmd := exec.Command("docker", "exec", containerID)
		cmd.Args = append(cmd.Args, command...)
		cmd.Debug = true
		cmd.InheritOutput = true
		return cmd.Run()
	}

	// make artifacts directory
	if err = execInBuild("mkdir", "/kind/"); err != nil {
		glog.Errorf("Image build Failed! %v", err)
		return err
	}

	// copy artifacts in
	if err = execInBuild("rsync", "-r", "/build/bits/", "/kind/"); err != nil {
		glog.Errorf("Image build Failed! %v", err)
		return err
	}

	// install the kube bits
	ic := &installContext{
		basePath:    "/kind/",
		containerID: containerID,
	}
	if err = c.Bits.Install(ic); err != nil {
		glog.Errorf("Image build Failed! %v", err)
		return err
	}

	// ensure we don't fail if swap is enabled on the host
	if err = execInBuild("/bin/sh", "-c",
		`echo "KUBELET_EXTRA_ARGS=--fail-swap-on=false" >> /etc/default/kubelet`,
	); err != nil {
		glog.Errorf("Image build Failed! %v", err)
		return err
	}

	// Save the image changes to a new image
	cmd := exec.Command("docker", "commit", containerID, c.ImageTag)
	cmd.Debug = true
	cmd.InheritOutput = true
	if err = cmd.Run(); err != nil {
		glog.Errorf("Image build Failed! %v", err)
		return err
	}

	glog.Info("Image build completed.")
	return nil
}

func (c *NodeImageBuildContext) createBuildContainer(buildDir string) (id string, err error) {
	cmd := exec.Command("docker", "run")
	cmd.Args = append(cmd.Args,
		"-d", // make the client exit while the container continues to run
		// label the container to make them easier to track
		"--label", fmt.Sprintf("%s=%s", BuildContainerLabelKey, time.Now().Format(time.RFC3339Nano)),
		"-v", fmt.Sprintf("%s:/build", buildDir),
		c.BaseImage,
	)
	cmd.Debug = true
	lines, err := cmd.CombinedOutputLines()
	if err != nil {
		return "", errors.Wrap(err, "failed to create build container")
	}
	if len(lines) != 1 {
		return "", fmt.Errorf("invalid container creation output: %v", lines)
	}
	return lines[0], nil
}
<|endoftext|>


<commit_before>package v1payload

//ClaimCapacityInput is input for capacity creation
type ClaimCapacityInput struct {
	ClusterID string `json:"cluster_id" valid:"required"`
}

//ClaimCapacityOutput is output for capacity creation
type ClaimCapacityOutput struct {
}

//ReleaseCapacityInput is input for capacity creation
type ReleaseCapacityInput struct {
	ClusterID  string `json:"cluster_id" valid:"required"`
	CapacityID string `json:"capacity_id" valid:"required"`
}

//ReleaseCapacityOutput is output for capacity creation
type ReleaseCapacityOutput struct {
}
<commit_msg>completed capacity output<commit_after>package v1payload

//ClaimCapacityInput is input for capacity creation
type ClaimCapacityInput struct {
	ClusterID string `json:"cluster_id" valid:"required"`
}

//ClaimCapacityOutput is output for capacity creation
type ClaimCapacityOutput struct {
	ClusterID  string `json:"cluster_id"`
	CapacityID string `json:"capacity_id"`
}

//ReleaseCapacityInput is input for capacity creation
type ReleaseCapacityInput struct {
	ClusterID  string `json:"cluster_id" valid:"required"`
	CapacityID string `json:"capacity_id" valid:"required"`
}

//ReleaseCapacityOutput is output for capacity creation
type ReleaseCapacityOutput struct {
}
<|endoftext|>


<commit_before>package cmds

import (
	"fmt"
	"math"
	"os"
	"path"
	"strings"
	"text/tabwriter"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/drive"
	"github.com/pachyderm/pachyderm/src/pfs/fuse"
	"github.com/pachyderm/pachyderm/src/pfs/pfsutil"
	"github.com/pachyderm/pachyderm/src/pfs/pretty"
	"github.com/spf13/cobra"
	"go.pedge.io/pkg/cobra"
	"google.golang.org/grpc"
)

func Cmds(address string) []*cobra.Command {
	var fileNumber int
	var fileModulus int
	var blockNumber int
	var blockModulus int
	shard := func() *pfs.Shard {
		return &pfs.Shard{
			FileNumber:   uint64(fileNumber),
			FileModulus:  uint64(fileModulus),
			BlockNumber:  uint64(blockNumber),
			BlockModulus: uint64(blockModulus),
		}
	}

	addShardFlags := func(cmd *cobra.Command) {
		cmd.Flags().IntVarP(&fileNumber, "file-shard", "s", 0, "file shard to read")
		cmd.Flags().IntVarP(&fileModulus, "file-modulus", "m", 1, "modulus of file shard")
		cmd.Flags().IntVarP(&blockNumber, "block-shard", "b", 0, "block shard to read")
		cmd.Flags().IntVarP(&blockModulus, "block-modulus", "n", 1, "modulus of block shard")
	}

	repo := &cobra.Command{
		Use:   "repo",
		Short: "Docs for repos.",
		Long: `Repos, short for repository, are the top level data object in Pachyderm.

Repos are created with create-repo.`,
	}

	createRepo := &cobra.Command{
		Use:   "create-repo repo-name",
		Short: "Create a new repo.",
		Long:  "Create a new repo.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			return pfsutil.CreateRepo(apiClient, args[0])
		}),
	}

	inspectRepo := &cobra.Command{
		Use:   "inspect-repo repo-name",
		Short: "Return info about a repo.",
		Long:  "Return info about a repo.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			repoInfo, err := pfsutil.InspectRepo(apiClient, args[0])
			if err != nil {
				return err
			}
			if repoInfo == nil {
				return fmt.Errorf("repo %s not found", args[0])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintRepoHeader(writer)
			pretty.PrintRepoInfo(writer, repoInfo)
			return writer.Flush()
		}),
	}

	listRepo := &cobra.Command{
		Use:   "list-repo",
		Short: "Return all repos.",
		Long:  "Reutrn all repos.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			repoInfos, err := pfsutil.ListRepo(apiClient)
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintRepoHeader(writer)
			for _, repoInfo := range repoInfos {
				pretty.PrintRepoInfo(writer, repoInfo)
			}
			return writer.Flush()
		}),
	}

	deleteRepo := &cobra.Command{
		Use:   "delete-repo repo-name",
		Short: "Delete a repo.",
		Long:  "Delete a repo.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			return pfsutil.DeleteRepo(apiClient, args[0])
		}),
	}

	commit := &cobra.Command{
		Use:   "commit",
		Short: "Docs for commits.",
		Long: `Commits are atomic transactions on the content of a repo.

Creating a commit is a multistep process:
- start a new commit with start-commit
- write files to it through fuse or with put-file
- finish the new commit with finish-commit

Commits that have been started but not finished are NOT durable storage.
Commits become reliable (and immutable) when they are finished.

Commits can be created with another commit as a parent.
This layers the data in the commit over the data in the parent.`,
	}

	startCommit := &cobra.Command{
		Use:   "start-commit repo-name [parent-commit-id]",
		Short: "Start a new commit.",
		Long:  "Start a new commit with parent-commit-id as the parent.",
		Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 1, Max: 2}, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			parentCommitID := ""
			if len(args) == 2 {
				parentCommitID = args[1]
			}
			commit, err := pfsutil.StartCommit(apiClient, args[0], parentCommitID)
			if err != nil {
				return err
			}
			fmt.Println(commit.Id)
			return nil
		}),
	}

	finishCommit := &cobra.Command{
		Use:   "finish-commit repo-name commit-id",
		Short: "Finish a started commit.",
		Long:  "Finish a started commit. Commit-id must be a writeable commit.",
		Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			return pfsutil.FinishCommit(apiClient, args[0], args[1])
		}),
	}

	inspectCommit := &cobra.Command{
		Use:   "inspect-commit repo-name commit-id",
		Short: "Return info about a commit.",
		Long:  "Return info about a commit.",
		Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			commitInfo, err := pfsutil.InspectCommit(apiClient, args[0], args[1])
			if err != nil {
				return err
			}
			if commitInfo == nil {
				return fmt.Errorf("commit %s not found", args[1])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintCommitInfoHeader(writer)
			pretty.PrintCommitInfo(writer, commitInfo)
			return writer.Flush()
		}),
	}

	listCommit := &cobra.Command{
		Use:   "list-commit repo-name",
		Short: "Return all commits on a repo.",
		Long:  "Return all commits on a repo.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			commitInfos, err := pfsutil.ListCommit(apiClient, args)
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintCommitInfoHeader(writer)
			for _, commitInfo := range commitInfos {
				pretty.PrintCommitInfo(writer, commitInfo)
			}
			return writer.Flush()
		}),
	}

	deleteCommit := &cobra.Command{
		Use:   "delete-commit repo-name commit-id",
		Short: "Delete a commit.",
		Long:  "Delete a commit.",
		Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			return pfsutil.DeleteCommit(apiClient, args[0], args[1])
		}),
	}

	mkdir := &cobra.Command{
		Use:   "mkdir repo-name commit-id path/to/dir",
		Short: "Make a directory.",
		Long:  "Make a directory. Parent directories need not exist.",
		Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			return pfsutil.MakeDirectory(apiClient, args[0], args[1], args[2])
		}),
	}

	file := &cobra.Command{
		Use:   "file",
		Short: "Docs for files.",
		Long: `Files are the lowest level data object in Pachyderm.

Files can be written to started (but not finished) commits with put-file.
Files can be read from finished commits with get-file.`,
	}

	putFile := &cobra.Command{
		Use:   "put-file repo-name commit-id path/to/file",
		Short: "Put a file from stdin",
		Long:  "Put a file from stdin. Directories must exist. commit-id must be a writeable commit.",
		Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			_, err = pfsutil.PutFile(apiClient, args[0], args[1], args[2], 0, os.Stdin)
			return err
		}),
	}

	getFile := &cobra.Command{
		Use:   "get-file repo-name commit-id path/to/file",
		Short: "Return the contents of a file.",
		Long:  "Return the contents of a file.",
		Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			return pfsutil.GetFile(apiClient, args[0], args[1], args[2], 0, math.MaxInt64, shard(), os.Stdout)
		}),
	}
	addShardFlags(getFile)

	inspectFile := &cobra.Command{
		Use:   "inspect-file repo-name commit-id path/to/file",
		Short: "Return info about a file.",
		Long:  "Return info about a file.",
		Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			fileInfo, err := pfsutil.InspectFile(apiClient, args[0], args[1], args[2], shard())
			if err != nil {
				return err
			}
			if fileInfo == nil {
				return fmt.Errorf("file %s not found", args[2])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintFileInfoHeader(writer)
			pretty.PrintFileInfo(writer, fileInfo)
			return writer.Flush()
		}),
	}
	addShardFlags(inspectFile)

	listFile := &cobra.Command{
		Use:   "list-file repo-name commit-id path/to/dir",
		Short: "Return the files in a directory.",
		Long:  "Return the files in a directory.",
		Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 2, Max: 3}, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			var path string
			if len(args) == 3 {
				path = args[2]
			}
			fileInfos, err := pfsutil.ListFile(apiClient, args[0], args[1], path, shard())
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintFileInfoHeader(writer)
			for _, fileInfo := range fileInfos {
				pretty.PrintFileInfo(writer, fileInfo)
			}
			return writer.Flush()
		}),
	}
	addShardFlags(listFile)

	deleteFile := &cobra.Command{
		Use:   "delete-file repo-name commit-id path/to/file",
		Short: "Delete a file.",
		Long:  "Delete a file.",
		Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			return pfsutil.DeleteFile(apiClient, args[0], args[1], args[2])
		}),
	}

	var mountPoint string
	mount := &cobra.Command{
		Use:   "mount [repo/commit:alias...]",
		Short: "Mount pfs locally.",
		Long:  "Mount pfs locally.",
		Run: pkgcobra.Run(func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			mounter := fuse.NewMounter(address, apiClient)
			return mounter.Mount(mountPoint, parseCommitMounts(args), nil)
		}),
	}
	mount.Flags().StringVarP(&mountPoint, "mount-point", "p", "/pfs", "root of mounted filesystem")

	var result []*cobra.Command
	result = append(result, repo)
	result = append(result, createRepo)
	result = append(result, inspectRepo)
	result = append(result, listRepo)
	result = append(result, deleteRepo)
	result = append(result, commit)
	result = append(result, startCommit)
	result = append(result, finishCommit)
	result = append(result, inspectCommit)
	result = append(result, listCommit)
	result = append(result, deleteCommit)
	result = append(result, mkdir)
	result = append(result, file)
	result = append(result, putFile)
	result = append(result, getFile)
	result = append(result, inspectFile)
	result = append(result, listFile)
	result = append(result, deleteFile)
	result = append(result, mount)
	return result
}

func getAPIClient(address string) (pfs.APIClient, error) {
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}
	return pfs.NewAPIClient(clientConn), nil
}

func getDriveAPIClient(address string) (drive.APIClient, error) {
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}
	return drive.NewAPIClient(clientConn), nil
}

func parseCommitMounts(args []string) []*fuse.CommitMount {
	var result []*fuse.CommitMount
	for _, arg := range args {
		commitMount := &fuse.CommitMount{Commit: pfsutil.NewCommit("", "")}
		repo, commitAlias := path.Split(arg)
		commitMount.Commit.Repo.Name = path.Clean(repo)
		split := strings.Split(commitAlias, ":")
		if len(split) > 0 {
			commitMount.Commit.Id = split[0]
		}
		if len(split) > 1 {
			commitMount.Alias = split[1]
		}
		result = append(result, commitMount)
	}
	return result
}
<commit_msg>Remove mkdir.<commit_after>package cmds

import (
	"fmt"
	"math"
	"os"
	"path"
	"strings"
	"text/tabwriter"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/drive"
	"github.com/pachyderm/pachyderm/src/pfs/fuse"
	"github.com/pachyderm/pachyderm/src/pfs/pfsutil"
	"github.com/pachyderm/pachyderm/src/pfs/pretty"
	"github.com/spf13/cobra"
	"go.pedge.io/pkg/cobra"
	"google.golang.org/grpc"
)

func Cmds(address string) []*cobra.Command {
	var fileNumber int
	var fileModulus int
	var blockNumber int
	var blockModulus int
	shard := func() *pfs.Shard {
		return &pfs.Shard{
			FileNumber:   uint64(fileNumber),
			FileModulus:  uint64(fileModulus),
			BlockNumber:  uint64(blockNumber),
			BlockModulus: uint64(blockModulus),
		}
	}

	addShardFlags := func(cmd *cobra.Command) {
		cmd.Flags().IntVarP(&fileNumber, "file-shard", "s", 0, "file shard to read")
		cmd.Flags().IntVarP(&fileModulus, "file-modulus", "m", 1, "modulus of file shard")
		cmd.Flags().IntVarP(&blockNumber, "block-shard", "b", 0, "block shard to read")
		cmd.Flags().IntVarP(&blockModulus, "block-modulus", "n", 1, "modulus of block shard")
	}

	repo := &cobra.Command{
		Use:   "repo",
		Short: "Docs for repos.",
		Long: `Repos, short for repository, are the top level data object in Pachyderm.

Repos are created with create-repo.`,
	}

	createRepo := &cobra.Command{
		Use:   "create-repo repo-name",
		Short: "Create a new repo.",
		Long:  "Create a new repo.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			return pfsutil.CreateRepo(apiClient, args[0])
		}),
	}

	inspectRepo := &cobra.Command{
		Use:   "inspect-repo repo-name",
		Short: "Return info about a repo.",
		Long:  "Return info about a repo.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			repoInfo, err := pfsutil.InspectRepo(apiClient, args[0])
			if err != nil {
				return err
			}
			if repoInfo == nil {
				return fmt.Errorf("repo %s not found", args[0])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintRepoHeader(writer)
			pretty.PrintRepoInfo(writer, repoInfo)
			return writer.Flush()
		}),
	}

	listRepo := &cobra.Command{
		Use:   "list-repo",
		Short: "Return all repos.",
		Long:  "Reutrn all repos.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			repoInfos, err := pfsutil.ListRepo(apiClient)
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintRepoHeader(writer)
			for _, repoInfo := range repoInfos {
				pretty.PrintRepoInfo(writer, repoInfo)
			}
			return writer.Flush()
		}),
	}

	deleteRepo := &cobra.Command{
		Use:   "delete-repo repo-name",
		Short: "Delete a repo.",
		Long:  "Delete a repo.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			return pfsutil.DeleteRepo(apiClient, args[0])
		}),
	}

	commit := &cobra.Command{
		Use:   "commit",
		Short: "Docs for commits.",
		Long: `Commits are atomic transactions on the content of a repo.

Creating a commit is a multistep process:
- start a new commit with start-commit
- write files to it through fuse or with put-file
- finish the new commit with finish-commit

Commits that have been started but not finished are NOT durable storage.
Commits become reliable (and immutable) when they are finished.

Commits can be created with another commit as a parent.
This layers the data in the commit over the data in the parent.`,
	}

	startCommit := &cobra.Command{
		Use:   "start-commit repo-name [parent-commit-id]",
		Short: "Start a new commit.",
		Long:  "Start a new commit with parent-commit-id as the parent.",
		Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 1, Max: 2}, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			parentCommitID := ""
			if len(args) == 2 {
				parentCommitID = args[1]
			}
			commit, err := pfsutil.StartCommit(apiClient, args[0], parentCommitID)
			if err != nil {
				return err
			}
			fmt.Println(commit.Id)
			return nil
		}),
	}

	finishCommit := &cobra.Command{
		Use:   "finish-commit repo-name commit-id",
		Short: "Finish a started commit.",
		Long:  "Finish a started commit. Commit-id must be a writeable commit.",
		Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			return pfsutil.FinishCommit(apiClient, args[0], args[1])
		}),
	}

	inspectCommit := &cobra.Command{
		Use:   "inspect-commit repo-name commit-id",
		Short: "Return info about a commit.",
		Long:  "Return info about a commit.",
		Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			commitInfo, err := pfsutil.InspectCommit(apiClient, args[0], args[1])
			if err != nil {
				return err
			}
			if commitInfo == nil {
				return fmt.Errorf("commit %s not found", args[1])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintCommitInfoHeader(writer)
			pretty.PrintCommitInfo(writer, commitInfo)
			return writer.Flush()
		}),
	}

	listCommit := &cobra.Command{
		Use:   "list-commit repo-name",
		Short: "Return all commits on a repo.",
		Long:  "Return all commits on a repo.",
		Run: pkgcobra.RunFixedArgs(1, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			commitInfos, err := pfsutil.ListCommit(apiClient, args)
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintCommitInfoHeader(writer)
			for _, commitInfo := range commitInfos {
				pretty.PrintCommitInfo(writer, commitInfo)
			}
			return writer.Flush()
		}),
	}

	deleteCommit := &cobra.Command{
		Use:   "delete-commit repo-name commit-id",
		Short: "Delete a commit.",
		Long:  "Delete a commit.",
		Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			return pfsutil.DeleteCommit(apiClient, args[0], args[1])
		}),
	}

	file := &cobra.Command{
		Use:   "file",
		Short: "Docs for files.",
		Long: `Files are the lowest level data object in Pachyderm.

Files can be written to started (but not finished) commits with put-file.
Files can be read from finished commits with get-file.`,
	}

	putFile := &cobra.Command{
		Use:   "put-file repo-name commit-id path/to/file",
		Short: "Put a file from stdin",
		Long:  "Put a file from stdin. Directories must exist. commit-id must be a writeable commit.",
		Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			_, err = pfsutil.PutFile(apiClient, args[0], args[1], args[2], 0, os.Stdin)
			return err
		}),
	}

	getFile := &cobra.Command{
		Use:   "get-file repo-name commit-id path/to/file",
		Short: "Return the contents of a file.",
		Long:  "Return the contents of a file.",
		Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			return pfsutil.GetFile(apiClient, args[0], args[1], args[2], 0, math.MaxInt64, shard(), os.Stdout)
		}),
	}
	addShardFlags(getFile)

	inspectFile := &cobra.Command{
		Use:   "inspect-file repo-name commit-id path/to/file",
		Short: "Return info about a file.",
		Long:  "Return info about a file.",
		Run: pkgcobra.RunFixedArgs(3, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			fileInfo, err := pfsutil.InspectFile(apiClient, args[0], args[1], args[2], shard())
			if err != nil {
				return err
			}
			if fileInfo == nil {
				return fmt.Errorf("file %s not found", args[2])
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintFileInfoHeader(writer)
			pretty.PrintFileInfo(writer, fileInfo)
			return writer.Flush()
		}),
	}
	addShardFlags(inspectFile)

	listFile := &cobra.Command{
		Use:   "list-file repo-name commit-id path/to/dir",
		Short: "Return the files in a directory.",
		Long:  "Return the files in a directory.",
		Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 2, Max: 3}, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			var path string
			if len(args) == 3 {
				path = args[2]
			}
			fileInfos, err := pfsutil.ListFile(apiClient, args[0], args[1], path, shard())
			if err != nil {
				return err
			}
			writer := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
			pretty.PrintFileInfoHeader(writer)
			for _, fileInfo := range fileInfos {
				pretty.PrintFileInfo(writer, fileInfo)
			}
			return writer.Flush()
		}),
	}
	addShardFlags(listFile)

	deleteFile := &cobra.Command{
		Use:   "delete-file repo-name commit-id path/to/file",
		Short: "Delete a file.",
		Long:  "Delete a file.",
		Run: pkgcobra.RunFixedArgs(2, func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			return pfsutil.DeleteFile(apiClient, args[0], args[1], args[2])
		}),
	}

	var mountPoint string
	mount := &cobra.Command{
		Use:   "mount [repo/commit:alias...]",
		Short: "Mount pfs locally.",
		Long:  "Mount pfs locally.",
		Run: pkgcobra.Run(func(args []string) error {
			apiClient, err := getAPIClient(address)
			if err != nil {
				return err
			}
			mounter := fuse.NewMounter(address, apiClient)
			return mounter.Mount(mountPoint, parseCommitMounts(args), nil)
		}),
	}
	mount.Flags().StringVarP(&mountPoint, "mount-point", "p", "/pfs", "root of mounted filesystem")

	var result []*cobra.Command
	result = append(result, repo)
	result = append(result, createRepo)
	result = append(result, inspectRepo)
	result = append(result, listRepo)
	result = append(result, deleteRepo)
	result = append(result, commit)
	result = append(result, startCommit)
	result = append(result, finishCommit)
	result = append(result, inspectCommit)
	result = append(result, listCommit)
	result = append(result, deleteCommit)
	result = append(result, file)
	result = append(result, putFile)
	result = append(result, getFile)
	result = append(result, inspectFile)
	result = append(result, listFile)
	result = append(result, deleteFile)
	result = append(result, mount)
	return result
}

func getAPIClient(address string) (pfs.APIClient, error) {
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}
	return pfs.NewAPIClient(clientConn), nil
}

func getDriveAPIClient(address string) (drive.APIClient, error) {
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}
	return drive.NewAPIClient(clientConn), nil
}

func parseCommitMounts(args []string) []*fuse.CommitMount {
	var result []*fuse.CommitMount
	for _, arg := range args {
		commitMount := &fuse.CommitMount{Commit: pfsutil.NewCommit("", "")}
		repo, commitAlias := path.Split(arg)
		commitMount.Commit.Repo.Name = path.Clean(repo)
		split := strings.Split(commitAlias, ":")
		if len(split) > 0 {
			commitMount.Commit.Id = split[0]
		}
		if len(split) > 1 {
			commitMount.Alias = split[1]
		}
		result = append(result, commitMount)
	}
	return result
}
<|endoftext|>


<commit_before>package sippy_header

type normalName struct {
    name    string
}

func newNormalName(name string) normalName {
    return normalName{
        name : name,
    }
}

func (self normalName) Name() string {
    return self.name
}

func (self normalName) CompactName() string {
    return self.name
}

type compactName struct {
    name            string
    compact_name    string
}

func newCompactName(name, compact_name string) compactName {
    return compactName{
        name         : name,
        compact_name : compact_name,
    }
}

func (self compactName) Name() string {
    return self.name
}

func (self compactName) CompactName() string {
    return self.compact_name
}
<commit_msg>Add copyright header.<commit_after>// Copyright (c) 2015 Andrii Pylypenko. All rights reserved.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation and/or
// other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package sippy_header

type normalName struct {
    name    string
}

func newNormalName(name string) normalName {
    return normalName{
        name : name,
    }
}

func (self normalName) Name() string {
    return self.name
}

func (self normalName) CompactName() string {
    return self.name
}

type compactName struct {
    name            string
    compact_name    string
}

func newCompactName(name, compact_name string) compactName {
    return compactName{
        name         : name,
        compact_name : compact_name,
    }
}

func (self compactName) Name() string {
    return self.name
}

func (self compactName) CompactName() string {
    return self.compact_name
}
<|endoftext|>


<commit_before>// Package tinystat provides the ability to compare data sets using Student's
// t-test at various levels of confidence.
package tinystat

import (
	"math"

	"gonum.org/v1/gonum/stat/distuv"
)

// A Summary is a statistical summary of a normally distributed data set.
type Summary struct {
	N        float64 // N is the number of measurements in the set.
	Mean     float64 // Mean is the arithmetic mean of the measurements.
	Variance float64 // Variance is the sample variance of the data set.
}

// Summarize analyzes the given data set and returns a Summary.
func Summarize(data []float64) Summary {
	// Welford algorithm for corrected variance
	var m, m2 float64

	for n, x := range data {
		delta := x - m
		m += delta / float64(n+1)
		m2 += delta * (x - m)
	}

	return Summary{
		Mean:     m,
		Variance: m2 / float64(len(data)-1), // Bessel's correction
		N:        float64(len(data)),
	}
}

// Difference represents the statistical difference between two samples.
type Difference struct {
	Delta    float64 // Delta is the difference between the samples' means.
	Error    float64 // Error is the margin of error at the given confidence level.
	RelDelta float64 // RelDelta is the ratio of Delta to the control mean.
	RelError float64 // RelError is the ratio of Error to the control mean.
	StdDev   float64 // StdDev is the pooled standard deviation of the two samples.
}

// Significant returns true if the difference is statistically significant.
func (d Difference) Significant() bool {
	return d.Delta > d.Error
}

// Compare returns the statistical difference between the two summaries using a
// two-tailed Student's t-test. The confidence level must be in the range (0,
// 100).
func Compare(control, experiment Summary, confidence float64) Difference {
	a, b := control, experiment
	nu := a.N + b.N - 2

	dist := distuv.StudentsT{
		Mu:    0,
		Sigma: 1,
		Nu:    nu,
	}
	t := dist.Quantile(1 - ((1 - (confidence / 100)) / 2))
	s := math.Sqrt(
		((a.N-1)*a.Variance + (b.N-1)*b.Variance) /
			(a.N + b.N - 2),
	)
	d := math.Abs(a.Mean - b.Mean)
	e := t * s * math.Sqrt(1.0/a.N+1.0/b.N)

	return Difference{
		Delta:    d,
		Error:    e,
		RelDelta: d / control.Mean,
		RelError: e / control.Mean,
		StdDev:   s,
	}
}
<commit_msg>Refactor things a little.<commit_after>// Package tinystat provides the ability to compare data sets using Student's
// t-test at various levels of confidence.
package tinystat

import (
	"math"

	"gonum.org/v1/gonum/stat/distuv"
)

// A Summary is a statistical summary of a normally distributed data set.
type Summary struct {
	N        float64 // N is the number of measurements in the set.
	Mean     float64 // Mean is the arithmetic mean of the measurements.
	Variance float64 // Variance is the sample variance of the data set.
}

// Summarize analyzes the given data set and returns a Summary.
func Summarize(data []float64) Summary {
	n := float64(len(data))
	m, v := meanAndVariance(data, n)

	return Summary{
		Mean:     m,
		Variance: v,
		N:        n,
	}
}

// Difference represents the statistical difference between two samples.
type Difference struct {
	Delta    float64 // Delta is the difference between the samples' means.
	Error    float64 // Error is the margin of error at the given confidence level.
	RelDelta float64 // RelDelta is the ratio of Delta to the control mean.
	RelError float64 // RelError is the ratio of Error to the control mean.
	StdDev   float64 // StdDev is the pooled standard deviation of the two samples.
}

// Significant returns true if the difference is statistically significant.
func (d Difference) Significant() bool {
	return d.Delta > d.Error
}

// Compare returns the statistical difference between the two summaries using a
// two-tailed Student's t-test. The confidence level must be in the range (0,
// 100).
func Compare(control, experiment Summary, confidence float64) Difference {
	a, b := control, experiment
	nu := a.N + b.N - 2
	t := distuv.StudentsT{
		Mu:    0, // Standard parameters for Student's T.
		Sigma: 1,
		Nu:    nu,
	}.Quantile(1 - ((1 - (confidence / 100)) / 2))
	s := math.Sqrt(((a.N-1)*a.Variance + (b.N-1)*b.Variance) / nu)
	d := math.Abs(a.Mean - b.Mean)
	e := t * s * math.Sqrt(1.0/a.N+1.0/b.N)

	return Difference{
		Delta:    d,
		Error:    e,
		RelDelta: d / control.Mean,
		RelError: e / control.Mean,
		StdDev:   s,
	}
}

func meanAndVariance(data []float64, n float64) (float64, float64) {
	var m, m2 float64

	// Welford algorithm for corrected variance
	for i, x := range data {
		delta := x - m
		m += delta / float64(i+1)
		m2 += delta * (x - m)
	}

	return m, m2 / (n - 1) // Bessel's correction
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

/*
	Floating-point hyperbolic tangent.

	Sinh and Cosh are called except for large arguments, which
	would cause overflow improperly.
*/

// Tanh computes the hyperbolic tangent of x.
//
// Special cases are:
//	Tanh(±0) = ±0
//	Tanh(±Inf) = ±1
//	Tanh(NaN) = NaN
func Tanh(x float64) float64 {
	if x < 0 {
		x = -x
		if x > 21 {
			return -1
		}
		return -Sinh(x) / Cosh(x)
	}
	if x > 21 {
		return 1
	}
	return Sinh(x) / Cosh(x)
}
<commit_msg>math: Faster Tanh<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

// The original C code, the long comment, and the constants
// below were from http://netlib.sandia.gov/cephes/cmath/sin.c,
// available from http://www.netlib.org/cephes/cmath.tgz.
// The go code is a simplified version of the original C.
//      tanh.c
//
//      Hyperbolic tangent
//
// SYNOPSIS:
//
// double x, y, tanh();
//
// y = tanh( x );
//
// DESCRIPTION:
//
// Returns hyperbolic tangent of argument in the range MINLOG to MAXLOG.
//      MAXLOG = 8.8029691931113054295988e+01 = log(2**127)
//      MINLOG = -8.872283911167299960540e+01 = log(2**-128)
//
// A rational function is used for |x| < 0.625.  The form
// x + x**3 P(x)/Q(x) of Cody & Waite is employed.
// Otherwise,
//      tanh(x) = sinh(x)/cosh(x) = 1  -  2/(exp(2x) + 1).
//
// ACCURACY:
//
//                      Relative error:
// arithmetic   domain     # trials      peak         rms
//    IEEE      -2,2        30000       2.5e-16     5.8e-17
//
// Cephes Math Library Release 2.8:  June, 2000
// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier
//
// The readme file at http://netlib.sandia.gov/cephes/ says:
//    Some software in this archive may be from the book _Methods and
// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster
// International, 1989) or from the Cephes Mathematical Library, a
// commercial product. In either event, it is copyrighted by the author.
// What you see here may be used freely but it comes with no support or
// guarantee.
//
//   The two known misprints in the book are repaired here in the
// source listings for the gamma function and the incomplete beta
// integral.
//
//   Stephen L. Moshier
//   moshier@na-net.ornl.gov
//

var tanhP = [...]float64{
	-9.64399179425052238628E-1,
	-9.92877231001918586564E1,
	-1.61468768441708447952E3,
}
var tanhQ = [...]float64{
	1.12811678491632931402E2,
	2.23548839060100448583E3,
	4.84406305325125486048E3,
}

// Tanh computes the hyperbolic tangent of x.
//
// Special cases are:
//	Tanh(±0) = ±0
//	Tanh(±Inf) = ±1
//	Tanh(NaN) = NaN
func Tanh(x float64) float64 {
	const MAXLOG = 8.8029691931113054295988e+01 // log(2**127)
	z := Abs(x)
	switch {
	case z > 0.5*MAXLOG:
		if x < 0 {
			return -1
		}
		return 1
	case z >= 0.625:
		s := Exp(2 * z)
		z = 1 - 2/(s+1)
		if x < 0 {
			z = -z
		}
	default:
		if x == 0 {
			return x
		}
		s := x * x
		z = x + x*s*((tanhP[0]*s+tanhP[1])*s+tanhP[2])/(((s+tanhQ[0])*s+tanhQ[1])*s+tanhQ[2])
	}
	return z
}
<|endoftext|>


<commit_before>package plugin

/*

Here are a bunch of frameworky-helper functions for use when creating a new backup/restore plugin. Important things to remember:

Use plugin.Run()  for starting your plugin execution.

Use plugin.PluginInfo to fill out the info for your plugin.
Make your plugin conform to the Plugin interface, by implementing Backup(), Restore(), Retrieve(), and Store(). If they don't make sense, just return plugin.UNSUPPORTED_ACTION, and a helpful errorm essage

plugin.Exec() can be used to easily run external commands sending their stdin/stdout to that of the plugin command. Keep in mind the commands don't get run in a shell, so things like '>', '<', '|' won't work the way you want them to, but you can just run /bin/bash -c <command> to solve that, right?

*/

import (
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"fmt"
	"github.com/voxelbrain/goptions"
	"os"
	"strings"
)

type PluginOpts struct {
	Debug   bool `goptions:"-D, --debug, description='Enable debugging'"`
	Version bool `goptions:"-v, --version, description='Display version information'"`
	Action  goptions.Verbs
	Info    struct {
	} `goptions:"info"`
	Backup struct {
		Endpoint string `goptions:"-e, --endpoint, obligatory, description='JSON string representing backup target'"`
	} `goptions:"backup"`
	Restore struct {
		Endpoint string `goptions:"-e, --endpoint, obligatory, description='JSON string representing backup target'"`
	} `goptions:"restore"`
	Store struct {
		Endpoint string `goptions:"-e, --endpoint, obligatory, description='JSON string representing store endpoint'"`
	} `goptions:"store"`
	Retrieve struct {
		Endpoint string `goptions:"-e, --endpoint, obligatory, description='JSON string representing retrieve endpoint'"`
		Key      string `goptions:"-k, --key, obligatory, description='Key of blob to retrieve from storage'"`
	} `goptions:"retrieve"`
	Purge struct {
		Endpoint string `goptions:"-e, --endpoint, obligatory, description='JSON string representing purge endpoint'"`
		Key      string `goptions:"-k, --key, obligatory, description='Key of blob to purge from storage'"`
	} `goptions:"purge"`
}

type Plugin interface {
	Backup(ShieldEndpoint) (int, error)
	Restore(ShieldEndpoint) (int, error)
	Store(ShieldEndpoint) (string, int, error)
	Retrieve(ShieldEndpoint, string) (int, error)
	Purge(ShieldEndpoint, string) (int, error)
	Meta() PluginInfo
}

type PluginInfo struct {
	Name     string         `json:"name"`
	Author   string         `json:"author"`
	Version  string         `json:"version"`
	Features PluginFeatures `json:"features"`
}

type PluginFeatures struct {
	Target string `json:"target"`
	Store  string `json:"store"`
}

var debug bool

func DEBUG(format string, args ...interface{}) {
	if debug {
		content := fmt.Sprintf(format, args...)
		lines := strings.Split(content, "\n")
		for i, line := range lines {
			lines[i] = "DEBUG> " + line
		}
		content = strings.Join(lines, "\n")
		fmt.Fprintf(stderr, "%s\n", content)
	}
}

var stdout = os.Stdout
var stderr = os.Stderr

var usage = func(err error) {
	fmt.Fprintf(stderr, "%s\n", err.Error())
	goptions.PrintHelp()
}
var exit = func(code int) {
	os.Exit(code)
}

func Run(p Plugin) {
	var code int
	var action string

	opts, err := getPluginOptions()
	if err != nil {
		usage(err)
		code = USAGE
	} else {
		action = string(opts.Action)

		if action == "info" {
			code, err = pluginInfo(p)
		} else if action != "" {
			code, err = dispatch(p, action, opts)
		} else {
			code = USAGE
			usage(fmt.Errorf("No plugin action was provided"))
		}

		if err != nil {
			fmt.Fprintf(stderr, "%s\n", err.Error())
		}
	}
	exit(code)
}

func dispatch(p Plugin, mode string, opts PluginOpts) (int, error) {
	var code int
	var err error
	var key string

	DEBUG("'%s' action requested with options %#v", mode, opts)

	switch mode {
	case "backup":
		endpoint, err := getEndpoint(opts.Backup.Endpoint)
		if err != nil {
			return JSON_FAILURE, fmt.Errorf("Error trying parse --endpoint value as JSON: %s", err.Error())
		}
		code, err = p.Backup(endpoint)
	case "restore":
		endpoint, err := getEndpoint(opts.Restore.Endpoint)
		if err != nil {
			return JSON_FAILURE, fmt.Errorf("Error trying parse --endpoint value as JSON: %s", err.Error())
		}
		code, err = p.Restore(endpoint)
	case "store":
		endpoint, err := getEndpoint(opts.Store.Endpoint)
		if err != nil {
			return JSON_FAILURE, fmt.Errorf("Error trying parse --endpoint value as JSON: %s", err.Error())
		}
		key, code, err = p.Store(endpoint)
		output, err := json.Marshal(struct {
			Key string `json:"key"`
		}{Key: key})
		if err != nil {
			return JSON_FAILURE, err
		}
		fmt.Fprintf(stdout, "%s\n", string(output))
	case "retrieve":
		endpoint, err := getEndpoint(opts.Retrieve.Endpoint)
		if err != nil {
			return JSON_FAILURE, fmt.Errorf("Error trying parse --endpoint value as JSON: %s", err.Error())
		}
		if opts.Retrieve.Key == "" {
			return RESTORE_KEY_REQUIRED, fmt.Errorf("retrieving requires --key, but it was not provided")
		}
		code, err = p.Retrieve(endpoint, opts.Retrieve.Key)
	case "purge":
		endpoint, err := getEndpoint(opts.Purge.Endpoint)
		if err != nil {
			return JSON_FAILURE, fmt.Errorf("Error trying parse --endpoint value as JSON: %s", err.Error())
		}
		if opts.Purge.Key == "" {
			return RESTORE_KEY_REQUIRED, fmt.Errorf("purging requires --key, but it was not provided")
		}
		code, err = p.Purge(endpoint, opts.Purge.Key)
	default:
		return UNSUPPORTED_ACTION, fmt.Errorf("Sorry, '%s' is not a supported action for S.H.I.E.L.D plugins", mode)
	}

	DEBUG("'%s' action returned %d", mode, code)
	return code, err
}

func getPluginOptions() (PluginOpts, error) {
	var opts PluginOpts
	err := goptions.Parse(&opts)
	if err != nil {
		return opts, err
	}

	if os.Getenv("DEBUG") != "" && strings.ToLower(os.Getenv("DEBUG")) != "false" && os.Getenv("DEBUG") != "0" {
		debug = true
	}

	if opts.Debug {
		debug = true
	}

	return opts, err
}

func pluginInfo(p Plugin) (int, error) {
	json, err := json.Marshal(p.Meta())
	if err != nil {
		return JSON_FAILURE, fmt.Errorf("Could not create plugin metadata output: %s", err.Error())
	}
	fmt.Fprintf(stdout, "%s\n", json)
	return SUCCESS, nil
}

func GenUUID() string {
	return uuid.New()
}
<commit_msg>Fixed scope issue preventing proper plugin error output, Fixed JSON formatting<commit_after>package plugin

/*

Here are a bunch of frameworky-helper functions for use when creating a new backup/restore plugin. Important things to remember:

Use plugin.Run()  for starting your plugin execution.

Use plugin.PluginInfo to fill out the info for your plugin.
Make your plugin conform to the Plugin interface, by implementing Backup(), Restore(), Retrieve(), and Store(). If they don't make sense, just return plugin.UNSUPPORTED_ACTION, and a helpful errorm essage

plugin.Exec() can be used to easily run external commands sending their stdin/stdout to that of the plugin command. Keep in mind the commands don't get run in a shell, so things like '>', '<', '|' won't work the way you want them to, but you can just run /bin/bash -c <command> to solve that, right?

*/

import (
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"fmt"
	"github.com/voxelbrain/goptions"
	"os"
	"strings"
)

type PluginOpts struct {
	Debug   bool `goptions:"-D, --debug, description='Enable debugging'"`
	Version bool `goptions:"-v, --version, description='Display version information'"`
	Action  goptions.Verbs
	Info    struct {
	} `goptions:"info"`
	Backup struct {
		Endpoint string `goptions:"-e, --endpoint, obligatory, description='JSON string representing backup target'"`
	} `goptions:"backup"`
	Restore struct {
		Endpoint string `goptions:"-e, --endpoint, obligatory, description='JSON string representing backup target'"`
	} `goptions:"restore"`
	Store struct {
		Endpoint string `goptions:"-e, --endpoint, obligatory, description='JSON string representing store endpoint'"`
	} `goptions:"store"`
	Retrieve struct {
		Endpoint string `goptions:"-e, --endpoint, obligatory, description='JSON string representing retrieve endpoint'"`
		Key      string `goptions:"-k, --key, obligatory, description='Key of blob to retrieve from storage'"`
	} `goptions:"retrieve"`
	Purge struct {
		Endpoint string `goptions:"-e, --endpoint, obligatory, description='JSON string representing purge endpoint'"`
		Key      string `goptions:"-k, --key, obligatory, description='Key of blob to purge from storage'"`
	} `goptions:"purge"`
}

type Plugin interface {
	Backup(ShieldEndpoint) (int, error)
	Restore(ShieldEndpoint) (int, error)
	Store(ShieldEndpoint) (string, int, error)
	Retrieve(ShieldEndpoint, string) (int, error)
	Purge(ShieldEndpoint, string) (int, error)
	Meta() PluginInfo
}

type PluginInfo struct {
	Name     string         `json:"name"`
	Author   string         `json:"author"`
	Version  string         `json:"version"`
	Features PluginFeatures `json:"features"`
}

type PluginFeatures struct {
	Target string `json:"target"`
	Store  string `json:"store"`
}

var debug bool

func DEBUG(format string, args ...interface{}) {
	if debug {
		content := fmt.Sprintf(format, args...)
		lines := strings.Split(content, "\n")
		for i, line := range lines {
			lines[i] = "DEBUG> " + line
		}
		content = strings.Join(lines, "\n")
		fmt.Fprintf(stderr, "%s\n", content)
	}
}

var stdout = os.Stdout
var stderr = os.Stderr

var usage = func(err error) {
	fmt.Fprintf(stderr, "%s\n", err.Error())
	goptions.PrintHelp()
}
var exit = func(code int) {
	os.Exit(code)
}

func Run(p Plugin) {
	var code int
	var action string

	opts, err := getPluginOptions()
	if err != nil {
		usage(err)
		code = USAGE
	} else {
		action = string(opts.Action)

		if action == "info" {
			code, err = pluginInfo(p)
			if err != nil {
				fmt.Fprintf(stderr, "%s\n", err.Error())
			}
		} else if action != "" {
			code, err = dispatch(p, action, opts)
			if err != nil {
				fmt.Fprintf(stderr, "%s\n", err.Error())
			}
		} else {
			code = USAGE
			usage(fmt.Errorf("No plugin action was provided"))
		}
	}
	exit(code)
}

func dispatch(p Plugin, mode string, opts PluginOpts) (int, error) {
	var code int
	var err error
	var key string
	var endpoint ShieldEndpoint

	DEBUG("'%s' action requested with options %#v", mode, opts)

	switch mode {
	case "backup":
		endpoint, err = getEndpoint(opts.Backup.Endpoint)
		if err != nil {
			return JSON_FAILURE, fmt.Errorf("Error trying parse --endpoint value as JSON: %s", err.Error())
		}
		code, err = p.Backup(endpoint)
	case "restore":
		endpoint, err = getEndpoint(opts.Restore.Endpoint)
		if err != nil {
			return JSON_FAILURE, fmt.Errorf("Error trying parse --endpoint value as JSON: %s", err.Error())
		}
		code, err = p.Restore(endpoint)
	case "store":
		endpoint, err = getEndpoint(opts.Store.Endpoint)
		if err != nil {
			return JSON_FAILURE, fmt.Errorf("Error trying parse --endpoint value as JSON: %s", err.Error())
		}
		key, code, err = p.Store(endpoint)
		var output []byte
		output, err = json.MarshalIndent(struct {
			Key string `json:"key"`
		}{Key: key}, "", "    ")
		if err != nil {
			return JSON_FAILURE, err
		}
		fmt.Fprintf(stdout, "%s\n", string(output))
	case "retrieve":
		endpoint, err = getEndpoint(opts.Retrieve.Endpoint)
		if err != nil {
			return JSON_FAILURE, fmt.Errorf("Error trying parse --endpoint value as JSON: %s", err.Error())
		}
		if opts.Retrieve.Key == "" {
			return RESTORE_KEY_REQUIRED, fmt.Errorf("retrieving requires --key, but it was not provided")
		}
		code, err = p.Retrieve(endpoint, opts.Retrieve.Key)
	case "purge":
		endpoint, err = getEndpoint(opts.Purge.Endpoint)
		if err != nil {
			return JSON_FAILURE, fmt.Errorf("Error trying parse --endpoint value as JSON: %s", err.Error())
		}
		if opts.Purge.Key == "" {
			return RESTORE_KEY_REQUIRED, fmt.Errorf("purging requires --key, but it was not provided")
		}
		code, err = p.Purge(endpoint, opts.Purge.Key)
	default:
		return UNSUPPORTED_ACTION, fmt.Errorf("Sorry, '%s' is not a supported action for S.H.I.E.L.D plugins", mode)
	}

	DEBUG("'%s' action returned %d", mode, code)
	return code, err
}

func getPluginOptions() (PluginOpts, error) {
	var opts PluginOpts
	err := goptions.Parse(&opts)
	if err != nil {
		return opts, err
	}

	if os.Getenv("DEBUG") != "" && strings.ToLower(os.Getenv("DEBUG")) != "false" && os.Getenv("DEBUG") != "0" {
		debug = true
	}

	if opts.Debug {
		debug = true
	}

	return opts, err
}

func pluginInfo(p Plugin) (int, error) {
	json, err := json.MarshalIndent(p.Meta(), "", "    ")
	if err != nil {
		return JSON_FAILURE, fmt.Errorf("Could not create plugin metadata output: %s", err.Error())
	}
	fmt.Fprintf(stdout, "%s\n", json)
	return SUCCESS, nil
}

func GenUUID() string {
	return uuid.New()
}
<|endoftext|>