<commit_before>package slave

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/codeskyblue/go-sh"
	"github.com/gobuild/gobuild2/models"
	"github.com/gobuild/gobuild2/pkg/xrpc"
	"github.com/gobuild/log"
)

var (
	TMPDIR     = "./tmp"
	PROGRAM, _ = filepath.Abs(os.Args[0])
	HOSTNAME   = "localhost"
	HOSTINFO   = &xrpc.HostInfo{Os: runtime.GOOS, Arch: runtime.GOARCH, Host: HOSTNAME}
)

func checkError(err error) {
	if err != nil {
		log.Errorf("err: %v", err)
	}
}

type NTMsg struct {
	Status string
	Output string
	Extra  string
}

func GoInterval(dur time.Duration, f func()) chan bool {
	done := make(chan bool)
	go func() {
		for {
			select {
			case <-done:
				return
			case <-time.After(dur):
				f()
			}
		}
	}()
	return done
}

func work(m *xrpc.Mission) (err error) {
	notify := func(status string, output string, extra ...string) {
		mstatus := &xrpc.MissionStatus{Mid: m.Mid, Status: status,
			Output: output,
			Extra:  strings.Join(extra, ""),
		}
		ok := false
		err := xrpc.Call("UpdateMissionStatus", mstatus, &ok)
		checkError(err)
	}
	defer func() {
		fmt.Println("DONE", err)
		if err != nil {
			notify(models.ST_ERROR, err.Error())
		}
	}()
	// prepare shell session
	sess := sh.NewSession()
	buffer := bytes.NewBuffer(nil)
	sess.Stdout = io.MultiWriter(buffer, os.Stdout)
	sess.Stderr = io.MultiWriter(buffer, os.Stderr)
	sess.ShowCMD = true
	var gopath, _ = filepath.Abs(TMPDIR)
	if !sh.Test("dir", gopath) {
		os.MkdirAll(gopath, 0755)
	}
	sess.SetEnv("GOPATH", gopath)
	sess.SetEnv("CGO_ENABLE", "0")
	if m.CgoEnable {
		sess.SetEnv("CGO_ENABLE", "1")
	}
	sess.SetEnv("GOOS", m.Os)
	sess.SetEnv("GOARCH", m.Arch)

	var repoName = m.Repo
	var srcPath = filepath.Join(gopath, "src", repoName)

	getsrc := func() (err error) {
		var params []interface{}
		params = append(params, "get", "-v", "-g")
		if m.Sha != "" {
			params = append(params, repoName+"@commit:"+m.Sha)
		} else {
			params = append(params, repoName+"@branch:"+m.Branch)
		}
		params = append(params, sh.Dir(gopath))
		if err = sess.Command("gopm", params...).Run(); err != nil {
			return
		}
		return nil
	}

	newNotify := func(status string, buf *bytes.Buffer) chan bool {
		return GoInterval(time.Second*2, func() {
			notify(status, string(buf.Bytes()))
		})
	}

	notify(models.ST_RETRIVING, "start get source")
	var done chan bool
	done = newNotify(models.ST_RETRIVING, buffer)
	err = getsrc()
	done <- true
	notify(models.ST_RETRIVING, string(buffer.Bytes()))
	if err != nil {
		log.Errorf("getsource err: %v", err)
		return
	}
	buffer.Reset()

	// extention := "zip"
	// var outFile = m.UpKey // fmt.Sprintf("%s-%s-%s.%s", filepath.Base(repoName), m.Os, m.Arch, extention)
	var outFile = filepath.Base(m.UpKey)
	var outFullPath = filepath.Join(srcPath, outFile)

	// notify(models.ST_BUILDING, "start building")
	done = newNotify(models.ST_BUILDING, buffer)
	err = sess.Command("gopm", "build", "-u", "-v", sh.Dir(srcPath)).Run()
	done <- true
	notify(models.ST_BUILDING, string(buffer.Bytes()))
	if err != nil {
		log.Errorf("gopm build error: %v", err)
		return
	}
	buffer.Reset()

	// write extra pkginfo
	pkginfo := "pkginfo.json"
	ioutil.WriteFile(filepath.Join(srcPath, pkginfo), m.PkgInfo, 0644)

	err = sess.Command(PROGRAM, "pack",
		"--nobuild", "-a", pkginfo, "-o", outFile, sh.Dir(srcPath)).Run()
	notify(models.ST_PACKING, string(buffer.Bytes()))
	if err != nil {
		log.Error(err)
		return
	}

	var cdnPath = m.UpKey
	notify(models.ST_PUBLISHING, cdnPath)
	log.Infof("cdn path: %s", cdnPath)
	q := &Qiniu{m.UpToken, m.UpKey, m.Bulket} // uptoken, key}
	var pubAddr string
	if pubAddr, err = q.Upload(outFullPath); err != nil {
		checkError(err)
		return
	}

	log.Debugf("publish %s to %s", outFile, pubAddr)
	notify(models.ST_DONE, pubAddr)
	return nil
}

func init() {
	var err error
	HOSTNAME, err = os.Hostname()
	if err != nil {
		log.Fatalf("hostname retrive err: %v", err)
	}
}

var IsPrivateUpload bool //todo

func prepare() (err error) {
	qi := new(xrpc.QiniuInfo)
	if err = xrpc.Call("GetQiniuInfo", HOSTINFO, qi); err != nil {
		return
	}
	initQiniu(qi.AccessKey, qi.SecretKey, qi.Bulket)

	TMPDIR, err = filepath.Abs(TMPDIR)
	if err != nil {
		log.Errorf("tmpdir to abspath err: %v", err)
		return
	}
	if !sh.Test("dir", TMPDIR) {
		os.MkdirAll(TMPDIR, 0755)
	}
	startWork()
	return nil
}

func Action(c *cli.Context) {
	fmt.Println("this is slave daemon")
	webaddr := c.String("webaddr")
	xrpc.DefaultWebAddress = webaddr

	if err := prepare(); err != nil {
		log.Fatalf("slave prepare err: %v", err)
	}
	for {
		mission := &xrpc.Mission{}
		if err := xrpc.Call("GetMission", HOSTINFO, mission); err != nil {
			log.Errorf("get mission failed: %v", err)
			time.Sleep(2 * time.Second)
			continue
		}

		if mission.Idle != 0 {
			fmt.Print(".")
			time.Sleep(mission.Idle)
			continue
		}
		log.Infof("new mission from xrpc: %v", mission)
		missionQueue <- mission
	}
}
<commit_msg>use fullpath<commit_after>package slave

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/codeskyblue/go-sh"
	"github.com/gobuild/gobuild2/models"
	"github.com/gobuild/gobuild2/pkg/xrpc"
	"github.com/gobuild/log"
)

var (
	TMPDIR     = "./tmp"
	PROGRAM, _ = filepath.Abs(os.Args[0])
	HOSTNAME   = "localhost"
	HOSTINFO   = &xrpc.HostInfo{Os: runtime.GOOS, Arch: runtime.GOARCH, Host: HOSTNAME}
)

func checkError(err error) {
	if err != nil {
		log.Errorf("err: %v", err)
	}
}

type NTMsg struct {
	Status string
	Output string
	Extra  string
}

func GoInterval(dur time.Duration, f func()) chan bool {
	done := make(chan bool)
	go func() {
		for {
			select {
			case <-done:
				return
			case <-time.After(dur):
				f()
			}
		}
	}()
	return done
}

func work(m *xrpc.Mission) (err error) {
	notify := func(status string, output string, extra ...string) {
		mstatus := &xrpc.MissionStatus{Mid: m.Mid, Status: status,
			Output: output,
			Extra:  strings.Join(extra, ""),
		}
		ok := false
		err := xrpc.Call("UpdateMissionStatus", mstatus, &ok)
		checkError(err)
	}
	defer func() {
		fmt.Println("DONE", err)
		if err != nil {
			notify(models.ST_ERROR, err.Error())
		}
	}()
	// prepare shell session
	sess := sh.NewSession()
	buffer := bytes.NewBuffer(nil)
	sess.Stdout = io.MultiWriter(buffer, os.Stdout)
	sess.Stderr = io.MultiWriter(buffer, os.Stderr)
	sess.ShowCMD = true
	var gopath, _ = filepath.Abs(TMPDIR)
	if !sh.Test("dir", gopath) {
		os.MkdirAll(gopath, 0755)
	}
	sess.SetEnv("GOPATH", gopath)
	sess.SetEnv("CGO_ENABLE", "0")
	if m.CgoEnable {
		sess.SetEnv("CGO_ENABLE", "1")
	}
	sess.SetEnv("GOOS", m.Os)
	sess.SetEnv("GOARCH", m.Arch)

	var repoName = m.Repo
	var srcPath = filepath.Join(gopath, "src", repoName)

	getsrc := func() (err error) {
		var params []interface{}
		params = append(params, "get", "-v", "-g")
		if m.Sha != "" {
			params = append(params, repoName+"@commit:"+m.Sha)
		} else {
			params = append(params, repoName+"@branch:"+m.Branch)
		}
		params = append(params, sh.Dir(gopath))
		if err = sess.Command("gopm", params...).Run(); err != nil {
			return
		}
		return nil
	}

	newNotify := func(status string, buf *bytes.Buffer) chan bool {
		return GoInterval(time.Second*2, func() {
			notify(status, string(buf.Bytes()))
		})
	}

	notify(models.ST_RETRIVING, "start get source")
	var done chan bool
	done = newNotify(models.ST_RETRIVING, buffer)
	err = getsrc()
	done <- true
	notify(models.ST_RETRIVING, string(buffer.Bytes()))
	if err != nil {
		log.Errorf("getsource err: %v", err)
		return
	}
	buffer.Reset()

	// extention := "zip"
	// var outFile = m.UpKey // fmt.Sprintf("%s-%s-%s.%s", filepath.Base(repoName), m.Os, m.Arch, extention)
	var outFile = filepath.Base(m.UpKey)
	var outFullPath = filepath.Join(srcPath, outFile)

	// notify(models.ST_BUILDING, "start building")
	done = newNotify(models.ST_BUILDING, buffer)
	gopm, _ := exec.LookPath("gopm")
	err = sess.Command(gopm, "build", "-u", "-v", sh.Dir(srcPath)).Run()
	done <- true
	notify(models.ST_BUILDING, string(buffer.Bytes()))
	if err != nil {
		log.Errorf("gopm build error: %v", err)
		return
	}
	buffer.Reset()

	// write extra pkginfo
	pkginfo := "pkginfo.json"
	ioutil.WriteFile(filepath.Join(srcPath, pkginfo), m.PkgInfo, 0644)

	err = sess.Command(PROGRAM, "pack",
		"--nobuild", "-a", pkginfo, "-o", outFile, sh.Dir(srcPath)).Run()
	notify(models.ST_PACKING, string(buffer.Bytes()))
	if err != nil {
		log.Error(err)
		return
	}

	var cdnPath = m.UpKey
	notify(models.ST_PUBLISHING, cdnPath)
	log.Infof("cdn path: %s", cdnPath)
	q := &Qiniu{m.UpToken, m.UpKey, m.Bulket} // uptoken, key}
	var pubAddr string
	if pubAddr, err = q.Upload(outFullPath); err != nil {
		checkError(err)
		return
	}

	log.Debugf("publish %s to %s", outFile, pubAddr)
	notify(models.ST_DONE, pubAddr)
	return nil
}

func init() {
	var err error
	HOSTNAME, err = os.Hostname()
	if err != nil {
		log.Fatalf("hostname retrive err: %v", err)
	}
}

var IsPrivateUpload bool //todo

func prepare() (err error) {
	qi := new(xrpc.QiniuInfo)
	if err = xrpc.Call("GetQiniuInfo", HOSTINFO, qi); err != nil {
		return
	}
	initQiniu(qi.AccessKey, qi.SecretKey, qi.Bulket)

	TMPDIR, err = filepath.Abs(TMPDIR)
	if err != nil {
		log.Errorf("tmpdir to abspath err: %v", err)
		return
	}
	if !sh.Test("dir", TMPDIR) {
		os.MkdirAll(TMPDIR, 0755)
	}
	startWork()
	return nil
}

func Action(c *cli.Context) {
	fmt.Println("this is slave daemon")
	webaddr := c.String("webaddr")
	xrpc.DefaultWebAddress = webaddr

	if err := prepare(); err != nil {
		log.Fatalf("slave prepare err: %v", err)
	}
	for {
		mission := &xrpc.Mission{}
		if err := xrpc.Call("GetMission", HOSTINFO, mission); err != nil {
			log.Errorf("get mission failed: %v", err)
			time.Sleep(2 * time.Second)
			continue
		}

		if mission.Idle != 0 {
			fmt.Print(".")
			time.Sleep(mission.Idle)
			continue
		}
		log.Infof("new mission from xrpc: %v", mission)
		missionQueue <- mission
	}
}
<|endoftext|>


<commit_before>package command

const VERSION = "0.9.9"

var (
	GitCommit string
	BuildTime string
)
<commit_msg>Release 0.9.10<commit_after>package command

const VERSION = "0.9.10"

var (
	GitCommit string
	BuildTime string
)
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package commands

import (
	"context"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"

	"github.com/google/go-containerregistry/pkg/authn"
	"github.com/google/go-containerregistry/pkg/name"
	v1 "github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/remote"
	"github.com/spf13/viper"
)

var (
	defaultBaseImage   name.Reference
	baseImageOverrides map[string]name.Reference
)

func getBaseImage(s string) (v1.Image, error) {
	ref, ok := baseImageOverrides[s]
	if !ok {
		ref = defaultBaseImage
	}
	log.Printf("Using base %s for %s", ref, s)
	return remote.Image(ref,
		remote.WithTransport(defaultTransport()),
		remote.WithAuthFromKeychain(authn.DefaultKeychain))
}

func getCreationTime() (*v1.Time, error) {
	epoch := os.Getenv("SOURCE_DATE_EPOCH")
	if epoch == "" {
		return nil, nil
	}

	seconds, err := strconv.ParseInt(epoch, 10, 64)
	if err != nil {
		return nil, fmt.Errorf("the environment variable SOURCE_DATE_EPOCH should be the number of seconds since January 1st 1970, 00:00 UTC, got: %v", err)
	}
	return &v1.Time{time.Unix(seconds, 0)}, nil
}

func createCancellableContext() context.Context {
	signals := make(chan os.Signal)
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
	ctx, cancel := context.WithCancel(context.Background())

	go func() {
		<-signals
		cancel()
	}()

	return ctx
}

func init() {
	// If omitted, use this base image.
	viper.SetDefault("defaultBaseImage", "gcr.io/distroless/static:latest")
	viper.SetConfigName(".ko") // .yaml is implicit
	viper.SetEnvPrefix("KO")
	viper.AutomaticEnv()

	if override := os.Getenv("KO_CONFIG_PATH"); override != "" {
		viper.AddConfigPath(override)
	}

	viper.AddConfigPath("./")

	if err := viper.ReadInConfig(); err != nil {
		if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
			log.Fatalf("error reading config file: %v", err)
		}
	}

	ref := viper.GetString("defaultBaseImage")
	dbi, err := name.ParseReference(ref)
	if err != nil {
		log.Fatalf("'defaultBaseImage': error parsing %q as image reference: %v", ref, err)
	}
	defaultBaseImage = dbi

	baseImageOverrides = make(map[string]name.Reference)
	overrides := viper.GetStringMapString("baseImageOverrides")
	for k, v := range overrides {
		bi, err := name.ParseReference(v)
		if err != nil {
			log.Fatalf("'baseImageOverrides': error parsing %q as image reference: %v", v, err)
		}
		baseImageOverrides[k] = bi
	}
}
<commit_msg>Viper keys are case insensitive. (#150)<commit_after>// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package commands

import (
	"context"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/google/go-containerregistry/pkg/authn"
	"github.com/google/go-containerregistry/pkg/name"
	v1 "github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/remote"
	"github.com/spf13/viper"
)

var (
	defaultBaseImage   name.Reference
	baseImageOverrides map[string]name.Reference
)

func getBaseImage(s string) (v1.Image, error) {
	// Viper configuration file keys are case insensitive, and are
	// returned as all lowercase.  This means that import paths with
	// uppercase must be normalized for matching here, e.g.
	//    github.com/GoogleCloudPlatform/foo/cmd/bar
	// comes through as:
	//    github.com/googlecloudplatform/foo/cmd/bar
	ref, ok := baseImageOverrides[strings.ToLower(s)]
	if !ok {
		ref = defaultBaseImage
	}
	log.Printf("Using base %s for %s", ref, s)
	return remote.Image(ref,
		remote.WithTransport(defaultTransport()),
		remote.WithAuthFromKeychain(authn.DefaultKeychain))
}

func getCreationTime() (*v1.Time, error) {
	epoch := os.Getenv("SOURCE_DATE_EPOCH")
	if epoch == "" {
		return nil, nil
	}

	seconds, err := strconv.ParseInt(epoch, 10, 64)
	if err != nil {
		return nil, fmt.Errorf("the environment variable SOURCE_DATE_EPOCH should be the number of seconds since January 1st 1970, 00:00 UTC, got: %v", err)
	}
	return &v1.Time{time.Unix(seconds, 0)}, nil
}

func createCancellableContext() context.Context {
	signals := make(chan os.Signal)
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
	ctx, cancel := context.WithCancel(context.Background())

	go func() {
		<-signals
		cancel()
	}()

	return ctx
}

func init() {
	// If omitted, use this base image.
	viper.SetDefault("defaultBaseImage", "gcr.io/distroless/static:latest")
	viper.SetConfigName(".ko") // .yaml is implicit
	viper.SetEnvPrefix("KO")
	viper.AutomaticEnv()

	if override := os.Getenv("KO_CONFIG_PATH"); override != "" {
		viper.AddConfigPath(override)
	}

	viper.AddConfigPath("./")

	if err := viper.ReadInConfig(); err != nil {
		if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
			log.Fatalf("error reading config file: %v", err)
		}
	}

	ref := viper.GetString("defaultBaseImage")
	dbi, err := name.ParseReference(ref)
	if err != nil {
		log.Fatalf("'defaultBaseImage': error parsing %q as image reference: %v", ref, err)
	}
	defaultBaseImage = dbi

	baseImageOverrides = make(map[string]name.Reference)
	overrides := viper.GetStringMapString("baseImageOverrides")
	for k, v := range overrides {
		bi, err := name.ParseReference(v)
		if err != nil {
			log.Fatalf("'baseImageOverrides': error parsing %q as image reference: %v", v, err)
		}
		baseImageOverrides[k] = bi
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package debugutil

import (
	"net/http"
	"net/http/pprof"
	"runtime"
)

const HTTPPrefixPProf = "/debug/pprof"

// PProfHandlers returns a map of pprof handlers keyed by the HTTP path.
func PProfHandlers() map[string]http.Handler {
	// set only when there's no existing setting
	if runtime.SetMutexProfileFraction(-1) == 0 {
		// 1 out of 5 mutex events are reported, on average
		runtime.SetMutexProfileFraction(5)
	}

	m := make(map[string]http.Handler)

	m[HTTPPrefixPProf+"/"] = http.HandlerFunc(pprof.Index)
	m[HTTPPrefixPProf+"/profile"] = http.HandlerFunc(pprof.Profile)
	m[HTTPPrefixPProf+"/symbol"] = http.HandlerFunc(pprof.Symbol)
	m[HTTPPrefixPProf+"/cmdline"] = http.HandlerFunc(pprof.Cmdline)
	m[HTTPPrefixPProf+"/trace "] = http.HandlerFunc(pprof.Trace)
	m[HTTPPrefixPProf+"/heap"] = pprof.Handler("heap")
	m[HTTPPrefixPProf+"/goroutine"] = pprof.Handler("goroutine")
	m[HTTPPrefixPProf+"/threadcreate"] = pprof.Handler("threadcreate")
	m[HTTPPrefixPProf+"/block"] = pprof.Handler("block")
	m[HTTPPrefixPProf+"/mutex"] = pprof.Handler("mutex")

	return m
}
<commit_msg>debugutil: Remove extra space in trace handler route<commit_after>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package debugutil

import (
	"net/http"
	"net/http/pprof"
	"runtime"
)

const HTTPPrefixPProf = "/debug/pprof"

// PProfHandlers returns a map of pprof handlers keyed by the HTTP path.
func PProfHandlers() map[string]http.Handler {
	// set only when there's no existing setting
	if runtime.SetMutexProfileFraction(-1) == 0 {
		// 1 out of 5 mutex events are reported, on average
		runtime.SetMutexProfileFraction(5)
	}

	m := make(map[string]http.Handler)

	m[HTTPPrefixPProf+"/"] = http.HandlerFunc(pprof.Index)
	m[HTTPPrefixPProf+"/profile"] = http.HandlerFunc(pprof.Profile)
	m[HTTPPrefixPProf+"/symbol"] = http.HandlerFunc(pprof.Symbol)
	m[HTTPPrefixPProf+"/cmdline"] = http.HandlerFunc(pprof.Cmdline)
	m[HTTPPrefixPProf+"/trace"] = http.HandlerFunc(pprof.Trace)
	m[HTTPPrefixPProf+"/heap"] = pprof.Handler("heap")
	m[HTTPPrefixPProf+"/goroutine"] = pprof.Handler("goroutine")
	m[HTTPPrefixPProf+"/threadcreate"] = pprof.Handler("threadcreate")
	m[HTTPPrefixPProf+"/block"] = pprof.Handler("block")
	m[HTTPPrefixPProf+"/mutex"] = pprof.Handler("mutex")

	return m
}
<|endoftext|>


<commit_before>package executor

import (
	"errors"
	"fmt"
	"os"
	"path"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/intelsdi-x/swan/pkg/isolation"
	"golang.org/x/crypto/ssh"
	"strings"
)

// Remote provisioning is responsible for providing the execution environment
// on remote machine via ssh.
type Remote struct {
	sshConfig         *SSHConfig
	commandDecorators isolation.Decorators
}

// NewRemote returns a Remote instance.
func NewRemote(sshConfig *SSHConfig) Remote {
	return Remote{
		sshConfig:         sshConfig,
		commandDecorators: []isolation.Decorator{},
	}
}

// NewRemoteIsolated returns a Remote instance.
func NewRemoteIsolated(sshConfig *SSHConfig, decorators isolation.Decorators) Remote {
	return Remote{
		sshConfig:         sshConfig,
		commandDecorators: decorators,
	}
}

// Execute runs the command given as input.
// Returned Task Handle is able to stop & monitor the provisioned process.
func (remote Remote) Execute(command string) (TaskHandle, error) {
	connection, err := ssh.Dial(
		"tcp",
		fmt.Sprintf("%s:%d", remote.sshConfig.Host, remote.sshConfig.Port),
		remote.sshConfig.ClientConfig)
	if err != nil {
		return nil, err
	}

	session, err := connection.NewSession()
	if err != nil {
		return nil, err
	}

	terminal := ssh.TerminalModes{
		ssh.ECHO:          0,
		ssh.TTY_OP_ISPEED: 14400,
		ssh.TTY_OP_OSPEED: 14400,
	}

	err = session.RequestPty("xterm", 80, 40, terminal)
	if err != nil {
		session.Close()
		return nil, err
	}

	stdoutFile, stderrFile, err := createExecutorOutputFiles(command, "remote")
	if err != nil {
		return nil, err
	}

	log.Debug("Created temporary files: ",
		"stdout path:  ", stdoutFile.Name(), ", stderr path:  ", stderrFile.Name())

	session.Stdout = stdoutFile
	session.Stderr = stderrFile

	// Escape the quotes characters for `sh -c`.
	stringForSh := remote.commandDecorators.Decorate(command)
	stringForSh = strings.Replace(stringForSh, "'", "\\'", -1)
	stringForSh = strings.Replace(stringForSh, "\"", "\\\"", -1)

	log.Debug("Starting '", stringForSh, "' remotely")

	// huponexit` ensures that the process will be killed when ssh connection will be closed.
	err = session.Start(fmt.Sprintf("shopt -s huponexit; sh -c '%s'", stringForSh))
	if err != nil {
		return nil, err
	}

	log.Debug("Started remote command")

	// Wait End channel is for checking the status of the Wait. If this channel is closed,
	// it means that the wait is completed (either with error or not)
	// This channel will not be used for passing any message.
	waitEndChannel := make(chan struct{})

	// TODO(bplotka): Move exit code constants to global executor scope.
	const successExitCode = int(0)
	const errorExitCode = int(-1)

	exitCodeInt := errorExitCode
	var exitCode *int
	exitCode = &exitCodeInt

	// Wait for local task in go routine.
	go func() {
		defer close(waitEndChannel)
		defer session.Close()

		*exitCode = successExitCode
		// Wait for task completion.
		err := session.Wait()
		if err != nil {
			if exitError, ok := err.(*ssh.ExitError); !ok {
				// In case of NON Exit Errors we are not sure if task does
				// terminate so panic.
				log.Panic("Waiting for remote task failed. ", err)
			} else {
				*exitCode = exitError.Waitmsg.ExitStatus()
			}
		}

		log.Debug(
			"Ended ", command,
			" with output in file: ", stdoutFile.Name(),
			" with err output in file: ", stderrFile.Name(),
			" with status code: ", *exitCode)
	}()

	return newRemoteTaskHandle(session, stdoutFile, stderrFile,
		remote.sshConfig.Host, waitEndChannel, exitCode), nil
}

const killTimeout = 5 * time.Second

// remoteTaskHandle implements TaskHandle interface.
type remoteTaskHandle struct {
	session        *ssh.Session
	stdoutFile     *os.File
	stderrFile     *os.File
	host           string
	waitEndChannel chan struct{}
	exitCode       *int
}

// newRemoteTaskHandle returns a remoteTaskHandle instance.
func newRemoteTaskHandle(session *ssh.Session, stdoutFile *os.File, stderrFile *os.File,
	host string, waitEndChannel chan struct{}, exitCode *int) *remoteTaskHandle {
	return &remoteTaskHandle{
		session:        session,
		stdoutFile:     stdoutFile,
		stderrFile:     stderrFile,
		host:           host,
		waitEndChannel: waitEndChannel,
		exitCode:       exitCode,
	}
}

// isTerminated checks if waitEndChannel is closed. If it is closed, it means
// that wait ended and task is in terminated state.
func (taskHandle *remoteTaskHandle) isTerminated() bool {
	select {
	case <-taskHandle.waitEndChannel:
		// If waitEndChannel is closed then task is terminated.
		return true
	default:
		return false
	}
}

// Stop terminates the remote task.
func (taskHandle *remoteTaskHandle) Stop() error {
	if taskHandle.isTerminated() {
		return nil
	}

	// Kill session.
	// NOTE: We need to find here a better way to stop task, since
	// closing channel just close the ssh session and some processes can be still running.
	// Some other approaches:
	// - sending Ctrl+C (very time based and not working currently)
	// - session.Signal does not work.
	// - gathering PID & killing the pid in separate session
	err := taskHandle.session.Close()
	if err != nil {
		log.Error(err)
		return err
	}

	// Checking if kill was successful.
	isTerminated := taskHandle.Wait(killTimeout)
	if !isTerminated {
		log.Error("Cannot terminate task")
		return errors.New("Cannot terminate task")

	}
	// No error, task terminated.
	return nil
}

// Status returns a state of the task.
func (taskHandle *remoteTaskHandle) Status() TaskState {
	if !taskHandle.isTerminated() {
		return RUNNING
	}

	return TERMINATED
}

// ExitCode returns a exitCode. If task is not terminated it returns error.
func (taskHandle *remoteTaskHandle) ExitCode() (int, error) {
	if !taskHandle.isTerminated() {
		return -1, errors.New("Task is not terminated")
	}

	return *taskHandle.exitCode, nil
}

// StdoutFile returns a file handle for file to the task's stdout file.
func (taskHandle *remoteTaskHandle) StdoutFile() (*os.File, error) {
	if _, err := os.Stat(taskHandle.stdoutFile.Name()); err != nil {
		return nil, err
	}

	taskHandle.stdoutFile.Seek(0, os.SEEK_SET)
	return taskHandle.stdoutFile, nil
}

// StderrFile returns a file handle for file to the task's stderr file.
func (taskHandle *remoteTaskHandle) StderrFile() (*os.File, error) {
	if _, err := os.Stat(taskHandle.stderrFile.Name()); err != nil {
		return nil, err
	}

	taskHandle.stderrFile.Seek(0, os.SEEK_SET)
	return taskHandle.stderrFile, nil
}

// Clean removes files to which stdout and stderr of executed command was written.
func (taskHandle *remoteTaskHandle) Clean() error {
	// Close stdout.
	stdoutErr := taskHandle.stdoutFile.Close()

	// Close stderr.
	stderrErr := taskHandle.stderrFile.Close()

	if stdoutErr != nil {
		return stdoutErr
	}

	if stderrErr != nil {
		return stderrErr
	}

	return nil
}

// EraseOutput removes task's stdout & stderr files.
func (taskHandle *remoteTaskHandle) EraseOutput() error {
	outputDir, _ := path.Split(taskHandle.stdoutFile.Name())

	// Remove temporary directory created for stdout and stderr.
	if err := os.RemoveAll(outputDir); err != nil {
		return err
	}
	return nil
}

// Wait waits for the command to finish with the given timeout time.
// It returns true if task is terminated.
func (taskHandle *remoteTaskHandle) Wait(timeout time.Duration) bool {
	if taskHandle.isTerminated() {
		return true
	}

	var timeoutChannel <-chan time.Time
	if timeout != 0 {
		// In case of wait with timeout set the timeout channel.
		timeoutChannel = time.After(timeout)
	}

	select {
	case <-taskHandle.waitEndChannel:
		// If waitEndChannel is closed then task is terminated.
		return true
	case <-timeoutChannel:
		// If timeout time exceeded return then task did not terminate yet.
		return false
	}
}

func (taskHandle *remoteTaskHandle) Address() string {
	return taskHandle.host
}
<commit_msg>ssh: close SSH connection after finishing remote commands (#241)<commit_after>package executor

import (
	"errors"
	"fmt"
	"os"
	"path"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/intelsdi-x/swan/pkg/isolation"
	"golang.org/x/crypto/ssh"
)

// Remote provisioning is responsible for providing the execution environment
// on remote machine via ssh.
type Remote struct {
	sshConfig         *SSHConfig
	commandDecorators isolation.Decorators
}

// NewRemote returns a Remote instance.
func NewRemote(sshConfig *SSHConfig) Remote {
	return Remote{
		sshConfig:         sshConfig,
		commandDecorators: []isolation.Decorator{},
	}
}

// NewRemoteIsolated returns a Remote instance.
func NewRemoteIsolated(sshConfig *SSHConfig, decorators isolation.Decorators) Remote {
	return Remote{
		sshConfig:         sshConfig,
		commandDecorators: decorators,
	}
}

// Execute runs the command given as input.
// Returned Task Handle is able to stop & monitor the provisioned process.
func (remote Remote) Execute(command string) (TaskHandle, error) {
	connection, err := ssh.Dial(
		"tcp",
		fmt.Sprintf("%s:%d", remote.sshConfig.Host, remote.sshConfig.Port),
		remote.sshConfig.ClientConfig)
	if err != nil {
		return nil, err
	}

	session, err := connection.NewSession()
	if err != nil {
		return nil, err
	}

	terminal := ssh.TerminalModes{
		ssh.ECHO:          0,
		ssh.TTY_OP_ISPEED: 14400,
		ssh.TTY_OP_OSPEED: 14400,
	}

	err = session.RequestPty("xterm", 80, 40, terminal)
	if err != nil {
		session.Close()
		return nil, err
	}

	stdoutFile, stderrFile, err := createExecutorOutputFiles(command, "remote")
	if err != nil {
		return nil, err
	}

	log.Debug("Created temporary files: ",
		"stdout path:  ", stdoutFile.Name(), ", stderr path:  ", stderrFile.Name())

	session.Stdout = stdoutFile
	session.Stderr = stderrFile

	// Escape the quotes characters for `sh -c`.
	stringForSh := remote.commandDecorators.Decorate(command)
	stringForSh = strings.Replace(stringForSh, "'", "\\'", -1)
	stringForSh = strings.Replace(stringForSh, "\"", "\\\"", -1)

	log.Debug("Starting '", stringForSh, "' remotely")

	// huponexit` ensures that the process will be killed when ssh connection will be closed.
	err = session.Start(fmt.Sprintf("shopt -s huponexit; sh -c '%s'", stringForSh))
	if err != nil {
		return nil, err
	}

	log.Debug("Started remote command")

	// Wait End channel is for checking the status of the Wait. If this channel is closed,
	// it means that the wait is completed (either with error or not)
	// This channel will not be used for passing any message.
	waitEndChannel := make(chan struct{})

	// TODO(bplotka): Move exit code constants to global executor scope.
	const successExitCode = int(0)
	const errorExitCode = int(-1)

	exitCodeInt := errorExitCode
	var exitCode *int
	exitCode = &exitCodeInt

	// Wait for local task in go routine.
	go func() {
		defer close(waitEndChannel)
		defer session.Close() // Closing a session is not enough to close connection.
		defer connection.Close()

		*exitCode = successExitCode
		// Wait for task completion.
		err := session.Wait()
		if err != nil {
			if exitError, ok := err.(*ssh.ExitError); !ok {
				// In case of NON Exit Errors we are not sure if task does
				// terminate so panic.
				log.Panic("Waiting for remote task failed. ", err)
			} else {
				*exitCode = exitError.Waitmsg.ExitStatus()
			}
		}

		log.Debug(
			"Ended ", command,
			" with output in file: ", stdoutFile.Name(),
			" with err output in file: ", stderrFile.Name(),
			" with status code: ", *exitCode)
	}()

	return newRemoteTaskHandle(session, stdoutFile, stderrFile,
		remote.sshConfig.Host, waitEndChannel, exitCode), nil
}

const killTimeout = 5 * time.Second

// remoteTaskHandle implements TaskHandle interface.
type remoteTaskHandle struct {
	session        *ssh.Session
	stdoutFile     *os.File
	stderrFile     *os.File
	host           string
	waitEndChannel chan struct{}
	exitCode       *int
}

// newRemoteTaskHandle returns a remoteTaskHandle instance.
func newRemoteTaskHandle(session *ssh.Session, stdoutFile *os.File, stderrFile *os.File,
	host string, waitEndChannel chan struct{}, exitCode *int) *remoteTaskHandle {
	return &remoteTaskHandle{
		session:        session,
		stdoutFile:     stdoutFile,
		stderrFile:     stderrFile,
		host:           host,
		waitEndChannel: waitEndChannel,
		exitCode:       exitCode,
	}
}

// isTerminated checks if waitEndChannel is closed. If it is closed, it means
// that wait ended and task is in terminated state.
func (taskHandle *remoteTaskHandle) isTerminated() bool {
	select {
	case <-taskHandle.waitEndChannel:
		// If waitEndChannel is closed then task is terminated.
		return true
	default:
		return false
	}
}

// Stop terminates the remote task.
func (taskHandle *remoteTaskHandle) Stop() error {
	if taskHandle.isTerminated() {
		return nil
	}

	// Kill session.
	// NOTE: We need to find here a better way to stop task, since
	// closing channel just close the ssh session and some processes can be still running.
	// Some other approaches:
	// - sending Ctrl+C (very time based and not working currently)
	// - session.Signal does not work.
	// - gathering PID & killing the pid in separate session
	err := taskHandle.session.Close()
	if err != nil {
		log.Error(err)
		return err
	}

	// Checking if kill was successful.
	isTerminated := taskHandle.Wait(killTimeout)
	if !isTerminated {
		log.Error("Cannot terminate task")
		return errors.New("Cannot terminate task")

	}
	// No error, task terminated.
	return nil
}

// Status returns a state of the task.
func (taskHandle *remoteTaskHandle) Status() TaskState {
	if !taskHandle.isTerminated() {
		return RUNNING
	}

	return TERMINATED
}

// ExitCode returns a exitCode. If task is not terminated it returns error.
func (taskHandle *remoteTaskHandle) ExitCode() (int, error) {
	if !taskHandle.isTerminated() {
		return -1, errors.New("Task is not terminated")
	}

	return *taskHandle.exitCode, nil
}

// StdoutFile returns a file handle for file to the task's stdout file.
func (taskHandle *remoteTaskHandle) StdoutFile() (*os.File, error) {
	if _, err := os.Stat(taskHandle.stdoutFile.Name()); err != nil {
		return nil, err
	}

	taskHandle.stdoutFile.Seek(0, os.SEEK_SET)
	return taskHandle.stdoutFile, nil
}

// StderrFile returns a file handle for file to the task's stderr file.
func (taskHandle *remoteTaskHandle) StderrFile() (*os.File, error) {
	if _, err := os.Stat(taskHandle.stderrFile.Name()); err != nil {
		return nil, err
	}

	taskHandle.stderrFile.Seek(0, os.SEEK_SET)
	return taskHandle.stderrFile, nil
}

// Clean removes files to which stdout and stderr of executed command was written.
func (taskHandle *remoteTaskHandle) Clean() error {
	// Close stdout.
	stdoutErr := taskHandle.stdoutFile.Close()

	// Close stderr.
	stderrErr := taskHandle.stderrFile.Close()

	if stdoutErr != nil {
		return stdoutErr
	}

	if stderrErr != nil {
		return stderrErr
	}

	return nil
}

// EraseOutput removes task's stdout & stderr files.
func (taskHandle *remoteTaskHandle) EraseOutput() error {
	outputDir, _ := path.Split(taskHandle.stdoutFile.Name())

	// Remove temporary directory created for stdout and stderr.
	if err := os.RemoveAll(outputDir); err != nil {
		return err
	}
	return nil
}

// Wait waits for the command to finish with the given timeout time.
// It returns true if task is terminated.
func (taskHandle *remoteTaskHandle) Wait(timeout time.Duration) bool {
	if taskHandle.isTerminated() {
		return true
	}

	var timeoutChannel <-chan time.Time
	if timeout != 0 {
		// In case of wait with timeout set the timeout channel.
		timeoutChannel = time.After(timeout)
	}

	select {
	case <-taskHandle.waitEndChannel:
		// If waitEndChannel is closed then task is terminated.
		return true
	case <-timeoutChannel:
		// If timeout time exceeded return then task did not terminate yet.
		return false
	}
}

func (taskHandle *remoteTaskHandle) Address() string {
	return taskHandle.host
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ghcache

import (
	"context"
	"fmt"
	"time"

	"github.com/google/go-github/v33/github"
	"github.com/google/triage-party/pkg/persist"
	"github.com/sirupsen/logrus"
)

const (
	keyTime = "2006-01-02T150405"
)

type blob struct {
	PullRequest         github.PullRequest
	CommitFiles         []github.CommitFile
	PullRequestComments []github.PullRequestComment
	IssueComments       []github.IssueComment
	Issue               github.Issue
}

func PullRequestsGet(ctx context.Context, p persist.Cacher, c *github.Client, t time.Time, org string, project string, num int) (*github.PullRequest, error) {
	key := fmt.Sprintf("pr-%s-%s-%d", org, project, num)
	val := p.Get(key, t)

	if val != nil {
		return val.GHPullRequest, nil
	}

	if val == nil {
		logrus.Debugf("cache miss for %v", key)
		pr, _, err := c.PullRequests.Get(ctx, org, project, num)
		if err != nil {
			return nil, fmt.Errorf("get: %v", err)
		}
		return pr, p.Set(key, &persist.Blob{GHPullRequest: pr})
	}

	logrus.Debugf("cache hit: %v", key)
	return val.GHPullRequest, nil
}

func PullRequestsListFiles(ctx context.Context, p persist.Cacher, c *github.Client, t time.Time, org string, project string, num int) ([]*github.CommitFile, error) {
	key := fmt.Sprintf("pr-listfiles-%s-%s-%d", org, project, num)
	val := p.Get(key, t)

	if val != nil {
		return val.GHCommitFiles, nil
	}

	logrus.Debugf("cache miss for %v: %s", key)

	opts := &github.ListOptions{PerPage: 100}
	fs := []*github.CommitFile{}

	for {
		fsp, resp, err := c.PullRequests.ListFiles(ctx, org, project, num, opts)
		if err != nil {
			return nil, fmt.Errorf("get: %v", err)
		}
		fs = append(fs, fsp...)

		if resp.NextPage == 0 {
			break
		}

		opts.Page = resp.NextPage
	}

	return fs, p.Set(key, &persist.Blob{GHCommitFiles: fs})

}

func PullRequestsListComments(ctx context.Context, p persist.Cacher, c *github.Client, t time.Time, org string, project string, num int) ([]*github.PullRequestComment, error) {
	key := fmt.Sprintf("pr-comments-%s-%s-%d", org, project, num)
	val := p.Get(key, t)

	if val != nil {
		return val.GHPullRequestComments, nil
	}

	logrus.Debugf("cache miss for %v: %s", key)

	cs := []*github.PullRequestComment{}
	opts := &github.PullRequestListCommentsOptions{
		ListOptions: github.ListOptions{PerPage: 100},
	}

	for {
		csp, resp, err := c.PullRequests.ListComments(ctx, org, project, num, opts)
		if err != nil {
			return nil, fmt.Errorf("get: %v", err)
		}

		cs = append(cs, csp...)

		if resp.NextPage == 0 {
			break
		}
		opts.ListOptions.Page = resp.NextPage
	}

	return cs, p.Set(key, &persist.Blob{GHPullRequestComments: cs})
}

func IssuesGet(ctx context.Context, p persist.Cacher, c *github.Client, t time.Time, org string, project string, num int) (*github.Issue, error) {
	key := fmt.Sprintf("issue-%s-%s-%d", org, project, num)
	val := p.Get(key, t)

	if val != nil {
		return val.GHIssue, nil
	}

	logrus.Debugf("cache miss for %v: %s", key)

	i, _, err := c.Issues.Get(ctx, org, project, num)
	if err != nil {
		return nil, fmt.Errorf("get: %v", err)
	}

	return i, p.Set(key, &persist.Blob{GHIssue: i})
}

func IssuesListComments(ctx context.Context, p persist.Cacher, c *github.Client, t time.Time, org string, project string, num int) ([]*github.IssueComment, error) {
	key := fmt.Sprintf("issue-comments-%s-%s-%d", org, project, num)
	val := p.Get(key, t)

	if val != nil {
		return val.GHIssueComments, nil
	}

	opts := &github.IssueListCommentsOptions{
		ListOptions: github.ListOptions{PerPage: 100},
	}

	cs := []*github.IssueComment{}
	for {
		csp, resp, err := c.Issues.ListComments(ctx, org, project, num, opts)
		if err != nil {
			return nil, fmt.Errorf("get: %v", err)
		}

		cs = append(cs, csp...)

		if resp.NextPage == 0 {
			break
		}

		opts.ListOptions.Page = resp.NextPage
	}

	return cs, p.Set(key, &persist.Blob{GHIssueComments: cs})
}
<commit_msg>fix Debugf calls<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ghcache

import (
	"context"
	"fmt"
	"time"

	"github.com/google/go-github/v33/github"
	"github.com/google/triage-party/pkg/persist"
	"github.com/sirupsen/logrus"
)

const (
	keyTime = "2006-01-02T150405"
)

type blob struct {
	PullRequest         github.PullRequest
	CommitFiles         []github.CommitFile
	PullRequestComments []github.PullRequestComment
	IssueComments       []github.IssueComment
	Issue               github.Issue
}

func PullRequestsGet(ctx context.Context, p persist.Cacher, c *github.Client, t time.Time, org string, project string, num int) (*github.PullRequest, error) {
	key := fmt.Sprintf("pr-%s-%s-%d", org, project, num)
	val := p.Get(key, t)

	if val != nil {
		return val.GHPullRequest, nil
	}

	if val == nil {
		logrus.Debugf("cache miss for %v", key)
		pr, _, err := c.PullRequests.Get(ctx, org, project, num)
		if err != nil {
			return nil, fmt.Errorf("get: %v", err)
		}
		return pr, p.Set(key, &persist.Blob{GHPullRequest: pr})
	}

	logrus.Debugf("cache hit: %v", key)
	return val.GHPullRequest, nil
}

func PullRequestsListFiles(ctx context.Context, p persist.Cacher, c *github.Client, t time.Time, org string, project string, num int) ([]*github.CommitFile, error) {
	key := fmt.Sprintf("pr-listfiles-%s-%s-%d", org, project, num)
	val := p.Get(key, t)

	if val != nil {
		return val.GHCommitFiles, nil
	}

	logrus.Debugf("cache miss for %v", key)

	opts := &github.ListOptions{PerPage: 100}
	fs := []*github.CommitFile{}

	for {
		fsp, resp, err := c.PullRequests.ListFiles(ctx, org, project, num, opts)
		if err != nil {
			return nil, fmt.Errorf("get: %v", err)
		}
		fs = append(fs, fsp...)

		if resp.NextPage == 0 {
			break
		}

		opts.Page = resp.NextPage
	}

	return fs, p.Set(key, &persist.Blob{GHCommitFiles: fs})
}

func PullRequestsListComments(ctx context.Context, p persist.Cacher, c *github.Client, t time.Time, org string, project string, num int) ([]*github.PullRequestComment, error) {
	key := fmt.Sprintf("pr-comments-%s-%s-%d", org, project, num)
	val := p.Get(key, t)

	if val != nil {
		return val.GHPullRequestComments, nil
	}

	logrus.Debugf("cache miss for %v", key)

	cs := []*github.PullRequestComment{}
	opts := &github.PullRequestListCommentsOptions{
		ListOptions: github.ListOptions{PerPage: 100},
	}

	for {
		csp, resp, err := c.PullRequests.ListComments(ctx, org, project, num, opts)
		if err != nil {
			return nil, fmt.Errorf("get: %v", err)
		}

		cs = append(cs, csp...)

		if resp.NextPage == 0 {
			break
		}
		opts.ListOptions.Page = resp.NextPage
	}

	return cs, p.Set(key, &persist.Blob{GHPullRequestComments: cs})
}

func IssuesGet(ctx context.Context, p persist.Cacher, c *github.Client, t time.Time, org string, project string, num int) (*github.Issue, error) {
	key := fmt.Sprintf("issue-%s-%s-%d", org, project, num)
	val := p.Get(key, t)

	if val != nil {
		return val.GHIssue, nil
	}

	logrus.Debugf("cache miss for %v", key)

	i, _, err := c.Issues.Get(ctx, org, project, num)
	if err != nil {
		return nil, fmt.Errorf("get: %v", err)
	}

	return i, p.Set(key, &persist.Blob{GHIssue: i})
}

func IssuesListComments(ctx context.Context, p persist.Cacher, c *github.Client, t time.Time, org string, project string, num int) ([]*github.IssueComment, error) {
	key := fmt.Sprintf("issue-comments-%s-%s-%d", org, project, num)
	val := p.Get(key, t)

	if val != nil {
		return val.GHIssueComments, nil
	}

	opts := &github.IssueListCommentsOptions{
		ListOptions: github.ListOptions{PerPage: 100},
	}

	cs := []*github.IssueComment{}
	for {
		csp, resp, err := c.Issues.ListComments(ctx, org, project, num, opts)
		if err != nil {
			return nil, fmt.Errorf("get: %v", err)
		}

		cs = append(cs, csp...)

		if resp.NextPage == 0 {
			break
		}

		opts.ListOptions.Page = resp.NextPage
	}

	return cs, p.Set(key, &persist.Blob{GHIssueComments: cs})
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubelet

import (
	"fmt"
	"io/ioutil"
	"path"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/pkg/cloudprovider"
	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
	"k8s.io/kubernetes/pkg/types"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/io"
	"k8s.io/kubernetes/pkg/util/mount"
	"k8s.io/kubernetes/pkg/util/strings"
	"k8s.io/kubernetes/pkg/volume"
)

var errUnsupportedVolumeType = fmt.Errorf("unsupported volume type")

// This just exports required functions from kubelet proper, for use by volume
// plugins.
type volumeHost struct {
	kubelet *Kubelet
}

func (vh *volumeHost) GetPluginDir(pluginName string) string {
	return vh.kubelet.getPluginDir(pluginName)
}

func (vh *volumeHost) GetPodVolumeDir(podUID types.UID, pluginName string, volumeName string) string {
	return vh.kubelet.getPodVolumeDir(podUID, pluginName, volumeName)
}

func (vh *volumeHost) GetPodPluginDir(podUID types.UID, pluginName string) string {
	return vh.kubelet.getPodPluginDir(podUID, pluginName)
}

func (vh *volumeHost) GetKubeClient() clientset.Interface {
	return vh.kubelet.kubeClient
}

func (vh *volumeHost) NewWrapperMounter(volName string, spec volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
	// The name of wrapper volume is set to "wrapped_{wrapped_volume_name}"
	wrapperVolumeName := "wrapped_" + volName
	if spec.Volume != nil {
		spec.Volume.Name = wrapperVolumeName
	}

	b, err := vh.kubelet.newVolumeMounterFromPlugins(&spec, pod, opts)
	if err == nil && b == nil {
		return nil, errUnsupportedVolumeType
	}
	return b, nil
}

func (vh *volumeHost) NewWrapperUnmounter(volName string, spec volume.Spec, podUID types.UID) (volume.Unmounter, error) {
	// The name of wrapper volume is set to "wrapped_{wrapped_volume_name}"
	wrapperVolumeName := "wrapped_" + volName
	if spec.Volume != nil {
		spec.Volume.Name = wrapperVolumeName
	}

	plugin, err := vh.kubelet.volumePluginMgr.FindPluginBySpec(&spec)
	if err != nil {
		return nil, err
	}
	if plugin == nil {
		// Not found but not an error
		return nil, nil
	}
	c, err := plugin.NewUnmounter(spec.Name(), podUID)
	if err == nil && c == nil {
		return nil, errUnsupportedVolumeType
	}
	return c, nil
}

func (vh *volumeHost) GetCloudProvider() cloudprovider.Interface {
	return vh.kubelet.cloud
}

func (vh *volumeHost) GetMounter() mount.Interface {
	return vh.kubelet.mounter
}

func (vh *volumeHost) GetWriter() io.Writer {
	return vh.kubelet.writer
}

// Returns the hostname of the host kubelet is running on
func (vh *volumeHost) GetHostName() string {
	return vh.kubelet.hostname
}

func (kl *Kubelet) mountExternalVolumes(pod *api.Pod) (kubecontainer.VolumeMap, error) {
	podVolumes := make(kubecontainer.VolumeMap)
	for i := range pod.Spec.Volumes {
		volSpec := &pod.Spec.Volumes[i]
		var fsGroup *int64
		if pod.Spec.SecurityContext != nil && pod.Spec.SecurityContext.FSGroup != nil {
			fsGroup = pod.Spec.SecurityContext.FSGroup
		}

		rootContext, err := kl.getRootDirContext()
		if err != nil {
			return nil, err
		}

		// Try to use a plugin for this volume.
		internal := volume.NewSpecFromVolume(volSpec)
		mounter, err := kl.newVolumeMounterFromPlugins(internal, pod, volume.VolumeOptions{RootContext: rootContext})
		if err != nil {
			glog.Errorf("Could not create volume mounter for pod %s: %v", pod.UID, err)
			return nil, err
		}
		if mounter == nil {
			return nil, errUnsupportedVolumeType
		}

		// some volumes require attachment before mounter's setup.
		// The plugin can be nil, but non-nil errors are legitimate errors.
		// For non-nil plugins, Attachment to a node is required before Mounter's setup.
		attacher, err := kl.newVolumeAttacherFromPlugins(internal, pod, volume.VolumeOptions{RootContext: rootContext})
		if err != nil {
			glog.Errorf("Could not create volume attacher for pod %s: %v", pod.UID, err)
			return nil, err
		}
		if attacher != nil {
			err = attacher.Attach()
			if err != nil {
				return nil, err
			}
		}

		err = mounter.SetUp(fsGroup)
		if err != nil {
			return nil, err
		}
		podVolumes[volSpec.Name] = kubecontainer.VolumeInfo{Mounter: mounter}
	}
	return podVolumes, nil
}

type volumeTuple struct {
	Kind string
	Name string
}

// ListVolumesForPod returns a map of the volumes associated with the given pod
func (kl *Kubelet) ListVolumesForPod(podUID types.UID) (map[string]volume.Volume, bool) {
	result := map[string]volume.Volume{}
	vm, ok := kl.volumeManager.GetVolumes(podUID)
	if !ok {
		return result, false
	}
	for name, info := range vm {
		result[name] = info.Mounter
	}
	return result, true
}

func (kl *Kubelet) getPodVolumes(podUID types.UID) ([]*volumeTuple, error) {
	var volumes []*volumeTuple
	podVolDir := kl.getPodVolumesDir(podUID)
	volumeKindDirs, err := ioutil.ReadDir(podVolDir)
	if err != nil {
		glog.Errorf("Could not read directory %s: %v", podVolDir, err)
	}
	for _, volumeKindDir := range volumeKindDirs {
		volumeKind := volumeKindDir.Name()
		volumeKindPath := path.Join(podVolDir, volumeKind)
		// ioutil.ReadDir exits without returning any healthy dir when encountering the first lstat error
		// but skipping dirs means no cleanup for healthy volumes. switching to a no-exit api solves this problem
		volumeNameDirs, volumeNameDirsStat, err := util.ReadDirNoExit(volumeKindPath)
		if err != nil {
			return []*volumeTuple{}, fmt.Errorf("could not read directory %s: %v", volumeKindPath, err)
		}
		for i, volumeNameDir := range volumeNameDirs {
			if volumeNameDir != nil {
				volumes = append(volumes, &volumeTuple{Kind: volumeKind, Name: volumeNameDir.Name()})
			} else {
				glog.Errorf("Could not read directory %s: %v", podVolDir, volumeNameDirsStat[i])
			}
		}
	}
	return volumes, nil
}

// cleanerTuple is a union struct to allow separating detaching from the cleaner.
// some volumes require detachment but not all.  Unmounter cannot be nil but Detacher is optional.
type cleanerTuple struct {
	Unmounter volume.Unmounter
	Detacher  *volume.Detacher
}

// getPodVolumesFromDisk examines directory structure to determine volumes that
// are presently active and mounted. Returns a union struct containing a volume.Unmounter
// and potentially a volume.Detacher.
func (kl *Kubelet) getPodVolumesFromDisk() map[string]cleanerTuple {
	currentVolumes := make(map[string]cleanerTuple)
	podUIDs, err := kl.listPodsFromDisk()
	if err != nil {
		glog.Errorf("Could not get pods from disk: %v", err)
		return map[string]cleanerTuple{}
	}
	// Find the volumes for each on-disk pod.
	for _, podUID := range podUIDs {
		volumes, err := kl.getPodVolumes(podUID)
		if err != nil {
			glog.Errorf("%v", err)
			continue
		}
		for _, volume := range volumes {
			identifier := fmt.Sprintf("%s/%s", podUID, volume.Name)
			glog.V(4).Infof("Making a volume.Unmounter for volume %s/%s of pod %s", volume.Kind, volume.Name, podUID)
			// TODO(thockin) This should instead return a reference to an extant
			// volume object, except that we don't actually hold on to pod specs
			// or volume objects.

			// Try to use a plugin for this volume.
			unmounter, err := kl.newVolumeUnmounterFromPlugins(volume.Kind, volume.Name, podUID)
			if err != nil {
				glog.Errorf("Could not create volume unmounter for %s: %v", volume.Name, err)
				continue
			}
			if unmounter == nil {
				glog.Errorf("Could not create volume unmounter for %s: %v", volume.Name, errUnsupportedVolumeType)
				continue
			}

			tuple := cleanerTuple{Unmounter: unmounter}
			detacher, err := kl.newVolumeDetacherFromPlugins(volume.Kind, volume.Name, podUID)
			// plugin can be nil but a non-nil error is a legitimate error
			if err != nil {
				glog.Errorf("Could not create volume detacher for %s: %v", volume.Name, err)
				continue
			}
			if detacher != nil {
				tuple.Detacher = &detacher
			}
			currentVolumes[identifier] = tuple
		}
	}
	return currentVolumes
}

func (kl *Kubelet) newVolumeMounterFromPlugins(spec *volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
	plugin, err := kl.volumePluginMgr.FindPluginBySpec(spec)
	if err != nil {
		return nil, fmt.Errorf("can't use volume plugins for %s: %v", spec.Name(), err)
	}
	if plugin == nil {
		// Not found but not an error
		return nil, nil
	}
	physicalMounter, err := plugin.NewMounter(spec, pod, opts)
	if err != nil {
		return nil, fmt.Errorf("failed to instantiate volume physicalMounter for %s: %v", spec.Name(), err)
	}
	glog.V(10).Infof("Used volume plugin %q to mount %s", plugin.Name(), spec.Name())
	return physicalMounter, nil
}

func (kl *Kubelet) newVolumeAttacherFromPlugins(spec *volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Attacher, error) {
	plugin, err := kl.volumePluginMgr.FindAttachablePluginBySpec(spec)
	if err != nil {
		return nil, fmt.Errorf("can't use volume plugins for %s: %v", spec.Name(), err)
	}
	if plugin == nil {
		// Not found but not an error.
		return nil, nil
	}

	attacher, err := plugin.NewAttacher(spec)
	if err != nil {
		return nil, fmt.Errorf("failed to instantiate volume attacher for %s: %v", spec.Name(), err)
	}
	glog.V(3).Infof("Used volume plugin %q to attach %s/%s", plugin.Name(), spec.Name())
	return attacher, nil
}

func (kl *Kubelet) newVolumeUnmounterFromPlugins(kind string, name string, podUID types.UID) (volume.Unmounter, error) {
	plugName := strings.UnescapeQualifiedNameForDisk(kind)
	plugin, err := kl.volumePluginMgr.FindPluginByName(plugName)
	if err != nil {
		// TODO: Maybe we should launch a cleanup of this dir?
		return nil, fmt.Errorf("can't use volume plugins for %s/%s: %v", podUID, kind, err)
	}
	if plugin == nil {
		// Not found but not an error.
		return nil, nil
	}
	unmounter, err := plugin.NewUnmounter(name, podUID)
	if err != nil {
		return nil, fmt.Errorf("failed to instantiate volume plugin for %s/%s: %v", podUID, kind, err)
	}
	glog.V(5).Infof("Used volume plugin %q to unmount %s/%s", plugin.Name(), podUID, kind)
	return unmounter, nil
}

func (kl *Kubelet) newVolumeDetacherFromPlugins(kind string, name string, podUID types.UID) (volume.Detacher, error) {
	plugName := strings.UnescapeQualifiedNameForDisk(kind)
	plugin, err := kl.volumePluginMgr.FindAttachablePluginByName(plugName)
	if err != nil {
		return nil, fmt.Errorf("can't use volume plugins for %s/%s: %v", podUID, kind, err)
	}
	if plugin == nil {
		// Not found but not an error.
		return nil, nil
	}

	detacher, err := plugin.NewDetacher(name, podUID)
	if err != nil {
		return nil, fmt.Errorf("failed to instantiate volume plugin for %s/%s: %v", podUID, kind, err)
	}
	glog.V(3).Infof("Used volume plugin %q to detach %s/%s", plugin.Name(), podUID, kind)
	return detacher, nil
}
<commit_msg>Add godoc to kubelet/volumes.go<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubelet

import (
	"fmt"
	"io/ioutil"
	"path"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/pkg/cloudprovider"
	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
	"k8s.io/kubernetes/pkg/types"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/io"
	"k8s.io/kubernetes/pkg/util/mount"
	"k8s.io/kubernetes/pkg/util/strings"
	"k8s.io/kubernetes/pkg/volume"
)

var errUnsupportedVolumeType = fmt.Errorf("unsupported volume type")

// This just exports required functions from kubelet proper, for use by volume
// plugins.
type volumeHost struct {
	kubelet *Kubelet
}

func (vh *volumeHost) GetPluginDir(pluginName string) string {
	return vh.kubelet.getPluginDir(pluginName)
}

func (vh *volumeHost) GetPodVolumeDir(podUID types.UID, pluginName string, volumeName string) string {
	return vh.kubelet.getPodVolumeDir(podUID, pluginName, volumeName)
}

func (vh *volumeHost) GetPodPluginDir(podUID types.UID, pluginName string) string {
	return vh.kubelet.getPodPluginDir(podUID, pluginName)
}

func (vh *volumeHost) GetKubeClient() clientset.Interface {
	return vh.kubelet.kubeClient
}

func (vh *volumeHost) NewWrapperMounter(volName string, spec volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
	// The name of wrapper volume is set to "wrapped_{wrapped_volume_name}"
	wrapperVolumeName := "wrapped_" + volName
	if spec.Volume != nil {
		spec.Volume.Name = wrapperVolumeName
	}

	b, err := vh.kubelet.newVolumeMounterFromPlugins(&spec, pod, opts)
	if err == nil && b == nil {
		return nil, errUnsupportedVolumeType
	}
	return b, nil
}

func (vh *volumeHost) NewWrapperUnmounter(volName string, spec volume.Spec, podUID types.UID) (volume.Unmounter, error) {
	// The name of wrapper volume is set to "wrapped_{wrapped_volume_name}"
	wrapperVolumeName := "wrapped_" + volName
	if spec.Volume != nil {
		spec.Volume.Name = wrapperVolumeName
	}

	plugin, err := vh.kubelet.volumePluginMgr.FindPluginBySpec(&spec)
	if err != nil {
		return nil, err
	}
	if plugin == nil {
		// Not found but not an error
		return nil, nil
	}
	c, err := plugin.NewUnmounter(spec.Name(), podUID)
	if err == nil && c == nil {
		return nil, errUnsupportedVolumeType
	}
	return c, nil
}

func (vh *volumeHost) GetCloudProvider() cloudprovider.Interface {
	return vh.kubelet.cloud
}

func (vh *volumeHost) GetMounter() mount.Interface {
	return vh.kubelet.mounter
}

func (vh *volumeHost) GetWriter() io.Writer {
	return vh.kubelet.writer
}

// Returns the hostname of the host kubelet is running on
func (vh *volumeHost) GetHostName() string {
	return vh.kubelet.hostname
}

// mountExternalVolumes mounts the volumes declared in a pod, attaching them
// to the host if necessary, and returns a map containing information about
// the volumes for the pod or an error.  This method is run multiple times,
// and requires that implementations of Attach() and SetUp() be idempotent.
//
// Note, in the future, the attach-detach controller will handle attaching and
// detaching volumes; this call site will be maintained for backward-
// compatibility with current behavior of static pods and pods created via the
// Kubelet's http API.
func (kl *Kubelet) mountExternalVolumes(pod *api.Pod) (kubecontainer.VolumeMap, error) {
	podVolumes := make(kubecontainer.VolumeMap)
	for i := range pod.Spec.Volumes {
		volSpec := &pod.Spec.Volumes[i]
		var fsGroup *int64
		if pod.Spec.SecurityContext != nil && pod.Spec.SecurityContext.FSGroup != nil {
			fsGroup = pod.Spec.SecurityContext.FSGroup
		}

		rootContext, err := kl.getRootDirContext()
		if err != nil {
			return nil, err
		}

		// Try to use a plugin for this volume.
		internal := volume.NewSpecFromVolume(volSpec)
		mounter, err := kl.newVolumeMounterFromPlugins(internal, pod, volume.VolumeOptions{RootContext: rootContext})
		if err != nil {
			glog.Errorf("Could not create volume mounter for pod %s: %v", pod.UID, err)
			return nil, err
		}
		if mounter == nil {
			return nil, errUnsupportedVolumeType
		}

		// some volumes require attachment before mounter's setup.
		// The plugin can be nil, but non-nil errors are legitimate errors.
		// For non-nil plugins, Attachment to a node is required before Mounter's setup.
		attacher, err := kl.newVolumeAttacherFromPlugins(internal, pod, volume.VolumeOptions{RootContext: rootContext})
		if err != nil {
			glog.Errorf("Could not create volume attacher for pod %s: %v", pod.UID, err)
			return nil, err
		}
		if attacher != nil {
			err = attacher.Attach()
			if err != nil {
				return nil, err
			}
		}

		err = mounter.SetUp(fsGroup)
		if err != nil {
			return nil, err
		}
		podVolumes[volSpec.Name] = kubecontainer.VolumeInfo{Mounter: mounter}
	}
	return podVolumes, nil
}

type volumeTuple struct {
	Kind string
	Name string
}

// ListVolumesForPod returns a map of the volumes associated with the given pod
func (kl *Kubelet) ListVolumesForPod(podUID types.UID) (map[string]volume.Volume, bool) {
	result := map[string]volume.Volume{}
	vm, ok := kl.volumeManager.GetVolumes(podUID)
	if !ok {
		return result, false
	}
	for name, info := range vm {
		result[name] = info.Mounter
	}
	return result, true
}

// getPodVolumes examines the directory structure for a pod and returns
// information about the name and kind of each presently mounted volume, or an
// error.
func (kl *Kubelet) getPodVolumes(podUID types.UID) ([]*volumeTuple, error) {
	var volumes []*volumeTuple
	podVolDir := kl.getPodVolumesDir(podUID)
	volumeKindDirs, err := ioutil.ReadDir(podVolDir)
	if err != nil {
		glog.Errorf("Could not read directory %s: %v", podVolDir, err)
	}
	for _, volumeKindDir := range volumeKindDirs {
		volumeKind := volumeKindDir.Name()
		volumeKindPath := path.Join(podVolDir, volumeKind)
		// ioutil.ReadDir exits without returning any healthy dir when encountering the first lstat error
		// but skipping dirs means no cleanup for healthy volumes. switching to a no-exit api solves this problem
		volumeNameDirs, volumeNameDirsStat, err := util.ReadDirNoExit(volumeKindPath)
		if err != nil {
			return []*volumeTuple{}, fmt.Errorf("could not read directory %s: %v", volumeKindPath, err)
		}
		for i, volumeNameDir := range volumeNameDirs {
			if volumeNameDir != nil {
				volumes = append(volumes, &volumeTuple{Kind: volumeKind, Name: volumeNameDir.Name()})
			} else {
				glog.Errorf("Could not read directory %s: %v", podVolDir, volumeNameDirsStat[i])
			}
		}
	}
	return volumes, nil
}

// cleanerTuple is a union struct to allow separating detaching from the cleaner.
// some volumes require detachment but not all.  Unmounter cannot be nil but Detacher is optional.
type cleanerTuple struct {
	Unmounter volume.Unmounter
	Detacher  *volume.Detacher
}

// getPodVolumesFromDisk examines directory structure to determine volumes that
// are presently active and mounted. Returns a union struct containing a volume.Unmounter
// and potentially a volume.Detacher.
func (kl *Kubelet) getPodVolumesFromDisk() map[string]cleanerTuple {
	currentVolumes := make(map[string]cleanerTuple)
	podUIDs, err := kl.listPodsFromDisk()
	if err != nil {
		glog.Errorf("Could not get pods from disk: %v", err)
		return map[string]cleanerTuple{}
	}
	// Find the volumes for each on-disk pod.
	for _, podUID := range podUIDs {
		volumes, err := kl.getPodVolumes(podUID)
		if err != nil {
			glog.Errorf("%v", err)
			continue
		}
		for _, volume := range volumes {
			identifier := fmt.Sprintf("%s/%s", podUID, volume.Name)
			glog.V(4).Infof("Making a volume.Unmounter for volume %s/%s of pod %s", volume.Kind, volume.Name, podUID)
			// TODO(thockin) This should instead return a reference to an extant
			// volume object, except that we don't actually hold on to pod specs
			// or volume objects.

			// Try to use a plugin for this volume.
			unmounter, err := kl.newVolumeUnmounterFromPlugins(volume.Kind, volume.Name, podUID)
			if err != nil {
				glog.Errorf("Could not create volume unmounter for %s: %v", volume.Name, err)
				continue
			}
			if unmounter == nil {
				glog.Errorf("Could not create volume unmounter for %s: %v", volume.Name, errUnsupportedVolumeType)
				continue
			}

			tuple := cleanerTuple{Unmounter: unmounter}
			detacher, err := kl.newVolumeDetacherFromPlugins(volume.Kind, volume.Name, podUID)
			// plugin can be nil but a non-nil error is a legitimate error
			if err != nil {
				glog.Errorf("Could not create volume detacher for %s: %v", volume.Name, err)
				continue
			}
			if detacher != nil {
				tuple.Detacher = &detacher
			}
			currentVolumes[identifier] = tuple
		}
	}
	return currentVolumes
}

func (kl *Kubelet) newVolumeMounterFromPlugins(spec *volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Mounter, error) {
	plugin, err := kl.volumePluginMgr.FindPluginBySpec(spec)
	if err != nil {
		return nil, fmt.Errorf("can't use volume plugins for %s: %v", spec.Name(), err)
	}
	if plugin == nil {
		// Not found but not an error
		return nil, nil
	}
	physicalMounter, err := plugin.NewMounter(spec, pod, opts)
	if err != nil {
		return nil, fmt.Errorf("failed to instantiate volume physicalMounter for %s: %v", spec.Name(), err)
	}
	glog.V(10).Infof("Used volume plugin %q to mount %s", plugin.Name(), spec.Name())
	return physicalMounter, nil
}

func (kl *Kubelet) newVolumeAttacherFromPlugins(spec *volume.Spec, pod *api.Pod, opts volume.VolumeOptions) (volume.Attacher, error) {
	plugin, err := kl.volumePluginMgr.FindAttachablePluginBySpec(spec)
	if err != nil {
		return nil, fmt.Errorf("can't use volume plugins for %s: %v", spec.Name(), err)
	}
	if plugin == nil {
		// Not found but not an error.
		return nil, nil
	}

	attacher, err := plugin.NewAttacher(spec)
	if err != nil {
		return nil, fmt.Errorf("failed to instantiate volume attacher for %s: %v", spec.Name(), err)
	}
	glog.V(3).Infof("Used volume plugin %q to attach %s/%s", plugin.Name(), spec.Name())
	return attacher, nil
}

func (kl *Kubelet) newVolumeUnmounterFromPlugins(kind string, name string, podUID types.UID) (volume.Unmounter, error) {
	plugName := strings.UnescapeQualifiedNameForDisk(kind)
	plugin, err := kl.volumePluginMgr.FindPluginByName(plugName)
	if err != nil {
		// TODO: Maybe we should launch a cleanup of this dir?
		return nil, fmt.Errorf("can't use volume plugins for %s/%s: %v", podUID, kind, err)
	}
	if plugin == nil {
		// Not found but not an error.
		return nil, nil
	}
	unmounter, err := plugin.NewUnmounter(name, podUID)
	if err != nil {
		return nil, fmt.Errorf("failed to instantiate volume plugin for %s/%s: %v", podUID, kind, err)
	}
	glog.V(5).Infof("Used volume plugin %q to unmount %s/%s", plugin.Name(), podUID, kind)
	return unmounter, nil
}

func (kl *Kubelet) newVolumeDetacherFromPlugins(kind string, name string, podUID types.UID) (volume.Detacher, error) {
	plugName := strings.UnescapeQualifiedNameForDisk(kind)
	plugin, err := kl.volumePluginMgr.FindAttachablePluginByName(plugName)
	if err != nil {
		return nil, fmt.Errorf("can't use volume plugins for %s/%s: %v", podUID, kind, err)
	}
	if plugin == nil {
		// Not found but not an error.
		return nil, nil
	}

	detacher, err := plugin.NewDetacher(name, podUID)
	if err != nil {
		return nil, fmt.Errorf("failed to instantiate volume plugin for %s/%s: %v", podUID, kind, err)
	}
	glog.V(3).Infof("Used volume plugin %q to detach %s/%s", plugin.Name(), podUID, kind)
	return detacher, nil
}
<|endoftext|>


<commit_before>package overlay

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/containers/storage/pkg/idtools"
	"github.com/containers/storage/pkg/system"
	"github.com/containers/storage/pkg/unshare"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/sys/unix"
)

// TempDir generates an overlay Temp directory in the container content
func TempDir(containerDir string, rootUID, rootGID int) (string, error) {
	contentDir := filepath.Join(containerDir, "overlay")
	if err := idtools.MkdirAllAs(contentDir, 0700, rootUID, rootGID); err != nil {
		return "", errors.Wrapf(err, "failed to create the overlay %s directory", contentDir)
	}

	contentDir, err := ioutil.TempDir(contentDir, "")
	if err != nil {
		return "", errors.Wrapf(err, "failed to create the overlay tmpdir in %s directory", contentDir)
	}

	return generateOverlayStructure(contentDir, rootUID, rootGID)
}

// GenerateStructure generates an overlay directory structure for container content
func GenerateStructure(containerDir, containerID, name string, rootUID, rootGID int) (string, error) {
	contentDir := filepath.Join(containerDir, "overlay-containers", containerID, name)
	if err := idtools.MkdirAllAs(contentDir, 0700, rootUID, rootGID); err != nil {
		return "", errors.Wrapf(err, "failed to create the overlay %s directory", contentDir)
	}

	return generateOverlayStructure(contentDir, rootUID, rootGID)
}

// generateOverlayStructure generates upper, work and merge directory structure for overlay directory
func generateOverlayStructure(containerDir string, rootUID, rootGID int) (string, error) {
	upperDir := filepath.Join(containerDir, "upper")
	workDir := filepath.Join(containerDir, "work")
	if err := idtools.MkdirAllAs(upperDir, 0700, rootUID, rootGID); err != nil {
		return "", errors.Wrapf(err, "failed to create the overlay %s directory", upperDir)
	}
	if err := idtools.MkdirAllAs(workDir, 0700, rootUID, rootGID); err != nil {
		return "", errors.Wrapf(err, "failed to create the overlay %s directory", workDir)
	}
	mergeDir := filepath.Join(containerDir, "merge")
	if err := idtools.MkdirAllAs(mergeDir, 0700, rootUID, rootGID); err != nil {
		return "", errors.Wrapf(err, "failed to create the overlay %s directory", mergeDir)
	}

	return containerDir, nil
}

// Mount creates a subdir of the contentDir based on the source directory
// from the source system.  It then mounts up the source directory on to the
// generated mount point and returns the mount point to the caller.
func Mount(contentDir, source, dest string, rootUID, rootGID int, graphOptions []string) (mount specs.Mount, Err error) {
	return mountHelper(contentDir, source, dest, rootUID, rootGID, graphOptions, false)
}

// MountReadOnly creates a subdir of the contentDir based on the source directory
// from the source system.  It then mounts up the source directory on to the
// generated mount point and returns the mount point to the caller.  Note that no
// upper layer will be created rendering it a read-only mount
func MountReadOnly(contentDir, source, dest string, rootUID, rootGID int, graphOptions []string) (mount specs.Mount, Err error) {
	return mountHelper(contentDir, source, dest, rootUID, rootGID, graphOptions, true)
}

// NOTE: rootUID and rootUID are not yet used.
func mountHelper(contentDir, source, dest string, _, _ int, graphOptions []string, readOnly bool) (mount specs.Mount, Err error) {
	mergeDir := filepath.Join(contentDir, "merge")

	// Create overlay mount options for rw/ro.
	var overlayOptions string
	if readOnly {
		// Read-only overlay mounts require two lower layer.
		lowerTwo := filepath.Join(contentDir, "lower")
		if err := os.Mkdir(lowerTwo, 0755); err != nil {
			return mount, err
		}
		overlayOptions = fmt.Sprintf("lowerdir=%s:%s,private", escapeColon(source), lowerTwo)
	} else {
		// Read-write overlay mounts want a lower, upper and a work layer.
		workDir := filepath.Join(contentDir, "work")
		upperDir := filepath.Join(contentDir, "upper")
		st, err := os.Stat(source)
		if err != nil {
			return mount, err
		}
		if err := os.Chmod(upperDir, st.Mode()); err != nil {
			return mount, err
		}
		if stat, ok := st.Sys().(*syscall.Stat_t); ok {
			if err := os.Chown(upperDir, int(stat.Uid), int(stat.Gid)); err != nil {
				return mount, err
			}
		}
		overlayOptions = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s,private", escapeColon(source), upperDir, workDir)
	}

	if unshare.IsRootless() {
		mountProgram := ""

		mountMap := map[string]bool{
			".mount_program":         true,
			"overlay.mount_program":  true,
			"overlay2.mount_program": true,
		}

		for _, i := range graphOptions {
			s := strings.SplitN(i, "=", 2)
			if len(s) != 2 {
				continue
			}
			key := s[0]
			val := s[1]
			if mountMap[key] {
				mountProgram = val
				break
			}
		}
		if mountProgram != "" {
			cmd := exec.Command(mountProgram, "-o", overlayOptions, mergeDir)

			if err := cmd.Run(); err != nil {
				return mount, errors.Wrapf(err, "exec %s", mountProgram)
			}

			mount.Source = mergeDir
			mount.Destination = dest
			mount.Type = "bind"
			mount.Options = []string{"bind", "slave"}
			return mount, nil
		}
		/* If a mount_program is not specified, fallback to try mount native overlay.  */
		overlayOptions = fmt.Sprintf("%s,userxattr", overlayOptions)
	}

	mount.Source = mergeDir
	mount.Destination = dest
	mount.Type = "overlay"
	mount.Options = strings.Split(overlayOptions, ",")

	return mount, nil
}

// Convert ":" to "\:", the path which will be overlay mounted need to be escaped
func escapeColon(source string) string {
	return strings.ReplaceAll(source, ":", "\\:")
}

// RemoveTemp removes temporary mountpoint and all content from its parent
// directory
func RemoveTemp(contentDir string) error {
	if err := Unmount(contentDir); err != nil {
		return err
	}

	return os.RemoveAll(contentDir)
}

// Unmount the overlay mountpoint
func Unmount(contentDir string) error {
	mergeDir := filepath.Join(contentDir, "merge")

	if unshare.IsRootless() {
		// Attempt to unmount the FUSE mount using either fusermount or fusermount3.
		// If they fail, fallback to unix.Unmount
		for _, v := range []string{"fusermount3", "fusermount"} {
			err := exec.Command(v, "-u", mergeDir).Run()
			if err != nil && errors.Cause(err) != exec.ErrNotFound {
				logrus.Debugf("Error unmounting %s with %s - %v", mergeDir, v, err)
			}
			if err == nil {
				return nil
			}
		}
		// If fusermount|fusermount3 failed to unmount the FUSE file system, attempt unmount
	}

	// Ignore EINVAL as the specified merge dir is not a mount point
	if err := unix.Unmount(mergeDir, 0); err != nil && !os.IsNotExist(err) && err != unix.EINVAL {
		return errors.Wrapf(err, "unmount overlay %s", mergeDir)
	}
	return nil
}

func recreate(contentDir string) error {
	st, err := system.Stat(contentDir)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return errors.Wrap(err, "failed to stat overlay upper directory")
	}

	if err := os.RemoveAll(contentDir); err != nil {
		return errors.WithStack(err)
	}

	if err := idtools.MkdirAllAs(contentDir, os.FileMode(st.Mode()), int(st.UID()), int(st.GID())); err != nil {
		return errors.Wrap(err, "failed to create overlay directory")
	}
	return nil
}

// CleanupMount removes all temporary mountpoint content
func CleanupMount(contentDir string) (Err error) {
	if err := recreate(filepath.Join(contentDir, "upper")); err != nil {
		return err
	}
	if err := recreate(filepath.Join(contentDir, "work")); err != nil {
		return err
	}
	return nil
}

// CleanupContent removes all temporary mountpoint and all content from
// directory
func CleanupContent(containerDir string) (Err error) {
	contentDir := filepath.Join(containerDir, "overlay")

	files, err := ioutil.ReadDir(contentDir)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return errors.Wrap(err, "read directory")
	}
	for _, f := range files {
		dir := filepath.Join(contentDir, f.Name())
		if err := Unmount(dir); err != nil {
			return err
		}
	}

	if err := os.RemoveAll(contentDir); err != nil && !os.IsNotExist(err) {
		return errors.Wrap(err, "failed to cleanup overlay directory")
	}
	return nil
}
<commit_msg>overlay: add MountWithOptions to API which extends support for advanced overlay<commit_after>package overlay

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/containers/storage/pkg/idtools"
	"github.com/containers/storage/pkg/system"
	"github.com/containers/storage/pkg/unshare"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/sys/unix"
)

// Overlay type holds various configuration options for overlay
// MountWithOptions accepts following type so its easier to specify
// more verbose configuration for overlay mount.
type Options struct {
	// The Upper directory is normally writable layer in an overlay mount.
	// Note!! : Following API does not handles escaping or validates correctness of the values
	// passed to UpperDirOptionFragment instead API will try to pass values as is it
	// to the `mount` command. Its user's responsibility to make sure they pre-validate
	// these values. Invalid inputs may lead to undefined behviour.
	// This is provided as-is, use it if it works for you, we can/will change/break that in the future.
	// See discussion here for more context: https://github.com/containers/buildah/pull/3715#discussion_r786036959
	// TODO: Should we address above comment and handle escaping of metacharacters like
	// `comma`, `backslash` ,`colon` and any other specical characters
	UpperDirOptionFragment string
	// The Workdir is used to prepare files as they are switched between the layers.
	// Note!! : Following API does not handles escaping or validates correctness of the values
	// passed to WorkDirOptionFragment instead API will try to pass values as is it
	// to the `mount` command. Its user's responsibility to make sure they pre-validate
	// these values. Invalid inputs may lead to undefined behviour.
	// This is provided as-is, use it if it works for you, we can/will change/break that in the future.
	// See discussion here for more context: https://github.com/containers/buildah/pull/3715#discussion_r786036959
	// TODO: Should we address above comment and handle escaping of metacharacters like
	// `comma`, `backslash` ,`colon` and any other specical characters
	WorkDirOptionFragment string
	// Graph options relayed from podman, will be responsible for choosing mount program
	GraphOpts []string
	// Mark if following overlay is read only
	ReadOnly bool
	// RootUID is not used yet but keeping it here for legacy reasons.
	RootUID int
	// RootGID is not used yet but keeping it here for legacy reasons.
	RootGID int
}

// TempDir generates an overlay Temp directory in the container content
func TempDir(containerDir string, rootUID, rootGID int) (string, error) {
	contentDir := filepath.Join(containerDir, "overlay")
	if err := idtools.MkdirAllAs(contentDir, 0700, rootUID, rootGID); err != nil {
		return "", errors.Wrapf(err, "failed to create the overlay %s directory", contentDir)
	}

	contentDir, err := ioutil.TempDir(contentDir, "")
	if err != nil {
		return "", errors.Wrapf(err, "failed to create the overlay tmpdir in %s directory", contentDir)
	}

	return generateOverlayStructure(contentDir, rootUID, rootGID)
}

// GenerateStructure generates an overlay directory structure for container content
func GenerateStructure(containerDir, containerID, name string, rootUID, rootGID int) (string, error) {
	contentDir := filepath.Join(containerDir, "overlay-containers", containerID, name)
	if err := idtools.MkdirAllAs(contentDir, 0700, rootUID, rootGID); err != nil {
		return "", errors.Wrapf(err, "failed to create the overlay %s directory", contentDir)
	}

	return generateOverlayStructure(contentDir, rootUID, rootGID)
}

// generateOverlayStructure generates upper, work and merge directory structure for overlay directory
func generateOverlayStructure(containerDir string, rootUID, rootGID int) (string, error) {
	upperDir := filepath.Join(containerDir, "upper")
	workDir := filepath.Join(containerDir, "work")
	if err := idtools.MkdirAllAs(upperDir, 0700, rootUID, rootGID); err != nil {
		return "", errors.Wrapf(err, "failed to create the overlay %s directory", upperDir)
	}
	if err := idtools.MkdirAllAs(workDir, 0700, rootUID, rootGID); err != nil {
		return "", errors.Wrapf(err, "failed to create the overlay %s directory", workDir)
	}
	mergeDir := filepath.Join(containerDir, "merge")
	if err := idtools.MkdirAllAs(mergeDir, 0700, rootUID, rootGID); err != nil {
		return "", errors.Wrapf(err, "failed to create the overlay %s directory", mergeDir)
	}

	return containerDir, nil
}

// Mount creates a subdir of the contentDir based on the source directory
// from the source system.  It then mounts up the source directory on to the
// generated mount point and returns the mount point to the caller.
func Mount(contentDir, source, dest string, rootUID, rootGID int, graphOptions []string) (mount specs.Mount, Err error) {
	overlayOpts := Options{GraphOpts: graphOptions, ReadOnly: false, RootUID: rootUID, RootGID: rootGID}
	return MountWithOptions(contentDir, source, dest, &overlayOpts)
}

// MountReadOnly creates a subdir of the contentDir based on the source directory
// from the source system.  It then mounts up the source directory on to the
// generated mount point and returns the mount point to the caller.  Note that no
// upper layer will be created rendering it a read-only mount
func MountReadOnly(contentDir, source, dest string, rootUID, rootGID int, graphOptions []string) (mount specs.Mount, Err error) {
	overlayOpts := Options{GraphOpts: graphOptions, ReadOnly: true, RootUID: rootUID, RootGID: rootGID}
	return MountWithOptions(contentDir, source, dest, &overlayOpts)
}

// MountWithOptions creates a subdir of the contentDir based on the source directory
// from the source system.  It then mounts up the source directory on to the
// generated mount point and returns the mount point to the caller.
// But allows api to set custom workdir, upperdir and other overlay options
// Following API is being used by podman at the moment
func MountWithOptions(contentDir, source, dest string, opts *Options) (mount specs.Mount, Err error) {
	mergeDir := filepath.Join(contentDir, "merge")

	// Create overlay mount options for rw/ro.
	var overlayOptions string
	if opts.ReadOnly {
		// Read-only overlay mounts require two lower layer.
		lowerTwo := filepath.Join(contentDir, "lower")
		if err := os.Mkdir(lowerTwo, 0755); err != nil {
			return mount, err
		}
		overlayOptions = fmt.Sprintf("lowerdir=%s:%s,private", escapeColon(source), lowerTwo)
	} else {
		// Read-write overlay mounts want a lower, upper and a work layer.
		workDir := filepath.Join(contentDir, "work")
		upperDir := filepath.Join(contentDir, "upper")

		if opts.WorkDirOptionFragment != "" && opts.UpperDirOptionFragment != "" {
			workDir = opts.WorkDirOptionFragment
			upperDir = opts.UpperDirOptionFragment
		}

		st, err := os.Stat(source)
		if err != nil {
			return mount, err
		}
		if err := os.Chmod(upperDir, st.Mode()); err != nil {
			return mount, err
		}
		if stat, ok := st.Sys().(*syscall.Stat_t); ok {
			if err := os.Chown(upperDir, int(stat.Uid), int(stat.Gid)); err != nil {
				return mount, err
			}
		}
		overlayOptions = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s,private", escapeColon(source), upperDir, workDir)
	}

	if unshare.IsRootless() {
		mountProgram := ""

		mountMap := map[string]bool{
			".mount_program":         true,
			"overlay.mount_program":  true,
			"overlay2.mount_program": true,
		}

		for _, i := range opts.GraphOpts {
			s := strings.SplitN(i, "=", 2)
			if len(s) != 2 {
				continue
			}
			key := s[0]
			val := s[1]
			if mountMap[key] {
				mountProgram = val
				break
			}
		}
		if mountProgram != "" {
			cmd := exec.Command(mountProgram, "-o", overlayOptions, mergeDir)

			if err := cmd.Run(); err != nil {
				return mount, errors.Wrapf(err, "exec %s", mountProgram)
			}

			mount.Source = mergeDir
			mount.Destination = dest
			mount.Type = "bind"
			mount.Options = []string{"bind", "slave"}
			return mount, nil
		}
		/* If a mount_program is not specified, fallback to try mount native overlay.  */
		overlayOptions = fmt.Sprintf("%s,userxattr", overlayOptions)
	}

	mount.Source = mergeDir
	mount.Destination = dest
	mount.Type = "overlay"
	mount.Options = strings.Split(overlayOptions, ",")

	return mount, nil
}

// Convert ":" to "\:", the path which will be overlay mounted need to be escaped
func escapeColon(source string) string {
	return strings.ReplaceAll(source, ":", "\\:")
}

// RemoveTemp removes temporary mountpoint and all content from its parent
// directory
func RemoveTemp(contentDir string) error {
	if err := Unmount(contentDir); err != nil {
		return err
	}

	return os.RemoveAll(contentDir)
}

// Unmount the overlay mountpoint
func Unmount(contentDir string) error {
	mergeDir := filepath.Join(contentDir, "merge")

	if unshare.IsRootless() {
		// Attempt to unmount the FUSE mount using either fusermount or fusermount3.
		// If they fail, fallback to unix.Unmount
		for _, v := range []string{"fusermount3", "fusermount"} {
			err := exec.Command(v, "-u", mergeDir).Run()
			if err != nil && errors.Cause(err) != exec.ErrNotFound {
				logrus.Debugf("Error unmounting %s with %s - %v", mergeDir, v, err)
			}
			if err == nil {
				return nil
			}
		}
		// If fusermount|fusermount3 failed to unmount the FUSE file system, attempt unmount
	}

	// Ignore EINVAL as the specified merge dir is not a mount point
	if err := unix.Unmount(mergeDir, 0); err != nil && !os.IsNotExist(err) && err != unix.EINVAL {
		return errors.Wrapf(err, "unmount overlay %s", mergeDir)
	}
	return nil
}

func recreate(contentDir string) error {
	st, err := system.Stat(contentDir)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return errors.Wrap(err, "failed to stat overlay upper directory")
	}

	if err := os.RemoveAll(contentDir); err != nil {
		return errors.WithStack(err)
	}

	if err := idtools.MkdirAllAs(contentDir, os.FileMode(st.Mode()), int(st.UID()), int(st.GID())); err != nil {
		return errors.Wrap(err, "failed to create overlay directory")
	}
	return nil
}

// CleanupMount removes all temporary mountpoint content
func CleanupMount(contentDir string) (Err error) {
	if err := recreate(filepath.Join(contentDir, "upper")); err != nil {
		return err
	}
	if err := recreate(filepath.Join(contentDir, "work")); err != nil {
		return err
	}
	return nil
}

// CleanupContent removes all temporary mountpoint and all content from
// directory
func CleanupContent(containerDir string) (Err error) {
	contentDir := filepath.Join(containerDir, "overlay")

	files, err := ioutil.ReadDir(contentDir)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return errors.Wrap(err, "read directory")
	}
	for _, f := range files {
		dir := filepath.Join(contentDir, f.Name())
		if err := Unmount(dir); err != nil {
			return err
		}
	}

	if err := os.RemoveAll(contentDir); err != nil && !os.IsNotExist(err) {
		return errors.Wrap(err, "failed to cleanup overlay directory")
	}
	return nil
}
<|endoftext|>


<commit_before>package project

var (
	description string = "The azure-operator manages Kubernetes clusters on Azure."
	gitSHA             = "n/a"
	name        string = "azure-operator"
	source      string = "https://github.com/giantswarm/azure-operator"
	version            = "5.7.0"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<commit_msg>Bump version to 5.7.1-dev (#1447)<commit_after>package project

var (
	description string = "The azure-operator manages Kubernetes clusters on Azure."
	gitSHA             = "n/a"
	name        string = "azure-operator"
	source      string = "https://github.com/giantswarm/azure-operator"
	version            = "5.7.1-dev"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<|endoftext|>


<commit_before>package proxy

import (
	"net"
	"net/http"
	"net/url"
	"strconv"
	"testing"
)

const NORMAL_PORT = 12354
const PROXY_PORT = 12355

type serverCb func(w http.ResponseWriter, r *http.Request)

type server struct {
	listener *net.Listener
	callback serverCb
}

func TestHttpRewriteHost(t *testing.T) {
	client := &http.Client{}
	settings := &Settings{false, true}

	incoming := make(chan string, 1)
	proxyHost := "localhost:" + strconv.Itoa(PROXY_PORT)
	normalHost := "localhost:" + strconv.Itoa(NORMAL_PORT)

	// Create normal server
	normal := newServer(func(w http.ResponseWriter, r *http.Request) {
		incoming <- r.Host
		w.Write([]byte("peace out"))
	})

	// Create proxy server
	proxy := newServer(func(w http.ResponseWriter, r *http.Request) {
		proxyURL := url.URL{Scheme: "http", Host: proxyHost, Path: "/foo"}
		destURL := url.URL{Scheme: "http", Host: normalHost, Path: "/bar"}
		ctx := Context{r, w, &proxyURL, &destURL, settings}
		ProxyHTTP(&ctx, client)
	})

	// Start servers
	if err := normal.start(NORMAL_PORT); err != nil {
		t.Fatalf("Failed to listen on port %d: %s", NORMAL_PORT, err.Error())
	}
	defer normal.stop()
	if err := proxy.start(PROXY_PORT); err != nil {
		t.Fatalf("Failed to listen on port %d: %s", PROXY_PORT, err.Error())
	}
	defer proxy.stop()

	// Host rewrite enabled
	res, err := http.Get("http://" + proxyHost + "/foo")
	if err != nil {
		t.Fatal("Failed to make request:", err)
	}
	res.Body.Close()
	if gotHost := <-incoming; gotHost != normalHost {
		t.Errorf("(Host rewrite enabled) expected %s got %s", normalHost,
			gotHost)
	}

	// Host rewrite disabled
	settings.RewriteHost = false
	res, err = http.Get("http://" + proxyHost + "/foo")
	if err != nil {
		t.Fatal("Failed to make request:", err)
	}
	res.Body.Close()
	if gotHost := <-incoming; gotHost != proxyHost {
		t.Errorf("(Host rewrite disabled) expected %s got %s", proxyHost,
			gotHost)
	}
}

func newServer(callback serverCb) *server {
	return &server{nil, callback}
}

func (self *server) start(port int) error {
	listener, err := net.Listen("tcp", ":"+strconv.Itoa(port))
	if err != nil {
		return err
	}

	self.listener = &listener
	go http.Serve(listener, self)
	return nil
}

func (self *server) stop() {
	(*self.listener).Close()
	self.listener = nil
}

func (self *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	self.callback(w, r)
}
<commit_msg>HTTP POST, status codes, content types work<commit_after>package proxy

import (
	"bytes"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strconv"
	"testing"
)

const NORMAL_PORT = 12354
const PROXY_PORT = 12355
const NORMAL_PORT_2 = 12356
const PROXY_PORT_2 = 12357

type serverCb func(w http.ResponseWriter, r *http.Request)

type server struct {
	listener *net.Listener
	callback serverCb
}

func TestHttpRewriteHost(t *testing.T) {
	client := new(http.Client)
	settings := &Settings{false, true}

	incoming := make(chan string, 1)
	proxyHost := "localhost:" + strconv.Itoa(PROXY_PORT)
	normalHost := "localhost:" + strconv.Itoa(NORMAL_PORT)

	// Create normal server
	normal := newServer(func(w http.ResponseWriter, r *http.Request) {
		incoming <- r.Host
		w.Write([]byte("peace out"))
	})

	// Create proxy server
	proxy := newServer(func(w http.ResponseWriter, r *http.Request) {
		proxyURL := url.URL{Scheme: "http", Host: proxyHost, Path: "/foo"}
		destURL := url.URL{Scheme: "http", Host: normalHost, Path: "/bar"}
		ctx := Context{r, w, &proxyURL, &destURL, settings}
		ProxyHTTP(&ctx, client)
	})

	// Start servers
	if err := normal.start(NORMAL_PORT); err != nil {
		t.Fatalf("Failed to listen on port %d: %s", NORMAL_PORT, err.Error())
	}
	defer normal.stop()
	if err := proxy.start(PROXY_PORT); err != nil {
		t.Fatalf("Failed to listen on port %d: %s", PROXY_PORT, err.Error())
	}
	defer proxy.stop()

	// Host rewrite enabled
	res, err := http.Get("http://" + proxyHost + "/foo")
	if err != nil {
		t.Fatal("Failed to make request:", err)
	}
	res.Body.Close()
	if gotHost := <-incoming; gotHost != normalHost {
		t.Errorf("(Host rewrite enabled) expected %s got %s", normalHost,
			gotHost)
	}

	// Host rewrite disabled
	settings.RewriteHost = false
	res, err = http.Get("http://" + proxyHost + "/foo")
	if err != nil {
		t.Fatal("Failed to make request:", err)
	}
	res.Body.Close()
	if gotHost := <-incoming; gotHost != proxyHost {
		t.Errorf("(Host rewrite disabled) expected %s got %s", proxyHost,
			gotHost)
	}
}

func TestHttpProxy(t *testing.T) {
	client := new(http.Client)
	settings := new(Settings)

	incoming := make(chan resInfo, 1)
	proxyHost := "localhost:" + strconv.Itoa(PROXY_PORT_2)
	normalHost := "localhost:" + strconv.Itoa(NORMAL_PORT_2)

	// Create normal server
	normal := newServer(func(w http.ResponseWriter, r *http.Request) {
		if data, err := ioutil.ReadAll(r.Body); err != nil {
			t.Fatal("Failed to read incoming data:", err)
		} else {
			incoming <- resInfo{data, r.Header}
		}
		w.Header().Set("Content-Type", "text/html")
		w.WriteHeader(404)
		w.Write([]byte("Some kind of 404 page!"))
	})

	// Create proxy server
	proxy := newServer(func(w http.ResponseWriter, r *http.Request) {
		proxyURL := url.URL{Scheme: "http", Host: proxyHost, Path: "/foo"}
		destURL := url.URL{Scheme: "http", Host: normalHost, Path: "/bar"}
		ctx := Context{r, w, &proxyURL, &destURL, settings}
		ProxyHTTP(&ctx, client)
	})

	// Start servers
	if err := normal.start(NORMAL_PORT_2); err != nil {
		t.Fatalf("Failed to listen on port %d: %s", NORMAL_PORT_2, err.Error())
	}
	defer normal.stop()
	if err := proxy.start(PROXY_PORT_2); err != nil {
		t.Fatalf("Failed to listen on port %d: %s", PROXY_PORT_2, err.Error())
	}
	defer proxy.stop()

	// Get data
	res, err := http.Post("http://" + proxyHost + "/foo", "text/plain",
		bytes.NewBuffer([]byte("Request data")))
	if err != nil {
		t.Fatal("Failed to make request:", err)
	}
	gotData, err := ioutil.ReadAll(res.Body)
	if err != nil {
		t.Fatal("Failed to read data:", gotData)
	}
	res.Body.Close()
	if !bytes.Equal(gotData, []byte("Some kind of 404 page!")) {
		t.Error("Got unexpected body:", string(gotData))
	}
	if res.StatusCode != 404 {
		t.Error("Expected status 404, got:", res.StatusCode)
	}

	info := <-incoming
	if !bytes.Equal(info.data, []byte("Request data")) {
		t.Error("Sent unexpected post data:", string(info.data))
	}
	if info.head.Get("Content-Type") != "text/plain" {
		t.Error("Invalid content-type:", info.head.Get("Content-Type"))
	}
}

func newServer(callback serverCb) *server {
	return &server{nil, callback}
}

func (self *server) start(port int) error {
	listener, err := net.Listen("tcp", ":"+strconv.Itoa(port))
	if err != nil {
		return err
	}

	self.listener = &listener
	go http.Serve(listener, self)
	return nil
}

func (self *server) stop() {
	(*self.listener).Close()
	self.listener = nil
}

func (self *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	self.callback(w, r)
}

type resInfo struct {
	data []byte
	head http.Header
}
<|endoftext|>