<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"strconv"

	"golang.org/x/net/html"
)

func main() {
	sr := os.Args[1]
	sn := os.Args[2]
	r, _ := strconv.Atoi(sr)
	n, _ := strconv.Atoi(sn)

	chr := make(chan string)   //任务
	csvch := make(chan string) //结果 作为csv的一行
	quitch := make(chan int)
	go Deal(chr, csvch)
	go Save(csvch, quitch, n)
	for i := 0; i < n; i++ {
		id := r - i
		url := fmt.Sprintf("http://www.bilibili.tv/video/av%d/index.html", id)
		fmt.Println("url:", url)
		chr <- url
	}

	<-quitch
}

func Deal(ch chan string, ch2 chan string) {
	for i := 0; i < 1; i++ {
		go func() {
			for {
				url := <-ch
				fmt.Println("get task:" + url)
				s, _ := fetchBody(url)
				//fmt.Println(s)
				url2 := parseCommentUrl(s)
				fmt.Println(url2)
				//sl, _ := getComments(url)
				//ch2 <- "save: " + strconv.Itoa(len(sl))
				ch2 <- "save: " + strconv.Itoa(len(url2))
			}
		}()
	}
}

func Save(ch chan string, quitch chan int, n int) {
	for {
		line := <-ch
		fmt.Println(line)

		n--
		if n == 0 {
			quitch <- -1
		}
	}
}

func fetchBody(url string) (string, error) {
	fmt.Println("fetch: " + url)
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		return "", err
	}
	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("getting %s: %s", url, resp.Status)
	}

	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	return string(contents), nil
}

func parseCommentUrl(s string) string {
	//fmt.Println(s)
	//pattern := `<div(.*?)>`
	pattern := `(?s)<div\sclass="scontent"\sid="bofqi">(.*?)</div>`
	reg := regexp.MustCompile(pattern)
	fmt.Printf("%q\n", reg.FindAllString(s, -1))
	return s[:20]
}

func getComments(url string) ([]string, error) {
	fmt.Println("fetch: " + url)
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("getting %s: %s", url, resp.Status)
	}

	doc, err := html.Parse(resp.Body)
	if err != nil {
		return nil, err
	}

	var comments []string
	visitNode := func(n *html.Node) {
		fmt.Println(n)
		if n.Type == html.ElementNode && n.Data == "d" {
			fmt.Println(n.Namespace)
			comments = append(comments, n.Namespace)
		}
	}

	//fmt.Println(doc)
	forEachNode(doc, visitNode, nil)
	fmt.Println(comments)
	return comments, nil
}

// Copied from gopl.io/ch5/outline2.
func forEachNode(n *html.Node, pre, post func(n *html.Node)) {
	if pre != nil {
		pre(n)
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		forEachNode(c, pre, post)
	}
	if post != nil {
		post(n)
	}
}
<commit_msg>regex ok<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"strconv"

	"golang.org/x/net/html"
)

func main() {
	sr := os.Args[1]
	sn := os.Args[2]
	r, _ := strconv.Atoi(sr)
	n, _ := strconv.Atoi(sn)

	chr := make(chan string)   //任务
	csvch := make(chan string) //结果 作为csv的一行
	quitch := make(chan int)
	go Deal(chr, csvch)
	go Save(csvch, quitch, n)
	for i := 0; i < n; i++ {
		id := r - i
		url := fmt.Sprintf("http://www.bilibili.tv/video/av%d/index.html", id)
		fmt.Println("url:", url)
		chr <- url
	}

	<-quitch
}

func Deal(ch chan string, ch2 chan string) {
	for i := 0; i < 1; i++ {
		go func() {
			for {
				url := <-ch
				fmt.Println("get task:" + url)
				s, _ := fetchBody(url)
				//fmt.Println(s)
				url2 := parseCommentUrl(s)
				fmt.Println(url2)
				//sl, _ := getComments(url)
				//ch2 <- "save: " + strconv.Itoa(len(sl))
				ch2 <- "save: " + strconv.Itoa(len(url2))
			}
		}()
	}
}

func Save(ch chan string, quitch chan int, n int) {
	for {
		line := <-ch
		fmt.Println(line)

		n--
		if n == 0 {
			quitch <- -1
		}
	}
}

func fetchBody(url string) (string, error) {
	fmt.Println("fetch: " + url)
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		return "", err
	}
	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("getting %s: %s", url, resp.Status)
	}

	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	return string(contents), nil
}

func parseCommentUrl(s string) string {
	pattern := `(?s)<div\sclass="scontent"\sid="bofqi">(.*?)</div>`
	reg := regexp.MustCompile(pattern)
	ss1 := reg.FindAllString(s, -1)
	fmt.Printf("%q\n", ss1[0])
	pattern2 := `[^a]id=(\d+)`
	reg2 := regexp.MustCompile(pattern2)
	ss2 := reg2.FindAllStringSubmatch(ss1[0], -1)
	fmt.Printf("%q\n", ss2)
	return ss2[0][1]
}

func getComments(url string) ([]string, error) {
	fmt.Println("fetch: " + url)
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("getting %s: %s", url, resp.Status)
	}

	doc, err := html.Parse(resp.Body)
	if err != nil {
		return nil, err
	}

	var comments []string
	visitNode := func(n *html.Node) {
		fmt.Println(n)
		if n.Type == html.ElementNode && n.Data == "d" {
			fmt.Println(n.Namespace)
			comments = append(comments, n.Namespace)
		}
	}

	//fmt.Println(doc)
	forEachNode(doc, visitNode, nil)
	fmt.Println(comments)
	return comments, nil
}

// Copied from gopl.io/ch5/outline2.
func forEachNode(n *html.Node, pre, post func(n *html.Node)) {
	if pre != nil {
		pre(n)
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		forEachNode(c, pre, post)
	}
	if post != nil {
		post(n)
	}
}
<|endoftext|>


<commit_before>package dogo

import (
	"net/http"
)

type pipelineSession struct {
}

func (s *pipelineSession) PipelineRun(response http.ResponseWriter, request *http.Request) bool {

	cookie, err := request.Cookie(RunTimeConfig.SessionName)
	if err != nil {
		if err == http.ErrNoCookie {
			DogoLog.Debug("SessionName not present in the cookie")
			SessionNameid := "wuciyourqyqqqq"
			DogoLog.Debugf("Reset SessionName[%s] to cookie ", SessionNameid)

			newCookie := &http.Cookie{}
			newCookie.Name = RunTimeConfig.SessionName
			newCookie.Value = SessionNameid

			http.SetCookie(response, newCookie)

		}
	}

	DogoLog.Debugf("Read SessionName success cookie:%+v", cookie)

	return true
}
<commit_msg>添加session模块（未完成）<commit_after>package dogo

import (
	"net/http"
)

type pipelineSession struct {
}

func (s *pipelineSession) PipelineRun(response http.ResponseWriter, request *http.Request) bool {

	cookie, err := request.Cookie(RunTimeConfig.SessionName)
	if err != nil {
		if err == http.ErrNoCookie {
			DogoLog.Debug("SessionName not present in the cookie ")
			SessionNameid := "wuciyourqyqqqq"
			DogoLog.Debugf("Reset SessionName[%s] to cookie ", SessionNameid)

			newCookie := &http.Cookie{}
			newCookie.Name = RunTimeConfig.SessionName
			newCookie.Value = SessionNameid

			http.SetCookie(response, newCookie)

		}
	}

	DogoLog.Debugf("Read SessionName success cookie:%+v", cookie)

	return true
}
<|endoftext|>


<commit_before>package archive

import (
	"archive/tar"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/pkg/pools"
	"github.com/docker/docker/pkg/system"
)

// UnpackLayer unpack `layer` to a `dest`. The stream `layer` can be
// compressed or uncompressed.
// Returns the size in bytes of the contents of the layer.
func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64, err error) {
	tr := tar.NewReader(layer)
	trBuf := pools.BufioReader32KPool.Get(tr)
	defer pools.BufioReader32KPool.Put(trBuf)

	var dirs []*tar.Header
	unpackedPaths := make(map[string]struct{})

	if options == nil {
		options = &TarOptions{}
	}
	if options.ExcludePatterns == nil {
		options.ExcludePatterns = []string{}
	}
	remappedRootUID, remappedRootGID, err := idtools.GetRootUIDGID(options.UIDMaps, options.GIDMaps)
	if err != nil {
		return 0, err
	}

	aufsTempdir := ""
	aufsHardlinks := make(map[string]*tar.Header)

	if options == nil {
		options = &TarOptions{}
	}
	// Iterate through the files in the archive.
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			return 0, err
		}

		size += hdr.Size

		// Normalize name, for safety and for a simple is-root check
		hdr.Name = filepath.Clean(hdr.Name)

		// Windows does not support filenames with colons in them. Ignore
		// these files. This is not a problem though (although it might
		// appear that it is). Let's suppose a client is running docker pull.
		// The daemon it points to is Windows. Would it make sense for the
		// client to be doing a docker pull Ubuntu for example (which has files
		// with colons in the name under /usr/share/man/man3)? No, absolutely
		// not as it would really only make sense that they were pulling a
		// Windows image. However, for development, it is necessary to be able
		// to pull Linux images which are in the repository.
		//
		// TODO Windows. Once the registry is aware of what images are Windows-
		// specific or Linux-specific, this warning should be changed to an error
		// to cater for the situation where someone does manage to upload a Linux
		// image but have it tagged as Windows inadvertently.
		if runtime.GOOS == "windows" {
			if strings.Contains(hdr.Name, ":") {
				logrus.Warnf("Windows: Ignoring %s (is this a Linux image?)", hdr.Name)
				continue
			}
		}

		// Note as these operations are platform specific, so must the slash be.
		if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) {
			// Not the root directory, ensure that the parent directory exists.
			// This happened in some tests where an image had a tarfile without any
			// parent directories.
			parent := filepath.Dir(hdr.Name)
			parentPath := filepath.Join(dest, parent)

			if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
				err = system.MkdirAll(parentPath, 0600)
				if err != nil {
					return 0, err
				}
			}
		}

		// Skip AUFS metadata dirs
		if strings.HasPrefix(hdr.Name, WhiteoutMetaPrefix) {
			// Regular files inside /.wh..wh.plnk can be used as hardlink targets
			// We don't want this directory, but we need the files in them so that
			// such hardlinks can be resolved.
			if strings.HasPrefix(hdr.Name, WhiteoutLinkDir) && hdr.Typeflag == tar.TypeReg {
				basename := filepath.Base(hdr.Name)
				aufsHardlinks[basename] = hdr
				if aufsTempdir == "" {
					if aufsTempdir, err = ioutil.TempDir("", "dockerplnk"); err != nil {
						return 0, err
					}
					defer os.RemoveAll(aufsTempdir)
				}
				if err := createTarFile(filepath.Join(aufsTempdir, basename), dest, hdr, tr, true, nil, options.InUserNS); err != nil {
					return 0, err
				}
			}

			if hdr.Name != WhiteoutOpaqueDir {
				continue
			}
		}
		path := filepath.Join(dest, hdr.Name)
		rel, err := filepath.Rel(dest, path)
		if err != nil {
			return 0, err
		}

		// Note as these operations are platform specific, so must the slash be.
		if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
			return 0, breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest))
		}
		base := filepath.Base(path)

		if strings.HasPrefix(base, WhiteoutPrefix) {
			dir := filepath.Dir(path)
			if base == WhiteoutOpaqueDir {
				_, err := os.Lstat(dir)
				if err != nil {
					return 0, err
				}
				err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
					if err != nil {
						if os.IsNotExist(err) {
							err = nil // parent was deleted
						}
						return err
					}
					if path == dir {
						return nil
					}
					if _, exists := unpackedPaths[path]; !exists {
						err := os.RemoveAll(path)
						return err
					}
					return nil
				})
				if err != nil {
					return 0, err
				}
			} else {
				originalBase := base[len(WhiteoutPrefix):]
				originalPath := filepath.Join(dir, originalBase)
				if err := os.RemoveAll(originalPath); err != nil {
					return 0, err
				}
			}
		} else {
			// If path exits we almost always just want to remove and replace it.
			// The only exception is when it is a directory *and* the file from
			// the layer is also a directory. Then we want to merge them (i.e.
			// just apply the metadata from the layer).
			if fi, err := os.Lstat(path); err == nil {
				if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
					if err := os.RemoveAll(path); err != nil {
						return 0, err
					}
				}
			}

			trBuf.Reset(tr)
			srcData := io.Reader(trBuf)
			srcHdr := hdr

			// Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so
			// we manually retarget these into the temporary files we extracted them into
			if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), WhiteoutLinkDir) {
				linkBasename := filepath.Base(hdr.Linkname)
				srcHdr = aufsHardlinks[linkBasename]
				if srcHdr == nil {
					return 0, fmt.Errorf("Invalid aufs hardlink")
				}
				tmpFile, err := os.Open(filepath.Join(aufsTempdir, linkBasename))
				if err != nil {
					return 0, err
				}
				defer tmpFile.Close()
				srcData = tmpFile
			}

			// if the options contain a uid & gid maps, convert header uid/gid
			// entries using the maps such that lchown sets the proper mapped
			// uid/gid after writing the file. We only perform this mapping if
			// the file isn't already owned by the remapped root UID or GID, as
			// that specific uid/gid has no mapping from container -> host, and
			// those files already have the proper ownership for inside the
			// container.
			if srcHdr.Uid != remappedRootUID {
				xUID, err := idtools.ToHost(srcHdr.Uid, options.UIDMaps)
				if err != nil {
					return 0, err
				}
				srcHdr.Uid = xUID
			}
			if srcHdr.Gid != remappedRootGID {
				xGID, err := idtools.ToHost(srcHdr.Gid, options.GIDMaps)
				if err != nil {
					return 0, err
				}
				srcHdr.Gid = xGID
			}
			if err := createTarFile(path, dest, srcHdr, srcData, true, nil, options.InUserNS); err != nil {
				return 0, err
			}

			// Directory mtimes must be handled at the end to avoid further
			// file creation in them to modify the directory mtime
			if hdr.Typeflag == tar.TypeDir {
				dirs = append(dirs, hdr)
			}
			unpackedPaths[path] = struct{}{}
		}
	}

	for _, hdr := range dirs {
		path := filepath.Join(dest, hdr.Name)
		if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil {
			return 0, err
		}
	}

	return size, nil
}

// ApplyLayer parses a diff in the standard layer format from `layer`,
// and applies it to the directory `dest`. The stream `layer` can be
// compressed or uncompressed.
// Returns the size in bytes of the contents of the layer.
func ApplyLayer(dest string, layer io.Reader) (int64, error) {
	return applyLayerHandler(dest, layer, &TarOptions{}, true)
}

// ApplyUncompressedLayer parses a diff in the standard layer format from
// `layer`, and applies it to the directory `dest`. The stream `layer`
// can only be uncompressed.
// Returns the size in bytes of the contents of the layer.
func ApplyUncompressedLayer(dest string, layer io.Reader, options *TarOptions) (int64, error) {
	return applyLayerHandler(dest, layer, options, false)
}

// do the bulk load of ApplyLayer, but allow for not calling DecompressStream
func applyLayerHandler(dest string, layer io.Reader, options *TarOptions, decompress bool) (int64, error) {
	dest = filepath.Clean(dest)

	// We need to be able to set any perms
	oldmask, err := system.Umask(0)
	if err != nil {
		return 0, err
	}
	defer system.Umask(oldmask) // ignore err, ErrNotSupportedPlatform

	if decompress {
		layer, err = DecompressStream(layer)
		if err != nil {
			return 0, err
		}
	}
	return UnpackLayer(dest, layer, options)
}
<commit_msg>pkg/archive/diff.go: avoid redundant options init<commit_after>package archive

import (
	"archive/tar"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/pkg/pools"
	"github.com/docker/docker/pkg/system"
)

// UnpackLayer unpack `layer` to a `dest`. The stream `layer` can be
// compressed or uncompressed.
// Returns the size in bytes of the contents of the layer.
func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64, err error) {
	tr := tar.NewReader(layer)
	trBuf := pools.BufioReader32KPool.Get(tr)
	defer pools.BufioReader32KPool.Put(trBuf)

	var dirs []*tar.Header
	unpackedPaths := make(map[string]struct{})

	if options == nil {
		options = &TarOptions{}
	}
	if options.ExcludePatterns == nil {
		options.ExcludePatterns = []string{}
	}
	remappedRootUID, remappedRootGID, err := idtools.GetRootUIDGID(options.UIDMaps, options.GIDMaps)
	if err != nil {
		return 0, err
	}

	aufsTempdir := ""
	aufsHardlinks := make(map[string]*tar.Header)

	// Iterate through the files in the archive.
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			return 0, err
		}

		size += hdr.Size

		// Normalize name, for safety and for a simple is-root check
		hdr.Name = filepath.Clean(hdr.Name)

		// Windows does not support filenames with colons in them. Ignore
		// these files. This is not a problem though (although it might
		// appear that it is). Let's suppose a client is running docker pull.
		// The daemon it points to is Windows. Would it make sense for the
		// client to be doing a docker pull Ubuntu for example (which has files
		// with colons in the name under /usr/share/man/man3)? No, absolutely
		// not as it would really only make sense that they were pulling a
		// Windows image. However, for development, it is necessary to be able
		// to pull Linux images which are in the repository.
		//
		// TODO Windows. Once the registry is aware of what images are Windows-
		// specific or Linux-specific, this warning should be changed to an error
		// to cater for the situation where someone does manage to upload a Linux
		// image but have it tagged as Windows inadvertently.
		if runtime.GOOS == "windows" {
			if strings.Contains(hdr.Name, ":") {
				logrus.Warnf("Windows: Ignoring %s (is this a Linux image?)", hdr.Name)
				continue
			}
		}

		// Note as these operations are platform specific, so must the slash be.
		if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) {
			// Not the root directory, ensure that the parent directory exists.
			// This happened in some tests where an image had a tarfile without any
			// parent directories.
			parent := filepath.Dir(hdr.Name)
			parentPath := filepath.Join(dest, parent)

			if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
				err = system.MkdirAll(parentPath, 0600)
				if err != nil {
					return 0, err
				}
			}
		}

		// Skip AUFS metadata dirs
		if strings.HasPrefix(hdr.Name, WhiteoutMetaPrefix) {
			// Regular files inside /.wh..wh.plnk can be used as hardlink targets
			// We don't want this directory, but we need the files in them so that
			// such hardlinks can be resolved.
			if strings.HasPrefix(hdr.Name, WhiteoutLinkDir) && hdr.Typeflag == tar.TypeReg {
				basename := filepath.Base(hdr.Name)
				aufsHardlinks[basename] = hdr
				if aufsTempdir == "" {
					if aufsTempdir, err = ioutil.TempDir("", "dockerplnk"); err != nil {
						return 0, err
					}
					defer os.RemoveAll(aufsTempdir)
				}
				if err := createTarFile(filepath.Join(aufsTempdir, basename), dest, hdr, tr, true, nil, options.InUserNS); err != nil {
					return 0, err
				}
			}

			if hdr.Name != WhiteoutOpaqueDir {
				continue
			}
		}
		path := filepath.Join(dest, hdr.Name)
		rel, err := filepath.Rel(dest, path)
		if err != nil {
			return 0, err
		}

		// Note as these operations are platform specific, so must the slash be.
		if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
			return 0, breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest))
		}
		base := filepath.Base(path)

		if strings.HasPrefix(base, WhiteoutPrefix) {
			dir := filepath.Dir(path)
			if base == WhiteoutOpaqueDir {
				_, err := os.Lstat(dir)
				if err != nil {
					return 0, err
				}
				err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
					if err != nil {
						if os.IsNotExist(err) {
							err = nil // parent was deleted
						}
						return err
					}
					if path == dir {
						return nil
					}
					if _, exists := unpackedPaths[path]; !exists {
						err := os.RemoveAll(path)
						return err
					}
					return nil
				})
				if err != nil {
					return 0, err
				}
			} else {
				originalBase := base[len(WhiteoutPrefix):]
				originalPath := filepath.Join(dir, originalBase)
				if err := os.RemoveAll(originalPath); err != nil {
					return 0, err
				}
			}
		} else {
			// If path exits we almost always just want to remove and replace it.
			// The only exception is when it is a directory *and* the file from
			// the layer is also a directory. Then we want to merge them (i.e.
			// just apply the metadata from the layer).
			if fi, err := os.Lstat(path); err == nil {
				if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
					if err := os.RemoveAll(path); err != nil {
						return 0, err
					}
				}
			}

			trBuf.Reset(tr)
			srcData := io.Reader(trBuf)
			srcHdr := hdr

			// Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so
			// we manually retarget these into the temporary files we extracted them into
			if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), WhiteoutLinkDir) {
				linkBasename := filepath.Base(hdr.Linkname)
				srcHdr = aufsHardlinks[linkBasename]
				if srcHdr == nil {
					return 0, fmt.Errorf("Invalid aufs hardlink")
				}
				tmpFile, err := os.Open(filepath.Join(aufsTempdir, linkBasename))
				if err != nil {
					return 0, err
				}
				defer tmpFile.Close()
				srcData = tmpFile
			}

			// if the options contain a uid & gid maps, convert header uid/gid
			// entries using the maps such that lchown sets the proper mapped
			// uid/gid after writing the file. We only perform this mapping if
			// the file isn't already owned by the remapped root UID or GID, as
			// that specific uid/gid has no mapping from container -> host, and
			// those files already have the proper ownership for inside the
			// container.
			if srcHdr.Uid != remappedRootUID {
				xUID, err := idtools.ToHost(srcHdr.Uid, options.UIDMaps)
				if err != nil {
					return 0, err
				}
				srcHdr.Uid = xUID
			}
			if srcHdr.Gid != remappedRootGID {
				xGID, err := idtools.ToHost(srcHdr.Gid, options.GIDMaps)
				if err != nil {
					return 0, err
				}
				srcHdr.Gid = xGID
			}
			if err := createTarFile(path, dest, srcHdr, srcData, true, nil, options.InUserNS); err != nil {
				return 0, err
			}

			// Directory mtimes must be handled at the end to avoid further
			// file creation in them to modify the directory mtime
			if hdr.Typeflag == tar.TypeDir {
				dirs = append(dirs, hdr)
			}
			unpackedPaths[path] = struct{}{}
		}
	}

	for _, hdr := range dirs {
		path := filepath.Join(dest, hdr.Name)
		if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil {
			return 0, err
		}
	}

	return size, nil
}

// ApplyLayer parses a diff in the standard layer format from `layer`,
// and applies it to the directory `dest`. The stream `layer` can be
// compressed or uncompressed.
// Returns the size in bytes of the contents of the layer.
func ApplyLayer(dest string, layer io.Reader) (int64, error) {
	return applyLayerHandler(dest, layer, &TarOptions{}, true)
}

// ApplyUncompressedLayer parses a diff in the standard layer format from
// `layer`, and applies it to the directory `dest`. The stream `layer`
// can only be uncompressed.
// Returns the size in bytes of the contents of the layer.
func ApplyUncompressedLayer(dest string, layer io.Reader, options *TarOptions) (int64, error) {
	return applyLayerHandler(dest, layer, options, false)
}

// do the bulk load of ApplyLayer, but allow for not calling DecompressStream
func applyLayerHandler(dest string, layer io.Reader, options *TarOptions, decompress bool) (int64, error) {
	dest = filepath.Clean(dest)

	// We need to be able to set any perms
	oldmask, err := system.Umask(0)
	if err != nil {
		return 0, err
	}
	defer system.Umask(oldmask) // ignore err, ErrNotSupportedPlatform

	if decompress {
		layer, err = DecompressStream(layer)
		if err != nil {
			return 0, err
		}
	}
	return UnpackLayer(dest, layer, options)
}
<|endoftext|>


<commit_before>// Copyright 2013 Beego Authors
// Copyright 2014 The Macaron Authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package toolbox

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"time"

	"github.com/Unknwon/com"
	"gopkg.in/macaron.v1"
)

var (
	profilePath  string
	pid          int
	startTime    = time.Now()
	inCPUProfile bool
)

// StartCPUProfile starts CPU profile monitor.
func StartCPUProfile() error {
	if inCPUProfile {
		return errors.New("CPU profile has alreday been started!")
	}
	inCPUProfile = true

	os.MkdirAll(profilePath, os.ModePerm)
	f, err := os.Create(path.Join(profilePath, "cpu-"+com.ToStr(pid)+".pprof"))
	if err != nil {
		panic("fail to record CPU profile: " + err.Error())
	}
	pprof.StartCPUProfile(f)
	return nil
}

// StopCPUProfile stops CPU profile monitor.
func StopCPUProfile() error {
	if !inCPUProfile {
		return errors.New("CPU profile hasn't been started!")
	}
	pprof.StopCPUProfile()
	inCPUProfile = false
	return nil
}

func init() {
	pid = os.Getpid()
}

// DumpMemProf dumps memory profile in pprof.
func DumpMemProf(w io.Writer) {
	pprof.WriteHeapProfile(w)
}

func dumpMemProf() {
	os.MkdirAll(profilePath, os.ModePerm)
	f, err := os.Create(path.Join(profilePath, "mem-"+com.ToStr(pid)+".memprof"))
	if err != nil {
		panic("fail to record memory profile: " + err.Error())
	}
	runtime.GC()
	DumpMemProf(f)
	f.Close()
}

func avg(items []time.Duration) time.Duration {
	var sum time.Duration
	for _, item := range items {
		sum += item
	}
	return time.Duration(int64(sum) / int64(len(items)))
}

func dumpGC(memStats *runtime.MemStats, gcstats *debug.GCStats, w io.Writer) {

	if gcstats.NumGC > 0 {
		lastPause := gcstats.Pause[0]
		elapsed := time.Now().Sub(startTime)
		overhead := float64(gcstats.PauseTotal) / float64(elapsed) * 100
		allocatedRate := float64(memStats.TotalAlloc) / elapsed.Seconds()

		fmt.Fprintf(w, "NumGC:%d Pause:%s Pause(Avg):%s Overhead:%3.2f%% Alloc:%s Sys:%s Alloc(Rate):%s/s Histogram:%s %s %s \n",
			gcstats.NumGC,
			com.ToStr(lastPause),
			com.ToStr(avg(gcstats.Pause)),
			overhead,
			com.HumaneFileSize(memStats.Alloc),
			com.HumaneFileSize(memStats.Sys),
			com.HumaneFileSize(uint64(allocatedRate)),
			com.ToStr(gcstats.PauseQuantiles[94]),
			com.ToStr(gcstats.PauseQuantiles[98]),
			com.ToStr(gcstats.PauseQuantiles[99]))
	} else {
		// while GC has disabled
		elapsed := time.Now().Sub(startTime)
		allocatedRate := float64(memStats.TotalAlloc) / elapsed.Seconds()

		fmt.Fprintf(w, "Alloc:%s Sys:%s Alloc(Rate):%s/s\n",
			com.HumaneFileSize(memStats.Alloc),
			com.HumaneFileSize(memStats.Sys),
			com.HumaneFileSize(uint64(allocatedRate)))
	}
}

// DumpGCSummary dumps GC information to io.Writer
func DumpGCSummary(w io.Writer) {
	memStats := &runtime.MemStats{}
	runtime.ReadMemStats(memStats)
	gcstats := &debug.GCStats{PauseQuantiles: make([]time.Duration, 100)}
	debug.ReadGCStats(gcstats)

	dumpGC(memStats, gcstats, w)
}

func handleProfile(ctx *macaron.Context) string {
	switch ctx.Query("op") {
	case "startcpu":
		if err := StartCPUProfile(); err != nil {
			return err.Error()
		}
	case "stopcpu":
		if err := StopCPUProfile(); err != nil {
			return err.Error()
		}
	case "mem":
		dumpMemProf()
	case "gc":
		var buf bytes.Buffer
		DumpGCSummary(&buf)
		return string(buf.Bytes())
	default:
		return fmt.Sprintf(`
Available operations:

Start CPU profile

Stop CPU profile

Dump memory profile

Dump GC summary

`, opt.ProfileURLPrefix, opt.ProfileURLPrefix, opt.ProfileURLPrefix, opt.ProfileURLPrefix)
	}
	ctx.Redirect(opt.ProfileURLPrefix)
	return ""
}
<commit_msg>minor improve<commit_after>// Copyright 2013 Beego Authors
// Copyright 2014 The Macaron Authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package toolbox

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"time"

	"github.com/Unknwon/com"
	"gopkg.in/macaron.v1"
)

var (
	profilePath  string
	pid          int
	startTime    = time.Now()
	inCPUProfile bool
)

// StartCPUProfile starts CPU profile monitor.
func StartCPUProfile() error {
	if inCPUProfile {
		return errors.New("CPU profile has alreday been started!")
	}
	inCPUProfile = true

	os.MkdirAll(profilePath, os.ModePerm)
	f, err := os.Create(path.Join(profilePath, "cpu-"+com.ToStr(pid)+".pprof"))
	if err != nil {
		panic("fail to record CPU profile: " + err.Error())
	}
	pprof.StartCPUProfile(f)
	return nil
}

// StopCPUProfile stops CPU profile monitor.
func StopCPUProfile() error {
	if !inCPUProfile {
		return errors.New("CPU profile hasn't been started!")
	}
	pprof.StopCPUProfile()
	inCPUProfile = false
	return nil
}

func init() {
	pid = os.Getpid()
}

// DumpMemProf dumps memory profile in pprof.
func DumpMemProf(w io.Writer) {
	pprof.WriteHeapProfile(w)
}

func dumpMemProf() {
	os.MkdirAll(profilePath, os.ModePerm)
	f, err := os.Create(path.Join(profilePath, "mem-"+com.ToStr(pid)+".memprof"))
	if err != nil {
		panic("fail to record memory profile: " + err.Error())
	}
	runtime.GC()
	DumpMemProf(f)
	f.Close()
}

func avg(items []time.Duration) time.Duration {
	var sum time.Duration
	for _, item := range items {
		sum += item
	}
	return time.Duration(int64(sum) / int64(len(items)))
}

func dumpGC(memStats *runtime.MemStats, gcstats *debug.GCStats, w io.Writer) {

	if gcstats.NumGC > 0 {
		lastPause := gcstats.Pause[0]
		elapsed := time.Now().Sub(startTime)
		overhead := float64(gcstats.PauseTotal) / float64(elapsed) * 100
		allocatedRate := float64(memStats.TotalAlloc) / elapsed.Seconds()

		fmt.Fprintf(w, "NumGC:%d Pause:%s Pause(Avg):%s Overhead:%3.2f%% Alloc:%s Sys:%s Alloc(Rate):%s/s Histogram:%s %s %s \n",
			gcstats.NumGC,
			com.ToStr(lastPause),
			com.ToStr(avg(gcstats.Pause)),
			overhead,
			com.HumaneFileSize(memStats.Alloc),
			com.HumaneFileSize(memStats.Sys),
			com.HumaneFileSize(uint64(allocatedRate)),
			com.ToStr(gcstats.PauseQuantiles[94]),
			com.ToStr(gcstats.PauseQuantiles[98]),
			com.ToStr(gcstats.PauseQuantiles[99]))
	} else {
		// while GC has disabled
		elapsed := time.Now().Sub(startTime)
		allocatedRate := float64(memStats.TotalAlloc) / elapsed.Seconds()

		fmt.Fprintf(w, "Alloc:%s Sys:%s Alloc(Rate):%s/s\n",
			com.HumaneFileSize(memStats.Alloc),
			com.HumaneFileSize(memStats.Sys),
			com.HumaneFileSize(uint64(allocatedRate)))
	}
}

// DumpGCSummary dumps GC information to io.Writer
func DumpGCSummary(w io.Writer) {
	memStats := &runtime.MemStats{}
	runtime.ReadMemStats(memStats)
	gcstats := &debug.GCStats{PauseQuantiles: make([]time.Duration, 100)}
	debug.ReadGCStats(gcstats)

	dumpGC(memStats, gcstats, w)
}

func handleProfile(ctx *macaron.Context) string {
	switch ctx.Query("op") {
	case "startcpu":
		if err := StartCPUProfile(); err != nil {
			return err.Error()
		}
	case "stopcpu":
		if err := StopCPUProfile(); err != nil {
			return err.Error()
		}
	case "mem":
		dumpMemProf()
	case "gc":
		var buf bytes.Buffer
		DumpGCSummary(&buf)
		return string(buf.Bytes())
	default:
		return fmt.Sprintf(`
Available operations:

Start CPU profile

Stop CPU profile

Dump memory profile

Dump GC summary

`, opt.ProfileURLPrefix)
	}
	ctx.Redirect(opt.ProfileURLPrefix)
	return ""
}
<|endoftext|>


<commit_before>package backup

import (
	"fmt"
	"io/ioutil"
	"strings"
	"time"

	"github.com/codeskyblue/go-sh"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"

	"github.com/stefanprodan/mgob/pkg/config"
)

func dump(plan config.Plan, tmpPath string, ts time.Time) (string, string, error) {
	archive := fmt.Sprintf("%v/%v-%v.gz", tmpPath, plan.Name, ts.Unix())
	mlog := fmt.Sprintf("%v/%v-%v.log", tmpPath, plan.Name, ts.Unix())
	dump := fmt.Sprintf("mongodump --archive=%v --gzip ", archive)

	if plan.Target.Uri != "" {
		// using uri (New in version 3.4.6)
		// host/port/username/password are incompatible with uri
		// https://docs.mongodb.com/manual/reference/program/mongodump/#cmdoption-mongodump-uri
		dump += fmt.Sprintf(`--uri "%v" `, plan.Target.Uri)
	} else {
		// use older host/port
		dump += fmt.Sprintf("--host %v --port %v ", plan.Target.Host, plan.Target.Port)

		if plan.Target.Username != "" && plan.Target.Password != "" {
			dump += fmt.Sprintf("-u %v -p %v ", plan.Target.Username, plan.Target.Password)
		}
	}

	if plan.Target.Database != "" {
		dump += fmt.Sprintf("--db %v ", plan.Target.Database)
	}

	if plan.Target.Params != "" {
		dump += fmt.Sprintf("%v", plan.Target.Params)
	}

	// TODO: mask password
	log.Debugf("dump cmd: %v", dump)
	output, err := sh.Command("/bin/sh", "-c", dump).SetTimeout(time.Duration(plan.Scheduler.Timeout) * time.Minute).CombinedOutput()
	if err != nil {
		ex := ""
		if len(output) > 0 {
			ex = strings.Replace(string(output), "\n", " ", -1)
		}
		return "", "", errors.Wrapf(err, "mongodump log %v", ex)
	}
	logToFile(mlog, output)

	return archive, mlog, nil
}

func logToFile(file string, data []byte) error {
	if len(data) > 0 {
		err := ioutil.WriteFile(file, data, 0644)
		if err != nil {
			return errors.Wrapf(err, "writing log %v failed", file)
		}
	}

	return nil
}

func applyRetention(path string, retention int) error {
	gz := fmt.Sprintf("cd %v && rm -f $(ls -1t *.gz | tail -n +%v)", path, retention+1)
	err := sh.Command("/bin/sh", "-c", gz).Run()
	if err != nil {
		return errors.Wrapf(err, "removing old gz files from %v failed", path)
	}

	log.Debug("apply retention")
	log := fmt.Sprintf("cd %v && rm -f $(ls -1t *.log | tail -n +%v)", path, retention+1)
	err = sh.Command("/bin/sh", "-c", log).Run()
	if err != nil {
		return errors.Wrapf(err, "removing old log files from %v failed", path)
	}

	return nil
}

// TmpCleanup remove files older than one day
func TmpCleanup(path string) error {
	rm := fmt.Sprintf("find %v -not -name \"mgob.db\" -mtime +%v -type f -delete", path, 1)
	err := sh.Command("/bin/sh", "-c", rm).Run()
	if err != nil {
		return errors.Wrapf(err, "%v cleanup failed", path)
	}

	return nil
}
<commit_msg>username and password need to be encapsuled by double quote<commit_after>package backup

import (
	"fmt"
	"io/ioutil"
	"strings"
	"time"

	"github.com/codeskyblue/go-sh"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"

	"github.com/stefanprodan/mgob/pkg/config"
)

func dump(plan config.Plan, tmpPath string, ts time.Time) (string, string, error) {
	archive := fmt.Sprintf("%v/%v-%v.gz", tmpPath, plan.Name, ts.Unix())
	mlog := fmt.Sprintf("%v/%v-%v.log", tmpPath, plan.Name, ts.Unix())
	dump := fmt.Sprintf("mongodump --archive=%v --gzip ", archive)

	if plan.Target.Uri != "" {
		// using uri (New in version 3.4.6)
		// host/port/username/password are incompatible with uri
		// https://docs.mongodb.com/manual/reference/program/mongodump/#cmdoption-mongodump-uri
		dump += fmt.Sprintf(`--uri "%v" `, plan.Target.Uri)
	} else {
		// use older host/port
		dump += fmt.Sprintf("--host %v --port %v ", plan.Target.Host, plan.Target.Port)

		if plan.Target.Username != "" && plan.Target.Password != "" {
			dump += fmt.Sprintf(`-u "%v" -p "%v" `, plan.Target.Username, plan.Target.Password)
		}
	}

	if plan.Target.Database != "" {
		dump += fmt.Sprintf("--db %v ", plan.Target.Database)
	}

	if plan.Target.Params != "" {
		dump += fmt.Sprintf("%v", plan.Target.Params)
	}

	// TODO: mask password
	log.Debugf("dump cmd: %v", dump)
	output, err := sh.Command("/bin/sh", "-c", dump).SetTimeout(time.Duration(plan.Scheduler.Timeout) * time.Minute).CombinedOutput()
	if err != nil {
		ex := ""
		if len(output) > 0 {
			ex = strings.Replace(string(output), "\n", " ", -1)
		}
		return "", "", errors.Wrapf(err, "mongodump log %v", ex)
	}
	logToFile(mlog, output)

	return archive, mlog, nil
}

func logToFile(file string, data []byte) error {
	if len(data) > 0 {
		err := ioutil.WriteFile(file, data, 0644)
		if err != nil {
			return errors.Wrapf(err, "writing log %v failed", file)
		}
	}

	return nil
}

func applyRetention(path string, retention int) error {
	gz := fmt.Sprintf("cd %v && rm -f $(ls -1t *.gz | tail -n +%v)", path, retention+1)
	err := sh.Command("/bin/sh", "-c", gz).Run()
	if err != nil {
		return errors.Wrapf(err, "removing old gz files from %v failed", path)
	}

	log.Debug("apply retention")
	log := fmt.Sprintf("cd %v && rm -f $(ls -1t *.log | tail -n +%v)", path, retention+1)
	err = sh.Command("/bin/sh", "-c", log).Run()
	if err != nil {
		return errors.Wrapf(err, "removing old log files from %v failed", path)
	}

	return nil
}

// TmpCleanup remove files older than one day
func TmpCleanup(path string) error {
	rm := fmt.Sprintf("find %v -not -name \"mgob.db\" -mtime +%v -type f -delete", path, 1)
	err := sh.Command("/bin/sh", "-c", rm).Run()
	if err != nil {
		return errors.Wrapf(err, "%v cleanup failed", path)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Kohei YOSHIDA <https://yosida95.com/>.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package chame

import (
	"context"
	"fmt"
	"net/url"
	"path"
	"time"
)

type Client struct {
	baseUrl string
	issuer  string
	store   Store
}

func NewClient(baseUrl string, issuer string, store Store) (*Client, error) {
	parsed, err := url.ParseRequestURI(baseUrl)
	if err != nil {
		return nil, fmt.Errorf("unrecognized base URL: %w", err)
	}
	if parsed.ForceQuery ||
		parsed.RawQuery != "" ||
		parsed.Fragment != "" {
		return nil, fmt.Errorf("Base URL must not contain query nor fragment")
	}
	// path.Join trims the ending slash.
	parsed.Path = path.Join(parsed.Path, proxyPrefix) + "/"

	return &Client{
		baseUrl: parsed.String(),
		issuer:  issuer,
		store:   store,
	}, nil
}

func (cli *Client) Sign(ctx context.Context, url string, opts SignOption) (string, error) {
	var exp JWTEpoch
	if !opts.Expiry.IsZero() {
		exp = JWTEpoch(opts.Expiry.Unix())
	}

	var nbf JWTEpoch
	if !opts.Expiry.IsZero() {
		nbf = JWTEpoch(opts.Expiry.Unix())
	}

	signed, err := EncodeToken(ctx, cli.store, &Token{
		Issuer:    cli.issuer,
		Subject:   url,
		NotBefore: nbf,
		Expiry:    exp,
	}, opts.JwtKid)
	if err != nil {
		return "", err
	}

	return cli.baseUrl + signed, nil
}

type SignOption struct {
	JwtKid    string
	NotBefore time.Time
	Expiry    time.Time
}
<commit_msg>fix SignOption.NotBefore has no effect<commit_after>// Copyright 2020 Kohei YOSHIDA <https://yosida95.com/>.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package chame

import (
	"context"
	"fmt"
	"net/url"
	"path"
	"time"
)

type Client struct {
	baseUrl string
	issuer  string
	store   Store
}

func NewClient(baseUrl string, issuer string, store Store) (*Client, error) {
	parsed, err := url.ParseRequestURI(baseUrl)
	if err != nil {
		return nil, fmt.Errorf("unrecognized base URL: %w", err)
	}
	if parsed.ForceQuery ||
		parsed.RawQuery != "" ||
		parsed.Fragment != "" {
		return nil, fmt.Errorf("Base URL must not contain query nor fragment")
	}
	// path.Join trims the ending slash.
	parsed.Path = path.Join(parsed.Path, proxyPrefix) + "/"

	return &Client{
		baseUrl: parsed.String(),
		issuer:  issuer,
		store:   store,
	}, nil
}

func (cli *Client) Sign(ctx context.Context, url string, opts SignOption) (string, error) {
	var exp JWTEpoch
	if !opts.Expiry.IsZero() {
		exp = JWTEpoch(opts.Expiry.Unix())
	}

	var nbf JWTEpoch
	if !opts.NotBefore.IsZero() {
		nbf = JWTEpoch(opts.NotBefore.Unix())
	}

	signed, err := EncodeToken(ctx, cli.store, &Token{
		Issuer:    cli.issuer,
		Subject:   url,
		NotBefore: nbf,
		Expiry:    exp,
	}, opts.JwtKid)
	if err != nil {
		return "", err
	}

	return cli.baseUrl + signed, nil
}

type SignOption struct {
	JwtKid    string
	NotBefore time.Time
	Expiry    time.Time
}
<|endoftext|>


<commit_before>package etcd

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"

	"github.com/coreos/etcd/config"
	etcdErr "github.com/coreos/etcd/error"
)

// v2client sends various requests using HTTP API.
// It is different from raft communication, and doesn't record anything in the log.
// The argument url is required to contain scheme and host only, and
// there is no trailing slash in it.
// Public functions return "etcd/error".Error intentionally to figure out
// etcd error code easily.
type v2client struct {
	http.Client
}

func newClient(tc *tls.Config) *v2client {
	tr := new(http.Transport)
	tr.TLSClientConfig = tc
	return &v2client{http.Client{Transport: tr}}
}

// CheckVersion returns true when the version check on the server returns 200.
func (c *v2client) CheckVersion(url string, version int) (bool, *etcdErr.Error) {
	resp, err := c.Get(url + fmt.Sprintf("/version/%d/check", version))
	if err != nil {
		return false, clientError(err)
	}

	defer resp.Body.Close()

	return resp.StatusCode == 200, nil
}

// GetVersion fetches the peer version of a cluster.
func (c *v2client) GetVersion(url string) (int, *etcdErr.Error) {
	resp, err := c.Get(url + "/version")
	if err != nil {
		return 0, clientError(err)
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return 0, clientError(err)
	}

	// Parse version number.
	version, err := strconv.Atoi(string(body))
	if err != nil {
		return 0, clientError(err)
	}
	return version, nil
}

func (c *v2client) GetMachines(url string) ([]*machineMessage, *etcdErr.Error) {
	resp, err := c.Get(url + "/v2/admin/machines/")
	if err != nil {
		return nil, clientError(err)
	}

	msgs := new([]*machineMessage)
	if uerr := c.parseJSONResponse(resp, msgs); uerr != nil {
		return nil, uerr
	}
	return *msgs, nil
}

func (c *v2client) GetClusterConfig(url string) (*config.ClusterConfig, *etcdErr.Error) {
	resp, err := c.Get(url + "/v2/admin/config")
	if err != nil {
		return nil, clientError(err)
	}

	config := new(config.ClusterConfig)
	if uerr := c.parseJSONResponse(resp, config); uerr != nil {
		return nil, uerr
	}
	return config, nil
}

// AddMachine adds machine to the cluster.
// The first return value is the commit index of join command.
func (c *v2client) AddMachine(url string, name string, info *context) *etcdErr.Error {
	b, _ := json.Marshal(info)
	url = url + "/v2/admin/machines/" + name

	log.Printf("Send Join Request to %s", url)
	resp, err := c.put(url, b)
	if err != nil {
		return clientError(err)
	}
	defer resp.Body.Close()

	if err := c.checkErrorResponse(resp); err != nil {
		return err
	}
	return nil
}

func (c *v2client) parseJSONResponse(resp *http.Response, val interface{}) *etcdErr.Error {
	defer resp.Body.Close()

	if err := c.checkErrorResponse(resp); err != nil {
		return err
	}
	if err := json.NewDecoder(resp.Body).Decode(val); err != nil {
		log.Printf("Error parsing join response: %v", err)
		return clientError(err)
	}
	return nil
}

func (c *v2client) checkErrorResponse(resp *http.Response) *etcdErr.Error {
	if resp.StatusCode != http.StatusOK {
		uerr := &etcdErr.Error{}
		if err := json.NewDecoder(resp.Body).Decode(uerr); err != nil {
			log.Printf("Error parsing response to etcd error: %v", err)
			return clientError(err)
		}
		return uerr
	}
	return nil
}

// put sends server side PUT request.
// It always follows redirects instead of stopping according to RFC 2616.
func (c *v2client) put(urlStr string, body []byte) (*http.Response, error) {
	return c.doAlwaysFollowingRedirects("PUT", urlStr, body)
}

func (c *v2client) doAlwaysFollowingRedirects(method string, urlStr string, body []byte) (resp *http.Response, err error) {
	var req *http.Request

	for redirect := 0; redirect < 10; redirect++ {
		req, err = http.NewRequest(method, urlStr, bytes.NewBuffer(body))
		if err != nil {
			return
		}

		if resp, err = c.Do(req); err != nil {
			if resp != nil {
				resp.Body.Close()
			}
			return
		}

		if resp.StatusCode == http.StatusMovedPermanently || resp.StatusCode == http.StatusTemporaryRedirect {
			resp.Body.Close()
			if urlStr = resp.Header.Get("Location"); urlStr == "" {
				err = errors.New(fmt.Sprintf("%d response missing Location header", resp.StatusCode))
				return
			}
			continue
		}
		return
	}

	err = errors.New("stopped after 10 redirects")
	return
}

func clientError(err error) *etcdErr.Error {
	return etcdErr.NewError(etcdErr.EcodeClientInternal, err.Error(), 0)
}
<commit_msg>v2_client: read whole response body before close<commit_after>package etcd

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"

	"github.com/coreos/etcd/config"
	etcdErr "github.com/coreos/etcd/error"
)

// v2client sends various requests using HTTP API.
// It is different from raft communication, and doesn't record anything in the log.
// The argument url is required to contain scheme and host only, and
// there is no trailing slash in it.
// Public functions return "etcd/error".Error intentionally to figure out
// etcd error code easily.
type v2client struct {
	http.Client
}

func newClient(tc *tls.Config) *v2client {
	tr := new(http.Transport)
	tr.TLSClientConfig = tc
	return &v2client{http.Client{Transport: tr}}
}

// CheckVersion returns true when the version check on the server returns 200.
func (c *v2client) CheckVersion(url string, version int) (bool, *etcdErr.Error) {
	resp, err := c.Get(url + fmt.Sprintf("/version/%d/check", version))
	if err != nil {
		return false, clientError(err)
	}

	defer resp.Body.Close()

	return resp.StatusCode == 200, nil
}

// GetVersion fetches the peer version of a cluster.
func (c *v2client) GetVersion(url string) (int, *etcdErr.Error) {
	resp, err := c.Get(url + "/version")
	if err != nil {
		return 0, clientError(err)
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return 0, clientError(err)
	}

	// Parse version number.
	version, err := strconv.Atoi(string(body))
	if err != nil {
		return 0, clientError(err)
	}
	return version, nil
}

func (c *v2client) GetMachines(url string) ([]*machineMessage, *etcdErr.Error) {
	resp, err := c.Get(url + "/v2/admin/machines/")
	if err != nil {
		return nil, clientError(err)
	}

	msgs := new([]*machineMessage)
	if uerr := c.readJSONResponse(resp, msgs); uerr != nil {
		return nil, uerr
	}
	return *msgs, nil
}

func (c *v2client) GetClusterConfig(url string) (*config.ClusterConfig, *etcdErr.Error) {
	resp, err := c.Get(url + "/v2/admin/config")
	if err != nil {
		return nil, clientError(err)
	}

	config := new(config.ClusterConfig)
	if uerr := c.readJSONResponse(resp, config); uerr != nil {
		return nil, uerr
	}
	return config, nil
}

// AddMachine adds machine to the cluster.
// The first return value is the commit index of join command.
func (c *v2client) AddMachine(url string, name string, info *context) *etcdErr.Error {
	b, _ := json.Marshal(info)
	url = url + "/v2/admin/machines/" + name

	log.Printf("Send Join Request to %s", url)
	resp, err := c.put(url, b)
	if err != nil {
		return clientError(err)
	}

	if err := c.readErrorResponse(resp); err != nil {
		return err
	}
	return nil
}

func (c *v2client) readJSONResponse(resp *http.Response, val interface{}) *etcdErr.Error {
	if err := c.readErrorResponse(resp); err != nil {
		return err
	}
	defer resp.Body.Close()
	defer ioutil.ReadAll(resp.Body)

	if err := json.NewDecoder(resp.Body).Decode(val); err != nil {
		log.Printf("Error parsing join response: %v", err)
		return clientError(err)
	}
	return nil
}

func (c *v2client) readErrorResponse(resp *http.Response) *etcdErr.Error {
	if resp.StatusCode == http.StatusOK {
		return nil
	}
	defer resp.Body.Close()
	defer ioutil.ReadAll(resp.Body)

	uerr := &etcdErr.Error{}
	if err := json.NewDecoder(resp.Body).Decode(uerr); err != nil {
		log.Printf("Error parsing response to etcd error: %v", err)
		return clientError(err)
	}
	return uerr
}

// put sends server side PUT request.
// It always follows redirects instead of stopping according to RFC 2616.
func (c *v2client) put(urlStr string, body []byte) (*http.Response, error) {
	return c.doAlwaysFollowingRedirects("PUT", urlStr, body)
}

func (c *v2client) doAlwaysFollowingRedirects(method string, urlStr string, body []byte) (resp *http.Response, err error) {
	var req *http.Request

	for redirect := 0; redirect < 10; redirect++ {
		req, err = http.NewRequest(method, urlStr, bytes.NewBuffer(body))
		if err != nil {
			return
		}

		if resp, err = c.Do(req); err != nil {
			if resp != nil {
				resp.Body.Close()
			}
			return
		}

		if resp.StatusCode == http.StatusMovedPermanently || resp.StatusCode == http.StatusTemporaryRedirect {
			resp.Body.Close()
			if urlStr = resp.Header.Get("Location"); urlStr == "" {
				err = errors.New(fmt.Sprintf("%d response missing Location header", resp.StatusCode))
				return
			}
			continue
		}
		return
	}

	err = errors.New("stopped after 10 redirects")
	return
}

func clientError(err error) *etcdErr.Error {
	return etcdErr.NewError(etcdErr.EcodeClientInternal, err.Error(), 0)
}
<|endoftext|>


<commit_before>package stub

import (
	"encoding/json"
	"fmt"
	"reflect"

	"github.com/banzaicloud/bank-vaults/pkg/apis/vault/v1alpha1"
	"github.com/banzaicloud/bank-vaults/pkg/kv/k8s"
	"github.com/banzaicloud/bank-vaults/pkg/vault"
	"github.com/hashicorp/vault/api"
	"github.com/operator-framework/operator-sdk/pkg/sdk/action"
	"github.com/operator-framework/operator-sdk/pkg/sdk/handler"
	"github.com/operator-framework/operator-sdk/pkg/sdk/query"
	"github.com/operator-framework/operator-sdk/pkg/sdk/types"
	appsv1 "k8s.io/api/apps/v1"
	"k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
)

func NewHandler() handler.Handler {
	return &Handler{}
}

type Handler struct {
}

func (h *Handler) Handle(ctx types.Context, event types.Event) error {
	switch o := event.Object.(type) {
	case *v1alpha1.Vault:
		v := o

		// Ignore the delete event since the garbage collector will clean up all secondary resources for the CR
		// All secondary resources must have the CR set as their OwnerReference for this to be the case
		if event.Deleted {
			return nil
		}

		// Create the deployment if it doesn't exist
		dep, err := deploymentForVault(v)
		if err != nil {
			return fmt.Errorf("failed to fabricate deployment: %v", err)
		}
		err = action.Create(dep)
		if err != nil && !apierrors.IsAlreadyExists(err) {
			return fmt.Errorf("failed to create deployment: %v", err)
		}

		// Ensure the deployment size is the same as the spec
		err = query.Get(dep)
		if err != nil {
			return fmt.Errorf("failed to get deployment: %v", err)
		}
		size := v.Spec.Size
		if *dep.Spec.Replicas != size {
			dep.Spec.Replicas = &size
			err = action.Update(dep)
			if err != nil {
				return fmt.Errorf("failed to update deployment: %v", err)
			}
		}

		// Update the Vault status with the pod names
		podList := podList()
		labelSelector := labels.SelectorFromSet(labelsForVault(v.Name)).String()
		listOps := &metav1.ListOptions{LabelSelector: labelSelector}
		err = query.List(v.Namespace, podList, query.WithListOptions(listOps))
		if err != nil {
			return fmt.Errorf("failed to list pods: %v", err)
		}
		podNames := getPodNames(podList.Items)
		if !reflect.DeepEqual(podNames, v.Status.Nodes) {
			v.Status.Nodes = podNames
			err := action.Update(v)
			if err != nil {
				return fmt.Errorf("failed to update vault status: %v", err)
			}
		}

		// Create the service if it doesn't exist
		ser := serviceForVault(v)
		err = action.Create(ser)
		if err != nil && !apierrors.IsAlreadyExists(err) {
			return fmt.Errorf("failed to create service: %v", err)
		}

		// Create the deployment if it doesn't exist
		dep = deploymentForConfigurer(v)
		err = action.Create(dep)
		if err != nil && !apierrors.IsAlreadyExists(err) {
			return fmt.Errorf("failed to create configurer deployment: %v", err)
		}

		// Create the configmap if it doesn't exist
		cm := configMapForConfigurer(v)
		err = action.Create(cm)
		if err != nil && !apierrors.IsAlreadyExists(err) {
			return fmt.Errorf("failed to create configurer configmap: %v", err)
		}

		// Ensure the configmap is the same as the spec
		err = query.Get(cm)
		if err != nil {
			return fmt.Errorf("failed to get deployment: %v", err)
		}

		externalConfig := v.Spec.ExternalConfigJSON()
		if cm.Data[vault.DefaultConfigFile] != externalConfig {
			cm.Data[vault.DefaultConfigFile] = externalConfig
			err = action.Update(cm)
			if err != nil {
				return fmt.Errorf("failed to update configurer configmap: %v", err)
			}
		}

	}
	return nil
}

// deploymentForVault returns a vault Deployment object
func deploymentForVault(v *v1alpha1.Vault) (*appsv1.Deployment, error) {
	ls := labelsForVault(v.Name)
	replicas := v.Spec.Size
	configJSON := v.Spec.ConfigJSON()
	owner := asOwner(v)
	ownerJSON, err := json.Marshal(owner)
	if err != nil {
		return nil, err
	}

	readOnlyRootFilesystem := true

	dep := &appsv1.Deployment{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "apps/v1",
			Kind:       "Deployment",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      v.Name,
			Namespace: v.Namespace,
		},
		Spec: appsv1.DeploymentSpec{
			Replicas: &replicas,
			Selector: &metav1.LabelSelector{
				MatchLabels: ls,
			},
			Template: v1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: ls,
				},
				Spec: v1.PodSpec{
					Containers: []v1.Container{
						{
							Image: v.Spec.Image,
							Name:  "vault",
							Args:  []string{"server"},
							Ports: []v1.ContainerPort{{
								ContainerPort: 8200,
								Name:          "vault",
							}},
							Env: []v1.EnvVar{{
								Name:  "VAULT_LOCAL_CONFIG",
								Value: configJSON,
							}},
							SecurityContext: &v1.SecurityContext{
								ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
								Capabilities: &v1.Capabilities{
									Add: []v1.Capability{"IPC_LOCK"},
								},
							},
						},
						{
							Image:           v.Spec.GetBankVaultsImage(),
							ImagePullPolicy: v1.PullIfNotPresent,
							Name:            "bank-vaults",
							Command:         []string{"bank-vaults", "unseal", "--init"},
							Args:            v.Spec.UnsealConfig.ToArgs(v),
							Env: []v1.EnvVar{
								{
									Name:  api.EnvVaultAddress,
									Value: "http://127.0.0.1:8200",
								},
								{
									Name:  k8s.EnvK8SOwnerReference,
									Value: string(ownerJSON),
								},
							},
							VolumeMounts: []v1.VolumeMount{{
								Name:      "file",
								MountPath: "/vault/file",
							}},
						},
					},
					Volumes: []v1.Volume{{
						Name: "file",
						VolumeSource: v1.VolumeSource{
							EmptyDir: &v1.EmptyDirVolumeSource{}, // TODO This should depend on the Vault configuration later on
						},
					}},
				},
			},
		},
	}
	addOwnerRefToObject(dep, owner)
	return dep, nil
}

func serviceForVault(v *v1alpha1.Vault) *v1.Service {
	ls := labelsForVault(v.Name)
	service := &v1.Service{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "v1",
			Kind:       "Service",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      v.Name,
			Namespace: v.Namespace,
		},
		Spec: v1.ServiceSpec{
			Type:     v1.ServiceTypeNodePort,
			Selector: ls,
			Ports: []v1.ServicePort{
				{
					Name: "vault",
					Port: 8200,
				},
			},
		},
	}
	addOwnerRefToObject(service, asOwner(v))
	return service
}

func deploymentForConfigurer(v *v1alpha1.Vault) *appsv1.Deployment {
	ls := labelsForVaultConfigurer(v.Name)
	dep := &appsv1.Deployment{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "apps/v1",
			Kind:       "Deployment",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      v.Name + "-configurer",
			Namespace: v.Namespace,
		},
		Spec: appsv1.DeploymentSpec{
			Selector: &metav1.LabelSelector{
				MatchLabels: ls,
			},
			Template: v1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: ls,
				},
				Spec: v1.PodSpec{
					Containers: []v1.Container{
						{
							Image:           v.Spec.GetBankVaultsImage(),
							ImagePullPolicy: v1.PullIfNotPresent,
							Name:            "bank-vaults",
							Command:         []string{"bank-vaults", "configure"},
							Args:            v.Spec.UnsealConfig.ToArgs(v),
							Env: []v1.EnvVar{
								{
									Name:  api.EnvVaultAddress,
									Value: fmt.Sprintf("http://%s:8200", v.Name),
								},
							},
							VolumeMounts: []v1.VolumeMount{{
								Name:      "config",
								MountPath: "/config",
							}},
							WorkingDir: "/config",
						},
					},
					Volumes: []v1.Volume{{
						Name: "config",
						VolumeSource: v1.VolumeSource{
							ConfigMap: &v1.ConfigMapVolumeSource{
								LocalObjectReference: v1.LocalObjectReference{Name: v.Name + "-configurer"},
							},
						}},
					},
				},
			},
		},
	}
	addOwnerRefToObject(dep, asOwner(v))
	return dep
}

func configMapForConfigurer(v *v1alpha1.Vault) *v1.ConfigMap {
	ls := labelsForVaultConfigurer(v.Name)
	cm := &v1.ConfigMap{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "v1",
			Kind:       "ConfigMap",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      v.Name + "-configurer",
			Namespace: v.Namespace,
			Labels:    ls,
		},
		Data: map[string]string{vault.DefaultConfigFile: v.Spec.ExternalConfigJSON()},
	}
	addOwnerRefToObject(cm, asOwner(v))
	return cm
}

// labelsForVault returns the labels for selecting the resources
// belonging to the given vault CR name.
func labelsForVault(name string) map[string]string {
	return map[string]string{"app": "vault", "vault_cr": name}
}

// labelsForVaultConfigurer returns the labels for selecting the resources
// belonging to the given vault CR name.
func labelsForVaultConfigurer(name string) map[string]string {
	return map[string]string{"app": "vault-configurator", "vault_cr": name}
}

// addOwnerRefToObject appends the desired OwnerReference to the object
func addOwnerRefToObject(obj metav1.Object, ownerRef metav1.OwnerReference) {
	obj.SetOwnerReferences(append(obj.GetOwnerReferences(), ownerRef))
}

// asOwner returns an OwnerReference set as the vault CR
func asOwner(v *v1alpha1.Vault) metav1.OwnerReference {
	trueVar := true
	return metav1.OwnerReference{
		APIVersion: v.APIVersion,
		Kind:       v.Kind,
		Name:       v.Name,
		UID:        v.UID,
		Controller: &trueVar,
	}
}

// podList returns a v1.PodList object
func podList() *v1.PodList {
	return &v1.PodList{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Pod",
			APIVersion: "v1",
		},
	}
}

// getPodNames returns the pod names of the array of pods passed in
func getPodNames(pods []v1.Pod) []string {
	var podNames []string
	for _, pod := range pods {
		podNames = append(podNames, pod.Name)
	}
	return podNames
}
<commit_msg>LivenessProbe and ReadinessProbe for operator<commit_after>package stub

import (
	"encoding/json"
	"fmt"
	"reflect"

	"k8s.io/apimachinery/pkg/util/intstr"

	"github.com/banzaicloud/bank-vaults/pkg/apis/vault/v1alpha1"
	"github.com/banzaicloud/bank-vaults/pkg/kv/k8s"
	"github.com/banzaicloud/bank-vaults/pkg/vault"
	"github.com/hashicorp/vault/api"
	"github.com/operator-framework/operator-sdk/pkg/sdk/action"
	"github.com/operator-framework/operator-sdk/pkg/sdk/handler"
	"github.com/operator-framework/operator-sdk/pkg/sdk/query"
	"github.com/operator-framework/operator-sdk/pkg/sdk/types"
	appsv1 "k8s.io/api/apps/v1"
	"k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
)

func NewHandler() handler.Handler {
	return &Handler{}
}

type Handler struct {
}

func (h *Handler) Handle(ctx types.Context, event types.Event) error {
	switch o := event.Object.(type) {
	case *v1alpha1.Vault:
		v := o

		// Ignore the delete event since the garbage collector will clean up all secondary resources for the CR
		// All secondary resources must have the CR set as their OwnerReference for this to be the case
		if event.Deleted {
			return nil
		}

		// Create the deployment if it doesn't exist
		dep, err := deploymentForVault(v)
		if err != nil {
			return fmt.Errorf("failed to fabricate deployment: %v", err)
		}
		err = action.Create(dep)
		if err != nil && !apierrors.IsAlreadyExists(err) {
			return fmt.Errorf("failed to create deployment: %v", err)
		}

		// Ensure the deployment size is the same as the spec
		err = query.Get(dep)
		if err != nil {
			return fmt.Errorf("failed to get deployment: %v", err)
		}
		size := v.Spec.Size
		if *dep.Spec.Replicas != size {
			dep.Spec.Replicas = &size
			err = action.Update(dep)
			if err != nil {
				return fmt.Errorf("failed to update deployment: %v", err)
			}
		}

		// Update the Vault status with the pod names
		podList := podList()
		labelSelector := labels.SelectorFromSet(labelsForVault(v.Name)).String()
		listOps := &metav1.ListOptions{LabelSelector: labelSelector}
		err = query.List(v.Namespace, podList, query.WithListOptions(listOps))
		if err != nil {
			return fmt.Errorf("failed to list pods: %v", err)
		}
		podNames := getPodNames(podList.Items)
		if !reflect.DeepEqual(podNames, v.Status.Nodes) {
			v.Status.Nodes = podNames
			err := action.Update(v)
			if err != nil {
				return fmt.Errorf("failed to update vault status: %v", err)
			}
		}

		// Create the service if it doesn't exist
		ser := serviceForVault(v)
		err = action.Create(ser)
		if err != nil && !apierrors.IsAlreadyExists(err) {
			return fmt.Errorf("failed to create service: %v", err)
		}

		// Create the deployment if it doesn't exist
		dep = deploymentForConfigurer(v)
		err = action.Create(dep)
		if err != nil && !apierrors.IsAlreadyExists(err) {
			return fmt.Errorf("failed to create configurer deployment: %v", err)
		}

		// Create the configmap if it doesn't exist
		cm := configMapForConfigurer(v)
		err = action.Create(cm)
		if err != nil && !apierrors.IsAlreadyExists(err) {
			return fmt.Errorf("failed to create configurer configmap: %v", err)
		}

		// Ensure the configmap is the same as the spec
		err = query.Get(cm)
		if err != nil {
			return fmt.Errorf("failed to get deployment: %v", err)
		}

		externalConfig := v.Spec.ExternalConfigJSON()
		if cm.Data[vault.DefaultConfigFile] != externalConfig {
			cm.Data[vault.DefaultConfigFile] = externalConfig
			err = action.Update(cm)
			if err != nil {
				return fmt.Errorf("failed to update configurer configmap: %v", err)
			}
		}

	}
	return nil
}

// deploymentForVault returns a vault Deployment object
func deploymentForVault(v *v1alpha1.Vault) (*appsv1.Deployment, error) {
	ls := labelsForVault(v.Name)
	replicas := v.Spec.Size
	configJSON := v.Spec.ConfigJSON()
	owner := asOwner(v)
	ownerJSON, err := json.Marshal(owner)
	if err != nil {
		return nil, err
	}

	readOnlyRootFilesystem := true

	dep := &appsv1.Deployment{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "apps/v1",
			Kind:       "Deployment",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      v.Name,
			Namespace: v.Namespace,
		},
		Spec: appsv1.DeploymentSpec{
			Replicas: &replicas,
			Selector: &metav1.LabelSelector{
				MatchLabels: ls,
			},
			Template: v1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: ls,
				},
				Spec: v1.PodSpec{
					Containers: []v1.Container{
						{
							Image: v.Spec.Image,
							Name:  "vault",
							Args:  []string{"server"},
							Ports: []v1.ContainerPort{{
								ContainerPort: 8200,
								Name:          "vault",
							}},
							Env: []v1.EnvVar{{
								Name:  "VAULT_LOCAL_CONFIG",
								Value: configJSON,
							}},
							SecurityContext: &v1.SecurityContext{
								ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
								Capabilities: &v1.Capabilities{
									Add: []v1.Capability{"IPC_LOCK"},
								},
							},
							// This probe makes sure Vault is responsive in a HTTPS manner
							// See: https://www.vaultproject.io/api/system/init.html
							LivenessProbe: &v1.Probe{
								Handler: v1.Handler{
									HTTPGet: &v1.HTTPGetAction{
										Scheme: v1.URISchemeHTTPS,
										Port:   intstr.FromString("vault"),
										Path:   "/v1/sys/init",
									}},
							},
							// This probe makes sure that only the active Vault instance gets traffic
							// See: https://www.vaultproject.io/api/system/health.html
							ReadinessProbe: &v1.Probe{
								Handler: v1.Handler{
									HTTPGet: &v1.HTTPGetAction{
										Scheme: v1.URISchemeHTTPS,
										Port:   intstr.FromString("vault"),
										Path:   "/v1/sys/health",
									}},
							},
						},
						{
							Image:           v.Spec.GetBankVaultsImage(),
							ImagePullPolicy: v1.PullIfNotPresent,
							Name:            "bank-vaults",
							Command:         []string{"bank-vaults", "unseal", "--init"},
							Args:            v.Spec.UnsealConfig.ToArgs(v),
							Env: []v1.EnvVar{
								{
									Name:  api.EnvVaultAddress,
									Value: "http://127.0.0.1:8200",
								},
								{
									Name:  k8s.EnvK8SOwnerReference,
									Value: string(ownerJSON),
								},
							},
							VolumeMounts: []v1.VolumeMount{{
								Name:      "file",
								MountPath: "/vault/file",
							}},
						},
					},
					Volumes: []v1.Volume{{
						Name: "file",
						VolumeSource: v1.VolumeSource{
							EmptyDir: &v1.EmptyDirVolumeSource{}, // TODO This should depend on the Vault configuration later on
						},
					}},
				},
			},
		},
	}
	addOwnerRefToObject(dep, owner)
	return dep, nil
}

func serviceForVault(v *v1alpha1.Vault) *v1.Service {
	ls := labelsForVault(v.Name)
	service := &v1.Service{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "v1",
			Kind:       "Service",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      v.Name,
			Namespace: v.Namespace,
		},
		Spec: v1.ServiceSpec{
			Type:     v1.ServiceTypeNodePort,
			Selector: ls,
			Ports: []v1.ServicePort{
				{
					Name: "vault",
					Port: 8200,
				},
			},
		},
	}
	addOwnerRefToObject(service, asOwner(v))
	return service
}

func deploymentForConfigurer(v *v1alpha1.Vault) *appsv1.Deployment {
	ls := labelsForVaultConfigurer(v.Name)
	dep := &appsv1.Deployment{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "apps/v1",
			Kind:       "Deployment",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      v.Name + "-configurer",
			Namespace: v.Namespace,
		},
		Spec: appsv1.DeploymentSpec{
			Selector: &metav1.LabelSelector{
				MatchLabels: ls,
			},
			Template: v1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: ls,
				},
				Spec: v1.PodSpec{
					Containers: []v1.Container{
						{
							Image:           v.Spec.GetBankVaultsImage(),
							ImagePullPolicy: v1.PullIfNotPresent,
							Name:            "bank-vaults",
							Command:         []string{"bank-vaults", "configure"},
							Args:            v.Spec.UnsealConfig.ToArgs(v),
							Env: []v1.EnvVar{
								{
									Name:  api.EnvVaultAddress,
									Value: fmt.Sprintf("http://%s:8200", v.Name),
								},
							},
							VolumeMounts: []v1.VolumeMount{{
								Name:      "config",
								MountPath: "/config",
							}},
							WorkingDir: "/config",
						},
					},
					Volumes: []v1.Volume{{
						Name: "config",
						VolumeSource: v1.VolumeSource{
							ConfigMap: &v1.ConfigMapVolumeSource{
								LocalObjectReference: v1.LocalObjectReference{Name: v.Name + "-configurer"},
							},
						}},
					},
				},
			},
		},
	}
	addOwnerRefToObject(dep, asOwner(v))
	return dep
}

func configMapForConfigurer(v *v1alpha1.Vault) *v1.ConfigMap {
	ls := labelsForVaultConfigurer(v.Name)
	cm := &v1.ConfigMap{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "v1",
			Kind:       "ConfigMap",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      v.Name + "-configurer",
			Namespace: v.Namespace,
			Labels:    ls,
		},
		Data: map[string]string{vault.DefaultConfigFile: v.Spec.ExternalConfigJSON()},
	}
	addOwnerRefToObject(cm, asOwner(v))
	return cm
}

// labelsForVault returns the labels for selecting the resources
// belonging to the given vault CR name.
func labelsForVault(name string) map[string]string {
	return map[string]string{"app": "vault", "vault_cr": name}
}

// labelsForVaultConfigurer returns the labels for selecting the resources
// belonging to the given vault CR name.
func labelsForVaultConfigurer(name string) map[string]string {
	return map[string]string{"app": "vault-configurator", "vault_cr": name}
}

// addOwnerRefToObject appends the desired OwnerReference to the object
func addOwnerRefToObject(obj metav1.Object, ownerRef metav1.OwnerReference) {
	obj.SetOwnerReferences(append(obj.GetOwnerReferences(), ownerRef))
}

// asOwner returns an OwnerReference set as the vault CR
func asOwner(v *v1alpha1.Vault) metav1.OwnerReference {
	trueVar := true
	return metav1.OwnerReference{
		APIVersion: v.APIVersion,
		Kind:       v.Kind,
		Name:       v.Name,
		UID:        v.UID,
		Controller: &trueVar,
	}
}

// podList returns a v1.PodList object
func podList() *v1.PodList {
	return &v1.PodList{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Pod",
			APIVersion: "v1",
		},
	}
}

// getPodNames returns the pod names of the array of pods passed in
func getPodNames(pods []v1.Pod) []string {
	var podNames []string
	for _, pod := range pods {
		podNames = append(podNames, pod.Name)
	}
	return podNames
}
<|endoftext|>


<commit_before>package eval

import (
	"os"
	"reflect"
	"strconv"
	"syscall"
	"testing"

	"github.com/elves/elvish/parse"
)

func TestNewEvaler(t *testing.T) {
	ev := NewEvaler(nil, "")
	pid := strconv.Itoa(syscall.Getpid())
	if toString(ev.global["pid"].Get()) != pid {
		t.Errorf(`ev.global["pid"] = %v, want %v`, ev.global["pid"], pid)
	}
}

func strs(ss ...string) []Value {
	vs := make([]Value, len(ss))
	for i, s := range ss {
		vs[i] = str(s)
	}
	return vs
}

type more struct {
	wantBytesOut []byte
	wantExitus   []Value
	wantError    bool
}

var nomore more

var evalTests = []struct {
	text    string
	wantOut []Value
	more
}{
	// Chunks
	// Empty chunk
	{"", []Value{}, nomore},
	// Outputs of pipelines in a chunk are concatenated
	{"put x; put y; put z", strs("x", "y", "z"), nomore},
	// A failed pipeline cause the whole chunk to fail
	{"put a; false; put b", strs("a"), more{wantExitus: []Value{newFailure("1")}}},

	// Pipelines
	// Pure byte pipeline
	{`echo "Albert\nAllan\nAlbraham\nBerlin" | sed s/l/1/g | grep e`,
		[]Value{}, more{wantBytesOut: []byte("A1bert\nBer1in\n")}},
	// TODO: Add a useful channel pipeline sample
	// TODO: Add a useful hybrid pipeline sample

	// Builtin functions
	// Arithmetics
	// TODO test more edge cases
	{"* 353 661", strs("233333"), nomore},
	{"+ 233100 233", strs("233333"), nomore},
	{"- 233333 233100", strs("233"), nomore},
	{"/ 233333 353", strs("661"), nomore},
	{"/ 1 0", strs("+Inf"), nomore},

	// String literal
	{`put 'such \"''literal'`, strs(`such \"'literal`), nomore},
	{`put "much \n\033[31;1m$cool\033[m"`,
		strs("much \n\033[31;1m$cool\033[m"), nomore},

	// Compounding
	{"put {fi,elvi}sh{1.0,1.1}",
		strs("fish1.0", "fish1.1", "elvish1.0", "elvish1.1"), nomore},

	// List, map and indexing
	{"println [a b c] [&key value] | from-lines",
		strs("[a b c] [&key value]"), nomore},
	{"put [a b c][2]", strs("c"), nomore},
	{"put [&key value][key]", strs("value"), nomore},

	// Output capture
	{"put (put lorem ipsum)", strs("lorem", "ipsum"), nomore},

	// Status capture
	{"put ?(true|false|false)",
		[]Value{ok, newFailure("1"), newFailure("1")}, nomore},

	// Variable and compounding
	{"set x = 'SHELL'\nput 'WOW, SUCH '$x', MUCH COOL'\n",
		strs("WOW, SUCH SHELL, MUCH COOL"), nomore},

	// Closure
	// Basics
	{"[]{ }", strs(), nomore},
	{"[x]{put $x} foo", strs("foo"), nomore},
	// Variable capture
	{"set x = lorem; []{set x = ipsum}; put $x", strs("ipsum"), nomore},
	{"set x = lorem; []{ put $x; set x = ipsum }; put $x",
		strs("lorem", "ipsum"), nomore},
	// Shadowing
	{"set x = ipsum; []{ set local:x = lorem; put $x }; put $x",
		strs("lorem", "ipsum"), nomore},
	// Shadowing by argument
	{"set x = ipsum; [x]{ put $x; set x = BAD } lorem; put $x",
		strs("lorem", "ipsum"), nomore},
	// Closure captures new local variables every time
	{`fn f []{ set x = 0; put []{set x = (+ $x 1)} []{put $x} }
set inc1 put1 = (f); $put1; $inc1; $put1
set inc2 put2 = (f); $put2; $inc2; $put2`,
		strs("0", "1", "0", "1"), nomore},

	// fn
	{"fn f [x]{ put $x ipsum }; f lorem",
		strs("lorem", "ipsum"), nomore},
	/*
		// if
		{"if true; then put x", strs("x"), nomore},
		{"if true; false; then put x; else put y",
			strs("y"), nomore},
		{"if true; false; then put x; else if false; put y; else put z",
			strs("z"), nomore},
	*/

	// Namespaces
	// Pseudo-namespaces local: and up:
	{"set x = lorem; []{set local:x = ipsum; put $up:x $local:x}",
		strs("lorem", "ipsum"), nomore},
	{"set x = lorem; []{set up:x = ipsum; put $x}; put $x",
		strs("ipsum", "ipsum"), nomore},
	// Pseudo-namespace env:
	{"set env:foo = lorem; put $env:foo", strs("lorem"), nomore},
	{"del env:foo; put $env:foo", strs(""), nomore},
	// TODO: Test module namespace
}

func mustParse(t *testing.T, name, text string) *parse.Chunk {
	n, err := parse.Parse("<eval_test>", text)
	if err != nil {
		t.Fatalf("Parser(%q) error: %s", text, err)
	}
	return n
}

func evalAndCollect(t *testing.T, texts []string, chsize int) ([]Value, []byte, []Value, error) {
	name := "<eval test>"
	ev := NewEvaler(nil, ".")

	// Collect byte output
	outBytes := []byte{}
	pr, pw, _ := os.Pipe()
	bytesExhausted := make(chan bool)
	go func() {
		for {
			var buf [64]byte
			nr, err := pr.Read(buf[:])
			outBytes = append(outBytes, buf[:nr]...)
			if err != nil {
				break
			}
		}
		bytesExhausted <- true
	}()

	// Channel output
	outs := []Value{}

	// Exitus. Only the exitus of the last text is saved.
	var ex []Value

	for _, text := range texts {
		n := mustParse(t, name, text)

		out := make(chan Value, chsize)
		exhausted := make(chan struct{})
		go func() {
			for v := range out {
				outs = append(outs, v)
			}
			exhausted <- struct{}{}
		}()

		var err error
		ex, err = ev.evalWithOut(name, text, n, &port{ch: out, closeCh: true, f: pw})
		if err != nil {
			return outs, outBytes, ex, err
		}
		<-exhausted
	}
	pw.Close()
	<-bytesExhausted
	return outs, outBytes, ex, nil
}

func TestEval(t *testing.T) {
	for _, tt := range evalTests {
		out, bytesOut, ex, err := evalAndCollect(t, []string{tt.text}, len(tt.wantOut))

		good := true
		errorf := func(format string, args ...interface{}) {
			if good {
				good = false
				t.Errorf("eval(%q) fails:", tt.text)
			}
			t.Errorf(format, args...)
		}

		if tt.wantError != (err != nil) {
			wantError := "nil"
			if tt.wantError {
				wantError = "non-nil"
			}
			errorf("got err=%v, want %s", err, wantError)
		}
		if tt.wantBytesOut != nil && !reflect.DeepEqual(tt.wantBytesOut, bytesOut) {
			errorf("got bytesOut=%q, want %q", bytesOut, tt.wantBytesOut)
		}
		if tt.wantExitus != nil && !reflect.DeepEqual(tt.wantExitus, ex) {
			errorf("got exitus=%v, want %v", ex, tt.wantExitus)
		}
		if tt.wantExitus == nil && HasFailure(ex) {
			errorf("got exitus=%v, want all ok", ex)
		}
		if !reflect.DeepEqual(tt.wantOut, out) {
			errorf("got out=%v, want %v", out, tt.wantOut)
		}
		if !good {
			t.Errorf("--------------")
		}
	}
}

func TestMultipleEval(t *testing.T) {
	outs, _, _, err := evalAndCollect(t, []string{"set x = hello", "put $x"}, 1)
	wanted := strs("hello")
	if err != nil {
		t.Errorf("eval %q => %v, want nil", err)
	}
	if !reflect.DeepEqual(outs, wanted) {
		t.Errorf("eval %q outputs %v, want %v", outs, wanted)
	}
}
<commit_msg>eval: test against pure channel pipeline<commit_after>package eval

import (
	"os"
	"reflect"
	"strconv"
	"syscall"
	"testing"

	"github.com/elves/elvish/parse"
)

func TestNewEvaler(t *testing.T) {
	ev := NewEvaler(nil, "")
	pid := strconv.Itoa(syscall.Getpid())
	if toString(ev.global["pid"].Get()) != pid {
		t.Errorf(`ev.global["pid"] = %v, want %v`, ev.global["pid"], pid)
	}
}

func strs(ss ...string) []Value {
	vs := make([]Value, len(ss))
	for i, s := range ss {
		vs[i] = str(s)
	}
	return vs
}

type more struct {
	wantBytesOut []byte
	wantExitus   []Value
	wantError    bool
}

var nomore more

var evalTests = []struct {
	text    string
	wantOut []Value
	more
}{
	// Chunks
	// Empty chunk
	{"", []Value{}, nomore},
	// Outputs of pipelines in a chunk are concatenated
	{"put x; put y; put z", strs("x", "y", "z"), nomore},
	// A failed pipeline cause the whole chunk to fail
	{"put a; false; put b", strs("a"), more{wantExitus: []Value{newFailure("1")}}},

	// Pipelines
	// Pure byte pipeline
	{`echo "Albert\nAllan\nAlbraham\nBerlin" | sed s/l/1/g | grep e`,
		[]Value{}, more{wantBytesOut: []byte("A1bert\nBer1in\n")}},
	// Pure channel pipeline
	{`put 233 42 19 | each [x]{+ $x 10}`, strs("243", "52", "29"), nomore},
	// TODO: Add a useful hybrid pipeline sample

	// Builtin functions
	// Arithmetics
	// TODO test more edge cases
	{"* 353 661", strs("233333"), nomore},
	{"+ 233100 233", strs("233333"), nomore},
	{"- 233333 233100", strs("233"), nomore},
	{"/ 233333 353", strs("661"), nomore},
	{"/ 1 0", strs("+Inf"), nomore},

	// String literal
	{`put 'such \"''literal'`, strs(`such \"'literal`), nomore},
	{`put "much \n\033[31;1m$cool\033[m"`,
		strs("much \n\033[31;1m$cool\033[m"), nomore},

	// Compounding
	{"put {fi,elvi}sh{1.0,1.1}",
		strs("fish1.0", "fish1.1", "elvish1.0", "elvish1.1"), nomore},

	// List, map and indexing
	{"println [a b c] [&key value] | from-lines",
		strs("[a b c] [&key value]"), nomore},
	{"put [a b c][2]", strs("c"), nomore},
	{"put [&key value][key]", strs("value"), nomore},

	// Output capture
	{"put (put lorem ipsum)", strs("lorem", "ipsum"), nomore},

	// Status capture
	{"put ?(true|false|false)",
		[]Value{ok, newFailure("1"), newFailure("1")}, nomore},

	// Variable and compounding
	{"set x = 'SHELL'\nput 'WOW, SUCH '$x', MUCH COOL'\n",
		strs("WOW, SUCH SHELL, MUCH COOL"), nomore},

	// Closure
	// Basics
	{"[]{ }", strs(), nomore},
	{"[x]{put $x} foo", strs("foo"), nomore},
	// Variable capture
	{"set x = lorem; []{set x = ipsum}; put $x", strs("ipsum"), nomore},
	{"set x = lorem; []{ put $x; set x = ipsum }; put $x",
		strs("lorem", "ipsum"), nomore},
	// Shadowing
	{"set x = ipsum; []{ set local:x = lorem; put $x }; put $x",
		strs("lorem", "ipsum"), nomore},
	// Shadowing by argument
	{"set x = ipsum; [x]{ put $x; set x = BAD } lorem; put $x",
		strs("lorem", "ipsum"), nomore},
	// Closure captures new local variables every time
	{`fn f []{ set x = 0; put []{set x = (+ $x 1)} []{put $x} }
set inc1 put1 = (f); $put1; $inc1; $put1
set inc2 put2 = (f); $put2; $inc2; $put2`,
		strs("0", "1", "0", "1"), nomore},

	// fn
	{"fn f [x]{ put $x ipsum }; f lorem",
		strs("lorem", "ipsum"), nomore},
	/*
		// if
		{"if true; then put x", strs("x"), nomore},
		{"if true; false; then put x; else put y",
			strs("y"), nomore},
		{"if true; false; then put x; else if false; put y; else put z",
			strs("z"), nomore},
	*/

	// Namespaces
	// Pseudo-namespaces local: and up:
	{"set x = lorem; []{set local:x = ipsum; put $up:x $local:x}",
		strs("lorem", "ipsum"), nomore},
	{"set x = lorem; []{set up:x = ipsum; put $x}; put $x",
		strs("ipsum", "ipsum"), nomore},
	// Pseudo-namespace env:
	{"set env:foo = lorem; put $env:foo", strs("lorem"), nomore},
	{"del env:foo; put $env:foo", strs(""), nomore},
	// TODO: Test module namespace
}

func mustParse(t *testing.T, name, text string) *parse.Chunk {
	n, err := parse.Parse("<eval_test>", text)
	if err != nil {
		t.Fatalf("Parser(%q) error: %s", text, err)
	}
	return n
}

func evalAndCollect(t *testing.T, texts []string, chsize int) ([]Value, []byte, []Value, error) {
	name := "<eval test>"
	ev := NewEvaler(nil, ".")

	// Collect byte output
	outBytes := []byte{}
	pr, pw, _ := os.Pipe()
	bytesExhausted := make(chan bool)
	go func() {
		for {
			var buf [64]byte
			nr, err := pr.Read(buf[:])
			outBytes = append(outBytes, buf[:nr]...)
			if err != nil {
				break
			}
		}
		bytesExhausted <- true
	}()

	// Channel output
	outs := []Value{}

	// Exitus. Only the exitus of the last text is saved.
	var ex []Value

	for _, text := range texts {
		n := mustParse(t, name, text)

		out := make(chan Value, chsize)
		exhausted := make(chan struct{})
		go func() {
			for v := range out {
				outs = append(outs, v)
			}
			exhausted <- struct{}{}
		}()

		var err error
		ex, err = ev.evalWithOut(name, text, n, &port{ch: out, closeCh: true, f: pw})
		if err != nil {
			return outs, outBytes, ex, err
		}
		<-exhausted
	}
	pw.Close()
	<-bytesExhausted
	return outs, outBytes, ex, nil
}

func TestEval(t *testing.T) {
	for _, tt := range evalTests {
		out, bytesOut, ex, err := evalAndCollect(t, []string{tt.text}, len(tt.wantOut))

		good := true
		errorf := func(format string, args ...interface{}) {
			if good {
				good = false
				t.Errorf("eval(%q) fails:", tt.text)
			}
			t.Errorf(format, args...)
		}

		if tt.wantError != (err != nil) {
			wantError := "nil"
			if tt.wantError {
				wantError = "non-nil"
			}
			errorf("got err=%v, want %s", err, wantError)
		}
		if tt.wantBytesOut != nil && !reflect.DeepEqual(tt.wantBytesOut, bytesOut) {
			errorf("got bytesOut=%q, want %q", bytesOut, tt.wantBytesOut)
		}
		if tt.wantExitus != nil && !reflect.DeepEqual(tt.wantExitus, ex) {
			errorf("got exitus=%v, want %v", ex, tt.wantExitus)
		}
		if tt.wantExitus == nil && HasFailure(ex) {
			errorf("got exitus=%v, want all ok", ex)
		}
		if !reflect.DeepEqual(tt.wantOut, out) {
			errorf("got out=%v, want %v", out, tt.wantOut)
		}
		if !good {
			t.Errorf("--------------")
		}
	}
}

func TestMultipleEval(t *testing.T) {
	outs, _, _, err := evalAndCollect(t, []string{"set x = hello", "put $x"}, 1)
	wanted := strs("hello")
	if err != nil {
		t.Errorf("eval %q => %v, want nil", err)
	}
	if !reflect.DeepEqual(outs, wanted) {
		t.Errorf("eval %q outputs %v, want %v", outs, wanted)
	}
}
<|endoftext|>


<commit_before>package server

import (
	"github.com/smancke/guble/protocol"

	"fmt"
	"net/http"
	"reflect"
	"time"
	"github.com/docker/distribution/health"
)

// Startable interface for modules which provide a start mechanism
type Startable interface {
	Start() error
}

// Stopable interface for modules which provide a stop mechanism
type Stopable interface {
	Stop() error
}

// Endpoint adds a HTTP handler for the `GetPrefix()` to the webserver
type Endpoint interface {
	http.Handler
	GetPrefix() string
}

// Service is the main class for simple control of a server
type Service struct {
	webServer  *WebServer
	router     Router
	stopables  []Stopable
	startables []Startable
	healthCheckers []health.Checker
	// The time given to each Module on Stop()
	StopGracePeriod time.Duration
}

// NewService registers the Main Router, where other modules can subscribe for messages
func NewService(
	addr string,
	router Router) *Service {
	service := &Service{
		stopables:       make([]Stopable, 0, 5),
		webServer:       NewWebServer(addr),
		router:          router,
		StopGracePeriod: time.Second * 2,
	}
	service.Register(service.webServer)
	service.Register(service.router)

	service.webServer.Handle("/health", http.HandlerFunc(health.StatusHandler))

	return service
}

// Register the supplied module on this service.
// This method checks the module for the following interfaces and
// does the expected registrations:
//   Startable:
//   Stopable: notify when the service stops
//   Endpoint: Register the handler function of the Endpoint in the http service at prefix
func (service *Service) Register(module interface{}) {
	name := reflect.TypeOf(module).String()

	if startable, ok := module.(Startable); ok {
		protocol.Info("register %v as StartListener", name)
		service.AddStartListener(startable)
	}

	if stopable, ok := module.(Stopable); ok {
		protocol.Info("register %v as StopListener", name)
		service.AddStopListener(stopable)
	}

	if endpoint, ok := module.(Endpoint); ok {
		protocol.Info("register %v as Endpoint to %v", name, endpoint.GetPrefix())
		service.webServer.Handle(endpoint.GetPrefix(), endpoint)
	}
}

func (service *Service) Start() error {
	el := protocol.NewErrorList("Errors occured while startup the service: ")

	for _, startable := range service.startables {
		name := reflect.TypeOf(startable).String()

		protocol.Debug("starting module %v", name)
		if err := startable.Start(); err != nil {
			protocol.Err("error on startup module %v", name)
			el.Add(err)
		}
	}
	return el.ErrorOrNil()
}

func (service *Service) Stop() error {
	errors := make(map[string]error)
	for _, stopable := range service.stopables {
		name := reflect.TypeOf(stopable).String()
		stoppedChan := make(chan bool)
		errorChan := make(chan error)
		protocol.Info("stopping %v ...", name)
		go func() {
			err := stopable.Stop()
			if err != nil {
				errorChan <- err
				return
			}
			stoppedChan <- true
		}()
		select {
		case err := <-errorChan:
			protocol.Err("error while stopping %v: %v", name, err.Error)
			errors[name] = err
		case <-stoppedChan:
			protocol.Info("stopped %v", name)
		case <-time.After(service.StopGracePeriod):
			errors[name] = fmt.Errorf("error while stopping %v: not returned after %v seconds", name, service.StopGracePeriod)
			protocol.Err(errors[name].Error())
		}
	}
	if len(errors) > 0 {
		return fmt.Errorf("Errors while stopping modules %q", errors)
	}
	return nil
}

func (service *Service) AddStopListener(stopable Stopable) {
	service.stopables = append(service.stopables, stopable)
}

func (service *Service) AddStartListener(startable Startable) {
	service.startables = append(service.startables, startable)
}

func (service *Service) GetWebServer() *WebServer {
	return service.webServer
}
<commit_msg>handling health.Checker interfaces in Service<commit_after>package server

import (
	"github.com/smancke/guble/protocol"

	"fmt"
	"net/http"
	"reflect"
	"time"
	"github.com/docker/distribution/health"
)

// Startable interface for modules which provide a start mechanism
type Startable interface {
	Start() error
}

// Stopable interface for modules which provide a stop mechanism
type Stopable interface {
	Stop() error
}

// Endpoint adds a HTTP handler for the `GetPrefix()` to the webserver
type Endpoint interface {
	http.Handler
	GetPrefix() string
}

// Service is the main class for simple control of a server
type Service struct {
	webServer  *WebServer
	router     Router
	stopables  []Stopable
	startables []Startable
	healthCheckers []health.Checker
	// The time given to each Module on Stop()
	StopGracePeriod time.Duration
}

// NewService registers the Main Router, where other modules can subscribe for messages
func NewService(
	addr string,
	router Router) *Service {
	service := &Service{
		stopables:       make([]Stopable, 0, 5),
		webServer:       NewWebServer(addr),
		router:          router,
		StopGracePeriod: time.Second * 2,
	}
	service.Register(service.webServer)
	service.Register(service.router)

	service.webServer.Handle("/health", http.HandlerFunc(health.StatusHandler))

	return service
}

// Register the supplied module on this service.
// This method checks the module for the following interfaces and
// does the expected registrations:
//   Startable:
//   Stopable: notify when the service stops
//   Endpoint: Register the handler function of the Endpoint in the http service at prefix
func (service *Service) Register(module interface{}) {
	name := reflect.TypeOf(module).String()

	if startable, ok := module.(Startable); ok {
		protocol.Info("register %v as StartListener", name)
		service.AddStartListener(startable)
	}

	if stopable, ok := module.(Stopable); ok {
		protocol.Info("register %v as StopListener", name)
		service.AddStopListener(stopable)
	}

	if checker, ok := module.(health.Checker); ok {
		protocol.Info("register %v as HealthChecker", name)
		service.AddHealthChecker(checker)
	}

	if endpoint, ok := module.(Endpoint); ok {
		protocol.Info("register %v as Endpoint to %v", name, endpoint.GetPrefix())
		service.webServer.Handle(endpoint.GetPrefix(), endpoint)
	}
}

func (service *Service) Start() error {
	el := protocol.NewErrorList("Errors occured while startup the service: ")

	for _, startable := range service.startables {
		name := reflect.TypeOf(startable).String()

		protocol.Debug("starting module %v", name)
		if err := startable.Start(); err != nil {
			protocol.Err("error on startup module %v", name)
			el.Add(err)
		}
	}
	return el.ErrorOrNil()
}

func (service *Service) Stop() error {
	errors := make(map[string]error)
	for _, stopable := range service.stopables {
		name := reflect.TypeOf(stopable).String()
		stoppedChan := make(chan bool)
		errorChan := make(chan error)
		protocol.Info("stopping %v ...", name)
		go func() {
			err := stopable.Stop()
			if err != nil {
				errorChan <- err
				return
			}
			stoppedChan <- true
		}()
		select {
		case err := <-errorChan:
			protocol.Err("error while stopping %v: %v", name, err.Error)
			errors[name] = err
		case <-stoppedChan:
			protocol.Info("stopped %v", name)
		case <-time.After(service.StopGracePeriod):
			errors[name] = fmt.Errorf("error while stopping %v: not returned after %v seconds", name, service.StopGracePeriod)
			protocol.Err(errors[name].Error())
		}
	}
	if len(errors) > 0 {
		return fmt.Errorf("Errors while stopping modules %q", errors)
	}
	return nil
}

func (service *Service) AddStopListener(stopable Stopable) {
	service.stopables = append(service.stopables, stopable)
}

func (service *Service) AddStartListener(startable Startable) {
	service.startables = append(service.startables, startable)
}

func (service *Service) AddHealthChecker(healthChecker health.Checker) {
	service.healthCheckers = append(service.healthCheckers, healthChecker)
}

func (service *Service) GetWebServer() *WebServer {
	return service.webServer
}
<|endoftext|>