<commit_before>package view

import (
	"github.com/ungerik/go-start/debug"
	"github.com/ungerik/go-start/utils"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"github.com/ungerik/web.go"
)

const PathFragmentPattern = "([a-zA-Z0-9_\\-\\.]+)"

var PathFragmentRegexp *regexp.Regexp = regexp.MustCompile(PathFragmentPattern)

///////////////////////////////////////////////////////////////////////////////
// ViewPath

// ViewPath holds all data necessary to define the URL path of a view,
// including the number of arguments parsed from the URL path,
// an Authenticator and sub paths.
type ViewPath struct {
	Name   string
	Args   int
	View   View
	Auth   Authenticator
	NoAuth URL
	Sub    []ViewPath // Only allowed when View is a Page or nil
}

// Pages and nil views will be registered with a trailing slash at their path
// and a permanent redirect from the path without trailing slash
// Nil views will be registered as NotFound404
// parentPath always ends with a slash
func (self *ViewPath) initAndRegisterViewsRecursive(parentPath string) {
	if parentPath == "" || parentPath[len(parentPath)-1] != '/' {
		panic("Parent path must end with a slash: " + parentPath)
	}
	if parentPath != "/" && self.Name == "" && self.Args == 0 {
		panic("Sub path of " + parentPath + " with no Name and Args")
	}
	if self.Name != "" && !PathFragmentRegexp.MatchString(self.Name) {
		panic("Invalid characters in view.ViewPath.Name: " + self.Name)
	}
	if self.View != nil && utils.IsDeepNil(self.View) {
		panic("Nil value wrapped with non nil view.View under parentPath: " + parentPath)
	}

	addSlash := self.Args > 0
	if self.View == nil {
		addSlash = true
		self.View = &NotFoundView{Message: "Invalid URL"}
	} else if _, isPage := self.View.(*Page); isPage {
		addSlash = true
		if self.Auth != nil {
			if self.NoAuth == nil && Config.LoginSignupPage != nil && *Config.LoginSignupPage != nil {
				self.NoAuth = &IndirectPageURL{Config.LoginSignupPage}
			}
		}
	}

	path := parentPath + self.Name
	if self.Args > 0 {
		if self.Name != "" {
			path += "/"
		}
		for i := 0; i < self.Args; i++ {
			path += PathFragmentPattern + "/"
		}
	}

	if addSlash {
		if path[len(path)-1] != '/' {
			path += "/"
		}
		if path != "/" {
			web.Get(path[:len(path)-1], func(webContext *web.Context, args ...string) string {
				webContext.Redirect(http.StatusMovedPermanently, webContext.Request.RequestURI+"/")
				return ""
			})
		}
	}

	if self.Args < 0 {
		panic("Negative Args at " + path)
	}
	if _, pathExists := viewsByPath[path]; pathExists {
		panic("View with path '" + path + "' already registered")
	}
	viewsByPath[path] = self.View

	if Config.DebugPrintPaths {
		debug.Print(path)
	}

	//debug.Print(path)
	self.View.Init(self.View)
	if viewWithURL, ok := self.View.(ViewWithURL); ok {
		viewWithURL.SetPath(path)
	}

	htmlFunc := func(webContext *web.Context, args ...string) string {
		context := NewContext(webContext, self.View, args)

		for _, subdomain := range Config.RedirectSubdomains {
			if len(subdomain) > 0 {
				if subdomain[len(subdomain)-1] != '.' {
					subdomain += "."
				}
				host := context.Request.Host
				if strings.Index(host, subdomain) == 0 {
					host = host[len(subdomain):]
					url := "http://" + host + context.Request.URL.Path
					context.Redirect(http.StatusMovedPermanently, url)
					return ""
				}
			}
		}

		handleErr := func(err error) string {
			switch err.(type) {
			case NotFound:
				context.NotFound(err.Error())
			case Redirect:
				context.Redirect(http.StatusFound, err.Error())
			case PermanentRedirect:
				context.Redirect(http.StatusMovedPermanently, err.Error())
			case Forbidden:
				context.Abort(http.StatusForbidden, err.Error())
			default:
				context.Abort(http.StatusInternalServerError, err.Error())
			}
			return ""
		}

		handleNoAuth := func(err error) string {
			switch {
			case err != nil:
				return handleErr(err)
			case self.NoAuth != nil:
				from := url.QueryEscape(context.Request.RequestURI)
				to := self.NoAuth.URL(context) + "?from=" + from
				return handleErr(Redirect(to))
			}
			return handleErr(Forbidden("403 Forbidden: authentication required"))
		}

		if Config.GlobalAuth != nil {
			if ok, err := Config.GlobalAuth.Authenticate(context); !ok {
				return handleNoAuth(err)
			}
		}

		if self.Auth == nil {
			self.Auth = Config.FallbackAuth
		}
		if self.Auth != nil {
			if ok, err := self.Auth.Authenticate(context); !ok {
				return handleNoAuth(err)
			}
		}

		//		numMiddlewares := len(Config.Middlewares)
		//		for i := 0; i < numMiddlewares; i++ {
		//			if abort := Config.Middlewares[i].PreRender(context); abort {
		//				return ""
		//			}
		//		}

		xmlBuffer := utils.NewXMLBuffer()

		err := self.View.Render(context, &xmlBuffer.XMLWriter)

		//		for i := numMiddlewares - 1; i >= 0; i-- {
		//			html, err = Config.Middlewares[i].PostRender(context, html, err)
		//		}

		if err != nil {
			return handleErr(err)
		}
		return xmlBuffer.String()
	}

	web.Get(path, htmlFunc)
	web.Post(path, htmlFunc)

	for i := range self.Sub {
		self.Sub[i].initAndRegisterViewsRecursive(path)
	}
}
<commit_msg>testing deferred runtime.GC() to fight memory consumption<commit_after>package view

import (
	"github.com/ungerik/go-start/debug"
	"github.com/ungerik/go-start/utils"
	"github.com/ungerik/web.go"
	"net/http"
	"net/url"
	"regexp"
	"runtime"
	"strings"
)

const PathFragmentPattern = "([a-zA-Z0-9_\\-\\.]+)"

var PathFragmentRegexp *regexp.Regexp = regexp.MustCompile(PathFragmentPattern)

///////////////////////////////////////////////////////////////////////////////
// ViewPath

// ViewPath holds all data necessary to define the URL path of a view,
// including the number of arguments parsed from the URL path,
// an Authenticator and sub paths.
type ViewPath struct {
	Name   string
	Args   int
	View   View
	Auth   Authenticator
	NoAuth URL
	Sub    []ViewPath // Only allowed when View is a Page or nil
}

// Pages and nil views will be registered with a trailing slash at their path
// and a permanent redirect from the path without trailing slash
// Nil views will be registered as NotFound404
// parentPath always ends with a slash
func (self *ViewPath) initAndRegisterViewsRecursive(parentPath string) {
	if parentPath == "" || parentPath[len(parentPath)-1] != '/' {
		panic("Parent path must end with a slash: " + parentPath)
	}
	if parentPath != "/" && self.Name == "" && self.Args == 0 {
		panic("Sub path of " + parentPath + " with no Name and Args")
	}
	if self.Name != "" && !PathFragmentRegexp.MatchString(self.Name) {
		panic("Invalid characters in view.ViewPath.Name: " + self.Name)
	}
	if self.View != nil && utils.IsDeepNil(self.View) {
		panic("Nil value wrapped with non nil view.View under parentPath: " + parentPath)
	}

	addSlash := self.Args > 0
	if self.View == nil {
		addSlash = true
		self.View = &NotFoundView{Message: "Invalid URL"}
	} else if _, isPage := self.View.(*Page); isPage {
		addSlash = true
		if self.Auth != nil {
			if self.NoAuth == nil && Config.LoginSignupPage != nil && *Config.LoginSignupPage != nil {
				self.NoAuth = &IndirectPageURL{Config.LoginSignupPage}
			}
		}
	}

	path := parentPath + self.Name
	if self.Args > 0 {
		if self.Name != "" {
			path += "/"
		}
		for i := 0; i < self.Args; i++ {
			path += PathFragmentPattern + "/"
		}
	}

	if addSlash {
		if path[len(path)-1] != '/' {
			path += "/"
		}
		if path != "/" {
			web.Get(path[:len(path)-1], func(webContext *web.Context, args ...string) string {
				webContext.Redirect(http.StatusMovedPermanently, webContext.Request.RequestURI+"/")
				return ""
			})
		}
	}

	if self.Args < 0 {
		panic("Negative Args at " + path)
	}
	if _, pathExists := viewsByPath[path]; pathExists {
		panic("View with path '" + path + "' already registered")
	}
	viewsByPath[path] = self.View

	if Config.DebugPrintPaths {
		debug.Print(path)
	}

	//debug.Print(path)
	self.View.Init(self.View)
	if viewWithURL, ok := self.View.(ViewWithURL); ok {
		viewWithURL.SetPath(path)
	}

	htmlFunc := func(webContext *web.Context, args ...string) string {
		defer func() {
			go runtime.GC()
		}()

		context := NewContext(webContext, self.View, args)

		for _, subdomain := range Config.RedirectSubdomains {
			if len(subdomain) > 0 {
				if subdomain[len(subdomain)-1] != '.' {
					subdomain += "."
				}
				host := context.Request.Host
				if strings.Index(host, subdomain) == 0 {
					host = host[len(subdomain):]
					url := "http://" + host + context.Request.URL.Path
					context.Redirect(http.StatusMovedPermanently, url)
					return ""
				}
			}
		}

		handleErr := func(err error) string {
			switch err.(type) {
			case NotFound:
				context.NotFound(err.Error())
			case Redirect:
				context.Redirect(http.StatusFound, err.Error())
			case PermanentRedirect:
				context.Redirect(http.StatusMovedPermanently, err.Error())
			case Forbidden:
				context.Abort(http.StatusForbidden, err.Error())
			default:
				context.Abort(http.StatusInternalServerError, err.Error())
			}
			return ""
		}

		handleNoAuth := func(err error) string {
			switch {
			case err != nil:
				return handleErr(err)
			case self.NoAuth != nil:
				from := url.QueryEscape(context.Request.RequestURI)
				to := self.NoAuth.URL(context) + "?from=" + from
				return handleErr(Redirect(to))
			}
			return handleErr(Forbidden("403 Forbidden: authentication required"))
		}

		if Config.GlobalAuth != nil {
			if ok, err := Config.GlobalAuth.Authenticate(context); !ok {
				return handleNoAuth(err)
			}
		}

		if self.Auth == nil {
			self.Auth = Config.FallbackAuth
		}
		if self.Auth != nil {
			if ok, err := self.Auth.Authenticate(context); !ok {
				return handleNoAuth(err)
			}
		}

		//		numMiddlewares := len(Config.Middlewares)
		//		for i := 0; i < numMiddlewares; i++ {
		//			if abort := Config.Middlewares[i].PreRender(context); abort {
		//				return ""
		//			}
		//		}

		xmlBuffer := utils.NewXMLBuffer()

		err := self.View.Render(context, &xmlBuffer.XMLWriter)

		//		for i := numMiddlewares - 1; i >= 0; i-- {
		//			html, err = Config.Middlewares[i].PostRender(context, html, err)
		//		}

		if err != nil {
			return handleErr(err)
		}
		return xmlBuffer.String()
	}

	web.Get(path, htmlFunc)
	web.Post(path, htmlFunc)

	for i := range self.Sub {
		self.Sub[i].initAndRegisterViewsRecursive(path)
	}
}
<|endoftext|>


<commit_before><commit_msg>increase timeout for flaky test<commit_after><|endoftext|>


<commit_before><commit_msg>rename "importToBind" => "importData"<commit_after><|endoftext|>


<commit_before><commit_msg>Add configuration merge & bug fix release<commit_after><|endoftext|>


<commit_before>/**
 * (c) 2014, Tonnerre Lombard <tonnerre@ancient-solutions.com>,
 *	     Ancient Solutions. All rights reserved.
 *
 * Redistribution and use in source  and binary forms, with or without
 * modification, are permitted  provided that the following conditions
 * are met:
 *
 * * Redistributions of  source code  must retain the  above copyright
 *   notice, this list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright
 *   notice, this  list of conditions and the  following disclaimer in
 *   the  documentation  and/or  other  materials  provided  with  the
 *   distribution.
 * * Neither  the  name  of  Ancient Solutions  nor  the  name  of its
 *   contributors may  be used to endorse or  promote products derived
 *   from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS"  AND ANY EXPRESS  OR IMPLIED WARRANTIES  OF MERCHANTABILITY
 * AND FITNESS  FOR A PARTICULAR  PURPOSE ARE DISCLAIMED. IN  NO EVENT
 * SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED  TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE,  DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT  LIABILITY,  OR  TORT  (INCLUDING NEGLIGENCE  OR  OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package main

import (
	"code.google.com/p/goprotobuf/proto"
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"hash"
	"io"
	"io/ioutil"
	"os"
	"strconv"
)

// Special "writer" which just counts the number of bytes passed through.
type CountingWriter struct {
	n uint64
}

// Add the length of p to the counter.
func (self *CountingWriter) Write(p []byte) (n int, err error) {
	n = len(p)
	self.n += uint64(n)
	return
}

// Return the full number of bytes passed through Write().
func (self *CountingWriter) BytesWritten() uint64 {
	return self.n
}

type blubberStore struct {
	blob_path string
	insecure  bool
	priv      *rsa.PrivateKey
}

// Construct the file name prefix for the blob with the given blob ID.
// Returns the full name prefix and the parent directory.
func (self *blubberStore) blobId2FileName(blobId []byte) (
	string, string) {
	var parent_dir, first_dir, second_dir string
	var path_name string = hex.EncodeToString(blobId)

	// Extract the first and second directory part piece.
	if len(blobId) > 0 {
		first_dir = strconv.FormatUint(uint64(blobId[0]), 16)
	} else {
		path_name = "zz"
		first_dir = "00"
	}
	if len(blobId) > 1 {
		second_dir = first_dir + strconv.FormatUint(uint64(blobId[1]), 16)
	} else {
		second_dir = first_dir + "00"
	}

	parent_dir = self.blob_path + "/" + first_dir + "/" + second_dir
	return parent_dir + "/" + path_name, parent_dir
}

// Create a new blob with the given blobId, or overwrite an existing one.
// The contents of the blob will be read from input.
func (self *blubberStore) StoreBlob(blobId []byte, input io.Reader) error {
	var outstream cipher.StreamWriter
	var bh BlubberBlockHeader
	var aescipher cipher.Block
	var cksum hash.Hash = sha256.New()
	var parent_dir, file_prefix string
	var counter CountingWriter
	var outfile *os.File
	var buf []byte
	var n int
	var err error

	// Create a block key and IV for the blob data.
	bh.BlockKey = make([]byte, aes.BlockSize)
	n, err = rand.Read(bh.BlockKey)
	if err != nil {
		return err
	}
	if n != aes.BlockSize {
		return errors.New("Unexpected length of random data")
	}

	bh.Iv = make([]byte, aes.BlockSize)
	n, err = rand.Read(bh.Iv)
	if err != nil {
		return err
	}
	if n != aes.BlockSize {
		return errors.New("Unexpected length of random data")
	}

	file_prefix, parent_dir = self.blobId2FileName(blobId)

	// Ensure we have the full directory path in place.
	err = os.MkdirAll(parent_dir, 0700)
	if err != nil {
		return err
	}

	// Now, write the actual data.
	outfile, err = os.Create(file_prefix + ".data")
	if err != nil {
		return err
	}
	aescipher, err = aes.NewCipher(bh.BlockKey)
	if err != nil {
		return err
	}
	outstream = cipher.StreamWriter{
		S: cipher.NewCTR(aescipher, bh.Iv),
		W: io.MultiWriter(outfile, cksum, &counter),
	}
	_, err = io.Copy(outstream, input)
	if err != nil {
		return err
	}
	err = outfile.Close()
	if err != nil {
		return err
	}

	// Fill in the last bits of the blob header.
	bh.Checksum = cksum.Sum(bh.Checksum)
	bh.Size = new(uint64)
	*bh.Size = counter.BytesWritten()
	buf, err = proto.Marshal(&bh)
	if err != nil {
		return err
	}
	if !self.insecure {
		// Encrypt the AES key and IV with the RSA key.
		buf, err = rsa.DecryptPKCS1v15(rand.Reader, self.priv, buf)
		if err != nil {
			return err
		}
	}

	// Write out the crypto head with the IV and the blob key.
	outfile, err = os.Create(file_prefix + ".crypthead")
	if err != nil {
		return err
	}
	n, err = outfile.Write(buf)
	if err != nil {
		return err
	}
	if n < len(buf) {
		return errors.New("Short write to file")
	}
	return outstream.Close()
}

// Extract the blubber block head for the given blob ID and return it.
func (self *blubberStore) extractBlockHead(blobId []byte) (
	bh *BlubberBlockHeader, err error) {
	var file_prefix string
	var data []byte

	file_prefix, _ = self.blobId2FileName(blobId)
	data, err = ioutil.ReadFile(file_prefix + ".crypthead")

	if !self.insecure {
		// Decrypt the AES key and IV with the RSA key.
		data, err = rsa.DecryptPKCS1v15(rand.Reader, self.priv, data)
		if err != nil {
			return
		}
	}

	bh = new(BlubberBlockHeader)
	err = proto.Unmarshal(data, bh)
	return
}

// Read the blob with the specified blob ID and return it to the caller.
// The contents will be sent as a regular HTTP response.
func (self *blubberStore) RetrieveBlob(blobId []byte, rw io.Writer) error {
	var file_prefix string
	var infile *os.File
	var bh *BlubberBlockHeader
	var instream cipher.StreamReader
	var aescipher cipher.Block
	var err error

	// Get the metadata.
	bh, err = self.extractBlockHead(blobId)
	if err != nil {
		return err
	}

	file_prefix, _ = self.blobId2FileName(blobId)
	infile, err = os.Open(file_prefix + ".data")
	if err != nil {
		return err
	}

	aescipher, err = aes.NewCipher(bh.BlockKey)
	if err != nil {
		infile.Close()
		return err
	}
	instream = cipher.StreamReader{
		S: cipher.NewCTR(aescipher, bh.Iv),
		R: infile,
	}

	_, err = io.Copy(rw, instream)
	if err != nil {
		infile.Close()
		return err
	}

	return infile.Close()
}

// Delete the blob with the given blob ID.
func (self *blubberStore) DeleteBlob(blobId []byte) error {
	return errors.New("Not yet implemented")
}

// Get some details about the specified blob.
func (self *blubberStore) StatBlob(blobId []byte) (
	ret BlubberStat, err error) {
	err = errors.New("Not yet implemented")
	return
}
<commit_msg>Implement a stat method invoked via HEAD.<commit_after>/**
 * (c) 2014, Tonnerre Lombard <tonnerre@ancient-solutions.com>,
 *	     Ancient Solutions. All rights reserved.
 *
 * Redistribution and use in source  and binary forms, with or without
 * modification, are permitted  provided that the following conditions
 * are met:
 *
 * * Redistributions of  source code  must retain the  above copyright
 *   notice, this list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright
 *   notice, this  list of conditions and the  following disclaimer in
 *   the  documentation  and/or  other  materials  provided  with  the
 *   distribution.
 * * Neither  the  name  of  Ancient Solutions  nor  the  name  of its
 *   contributors may  be used to endorse or  promote products derived
 *   from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS"  AND ANY EXPRESS  OR IMPLIED WARRANTIES  OF MERCHANTABILITY
 * AND FITNESS  FOR A PARTICULAR  PURPOSE ARE DISCLAIMED. IN  NO EVENT
 * SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED  TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE,  DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT  LIABILITY,  OR  TORT  (INCLUDING NEGLIGENCE  OR  OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package main

import (
	"code.google.com/p/goprotobuf/proto"
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"hash"
	"io"
	"io/ioutil"
	"os"
	"strconv"
)

// Special "writer" which just counts the number of bytes passed through.
type CountingWriter struct {
	n uint64
}

// Add the length of p to the counter.
func (self *CountingWriter) Write(p []byte) (n int, err error) {
	n = len(p)
	self.n += uint64(n)
	return
}

// Return the full number of bytes passed through Write().
func (self *CountingWriter) BytesWritten() uint64 {
	return self.n
}

type blubberStore struct {
	blob_path string
	insecure  bool
	priv      *rsa.PrivateKey
}

// Construct the file name prefix for the blob with the given blob ID.
// Returns the full name prefix and the parent directory.
func (self *blubberStore) blobId2FileName(blobId []byte) (
	string, string) {
	var parent_dir, first_dir, second_dir string
	var path_name string = hex.EncodeToString(blobId)

	// Extract the first and second directory part piece.
	if len(blobId) > 0 {
		first_dir = strconv.FormatUint(uint64(blobId[0]), 16)
	} else {
		path_name = "zz"
		first_dir = "00"
	}
	if len(blobId) > 1 {
		second_dir = first_dir + strconv.FormatUint(uint64(blobId[1]), 16)
	} else {
		second_dir = first_dir + "00"
	}

	parent_dir = self.blob_path + "/" + first_dir + "/" + second_dir
	return parent_dir + "/" + path_name, parent_dir
}

// Create a new blob with the given blobId, or overwrite an existing one.
// The contents of the blob will be read from input.
func (self *blubberStore) StoreBlob(blobId []byte, input io.Reader) error {
	var outstream cipher.StreamWriter
	var bh BlubberBlockHeader
	var aescipher cipher.Block
	var cksum hash.Hash = sha256.New()
	var parent_dir, file_prefix string
	var counter CountingWriter
	var outfile *os.File
	var buf []byte
	var n int
	var err error

	// Create a block key and IV for the blob data.
	bh.BlockKey = make([]byte, aes.BlockSize)
	n, err = rand.Read(bh.BlockKey)
	if err != nil {
		return err
	}
	if n != aes.BlockSize {
		return errors.New("Unexpected length of random data")
	}

	bh.Iv = make([]byte, aes.BlockSize)
	n, err = rand.Read(bh.Iv)
	if err != nil {
		return err
	}
	if n != aes.BlockSize {
		return errors.New("Unexpected length of random data")
	}

	file_prefix, parent_dir = self.blobId2FileName(blobId)

	// Ensure we have the full directory path in place.
	err = os.MkdirAll(parent_dir, 0700)
	if err != nil {
		return err
	}

	// Now, write the actual data.
	outfile, err = os.Create(file_prefix + ".data")
	if err != nil {
		return err
	}
	aescipher, err = aes.NewCipher(bh.BlockKey)
	if err != nil {
		return err
	}
	outstream = cipher.StreamWriter{
		S: cipher.NewCTR(aescipher, bh.Iv),
		W: io.MultiWriter(outfile, cksum, &counter),
	}
	_, err = io.Copy(outstream, input)
	if err != nil {
		return err
	}
	err = outfile.Close()
	if err != nil {
		return err
	}

	// Fill in the last bits of the blob header.
	bh.Checksum = cksum.Sum(bh.Checksum)
	bh.Size = new(uint64)
	*bh.Size = counter.BytesWritten()
	buf, err = proto.Marshal(&bh)
	if err != nil {
		return err
	}
	if !self.insecure {
		// Encrypt the AES key and IV with the RSA key.
		buf, err = rsa.DecryptPKCS1v15(rand.Reader, self.priv, buf)
		if err != nil {
			return err
		}
	}

	// Write out the crypto head with the IV and the blob key.
	outfile, err = os.Create(file_prefix + ".crypthead")
	if err != nil {
		return err
	}
	n, err = outfile.Write(buf)
	if err != nil {
		return err
	}
	if n < len(buf) {
		return errors.New("Short write to file")
	}
	return outstream.Close()
}

// Extract the blubber block head for the given blob ID and return it.
func (self *blubberStore) extractBlockHead(blobId []byte) (
	bh *BlubberBlockHeader, err error) {
	var file_prefix string
	var data []byte

	file_prefix, _ = self.blobId2FileName(blobId)
	data, err = ioutil.ReadFile(file_prefix + ".crypthead")

	if !self.insecure {
		// Decrypt the AES key and IV with the RSA key.
		data, err = rsa.DecryptPKCS1v15(rand.Reader, self.priv, data)
		if err != nil {
			return
		}
	}

	bh = new(BlubberBlockHeader)
	err = proto.Unmarshal(data, bh)
	return
}

// Read the blob with the specified blob ID and return it to the caller.
// The contents will be sent as a regular HTTP response.
func (self *blubberStore) RetrieveBlob(blobId []byte, rw io.Writer) error {
	var file_prefix string
	var infile *os.File
	var bh *BlubberBlockHeader
	var instream cipher.StreamReader
	var aescipher cipher.Block
	var err error

	// Get the metadata.
	bh, err = self.extractBlockHead(blobId)
	if err != nil {
		return err
	}

	file_prefix, _ = self.blobId2FileName(blobId)
	infile, err = os.Open(file_prefix + ".data")
	if err != nil {
		return err
	}

	aescipher, err = aes.NewCipher(bh.BlockKey)
	if err != nil {
		infile.Close()
		return err
	}
	instream = cipher.StreamReader{
		S: cipher.NewCTR(aescipher, bh.Iv),
		R: infile,
	}

	_, err = io.Copy(rw, instream)
	if err != nil {
		infile.Close()
		return err
	}

	return infile.Close()
}

// Delete the blob with the given blob ID.
func (self *blubberStore) DeleteBlob(blobId []byte) error {
	return errors.New("Not yet implemented")
}

// Get some details about the specified blob.
func (self *blubberStore) StatBlob(blobId []byte) (
	ret BlubberStat, err error) {
	var bh *BlubberBlockHeader

	bh, err = self.extractBlockHead(blobId)
	if err != nil {
		return
	}

	ret.BlockId = make([]byte, len(blobId))
	copy(ret.BlockId, blobId)

	ret.Checksum = make([]byte, len(bh.Checksum))
	copy(ret.Checksum, bh.Checksum)

	ret.Size = bh.Size
	return
}
<|endoftext|>


<commit_before>package targets

import (
	"github.com/go-kit/log"
	"github.com/go-kit/log/level"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"

	"github.com/grafana/loki/clients/pkg/promtail/api"
	"github.com/grafana/loki/clients/pkg/promtail/positions"
	"github.com/grafana/loki/clients/pkg/promtail/scrapeconfig"
	"github.com/grafana/loki/clients/pkg/promtail/targets/file"
	"github.com/grafana/loki/clients/pkg/promtail/targets/gcplog"
	"github.com/grafana/loki/clients/pkg/promtail/targets/journal"
	"github.com/grafana/loki/clients/pkg/promtail/targets/lokipush"
	"github.com/grafana/loki/clients/pkg/promtail/targets/stdin"
	"github.com/grafana/loki/clients/pkg/promtail/targets/syslog"
	"github.com/grafana/loki/clients/pkg/promtail/targets/target"
	"github.com/grafana/loki/clients/pkg/promtail/targets/windows"
)

const (
	FileScrapeConfigs    = "fileScrapeConfigs"
	JournalScrapeConfigs = "journalScrapeConfigs"
	SyslogScrapeConfigs  = "syslogScrapeConfigs"
	GcplogScrapeConfigs  = "gcplogScrapeConfigs"
	PushScrapeConfigs    = "pushScrapeConfigs"
	WindowsEventsConfigs = "windowsEventsConfigs"
)

type targetManager interface {
	Ready() bool
	Stop()
	ActiveTargets() map[string][]target.Target
	AllTargets() map[string][]target.Target
}

// TargetManagers manages a list of target managers.
type TargetManagers struct {
	targetManagers []targetManager
	positions      positions.Positions
}

// NewTargetManagers makes a new TargetManagers
func NewTargetManagers(
	app stdin.Shutdownable,
	reg prometheus.Registerer,
	logger log.Logger,
	positionsConfig positions.Config,
	client api.EntryHandler,
	scrapeConfigs []scrapeconfig.Config,
	targetConfig *file.Config,
) (*TargetManagers, error) {
	var targetManagers []targetManager
	targetScrapeConfigs := make(map[string][]scrapeconfig.Config, 4)

	if targetConfig.Stdin {
		level.Debug(logger).Log("msg", "configured to read from stdin")
		stdin, err := stdin.NewStdinTargetManager(reg, logger, app, client, scrapeConfigs)
		if err != nil {
			return nil, err
		}
		targetManagers = append(targetManagers, stdin)
		return &TargetManagers{targetManagers: targetManagers}, nil
	}

	for _, cfg := range scrapeConfigs {
		switch {
		case cfg.HasServiceDiscoveryConfig():
			targetScrapeConfigs[FileScrapeConfigs] = append(targetScrapeConfigs[FileScrapeConfigs], cfg)
		case cfg.JournalConfig != nil:
			targetScrapeConfigs[JournalScrapeConfigs] = append(targetScrapeConfigs[JournalScrapeConfigs], cfg)
		case cfg.SyslogConfig != nil:
			targetScrapeConfigs[SyslogScrapeConfigs] = append(targetScrapeConfigs[SyslogScrapeConfigs], cfg)
		case cfg.GcplogConfig != nil:
			targetScrapeConfigs[GcplogScrapeConfigs] = append(targetScrapeConfigs[GcplogScrapeConfigs], cfg)
		case cfg.PushConfig != nil:
			targetScrapeConfigs[PushScrapeConfigs] = append(targetScrapeConfigs[PushScrapeConfigs], cfg)
		case cfg.WindowsConfig != nil:
			targetScrapeConfigs[WindowsEventsConfigs] = append(targetScrapeConfigs[WindowsEventsConfigs], cfg)

		default:
			return nil, errors.New("unknown scrape config")
		}
	}

	var positionFile positions.Positions

	// position file is a singleton, we use a function to keep it so.
	getPositionFile := func() (positions.Positions, error) {
		if positionFile == nil {
			var err error
			positionFile, err = positions.New(logger, positionsConfig)
			if err != nil {
				return nil, err
			}
		}
		return positionFile, nil
	}

	var (
		fileMetrics   *file.Metrics
		syslogMetrics *syslog.Metrics
		gcplogMetrics *gcplog.Metrics
	)
	if len(targetScrapeConfigs[FileScrapeConfigs]) > 0 {
		fileMetrics = file.NewMetrics(reg)
	}
	if len(targetScrapeConfigs[SyslogScrapeConfigs]) > 0 {
		syslogMetrics = syslog.NewMetrics(reg)
	}
	if len(targetScrapeConfigs[GcplogScrapeConfigs]) > 0 {
		gcplogMetrics = gcplog.NewMetrics(reg)
	}

	for target, scrapeConfigs := range targetScrapeConfigs {
		switch target {
		case FileScrapeConfigs:
			pos, err := getPositionFile()
			if err != nil {
				return nil, err
			}
			fileTargetManager, err := file.NewFileTargetManager(
				fileMetrics,
				logger,
				pos,
				client,
				scrapeConfigs,
				targetConfig,
			)
			if err != nil {
				return nil, errors.Wrap(err, "failed to make file target manager")
			}
			targetManagers = append(targetManagers, fileTargetManager)
		case JournalScrapeConfigs:
			pos, err := getPositionFile()
			if err != nil {
				return nil, err
			}
			journalTargetManager, err := journal.NewJournalTargetManager(
				reg,
				logger,
				pos,
				client,
				scrapeConfigs,
			)
			if err != nil {
				return nil, errors.Wrap(err, "failed to make journal target manager")
			}
			targetManagers = append(targetManagers, journalTargetManager)
		case SyslogScrapeConfigs:
			syslogTargetManager, err := syslog.NewSyslogTargetManager(
				syslogMetrics,
				logger,
				client,
				scrapeConfigs,
			)
			if err != nil {
				return nil, errors.Wrap(err, "failed to make syslog target manager")
			}
			targetManagers = append(targetManagers, syslogTargetManager)
		case GcplogScrapeConfigs:
			pubsubTargetManager, err := gcplog.NewGcplogTargetManager(
				gcplogMetrics,
				logger,
				client,
				scrapeConfigs,
			)
			if err != nil {
				return nil, errors.Wrap(err, "failed to make syslog target manager")
			}
			targetManagers = append(targetManagers, pubsubTargetManager)
		case PushScrapeConfigs:
			pushTargetManager, err := lokipush.NewPushTargetManager(
				reg,
				logger,
				client,
				scrapeConfigs,
			)
			if err != nil {
				return nil, errors.Wrap(err, "failed to make Loki Push API target manager")
			}
			targetManagers = append(targetManagers, pushTargetManager)
		case WindowsEventsConfigs:
			windowsTargetManager, err := windows.NewTargetManager(reg, logger, client, scrapeConfigs)
			if err != nil {
				return nil, errors.Wrap(err, "failed to make windows target manager")
			}
			targetManagers = append(targetManagers, windowsTargetManager)
		default:
			return nil, errors.New("unknown scrape config")
		}
	}

	return &TargetManagers{
		targetManagers: targetManagers,
		positions:      positionFile,
	}, nil
}

// ActiveTargets returns active targets per jobs
func (tm *TargetManagers) ActiveTargets() map[string][]target.Target {
	result := map[string][]target.Target{}
	for _, t := range tm.targetManagers {
		for job, targets := range t.ActiveTargets() {
			result[job] = append(result[job], targets...)
		}
	}
	return result
}

// AllTargets returns all targets per jobs
func (tm *TargetManagers) AllTargets() map[string][]target.Target {
	result := map[string][]target.Target{}
	for _, t := range tm.targetManagers {
		for job, targets := range t.AllTargets() {
			result[job] = append(result[job], targets...)
		}
	}
	return result
}

// Ready if there's at least one ready target manager.
func (tm *TargetManagers) Ready() bool {
	for _, t := range tm.targetManagers {
		if t.Ready() {
			return true
		}
	}
	return false
}

// Stop the TargetManagers.
func (tm *TargetManagers) Stop() {
	for _, t := range tm.targetManagers {
		t.Stop()
	}
	if tm.positions != nil {
		tm.positions.Stop()
	}
}
<commit_msg>Clarify error message (#4524)<commit_after>package targets

import (
	"fmt"

	"github.com/go-kit/log"
	"github.com/go-kit/log/level"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"

	"github.com/grafana/loki/clients/pkg/promtail/api"
	"github.com/grafana/loki/clients/pkg/promtail/positions"
	"github.com/grafana/loki/clients/pkg/promtail/scrapeconfig"
	"github.com/grafana/loki/clients/pkg/promtail/targets/file"
	"github.com/grafana/loki/clients/pkg/promtail/targets/gcplog"
	"github.com/grafana/loki/clients/pkg/promtail/targets/journal"
	"github.com/grafana/loki/clients/pkg/promtail/targets/lokipush"
	"github.com/grafana/loki/clients/pkg/promtail/targets/stdin"
	"github.com/grafana/loki/clients/pkg/promtail/targets/syslog"
	"github.com/grafana/loki/clients/pkg/promtail/targets/target"
	"github.com/grafana/loki/clients/pkg/promtail/targets/windows"
)

const (
	FileScrapeConfigs    = "fileScrapeConfigs"
	JournalScrapeConfigs = "journalScrapeConfigs"
	SyslogScrapeConfigs  = "syslogScrapeConfigs"
	GcplogScrapeConfigs  = "gcplogScrapeConfigs"
	PushScrapeConfigs    = "pushScrapeConfigs"
	WindowsEventsConfigs = "windowsEventsConfigs"
)

type targetManager interface {
	Ready() bool
	Stop()
	ActiveTargets() map[string][]target.Target
	AllTargets() map[string][]target.Target
}

// TargetManagers manages a list of target managers.
type TargetManagers struct {
	targetManagers []targetManager
	positions      positions.Positions
}

// NewTargetManagers makes a new TargetManagers
func NewTargetManagers(
	app stdin.Shutdownable,
	reg prometheus.Registerer,
	logger log.Logger,
	positionsConfig positions.Config,
	client api.EntryHandler,
	scrapeConfigs []scrapeconfig.Config,
	targetConfig *file.Config,
) (*TargetManagers, error) {
	var targetManagers []targetManager
	targetScrapeConfigs := make(map[string][]scrapeconfig.Config, 4)

	if targetConfig.Stdin {
		level.Debug(logger).Log("msg", "configured to read from stdin")
		stdin, err := stdin.NewStdinTargetManager(reg, logger, app, client, scrapeConfigs)
		if err != nil {
			return nil, err
		}
		targetManagers = append(targetManagers, stdin)
		return &TargetManagers{targetManagers: targetManagers}, nil
	}

	for _, cfg := range scrapeConfigs {
		switch {
		case cfg.HasServiceDiscoveryConfig():
			targetScrapeConfigs[FileScrapeConfigs] = append(targetScrapeConfigs[FileScrapeConfigs], cfg)
		case cfg.JournalConfig != nil:
			targetScrapeConfigs[JournalScrapeConfigs] = append(targetScrapeConfigs[JournalScrapeConfigs], cfg)
		case cfg.SyslogConfig != nil:
			targetScrapeConfigs[SyslogScrapeConfigs] = append(targetScrapeConfigs[SyslogScrapeConfigs], cfg)
		case cfg.GcplogConfig != nil:
			targetScrapeConfigs[GcplogScrapeConfigs] = append(targetScrapeConfigs[GcplogScrapeConfigs], cfg)
		case cfg.PushConfig != nil:
			targetScrapeConfigs[PushScrapeConfigs] = append(targetScrapeConfigs[PushScrapeConfigs], cfg)
		case cfg.WindowsConfig != nil:
			targetScrapeConfigs[WindowsEventsConfigs] = append(targetScrapeConfigs[WindowsEventsConfigs], cfg)

		default:
			return nil, fmt.Errorf("no valid target scrape config defined for %q", cfg.JobName)
		}
	}

	var positionFile positions.Positions

	// position file is a singleton, we use a function to keep it so.
	getPositionFile := func() (positions.Positions, error) {
		if positionFile == nil {
			var err error
			positionFile, err = positions.New(logger, positionsConfig)
			if err != nil {
				return nil, err
			}
		}
		return positionFile, nil
	}

	var (
		fileMetrics   *file.Metrics
		syslogMetrics *syslog.Metrics
		gcplogMetrics *gcplog.Metrics
	)
	if len(targetScrapeConfigs[FileScrapeConfigs]) > 0 {
		fileMetrics = file.NewMetrics(reg)
	}
	if len(targetScrapeConfigs[SyslogScrapeConfigs]) > 0 {
		syslogMetrics = syslog.NewMetrics(reg)
	}
	if len(targetScrapeConfigs[GcplogScrapeConfigs]) > 0 {
		gcplogMetrics = gcplog.NewMetrics(reg)
	}

	for target, scrapeConfigs := range targetScrapeConfigs {
		switch target {
		case FileScrapeConfigs:
			pos, err := getPositionFile()
			if err != nil {
				return nil, err
			}
			fileTargetManager, err := file.NewFileTargetManager(
				fileMetrics,
				logger,
				pos,
				client,
				scrapeConfigs,
				targetConfig,
			)
			if err != nil {
				return nil, errors.Wrap(err, "failed to make file target manager")
			}
			targetManagers = append(targetManagers, fileTargetManager)
		case JournalScrapeConfigs:
			pos, err := getPositionFile()
			if err != nil {
				return nil, err
			}
			journalTargetManager, err := journal.NewJournalTargetManager(
				reg,
				logger,
				pos,
				client,
				scrapeConfigs,
			)
			if err != nil {
				return nil, errors.Wrap(err, "failed to make journal target manager")
			}
			targetManagers = append(targetManagers, journalTargetManager)
		case SyslogScrapeConfigs:
			syslogTargetManager, err := syslog.NewSyslogTargetManager(
				syslogMetrics,
				logger,
				client,
				scrapeConfigs,
			)
			if err != nil {
				return nil, errors.Wrap(err, "failed to make syslog target manager")
			}
			targetManagers = append(targetManagers, syslogTargetManager)
		case GcplogScrapeConfigs:
			pubsubTargetManager, err := gcplog.NewGcplogTargetManager(
				gcplogMetrics,
				logger,
				client,
				scrapeConfigs,
			)
			if err != nil {
				return nil, errors.Wrap(err, "failed to make syslog target manager")
			}
			targetManagers = append(targetManagers, pubsubTargetManager)
		case PushScrapeConfigs:
			pushTargetManager, err := lokipush.NewPushTargetManager(
				reg,
				logger,
				client,
				scrapeConfigs,
			)
			if err != nil {
				return nil, errors.Wrap(err, "failed to make Loki Push API target manager")
			}
			targetManagers = append(targetManagers, pushTargetManager)
		case WindowsEventsConfigs:
			windowsTargetManager, err := windows.NewTargetManager(reg, logger, client, scrapeConfigs)
			if err != nil {
				return nil, errors.Wrap(err, "failed to make windows target manager")
			}
			targetManagers = append(targetManagers, windowsTargetManager)
		default:
			return nil, errors.New("unknown scrape config")
		}
	}

	return &TargetManagers{
		targetManagers: targetManagers,
		positions:      positionFile,
	}, nil
}

// ActiveTargets returns active targets per jobs
func (tm *TargetManagers) ActiveTargets() map[string][]target.Target {
	result := map[string][]target.Target{}
	for _, t := range tm.targetManagers {
		for job, targets := range t.ActiveTargets() {
			result[job] = append(result[job], targets...)
		}
	}
	return result
}

// AllTargets returns all targets per jobs
func (tm *TargetManagers) AllTargets() map[string][]target.Target {
	result := map[string][]target.Target{}
	for _, t := range tm.targetManagers {
		for job, targets := range t.AllTargets() {
			result[job] = append(result[job], targets...)
		}
	}
	return result
}

// Ready if there's at least one ready target manager.
func (tm *TargetManagers) Ready() bool {
	for _, t := range tm.targetManagers {
		if t.Ready() {
			return true
		}
	}
	return false
}

// Stop the TargetManagers.
func (tm *TargetManagers) Stop() {
	for _, t := range tm.targetManagers {
		t.Stop()
	}
	if tm.positions != nil {
		tm.positions.Stop()
	}
}
<|endoftext|>


<commit_before>// Copyright © 2017 The Kubicorn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package resources

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/digitalocean/godo"
	"github.com/kris-nova/klone/pkg/local"
	"github.com/kris-nova/kubicorn/apis/cluster"
	"github.com/kris-nova/kubicorn/bootstrap"
	"github.com/kris-nova/kubicorn/cloud"
	"github.com/kris-nova/kubicorn/cutil/compare"
	"github.com/kris-nova/kubicorn/cutil/logger"
	"github.com/kris-nova/kubicorn/cutil/scp"
	"github.com/kris-nova/kubicorn/cutil/script"
)

type Droplet struct {
	Shared
	Region           string
	Size             string
	Image            string
	Count            int
	SSHFingerprint   string
	BootstrapScripts []string
	ServerPool       *cluster.ServerPool
}

const (
	MasterIPAttempts               = 100
	MasterIPSleepSecondsPerAttempt = 5
	DeleteAttempts = 25
	DeleteSleepSecondsPerAttempt = 3
)

func (r *Droplet) Actual(known *cluster.Cluster) (cloud.Resource, error) {
	logger.Debug("droplet.Actual")
	if r.CachedActual != nil {
		logger.Debug("Using cached droplet [actual]")
		return r.CachedActual, nil
	}
	actual := &Droplet{
		Shared: Shared{
			Name:    r.Name,
			CloudID: r.ServerPool.Identifier,
		},
	}

	droplets, _, err := Sdk.Client.Droplets.ListByTag(context.TODO(), r.Name, &godo.ListOptions{})
	if err != nil {
		return nil, err
	}
	ld := len(droplets)
	if ld > 0 {
		actual.Count = len(droplets)

		// Todo (@kris-nova) once we start to test these implementations we really need to work on the droplet logic. Right now we just pick the first one..
		droplet := droplets[0]
		id := strconv.Itoa(droplet.ID)
		actual.Name = droplet.Name
		actual.CloudID = id
		actual.Size = droplet.Size.Slug
		actual.Image = droplet.Image.Slug
		actual.Region = droplet.Region.Slug
	}
	actual.BootstrapScripts = r.ServerPool.BootstrapScripts
	actual.SSHFingerprint = known.SSH.PublicKeyFingerprint
	actual.Name = r.Name
	r.CachedActual = actual
	return actual, nil
}

func (r *Droplet) Expected(known *cluster.Cluster) (cloud.Resource, error) {
	logger.Debug("droplet.Expected")
	if r.CachedExpected != nil {
		logger.Debug("Using droplet subnet [expected]")
		return r.CachedExpected, nil
	}
	expected := &Droplet{
		Shared: Shared{
			Name:    r.Name,
			CloudID: r.ServerPool.Identifier,
		},
		Size:             r.ServerPool.Size,
		Region:           known.Location,
		Image:            r.ServerPool.Image,
		Count:            r.ServerPool.MaxCount,
		SSHFingerprint:   known.SSH.PublicKeyFingerprint,
		BootstrapScripts: r.ServerPool.BootstrapScripts,
	}
	r.CachedExpected = expected
	return expected, nil
}

func (r *Droplet) Apply(actual, expected cloud.Resource, applyCluster *cluster.Cluster) (cloud.Resource, error) {
	logger.Debug("droplet.Apply")
	applyResource := expected.(*Droplet)
	isEqual, err := compare.IsEqual(actual.(*Droplet), expected.(*Droplet))
	if err != nil {
		return nil, err
	}
	if isEqual {
		return applyResource, nil
	}

	userData, err := script.BuildBootstrapScript(r.ServerPool.BootstrapScripts)
	if err != nil {
		return nil, err
	}

	//masterIpPrivate := ""
	masterIPPublic := ""
	if r.ServerPool.Type == cluster.ServerPoolTypeNode {
		found := false
		for i := 0; i < MasterIPAttempts; i++ {
			masterTag := ""
			for _, serverPool := range applyCluster.ServerPools {
				if serverPool.Type == cluster.ServerPoolTypeMaster {
					masterTag = serverPool.Name
				}
			}
			if masterTag == "" {
				return nil, fmt.Errorf("Unable to find master tag for master IP")
			}
			droplets, _, err := Sdk.Client.Droplets.ListByTag(context.TODO(), masterTag, &godo.ListOptions{})
			if err != nil {
				logger.Debug("Hanging for master IP.. (%v)", err)
				time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
				continue
			}
			ld := len(droplets)
			if ld == 0 {
				logger.Debug("Hanging for master IP..")
				time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
				continue
			}
			if ld > 1 {
				return nil, fmt.Errorf("Found [%d] droplets for tag [%s]", ld, masterTag)
			}
			droplet := droplets[0]
			//masterIpPrivate, err = droplet.PrivateIPv4()
			//if err != nil {
			//	return nil, fmt.Errorf("Unable to detect private IP: %v", err)
			//}
			masterIPPublic, err = droplet.PublicIPv4()
			if err != nil {
				return nil, fmt.Errorf("Unable to detect public IP: %v", err)
			}

			logger.Info("Setting up VPN on Droplets... This could a little bit longer...")
			pubPath := local.Expand(applyCluster.SSH.PublicKeyPath)
			privPath := strings.Replace(pubPath, ".pub", "", 1)
			scp := scp.NewSecureCopier(applyCluster.SSH.User, masterIPPublic, "22", privPath)
			masterVpnIP, err := scp.ReadBytes("/tmp/.ip")
			if err != nil {
				logger.Debug("Hanging for VPN IP.. /tmp/.ip (%v)", err)
				time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
				continue
			}
			masterVpnIPStr := strings.Replace(string(masterVpnIP), "\n", "", -1)
			openvpnConfig, err := scp.ReadBytes("/tmp/clients.conf")
			if err != nil {
				logger.Debug("Hanging for VPN config.. /tmp/clients.ovpn (%v)", err)
				time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
				continue
			}
			openvpnConfigEscaped := strings.Replace(string(openvpnConfig), "\n", "\\n", -1)
			found = true
			applyCluster.Values.ItemMap["INJECTEDMASTER"] = fmt.Sprintf("%s:%s", masterVpnIPStr, applyCluster.KubernetesAPI.Port)
			applyCluster.Values.ItemMap["INJECTEDCONF"] = openvpnConfigEscaped
			break
		}
		if !found {
			return nil, fmt.Errorf("Unable to find Master IP after defined wait")
		}
	}

	applyCluster.Values.ItemMap["INJECTEDPORT"] = applyCluster.KubernetesAPI.Port
	userData, err = bootstrap.Inject(userData, applyCluster.Values.ItemMap)
	if err != nil {
		return nil, err
	}

	sshID, err := strconv.Atoi(applyCluster.SSH.Identifier)
	if err != nil {
		return nil, err
	}

	var droplet *godo.Droplet
	for j := 0; j < expected.(*Droplet).Count; j++ {
		createRequest := &godo.DropletCreateRequest{
			Name:   fmt.Sprintf("%s-%d", expected.(*Droplet).Name, j),
			Region: expected.(*Droplet).Region,
			Size:   expected.(*Droplet).Size,
			Image: godo.DropletCreateImage{
				Slug: expected.(*Droplet).Image,
			},
			Tags:              []string{expected.(*Droplet).Name},
			PrivateNetworking: true,
			SSHKeys: []godo.DropletCreateSSHKey{
				{
					ID:          sshID,
					Fingerprint: expected.(*Droplet).SSHFingerprint,
				},
			},
			UserData: string(userData),
		}
		droplet, _, err = Sdk.Client.Droplets.Create(context.TODO(), createRequest)
		if err != nil {
			return nil, err
		}
		logger.Info("Created Droplet [%d]", droplet.ID)
	}

	newResource := &Droplet{
		Shared: Shared{
			Name: r.ServerPool.Name,
			//CloudID: id,
		},
		Image:            droplet.Image.Slug,
		Size:             droplet.Size.Slug,
		Region:           droplet.Region.Slug,
		Count:            expected.(*Droplet).Count,
		BootstrapScripts: expected.(*Droplet).BootstrapScripts,
	}
	applyCluster.KubernetesAPI.Endpoint = masterIPPublic
	return newResource, nil
}
func (r *Droplet) Delete(actual cloud.Resource, known *cluster.Cluster) (cloud.Resource, error) {
	logger.Debug("droplet.Delete")
	deleteResource := actual.(*Droplet)
	if deleteResource.Name == "" {
		return nil, fmt.Errorf("Unable to delete droplet resource without Name [%s]", deleteResource.Name)
	}

	droplets, _, err := Sdk.Client.Droplets.ListByTag(context.TODO(), r.Name, &godo.ListOptions{})
	if err != nil {
		return nil, err
	}
	if len(droplets) != actual.(*Droplet).Count {
		for i := 0; i < DeleteAttempts; i++ {
			logger.Info("Droplet count mis-match, trying query again")
			time.Sleep(5 * time.Second)
			droplets, _, err = Sdk.Client.Droplets.ListByTag(context.TODO(), r.Name, &godo.ListOptions{})
			if err != nil {
				return nil, err
			}
			if len(droplets) == actual.(*Droplet).Count {
				break
			}
		}
	}

	for _, droplet := range droplets {
		for i := 0; i < DeleteAttempts; i++ {
			if droplet.Status == "new" {
				logger.Debug("Waiting for Droplet creation to finish [%d]...", droplet.ID)
				time.Sleep(DeleteSleepSecondsPerAttempt * time.Second)
			} else {
				break
			}
		}
		_, err = Sdk.Client.Droplets.Delete(context.TODO(), droplet.ID)
		if err != nil {
			return nil, err
		}
		logger.Info("Deleted Droplet [%d]", droplet.ID)
	}

	// Kubernetes API
	known.KubernetesAPI.Endpoint = ""

	newResource := &Droplet{}
	newResource.Name = actual.(*Droplet).Name
	newResource.Tags = actual.(*Droplet).Tags
	newResource.Image = actual.(*Droplet).Image
	newResource.Size = actual.(*Droplet).Size
	newResource.Count = actual.(*Droplet).Count
	newResource.Region = actual.(*Droplet).Region
	newResource.BootstrapScripts = actual.(*Droplet).BootstrapScripts
	return newResource, nil
}

func (r *Droplet) Render(renderResource cloud.Resource, renderCluster *cluster.Cluster) (*cluster.Cluster, error) {
	logger.Debug("droplet.Render")

	serverPool := &cluster.ServerPool{}
	serverPool.Type = r.ServerPool.Type
	serverPool.Image = renderResource.(*Droplet).Image
	serverPool.Size = renderResource.(*Droplet).Size
	serverPool.Name = renderResource.(*Droplet).Name
	serverPool.MaxCount = renderResource.(*Droplet).Count
	serverPool.BootstrapScripts = renderResource.(*Droplet).BootstrapScripts
	found := false
	for i := 0; i < len(renderCluster.ServerPools); i++ {
		if renderCluster.ServerPools[i].Name == renderResource.(*Droplet).Name {
			renderCluster.ServerPools[i].Image = renderResource.(*Droplet).Image
			renderCluster.ServerPools[i].Size = renderResource.(*Droplet).Size
			renderCluster.ServerPools[i].MaxCount = renderResource.(*Droplet).Count
			renderCluster.ServerPools[i].BootstrapScripts = renderResource.(*Droplet).BootstrapScripts
			found = true
		}
	}
	if !found {
		renderCluster.ServerPools = append(renderCluster.ServerPools, serverPool)
	}
	renderCluster.Location = renderResource.(*Droplet).Region
	return renderCluster, nil
}

func (r *Droplet) Tag(tags map[string]string) error {
	return nil
}
<commit_msg>do: gofmt code<commit_after>// Copyright © 2017 The Kubicorn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package resources

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/digitalocean/godo"
	"github.com/kris-nova/klone/pkg/local"
	"github.com/kris-nova/kubicorn/apis/cluster"
	"github.com/kris-nova/kubicorn/bootstrap"
	"github.com/kris-nova/kubicorn/cloud"
	"github.com/kris-nova/kubicorn/cutil/compare"
	"github.com/kris-nova/kubicorn/cutil/logger"
	"github.com/kris-nova/kubicorn/cutil/scp"
	"github.com/kris-nova/kubicorn/cutil/script"
)

type Droplet struct {
	Shared
	Region           string
	Size             string
	Image            string
	Count            int
	SSHFingerprint   string
	BootstrapScripts []string
	ServerPool       *cluster.ServerPool
}

const (
	MasterIPAttempts               = 100
	MasterIPSleepSecondsPerAttempt = 5
	DeleteAttempts                 = 25
	DeleteSleepSecondsPerAttempt   = 3
)

func (r *Droplet) Actual(known *cluster.Cluster) (cloud.Resource, error) {
	logger.Debug("droplet.Actual")
	if r.CachedActual != nil {
		logger.Debug("Using cached droplet [actual]")
		return r.CachedActual, nil
	}
	actual := &Droplet{
		Shared: Shared{
			Name:    r.Name,
			CloudID: r.ServerPool.Identifier,
		},
	}

	droplets, _, err := Sdk.Client.Droplets.ListByTag(context.TODO(), r.Name, &godo.ListOptions{})
	if err != nil {
		return nil, err
	}
	ld := len(droplets)
	if ld > 0 {
		actual.Count = len(droplets)

		// Todo (@kris-nova) once we start to test these implementations we really need to work on the droplet logic. Right now we just pick the first one..
		droplet := droplets[0]
		id := strconv.Itoa(droplet.ID)
		actual.Name = droplet.Name
		actual.CloudID = id
		actual.Size = droplet.Size.Slug
		actual.Image = droplet.Image.Slug
		actual.Region = droplet.Region.Slug
	}
	actual.BootstrapScripts = r.ServerPool.BootstrapScripts
	actual.SSHFingerprint = known.SSH.PublicKeyFingerprint
	actual.Name = r.Name
	r.CachedActual = actual
	return actual, nil
}

func (r *Droplet) Expected(known *cluster.Cluster) (cloud.Resource, error) {
	logger.Debug("droplet.Expected")
	if r.CachedExpected != nil {
		logger.Debug("Using droplet subnet [expected]")
		return r.CachedExpected, nil
	}
	expected := &Droplet{
		Shared: Shared{
			Name:    r.Name,
			CloudID: r.ServerPool.Identifier,
		},
		Size:             r.ServerPool.Size,
		Region:           known.Location,
		Image:            r.ServerPool.Image,
		Count:            r.ServerPool.MaxCount,
		SSHFingerprint:   known.SSH.PublicKeyFingerprint,
		BootstrapScripts: r.ServerPool.BootstrapScripts,
	}
	r.CachedExpected = expected
	return expected, nil
}

func (r *Droplet) Apply(actual, expected cloud.Resource, applyCluster *cluster.Cluster) (cloud.Resource, error) {
	logger.Debug("droplet.Apply")
	applyResource := expected.(*Droplet)
	isEqual, err := compare.IsEqual(actual.(*Droplet), expected.(*Droplet))
	if err != nil {
		return nil, err
	}
	if isEqual {
		return applyResource, nil
	}

	userData, err := script.BuildBootstrapScript(r.ServerPool.BootstrapScripts)
	if err != nil {
		return nil, err
	}

	//masterIpPrivate := ""
	masterIPPublic := ""
	if r.ServerPool.Type == cluster.ServerPoolTypeNode {
		found := false
		for i := 0; i < MasterIPAttempts; i++ {
			masterTag := ""
			for _, serverPool := range applyCluster.ServerPools {
				if serverPool.Type == cluster.ServerPoolTypeMaster {
					masterTag = serverPool.Name
				}
			}
			if masterTag == "" {
				return nil, fmt.Errorf("Unable to find master tag for master IP")
			}
			droplets, _, err := Sdk.Client.Droplets.ListByTag(context.TODO(), masterTag, &godo.ListOptions{})
			if err != nil {
				logger.Debug("Hanging for master IP.. (%v)", err)
				time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
				continue
			}
			ld := len(droplets)
			if ld == 0 {
				logger.Debug("Hanging for master IP..")
				time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
				continue
			}
			if ld > 1 {
				return nil, fmt.Errorf("Found [%d] droplets for tag [%s]", ld, masterTag)
			}
			droplet := droplets[0]
			//masterIpPrivate, err = droplet.PrivateIPv4()
			//if err != nil {
			//	return nil, fmt.Errorf("Unable to detect private IP: %v", err)
			//}
			masterIPPublic, err = droplet.PublicIPv4()
			if err != nil {
				return nil, fmt.Errorf("Unable to detect public IP: %v", err)
			}

			logger.Info("Setting up VPN on Droplets... This could a little bit longer...")
			pubPath := local.Expand(applyCluster.SSH.PublicKeyPath)
			privPath := strings.Replace(pubPath, ".pub", "", 1)
			scp := scp.NewSecureCopier(applyCluster.SSH.User, masterIPPublic, "22", privPath)
			masterVpnIP, err := scp.ReadBytes("/tmp/.ip")
			if err != nil {
				logger.Debug("Hanging for VPN IP.. /tmp/.ip (%v)", err)
				time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
				continue
			}
			masterVpnIPStr := strings.Replace(string(masterVpnIP), "\n", "", -1)
			openvpnConfig, err := scp.ReadBytes("/tmp/clients.conf")
			if err != nil {
				logger.Debug("Hanging for VPN config.. /tmp/clients.ovpn (%v)", err)
				time.Sleep(time.Duration(MasterIPSleepSecondsPerAttempt) * time.Second)
				continue
			}
			openvpnConfigEscaped := strings.Replace(string(openvpnConfig), "\n", "\\n", -1)
			found = true
			applyCluster.Values.ItemMap["INJECTEDMASTER"] = fmt.Sprintf("%s:%s", masterVpnIPStr, applyCluster.KubernetesAPI.Port)
			applyCluster.Values.ItemMap["INJECTEDCONF"] = openvpnConfigEscaped
			break
		}
		if !found {
			return nil, fmt.Errorf("Unable to find Master IP after defined wait")
		}
	}

	applyCluster.Values.ItemMap["INJECTEDPORT"] = applyCluster.KubernetesAPI.Port
	userData, err = bootstrap.Inject(userData, applyCluster.Values.ItemMap)
	if err != nil {
		return nil, err
	}

	sshID, err := strconv.Atoi(applyCluster.SSH.Identifier)
	if err != nil {
		return nil, err
	}

	var droplet *godo.Droplet
	for j := 0; j < expected.(*Droplet).Count; j++ {
		createRequest := &godo.DropletCreateRequest{
			Name:   fmt.Sprintf("%s-%d", expected.(*Droplet).Name, j),
			Region: expected.(*Droplet).Region,
			Size:   expected.(*Droplet).Size,
			Image: godo.DropletCreateImage{
				Slug: expected.(*Droplet).Image,
			},
			Tags:              []string{expected.(*Droplet).Name},
			PrivateNetworking: true,
			SSHKeys: []godo.DropletCreateSSHKey{
				{
					ID:          sshID,
					Fingerprint: expected.(*Droplet).SSHFingerprint,
				},
			},
			UserData: string(userData),
		}
		droplet, _, err = Sdk.Client.Droplets.Create(context.TODO(), createRequest)
		if err != nil {
			return nil, err
		}
		logger.Info("Created Droplet [%d]", droplet.ID)
	}

	newResource := &Droplet{
		Shared: Shared{
			Name: r.ServerPool.Name,
			//CloudID: id,
		},
		Image:            droplet.Image.Slug,
		Size:             droplet.Size.Slug,
		Region:           droplet.Region.Slug,
		Count:            expected.(*Droplet).Count,
		BootstrapScripts: expected.(*Droplet).BootstrapScripts,
	}
	applyCluster.KubernetesAPI.Endpoint = masterIPPublic
	return newResource, nil
}
func (r *Droplet) Delete(actual cloud.Resource, known *cluster.Cluster) (cloud.Resource, error) {
	logger.Debug("droplet.Delete")
	deleteResource := actual.(*Droplet)
	if deleteResource.Name == "" {
		return nil, fmt.Errorf("Unable to delete droplet resource without Name [%s]", deleteResource.Name)
	}

	droplets, _, err := Sdk.Client.Droplets.ListByTag(context.TODO(), r.Name, &godo.ListOptions{})
	if err != nil {
		return nil, err
	}
	if len(droplets) != actual.(*Droplet).Count {
		for i := 0; i < DeleteAttempts; i++ {
			logger.Info("Droplet count mis-match, trying query again")
			time.Sleep(5 * time.Second)
			droplets, _, err = Sdk.Client.Droplets.ListByTag(context.TODO(), r.Name, &godo.ListOptions{})
			if err != nil {
				return nil, err
			}
			if len(droplets) == actual.(*Droplet).Count {
				break
			}
		}
	}

	for _, droplet := range droplets {
		for i := 0; i < DeleteAttempts; i++ {
			if droplet.Status == "new" {
				logger.Debug("Waiting for Droplet creation to finish [%d]...", droplet.ID)
				time.Sleep(DeleteSleepSecondsPerAttempt * time.Second)
			} else {
				break
			}
		}
		_, err = Sdk.Client.Droplets.Delete(context.TODO(), droplet.ID)
		if err != nil {
			return nil, err
		}
		logger.Info("Deleted Droplet [%d]", droplet.ID)
	}

	// Kubernetes API
	known.KubernetesAPI.Endpoint = ""

	newResource := &Droplet{}
	newResource.Name = actual.(*Droplet).Name
	newResource.Tags = actual.(*Droplet).Tags
	newResource.Image = actual.(*Droplet).Image
	newResource.Size = actual.(*Droplet).Size
	newResource.Count = actual.(*Droplet).Count
	newResource.Region = actual.(*Droplet).Region
	newResource.BootstrapScripts = actual.(*Droplet).BootstrapScripts
	return newResource, nil
}

func (r *Droplet) Render(renderResource cloud.Resource, renderCluster *cluster.Cluster) (*cluster.Cluster, error) {
	logger.Debug("droplet.Render")

	serverPool := &cluster.ServerPool{}
	serverPool.Type = r.ServerPool.Type
	serverPool.Image = renderResource.(*Droplet).Image
	serverPool.Size = renderResource.(*Droplet).Size
	serverPool.Name = renderResource.(*Droplet).Name
	serverPool.MaxCount = renderResource.(*Droplet).Count
	serverPool.BootstrapScripts = renderResource.(*Droplet).BootstrapScripts
	found := false
	for i := 0; i < len(renderCluster.ServerPools); i++ {
		if renderCluster.ServerPools[i].Name == renderResource.(*Droplet).Name {
			renderCluster.ServerPools[i].Image = renderResource.(*Droplet).Image
			renderCluster.ServerPools[i].Size = renderResource.(*Droplet).Size
			renderCluster.ServerPools[i].MaxCount = renderResource.(*Droplet).Count
			renderCluster.ServerPools[i].BootstrapScripts = renderResource.(*Droplet).BootstrapScripts
			found = true
		}
	}
	if !found {
		renderCluster.ServerPools = append(renderCluster.ServerPools, serverPool)
	}
	renderCluster.Location = renderResource.(*Droplet).Region
	return renderCluster, nil
}

func (r *Droplet) Tag(tags map[string]string) error {
	return nil
}
<|endoftext|>


<commit_before><commit_msg>Skipping that test for now.<commit_after><|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package simulator

import (
	"fmt"
	"math"

	kube_api "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"
	cmd "k8s.io/kubernetes/pkg/kubectl/cmd"
	"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"

	"github.com/golang/glog"
)

// FindNodesToRemove finds nodes that can be removed.
func FindNodesToRemove(candidates []*kube_api.Node, allNodes []*kube_api.Node, pods []*kube_api.Pod,
	client *kube_client.Client, predicateChecker *PredicateChecker, maxCount int,
	fastCheck bool) ([]*kube_api.Node, error) {

	nodeNameToNodeInfo := schedulercache.CreateNodeNameToInfoMap(pods)
	for _, node := range allNodes {
		if nodeInfo, found := nodeNameToNodeInfo[node.Name]; found {
			nodeInfo.SetNode(node)
		}
	}
	result := make([]*kube_api.Node, 0)

	for _, node := range candidates {
		glog.V(2).Infof("Considering %s for removal", node.Name)

		var podsToRemove []*kube_api.Pod
		var err error

		if fastCheck {
			if nodeInfo, found := nodeNameToNodeInfo[node.Name]; found {
				podsToRemove, err = FastGetPodsToMove(nodeInfo, false, kube_api.Codecs.UniversalDecoder())
			}
		} else {
			drainResult, _, _, err := cmd.GetPodsForDeletionOnNodeDrain(client, node.Name,
				kube_api.Codecs.UniversalDecoder(), false, true)

			if err != nil {
				glog.V(2).Infof("Node %s cannot be removed: %v", node.Name, err)
				continue
			}
			podsToRemove = make([]*kube_api.Pod, 0, len(drainResult))
			for i := range drainResult {
				podsToRemove = append(podsToRemove, &drainResult[i])
			}
		}

		findProblems := findPlaceFor(node.Name, podsToRemove, allNodes, nodeNameToNodeInfo, predicateChecker)
		if findProblems == nil {
			result = append(result, node)
			if len(result) >= maxCount {
				break
			}
		}
		glog.V(2).Infof("Node %s is not suitable for removal %v", node.Name, err)
	}
	return result, nil
}

// CalculateReservation calculates reservation of a node.
func CalculateReservation(node *kube_api.Node, nodeInfo *schedulercache.NodeInfo) (float64, error) {
	cpu, err := calculateReservationOfResource(node, nodeInfo, kube_api.ResourceCPU)
	if err != nil {
		return 0, err
	}
	mem, err := calculateReservationOfResource(node, nodeInfo, kube_api.ResourceMemory)
	if err != nil {
		return 0, err
	}
	return math.Max(cpu, mem), nil
}

func calculateReservationOfResource(node *kube_api.Node, nodeInfo *schedulercache.NodeInfo, resourceName kube_api.ResourceName) (float64, error) {
	nodeCapacity, found := node.Status.Capacity[resourceName]
	if !found {
		return 0, fmt.Errorf("Failed to get %v from %s", resourceName, node.Name)
	}
	if nodeCapacity.MilliValue() == 0 {
		return 0, fmt.Errorf("%v is 0 at %s", resourceName, node.Name)
	}
	podsRequest := resource.MustParse("0")
	for _, pod := range nodeInfo.Pods() {
		for _, container := range pod.Spec.Containers {
			if resourceValue, found := container.Resources.Requests[resourceName]; found {
				podsRequest.Add(resourceValue)
			}
		}
	}
	return float64(podsRequest.MilliValue()) / float64(nodeCapacity.MilliValue()), nil
}

// TODO: We don't need to pass list of nodes here as they are already available in nodeInfos.
func findPlaceFor(bannedNode string, pods []*kube_api.Pod, nodes []*kube_api.Node, nodeInfos map[string]*schedulercache.NodeInfo,
	predicateChecker *PredicateChecker) error {

	newNodeInfos := make(map[string]*schedulercache.NodeInfo)

	for _, podptr := range pods {
		newpod := *podptr
		newpod.Spec.NodeName = ""
		pod := &newpod

		foundPlace := false
		glog.V(4).Infof("Looking for place for %s/%s", pod.Namespace, pod.Name)
		podKey := fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)

		// TODO: Sort nodes by reservation
	nodeloop:
		for _, node := range nodes {
			if node.Name == bannedNode {
				continue
			}

			node.Status.Allocatable = node.Status.Capacity
			nodeInfo, found := newNodeInfos[node.Name]
			if !found {
				nodeInfo, found = nodeInfos[node.Name]
			}

			if found {
				err := predicateChecker.CheckPredicates(pod, nodeInfo)
				glog.V(4).Infof("Evaluation %s for %s -> %v", node.Name, podKey, err)
				if err == nil {
					foundPlace = true
					// TODO(mwielgus): Optimize it.
					podsOnNode := nodeInfo.Pods()
					podsOnNode = append(podsOnNode, pod)
					newNodeInfo := schedulercache.NewNodeInfo(podsOnNode...)
					newNodeInfo.SetNode(node)
					newNodeInfos[node.Name] = newNodeInfo
					break nodeloop
				}
			}
		}
		if !foundPlace {
			return fmt.Errorf("failed to find place for %s", podKey)
		}
	}
	return nil
}
<commit_msg>Cluster-autoscaler: scaledown logging fix<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package simulator

import (
	"fmt"
	"math"

	kube_api "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"
	cmd "k8s.io/kubernetes/pkg/kubectl/cmd"
	"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"

	"github.com/golang/glog"
)

// FindNodesToRemove finds nodes that can be removed.
func FindNodesToRemove(candidates []*kube_api.Node, allNodes []*kube_api.Node, pods []*kube_api.Pod,
	client *kube_client.Client, predicateChecker *PredicateChecker, maxCount int,
	fastCheck bool) ([]*kube_api.Node, error) {

	nodeNameToNodeInfo := schedulercache.CreateNodeNameToInfoMap(pods)
	for _, node := range allNodes {
		if nodeInfo, found := nodeNameToNodeInfo[node.Name]; found {
			nodeInfo.SetNode(node)
		}
	}
	result := make([]*kube_api.Node, 0)

candidateloop:
	for _, node := range candidates {
		glog.V(2).Infof("Considering %s for removal", node.Name)

		var podsToRemove []*kube_api.Pod
		var err error

		if fastCheck {
			if nodeInfo, found := nodeNameToNodeInfo[node.Name]; found {
				podsToRemove, err = FastGetPodsToMove(nodeInfo, false, kube_api.Codecs.UniversalDecoder())
			}
		} else {
			drainResult, _, _, err := cmd.GetPodsForDeletionOnNodeDrain(client, node.Name,
				kube_api.Codecs.UniversalDecoder(), false, true)

			if err != nil {
				glog.V(2).Infof("Node %s cannot be removed: %v", node.Name, err)
				continue
			}
			podsToRemove = make([]*kube_api.Pod, 0, len(drainResult))
			for i := range drainResult {
				podsToRemove = append(podsToRemove, &drainResult[i])
			}
		}
		findProblems := findPlaceFor(node.Name, podsToRemove, allNodes, nodeNameToNodeInfo, predicateChecker)
		if findProblems == nil {
			result = append(result, node)
			if len(result) >= maxCount {
				break candidateloop
			}
		} else {
			glog.V(2).Infof("Node %s is not suitable for removal %v", node.Name, err)
		}
	}
	return result, nil
}

// CalculateReservation calculates reservation of a node.
func CalculateReservation(node *kube_api.Node, nodeInfo *schedulercache.NodeInfo) (float64, error) {
	cpu, err := calculateReservationOfResource(node, nodeInfo, kube_api.ResourceCPU)
	if err != nil {
		return 0, err
	}
	mem, err := calculateReservationOfResource(node, nodeInfo, kube_api.ResourceMemory)
	if err != nil {
		return 0, err
	}
	return math.Max(cpu, mem), nil
}

func calculateReservationOfResource(node *kube_api.Node, nodeInfo *schedulercache.NodeInfo, resourceName kube_api.ResourceName) (float64, error) {
	nodeCapacity, found := node.Status.Capacity[resourceName]
	if !found {
		return 0, fmt.Errorf("Failed to get %v from %s", resourceName, node.Name)
	}
	if nodeCapacity.MilliValue() == 0 {
		return 0, fmt.Errorf("%v is 0 at %s", resourceName, node.Name)
	}
	podsRequest := resource.MustParse("0")
	for _, pod := range nodeInfo.Pods() {
		for _, container := range pod.Spec.Containers {
			if resourceValue, found := container.Resources.Requests[resourceName]; found {
				podsRequest.Add(resourceValue)
			}
		}
	}
	return float64(podsRequest.MilliValue()) / float64(nodeCapacity.MilliValue()), nil
}

// TODO: We don't need to pass list of nodes here as they are already available in nodeInfos.
func findPlaceFor(bannedNode string, pods []*kube_api.Pod, nodes []*kube_api.Node, nodeInfos map[string]*schedulercache.NodeInfo,
	predicateChecker *PredicateChecker) error {

	newNodeInfos := make(map[string]*schedulercache.NodeInfo)

	for _, podptr := range pods {
		newpod := *podptr
		newpod.Spec.NodeName = ""
		pod := &newpod

		foundPlace := false
		glog.V(4).Infof("Looking for place for %s/%s", pod.Namespace, pod.Name)
		podKey := fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)

		// TODO: Sort nodes by reservation
	nodeloop:
		for _, node := range nodes {
			if node.Name == bannedNode {
				continue
			}

			node.Status.Allocatable = node.Status.Capacity
			nodeInfo, found := newNodeInfos[node.Name]
			if !found {
				nodeInfo, found = nodeInfos[node.Name]
			}

			if found {
				err := predicateChecker.CheckPredicates(pod, nodeInfo)
				glog.V(4).Infof("Evaluation %s for %s -> %v", node.Name, podKey, err)
				if err == nil {
					foundPlace = true
					// TODO(mwielgus): Optimize it.
					podsOnNode := nodeInfo.Pods()
					podsOnNode = append(podsOnNode, pod)
					newNodeInfo := schedulercache.NewNodeInfo(podsOnNode...)
					newNodeInfo.SetNode(node)
					newNodeInfos[node.Name] = newNodeInfo
					break nodeloop
				}
			}
		}
		if !foundPlace {
			return fmt.Errorf("failed to find place for %s", podKey)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/Microsoft/hcsshim/internal/hcs"
	"github.com/containerd/containerd/runtime/v2/task"
	"github.com/gogo/protobuf/proto"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
)

var deleteCommand = cli.Command{
	Name: "delete",
	Usage: `
This command allows containerd to delete any container resources created, mounted, and/or run by a shim when containerd can no longer communicate over rpc. This happens if a shim is SIGKILL'd with a running container. These resources will need to be cleaned up when containerd loses the connection to a shim. This is also used when containerd boots and reconnects to shims. If a bundle is still on disk but containerd cannot connect to a shim, the delete command is invoked.

The delete command will be executed in the container's bundle as its cwd.
`,
	SkipArgReorder: true,
	Action: func(context *cli.Context) error {
		// We cant write anything to stdout for this cmd other than the
		// task.DeleteResponse by protcol. We can write to stderr which will be
		// warning logged in containerd.
		logrus.SetOutput(ioutil.Discard)

		bundleFlag := context.GlobalString("bundle")
		if bundleFlag == "" {
			return errors.New("bundle is required")
		}

		// Attempt to find the hcssystem for this bundle and terminate it.
		if sys, _ := hcs.OpenComputeSystem(idFlag); sys != nil {
			if err := sys.Terminate(); err != nil {
				if hcs.IsPending(err) {
					if werr := sys.Wait(); err != nil {
						fmt.Fprintf(os.Stderr, "failed to wait for '%s' to terminate: %v", idFlag, werr)
					}
				} else {
					fmt.Fprintf(os.Stderr, "failed to terminate '%s': %v", idFlag, err)
				}
			}
			sys.Close()
		}

		// Determine if the config file was a POD and if so kill the whole POD.
		if s, err := getSpecAnnotations(bundleFlag); err != nil {
			if !os.IsNotExist(err) {
				return err
			}
		} else {
			if containerType := s["io.kubernetes.cri.container-type"]; containerType == "container" {
				if sandboxID := s["io.kubernetes.cri.sandbox-id"]; sandboxID != "" {
					if sys, _ := hcs.OpenComputeSystem(sandboxID); sys != nil {
						if err := sys.Terminate(); err != nil {
							if hcs.IsPending(err) {
								if werr := sys.Wait(); err != nil {
									fmt.Fprintf(os.Stderr, "failed to wait for '%s' to terminate: %v", idFlag, werr)
								}
							} else {
								fmt.Fprintf(os.Stderr, "failed to terminate '%s': %v", idFlag, err)
							}
						}
						sys.Close()
					}
				}
			}
		}

		// Remove the bundle on disk
		if err := os.RemoveAll(bundleFlag); err != nil && !os.IsNotExist(err) {
			return err
		}

		if data, err := proto.Marshal(&task.DeleteResponse{
			ExitedAt:   time.Now(),
			ExitStatus: 255,
		}); err != nil {
			return err
		} else {
			if _, err := os.Stdout.Write(data); err != nil {
				return err
			}
		}
		return nil
	},
}
<commit_msg>Fix delete hang when the platfrom does not return<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/Microsoft/hcsshim/internal/hcs"
	"github.com/containerd/containerd/runtime/v2/task"
	"github.com/gogo/protobuf/proto"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
)

var deleteCommand = cli.Command{
	Name: "delete",
	Usage: `
This command allows containerd to delete any container resources created, mounted, and/or run by a shim when containerd can no longer communicate over rpc. This happens if a shim is SIGKILL'd with a running container. These resources will need to be cleaned up when containerd loses the connection to a shim. This is also used when containerd boots and reconnects to shims. If a bundle is still on disk but containerd cannot connect to a shim, the delete command is invoked.

The delete command will be executed in the container's bundle as its cwd.
`,
	SkipArgReorder: true,
	Action: func(context *cli.Context) error {
		// We cant write anything to stdout for this cmd other than the
		// task.DeleteResponse by protcol. We can write to stderr which will be
		// warning logged in containerd.
		logrus.SetOutput(ioutil.Discard)

		bundleFlag := context.GlobalString("bundle")
		if bundleFlag == "" {
			return errors.New("bundle is required")
		}

		// Attempt to find the hcssystem for this bundle and terminate it.
		if sys, _ := hcs.OpenComputeSystem(idFlag); sys != nil {
			if err := sys.Terminate(); err != nil {
				if hcs.IsPending(err) {
					const terminateTimeout = time.Second * 30
					done := make(chan bool)
					go func() {
						if werr := sys.Wait(); err != nil {
							fmt.Fprintf(os.Stderr, "failed to wait for '%s' to terminate: %v", idFlag, werr)
						}
						done <- true
					}()
					select {
					case <-done:
					case <-time.After(terminateTimeout):
						sys.Close()
						return fmt.Errorf("timed out waiting for '%s' to terminate", idFlag)
					}
				} else {
					fmt.Fprintf(os.Stderr, "failed to terminate '%s': %v", idFlag, err)
				}
			}
			sys.Close()
		}

		// Determine if the config file was a POD and if so kill the whole POD.
		if s, err := getSpecAnnotations(bundleFlag); err != nil {
			if !os.IsNotExist(err) {
				return err
			}
		} else {
			if containerType := s["io.kubernetes.cri.container-type"]; containerType == "container" {
				if sandboxID := s["io.kubernetes.cri.sandbox-id"]; sandboxID != "" {
					if sys, _ := hcs.OpenComputeSystem(sandboxID); sys != nil {
						if err := sys.Terminate(); err != nil {
							if hcs.IsPending(err) {
								if werr := sys.Wait(); err != nil {
									fmt.Fprintf(os.Stderr, "failed to wait for '%s' to terminate: %v", idFlag, werr)
								}
							} else {
								fmt.Fprintf(os.Stderr, "failed to terminate '%s': %v", idFlag, err)
							}
						}
						sys.Close()
					}
				}
			}
		}

		// Remove the bundle on disk
		if err := os.RemoveAll(bundleFlag); err != nil && !os.IsNotExist(err) {
			return err
		}

		if data, err := proto.Marshal(&task.DeleteResponse{
			ExitedAt:   time.Now(),
			ExitStatus: 255,
		}); err != nil {
			return err
		} else {
			if _, err := os.Stdout.Write(data); err != nil {
				return err
			}
		}
		return nil
	},
}
<|endoftext|>


<commit_before>package bot

import (
	"fmt"
	"testing"
)

func newMeu() *Meu {
	return &Meu{nil}
}

func TestSimpleReactionMeu(t *testing.T) {
	meu := newMeu()

	cases := []struct {
		text     string
		username string
		want     interface{}
	}{
		{"메우, 멱살", "", "사람은 일을 하고 살아야한다. 메우"},
		{"메우메우 펫탄탄", "", `메메메 메메메메 메우메우
메메메 메우메우
펫땅펫땅펫땅펫땅 다이스키`},
		{"메우메우 펫탄", "", nil},
	}

	for _, c := range cases {
		ret := meuMessageProcess(meu, newMessageEvent(c.text, c.username))
		if ret != c.want {
			t.Errorf("Simple reaction test failed. expected \"%q\" but \"%q\"", c.want, ret)
		}
	}
}

func TestCalcMeu(t *testing.T) {
	meu := newMeu()

	ret := meuMessageProcess(meu, newMessageEvent("계산하라 메우 1+1", "meu"))
	switch ret.(type) {
	case string:
		if ret.(string) != fmt.Sprintf("%f", 2.0) {
			t.Errorf("Error calc error")
		}
	default:
		t.Errorf("Error type")
	}
}
<commit_msg>fix test error...<commit_after>package bot

import (
	"fmt"
	"testing"
)

func newMeu() *Meu {
	ret := &Meu{&BaseBot{nil, nil, "<@meu>", nil}, nil, nil}
	return ret
}

func TestSimpleReactionMeu(t *testing.T) {
	meu := newMeu()

	cases := []struct {
		text     string
		username string
		want     interface{}
	}{
		{"메우, 멱살", "", "사람은 일을 하고 살아야한다. 메우"},
		{"메우메우 펫탄탄", "", `메메메 메메메메 메우메우
메메메 메우메우
펫땅펫땅펫땅펫땅 다이스키`},
		{"메우메우 펫탄", "", nil},
	}

	for _, c := range cases {
		ret := meuMessageProcess(meu, newMessageEvent(c.text, c.username))
		if ret != c.want {
			t.Errorf("Simple reaction test failed. expected \"%q\" but \"%q\"", c.want, ret)
		}
	}
}

func TestCalcMeu(t *testing.T) {
	meu := newMeu()

	ret := meuMessageProcess(meu, newMessageEvent("계산하라 메우 1+1", "meu"))
	switch ret.(type) {
	case string:
		if ret.(string) != fmt.Sprintf("%f", 2.0) {
			t.Errorf("Error calc error")
		}
	default:
		t.Errorf("Error type")
	}
}
<|endoftext|>


<commit_before>package bootstrap

import (
	"encoding/json"
	"fmt"
	"reflect"
	"time"

	"github.com/flynn/flynn-controller/client"
	ct "github.com/flynn/flynn-controller/types"
	"github.com/flynn/go-flynn/cluster"
)

type State struct {
	StepData  map[string]interface{}
	Providers map[string]*ct.Provider

	clusterc    *cluster.Client
	controllerc *controller.Client
}

func (s *State) ClusterClient() (*cluster.Client, error) {
	if s.clusterc == nil {
		cc, err := cluster.NewClient()
		if err != nil {
			return nil, err
		}
		s.clusterc = cc
	}
	return s.clusterc, nil
}

func (s *State) ControllerClient() (*controller.Client, error) {
	if s.controllerc == nil {
		cc, err := controller.NewClient("discoverd+http://flynn-controller")
		if err != nil {
			return nil, err
		}
		s.controllerc = cc
	}
	return s.controllerc, nil
}

type Action interface {
	Run(*State) error
}

var registeredActions = make(map[string]reflect.Type)

func Register(name string, action Action) {
	registeredActions[name] = reflect.Indirect(reflect.ValueOf(action)).Type()
}

type StepAction struct {
	ID     string `json:"id"`
	Action string `json:"action"`
}

type StepInfo struct {
	StepAction
	StepData  interface{} `json:"data,omitempty"`
	State     string      `json:"state"`
	Error     error       `json:"error,omitempty"`
	Timestamp time.Time   `json:"ts"`
}

func Run(manifest []byte, ch chan<- *StepInfo) (err error) {
	steps := make([]json.RawMessage, 0)
	if err := json.Unmarshal(manifest, &steps); err != nil {
		return err
	}

	state := &State{
		StepData:  make(map[string]interface{}),
		Providers: make(map[string]*ct.Provider),
	}

	var a StepAction

	defer close(ch)
	defer func() {
		if err != nil {
			ch <- &StepInfo{StepAction: a, State: "error", Error: err, Timestamp: time.Now().UTC()}
		}
	}()

	for _, s := range steps {
		if err := json.Unmarshal(s, &a); err != nil {
			return err
		}
		actionType, ok := registeredActions[a.Action]
		if !ok {
			return fmt.Errorf("bootstrap: unknown action %q", a.Action)
		}
		action := reflect.New(actionType).Interface().(Action)

		if err := json.Unmarshal(s, action); err != nil {
			return err
		}

		ch <- &StepInfo{StepAction: a, State: "start", Timestamp: time.Now().UTC()}

		if err := action.Run(state); err != nil {
			return err
		}

		si := &StepInfo{StepAction: a, State: "done", Timestamp: time.Now().UTC()}
		if data, ok := state.StepData[a.ID]; ok {
			si.StepData = data
		}
		ch <- si
	}

	return nil
}
<commit_msg>bootstrap: Emit errors in JSON output properly<commit_after>package bootstrap

import (
	"encoding/json"
	"fmt"
	"reflect"
	"time"

	"github.com/flynn/flynn-controller/client"
	ct "github.com/flynn/flynn-controller/types"
	"github.com/flynn/go-flynn/cluster"
)

type State struct {
	StepData  map[string]interface{}
	Providers map[string]*ct.Provider

	clusterc    *cluster.Client
	controllerc *controller.Client
}

func (s *State) ClusterClient() (*cluster.Client, error) {
	if s.clusterc == nil {
		cc, err := cluster.NewClient()
		if err != nil {
			return nil, err
		}
		s.clusterc = cc
	}
	return s.clusterc, nil
}

func (s *State) ControllerClient() (*controller.Client, error) {
	if s.controllerc == nil {
		cc, err := controller.NewClient("discoverd+http://flynn-controller")
		if err != nil {
			return nil, err
		}
		s.controllerc = cc
	}
	return s.controllerc, nil
}

type Action interface {
	Run(*State) error
}

var registeredActions = make(map[string]reflect.Type)

func Register(name string, action Action) {
	registeredActions[name] = reflect.Indirect(reflect.ValueOf(action)).Type()
}

type StepAction struct {
	ID     string `json:"id"`
	Action string `json:"action"`
}

type StepInfo struct {
	StepAction
	StepData  interface{} `json:"data,omitempty"`
	State     string      `json:"state"`
	Error     string      `json:"error,omitempty"`
	Timestamp time.Time   `json:"ts"`
}

func Run(manifest []byte, ch chan<- *StepInfo) (err error) {
	steps := make([]json.RawMessage, 0)
	if err := json.Unmarshal(manifest, &steps); err != nil {
		return err
	}

	state := &State{
		StepData:  make(map[string]interface{}),
		Providers: make(map[string]*ct.Provider),
	}

	var a StepAction

	defer close(ch)
	defer func() {
		if err != nil {
			ch <- &StepInfo{StepAction: a, State: "error", Error: err.Error(), Timestamp: time.Now().UTC()}
		}
	}()

	for _, s := range steps {
		if err := json.Unmarshal(s, &a); err != nil {
			return err
		}
		actionType, ok := registeredActions[a.Action]
		if !ok {
			return fmt.Errorf("bootstrap: unknown action %q", a.Action)
		}
		action := reflect.New(actionType).Interface().(Action)

		if err := json.Unmarshal(s, action); err != nil {
			return err
		}

		ch <- &StepInfo{StepAction: a, State: "start", Timestamp: time.Now().UTC()}

		if err := action.Run(state); err != nil {
			return err
		}

		si := &StepInfo{StepAction: a, State: "done", Timestamp: time.Now().UTC()}
		if data, ok := state.StepData[a.ID]; ok {
			si.StepData = data
		}
		ch <- si
	}

	return nil
}
<|endoftext|>


<commit_before>package bootstrap

import (
	"encoding/json"
	"fmt"
	"log"

	"github.com/tinzenite/channel"
	"github.com/tinzenite/shared"
)

/*
Bootstrap is a temporary peer object that allows to bootstrap into an existing
Tinzenite network.
*/
type Bootstrap struct {
	// root path
	path string
	// internal hidden struct for channel callbacks
	cInterface *chaninterface
	// tox communication channel
	channel *channel.Channel
	// self peer
	peer *shared.Peer
	// stores address of peers we need to bootstrap
	bootstrap map[string]bool
}

/*
Start begins a bootstrap process to the given address.
*/
func (b *Bootstrap) Start(address string) error {
	// send own peer
	msg, err := json.Marshal(b.peer)
	if err != nil {
		return err
	}
	// send request
	return b.channel.RequestConnection(address, string(msg))
}

/*
Check looks if the bootstrapped address is online and initiates the boostrap
process.

TODO: we can do this in the background... sigh
*/
func (b *Bootstrap) Check() {
	addresses, err := b.channel.OnlineAddresses()
	if err != nil {
		log.Println("Check:", err)
		return
	}
	if len(addresses) != 1 {
		/*TODO pick one? Randomly?*/
		log.Println("Multiple online!")
	}
	/*TODO start bootstrap*/
}

/*
Store writes a bootstrapped .TINZENITEDIR to disk. Call this if you want
persistant bootstrapping (and why wouldn't you?).
*/
func (b *Bootstrap) Store() error {
	err := shared.MakeDotTinzenite(b.path)
	if err != nil {
		return err
	}
	// write self peer
	err = b.peer.Store(b.path)
	if err != nil {
		return err
	}
	// store local peer info with toxdata
	toxData, err := b.channel.ToxData()
	if err != nil {
		return err
	}
	toxPeerDump := &shared.ToxPeerDump{
		SelfPeer: b.peer,
		ToxData:  toxData}
	err = toxPeerDump.Store(b.path)
	if err != nil {
		return err
	}
	return nil
}

/*
PrintStatus returns a formatted string of the peer status.
*/
func (b *Bootstrap) PrintStatus() string {
	var out string
	out += "Online:\n"
	addresses, err := b.channel.FriendAddresses()
	if err != nil {
		out += "channel.FriendAddresses failed!"
	} else {
		var count int
		for _, address := range addresses {
			online, err := b.channel.IsOnline(address)
			var insert string
			if err != nil {
				insert = "ERROR"
			} else {
				insert = fmt.Sprintf("%v", online)
			}
			out += address[:16] + " :: " + insert + "\n"
			count++
		}
		out += "Total friends: " + fmt.Sprintf("%d", count)
	}
	return out
}

/*
Close cleanly closes everything underlying.
*/
func (b *Bootstrap) Close() {
	b.channel.Close()
}
<commit_msg>small improvement<commit_after>package bootstrap

import (
	"encoding/json"
	"fmt"
	"log"

	"github.com/tinzenite/channel"
	"github.com/tinzenite/shared"
)

/*
Bootstrap is a temporary peer object that allows to bootstrap into an existing
Tinzenite network.
*/
type Bootstrap struct {
	// root path
	path string
	// internal hidden struct for channel callbacks
	cInterface *chaninterface
	// tox communication channel
	channel *channel.Channel
	// self peer
	peer *shared.Peer
	// stores address of peers we need to bootstrap
	bootstrap map[string]bool
}

/*
Start begins a bootstrap process to the given address.
*/
func (b *Bootstrap) Start(address string) error {
	// send own peer
	msg, err := json.Marshal(b.peer)
	if err != nil {
		return err
	}
	// send request
	return b.channel.RequestConnection(address, string(msg))
}

/*
Check looks if the bootstrapped address is online and initiates the boostrap
process.

TODO: we can do this in the background... sigh
*/
func (b *Bootstrap) Check() {
	addresses, err := b.channel.OnlineAddresses()
	if err != nil {
		log.Println("Check:", err)
		return
	}
	if len(addresses) == 0 {
		log.Println("None available yet.")
		return
	}
	if len(addresses) > 1 {
		/*TODO pick one? Randomly?*/
		log.Println("Multiple online!")
	}
	/*TODO start bootstrap*/
	log.Println("Okay, TODO")
}

/*
Store writes a bootstrapped .TINZENITEDIR to disk. Call this if you want
persistant bootstrapping (and why wouldn't you?).
*/
func (b *Bootstrap) Store() error {
	err := shared.MakeDotTinzenite(b.path)
	if err != nil {
		return err
	}
	// write self peer
	err = b.peer.Store(b.path)
	if err != nil {
		return err
	}
	// store local peer info with toxdata
	toxData, err := b.channel.ToxData()
	if err != nil {
		return err
	}
	toxPeerDump := &shared.ToxPeerDump{
		SelfPeer: b.peer,
		ToxData:  toxData}
	err = toxPeerDump.Store(b.path)
	if err != nil {
		return err
	}
	return nil
}

/*
PrintStatus returns a formatted string of the peer status.
*/
func (b *Bootstrap) PrintStatus() string {
	var out string
	out += "Online:\n"
	addresses, err := b.channel.FriendAddresses()
	if err != nil {
		out += "channel.FriendAddresses failed!"
	} else {
		var count int
		for _, address := range addresses {
			online, err := b.channel.IsOnline(address)
			var insert string
			if err != nil {
				insert = "ERROR"
			} else {
				insert = fmt.Sprintf("%v", online)
			}
			out += address[:16] + " :: " + insert + "\n"
			count++
		}
		out += "Total friends: " + fmt.Sprintf("%d", count)
	}
	return out
}

/*
Close cleanly closes everything underlying.
*/
func (b *Bootstrap) Close() {
	b.channel.Close()
}
<|endoftext|>


<commit_before>package debug

/*
github.com/dokipen/debug is a port of visionmedia/debug nodejs lib. It is meant to be extremely quick to get going with. All configuration is done on the commandline with the DEBUG environmental variable.

The current differences with nodejs's debug are that timing is not supported and we include the debug callers file and line number in the output.
*/

import (
	"crypto/md5"
	"fmt"
	colorfmt "github.com/wsxiaoys/terminal/color"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
)

// Possible color values.
const COLORS = "rgbcmykw"

var (
	matchCache  = map[string]bool{}
	prefixCache = map[string]string{}
	colorCache  = map[string]string{}
	mutex       = new(sync.Mutex)
	globs       []string
)

/*
init prepares the environmental DEBUG variable.
*/
func init() {
	debug := os.Getenv("DEBUG")
	if debug == "" {
		globs = []string{}
	} else {
		globs = strings.Split(debug, " ")
	}
}

/*
match checks if the namespace is enabled for debug logging. It also seeds the
cache for the namespace to speed things up.
*/
func match(namespace string) (match bool) {
	var ok bool

	match = false

	// This might be a pointless opimization, but I'm hoping it helps with
	// multithreaded applications because we can skip the mutex if there
	// is no DEBUG env.
	if len(globs) == 0 {
		return
	}

	mutex.Lock()
	defer mutex.Unlock()
	if match, ok = matchCache[namespace]; ok {
		return
	}

	for _, glob := range globs {
		if ok, _ = filepath.Match(glob, namespace); ok {
			match = true
			matchCache[namespace] = match
			colorCache[namespace] = getcolor(namespace)
			prefixCache[namespace] = colorfmt.Sprint(colorize("@%s%s@|", colorCache[namespace], namespace))
			return
		}
	}
	matchCache[namespace] = match
	return
}

/*
getcolor returns the color of the namespace.
*/
func getcolor(namespace string) string {
	h := md5.New()
	io.WriteString(h, namespace)
	var sum int
	sumbytes := h.Sum(nil)
	for _, i := range sumbytes {
		sum += int(i)
	}
	return string(COLORS[sum%len(COLORS)])

}

/*
printNs prints the namespace and returns the namespaces color.
*/
func printNs(namespace string) (color string) {
	var ok bool
	var prefix string

	mutex.Lock()
	prefix, ok = prefixCache[namespace]
	color = colorCache[namespace]
	mutex.Unlock()
	if ok {
		fmt.Print(prefix)
	}
	return
}

/*
colorize formats the string before sending it to color.
*/
func colorize(format string, args ...interface{}) string {
	return fmt.Sprintf(format, args...)
}

/*
printCaller prints the caller's filepath and line number.
*/
func printCaller(path string, line int, color string) {
	filename := filepath.Base(path)
	colorfmt.Print(colorize("@%s%s:%d@|", color, filename, line))
}

/*
printMs prints how much time has elapsed since the last message was logged.
TODO: please implement.
*/
func printMs(color string) {
	colorfmt.Print(colorize("@%s+1ms", color))
}

/*
printMsg prints the actual log message.
*/
func printMsg(format string, args ...interface{}) {
	msg := fmt.Sprintf(format, args...)
	colored := colorize("@w%s@|", msg)
	colorfmt.Print(colored)
}

/*
Log logs your message in a namespace.
*/
func Log(namespace, msg string, args ...interface{}) {
	if match(namespace) {
		_, path, line, _ := runtime.Caller(2)

		fmt.Print("  ")
		color := printNs(namespace)

		fmt.Print(" ")
		printCaller(path, line, color)

		fmt.Print(" ")
		printMsg(msg, args...)

		//fmt.Print(" ")
		//printMs(color)

		fmt.Println()
	}
}

/*
Logger returns a closure for namespacing Log.
*/
func Logger(namespace string) func(string, ...interface{}) {
	return func(msg string, args ...interface{}) {
		Log(namespace, msg, args...)
	}
}
<commit_msg>Fixes godoc<commit_after>/*
github.com/dokipen/debug is a port of visionmedia/debug nodejs lib. It is meant to be extremely quick to get going with. All configuration is done on the commandline with the DEBUG environmental variable.

The current differences with nodejs's debug are that timing is not supported and we include the debug callers file and line number in the output.
*/
package debug

import (
	"crypto/md5"
	"fmt"
	colorfmt "github.com/wsxiaoys/terminal/color"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
)

// Possible color values.
const COLORS = "rgbcmykw"

var (
	matchCache  = map[string]bool{}
	prefixCache = map[string]string{}
	colorCache  = map[string]string{}
	mutex       = new(sync.Mutex)
	globs       []string
)

/*
init prepares the environmental DEBUG variable.
*/
func init() {
	debug := os.Getenv("DEBUG")
	if debug == "" {
		globs = []string{}
	} else {
		globs = strings.Split(debug, " ")
	}
}

/*
match checks if the namespace is enabled for debug logging. It also seeds the
cache for the namespace to speed things up.
*/
func match(namespace string) (match bool) {
	var ok bool

	match = false

	// This might be a pointless opimization, but I'm hoping it helps with
	// multithreaded applications because we can skip the mutex if there
	// is no DEBUG env.
	if len(globs) == 0 {
		return
	}

	mutex.Lock()
	defer mutex.Unlock()
	if match, ok = matchCache[namespace]; ok {
		return
	}

	for _, glob := range globs {
		if ok, _ = filepath.Match(glob, namespace); ok {
			match = true
			matchCache[namespace] = match
			colorCache[namespace] = getcolor(namespace)
			prefixCache[namespace] = colorfmt.Sprint(colorize("@%s%s@|", colorCache[namespace], namespace))
			return
		}
	}
	matchCache[namespace] = match
	return
}

/*
getcolor returns the color of the namespace.
*/
func getcolor(namespace string) string {
	h := md5.New()
	io.WriteString(h, namespace)
	var sum int
	sumbytes := h.Sum(nil)
	for _, i := range sumbytes {
		sum += int(i)
	}
	return string(COLORS[sum%len(COLORS)])

}

/*
printNs prints the namespace and returns the namespaces color.
*/
func printNs(namespace string) (color string) {
	var ok bool
	var prefix string

	mutex.Lock()
	prefix, ok = prefixCache[namespace]
	color = colorCache[namespace]
	mutex.Unlock()
	if ok {
		fmt.Print(prefix)
	}
	return
}

/*
colorize formats the string before sending it to color.
*/
func colorize(format string, args ...interface{}) string {
	return fmt.Sprintf(format, args...)
}

/*
printCaller prints the caller's filepath and line number.
*/
func printCaller(path string, line int, color string) {
	filename := filepath.Base(path)
	colorfmt.Print(colorize("@%s%s:%d@|", color, filename, line))
}

/*
printMs prints how much time has elapsed since the last message was logged.
TODO: please implement.
*/
func printMs(color string) {
	colorfmt.Print(colorize("@%s+1ms", color))
}

/*
printMsg prints the actual log message.
*/
func printMsg(format string, args ...interface{}) {
	msg := fmt.Sprintf(format, args...)
	colored := colorize("@w%s@|", msg)
	colorfmt.Print(colored)
}

/*
Log logs your message in a namespace.
*/
func Log(namespace, msg string, args ...interface{}) {
	if match(namespace) {
		_, path, line, _ := runtime.Caller(2)

		fmt.Print("  ")
		color := printNs(namespace)

		fmt.Print(" ")
		printCaller(path, line, color)

		fmt.Print(" ")
		printMsg(msg, args...)

		//fmt.Print(" ")
		//printMs(color)

		fmt.Println()
	}
}

/*
Logger returns a closure for namespacing Log.
*/
func Logger(namespace string) func(string, ...interface{}) {
	return func(msg string, args ...interface{}) {
		Log(namespace, msg, args...)
	}
}
<|endoftext|>