<commit_before>package kite

import (
	"flag"
	"fmt"
	"github.com/op/go-logging"
	"koding/newkite/dnode/rpc"
	"koding/newkite/protocol"
	"koding/newkite/utils"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"syscall"
	"time"
)

func init() {
	// Use all available CPUS.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Debugging helper: Prints stacktrace on SIGUSR1.
	c := make(chan os.Signal)
	signal.Notify(c, syscall.SIGUSR1)
	go func() {
		for {
			s := <-c
			fmt.Println("Got signal:", s)
			buf := make([]byte, 1<<16)
			runtime.Stack(buf, true)
			fmt.Println(string(buf))
			fmt.Print("Number of goroutines:", runtime.NumGoroutine())
			m := new(runtime.MemStats)
			runtime.GC()
			runtime.ReadMemStats(m)
			fmt.Printf(", Memory allocated: %+v\n", m.Alloc)
		}
	}()
}

// Kite defines a single process that enables distributed service messaging
// amongst the peers it is connected. A Kite process acts as a Client and as a
// Server. That means it can receive request, process them, but it also can
// make request to other kites. A Kite can be anything. It can be simple Image
// processing kite (which would process data), it could be a Chat kite that
// enables peer-to-peer chat. For examples we have FileSystem kite that expose
// the file system to a client, which in order build the filetree.
type Kite struct {
	protocol.Kite

	// KodingKey is used for authenticate to Kontrol.
	KodingKey string

	// Is this Kite Public or Private? Default is Private.
	Visibility protocol.Visibility

	// Points to the Kontrol instance if enabled
	Kontrol *Kontrol

	// Wheter we want to connect to Kontrol on startup, true by default.
	KontrolEnabled bool

	// Wheter we want to register our Kite to Kontrol, true by default.
	RegisterToKontrol bool

	// method map for exported methods
	handlers map[string]HandlerFunc

	// Dnode rpc server
	server *rpc.Server

	// Handlers to call when a Kite opens a connection to this Kite.
	onConnectHandlers []func(*RemoteKite)

	// Handlers to call when a client has disconnected.
	onDisconnectHandlers []func(*RemoteKite)

	// Contains different functions for authenticating user from request.
	// Keys are the authentication types (options.authentication.type).
	Authenticators map[string]func(*Request) error

	// Used to signal if the kite is ready to start and make calls to
	// other kites.
	ready chan bool

	// Prints logging messages to stderr and syslog.
	Log *logging.Logger
}

// New creates, initialize and then returns a new Kite instance. It accepts
// a single options argument that is a config struct that needs to be filled
// with several informations like Name, Port, IP and so on.
func New(options *Options) *Kite {
	var err error
	if options == nil {
		options, err = ReadKiteOptions("manifest.json")
		if err != nil {
			log.Fatal("error: could not read config file", err)
		}
	}

	options.validate() // exits if validating fails

	hostname, _ := os.Hostname()
	kiteID := utils.GenerateUUID()
	kodingKey, err := utils.GetKodingKey()
	if err != nil {
		log.Fatal("Couldn't find koding.key. Please run 'kd register'.")
	}

	k := &Kite{
		Kite: protocol.Kite{
			Name:        options.Kitename,
			Username:    options.Username,
			ID:          kiteID,
			Version:     options.Version,
			Hostname:    hostname,
			Port:        options.Port,
			Environment: options.Environment,
			Region:      options.Region,
			Visibility:  options.Visibility,

			// PublicIP will be set by Kontrol after registering if it is not set.
			PublicIP: options.PublicIP,
		},
		KodingKey:         kodingKey,
		server:            rpc.NewServer(),
		KontrolEnabled:    true,
		RegisterToKontrol: true,
		Authenticators:    make(map[string]func(*Request) error),
		handlers:          make(map[string]HandlerFunc),
		ready:             make(chan bool),
	}

	k.Log = newLogger(k.Name, k.hasDebugFlag())
	k.Kontrol = k.NewKontrol(options.KontrolAddr)

	// Call registered handlers when a client has disconnected.
	k.server.OnDisconnect(func(c *rpc.Client) {
		if r, ok := c.Properties()["remoteKite"]; ok {
			// Run OnDisconnect handlers.
			k.notifyRemoteKiteDisconnected(r.(*RemoteKite))
		}
	})

	// Every kite should be able to authenticate the user from token.
	k.Authenticators["token"] = k.AuthenticateFromToken
	// A kite accepts requests from Kontrol.
	k.Authenticators["kodingKey"] = k.AuthenticateFromKodingKey

	// Register our internal methods
	k.HandleFunc("systemInfo", new(Status).Info)
	k.HandleFunc("heartbeat", k.handleHeartbeat)
	k.HandleFunc("log", k.handleLog)

	return k
}

// Run is a blocking method. It runs the kite server and then accepts requests
// asynchronously.
func (k *Kite) Run() {
	k.Start()
	select {}
}

// Start is like Run(), but does not wait for it to complete. It's nonblocking.
func (k *Kite) Start() {
	k.parseVersionFlag()

	go func() {
		err := k.listenAndServe()
		if err != nil {
			k.Log.Fatal(err)
		}
	}()

	<-k.ready // wait until we are ready
}

func (k *Kite) handleHeartbeat(r *Request) (interface{}, error) {
	args := r.Args.MustSliceOfLength(2)
	seconds := args[0].MustFloat64()
	ping := args[1].MustFunction()

	go func() {
		for {
			time.Sleep(time.Duration(seconds) * time.Second)
			if ping() != nil {
				return
			}
		}
	}()

	return nil, nil
}

// handleLog prints a log message to stdout.
func (k *Kite) handleLog(r *Request) (interface{}, error) {
	msg := r.Args.MustString()
	k.Log.Info(fmt.Sprintf("%s: %s", r.RemoteKite.Name, msg))
	return nil, nil
}

func init() {
	// These logging related stuff needs to be called once because stupid
	// logging library uses global variables and resets the backends every time.
	logging.SetFormatter(logging.MustStringFormatter("%{level:-8s} ▶ %{message}"))
	stderrBackend := logging.NewLogBackend(os.Stderr, "", log.LstdFlags)
	stderrBackend.Color = true
	syslogBackend, _ := logging.NewSyslogBackend("")
	logging.SetBackend(stderrBackend, syslogBackend)
}

// newLogger returns a new logger object for desired name and level.
func newLogger(name string, debug bool) *logging.Logger {
	logger := logging.MustGetLogger(name)

	level := logging.INFO
	if debug {
		level = logging.DEBUG
	}

	logging.SetLevel(level, name)
	return logger
}

// If the user wants to call flag.Parse() the flag must be defined in advance.
var _ = flag.Bool("version", false, "show version")
var _ = flag.Bool("debug", false, "print debug logs")

// parseVersionFlag prints the version number of the kite and exits with 0
// if "-version" flag is enabled.
// We did not use the "flag" package because it causes trouble if the user
// also calls "flag.Parse()" in his code. flag.Parse() can be called only once.
func (k *Kite) parseVersionFlag() {
	for _, flag := range os.Args {
		if flag == "-version" {
			fmt.Println(k.Version)
			os.Exit(0)
		}
	}
}

// hasDebugFlag returns true if -debug flag is present in os.Args.
func (k *Kite) hasDebugFlag() bool {
	for _, flag := range os.Args {
		if flag == "-debug" {
			return true
		}
	}

	// We can't use flags when running "go test" command.
	// This is another way to print debug logs.
	if os.Getenv("DEBUG") != "" {
		return true
	}

	return false
}

// listenAndServe starts our rpc server with the given addr.
func (k *Kite) listenAndServe() error {
	listener, err := net.Listen("tcp4", ":"+k.Port)
	if err != nil {
		return err
	}

	k.Log.Info("Listening: %s", listener.Addr().String())

	// Port is known here if "0" is used as port number
	_, k.Port, _ = net.SplitHostPort(listener.Addr().String())

	// We must connect to Kontrol after starting to listen on port
	if k.KontrolEnabled {
		if k.RegisterToKontrol {
			k.Kontrol.OnConnect(k.registerToKontrol)
		}

		k.Kontrol.DialForever()
	}

	k.ready <- true // listener is ready, means we are ready too
	return http.Serve(listener, k.server)
}

func (k *Kite) registerToKontrol() {
	err := k.Kontrol.Register()
	if err != nil {
		k.Log.Fatalf("Cannot register to Kontrol: %s", err)
	}
}

// OnConnect registers a function to run when a Kite connects to this Kite.
func (k *Kite) OnConnect(handler func(*RemoteKite)) {
	k.onConnectHandlers = append(k.onConnectHandlers, handler)
}

// OnDisconnect registers a function to run when a connected Kite is disconnected.
func (k *Kite) OnDisconnect(handler func(*RemoteKite)) {
	k.onDisconnectHandlers = append(k.onDisconnectHandlers, handler)
}

// notifyRemoteKiteConnected runs the registered handlers with OnConnect().
func (k *Kite) notifyRemoteKiteConnected(r *RemoteKite) {
	k.Log.Info("Client is connected to us: [%s %s]", r.Name, r.Addr())

	for _, handler := range k.onConnectHandlers {
		go handler(r)
	}
}

func (k *Kite) notifyRemoteKiteDisconnected(r *RemoteKite) {
	k.Log.Info("Client has disconnected: [%s %s]", r.Name, r.Addr())

	for _, handler := range k.onDisconnectHandlers {
		go handler(r)
	}
}
<commit_msg>kite: add Kite.Close() method<commit_after>package kite

import (
	"flag"
	"fmt"
	"github.com/op/go-logging"
	"koding/newkite/dnode/rpc"
	"koding/newkite/protocol"
	"koding/newkite/utils"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"strings"
	"syscall"
	"time"
)

func init() {
	// Use all available CPUS.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Debugging helper: Prints stacktrace on SIGUSR1.
	c := make(chan os.Signal)
	signal.Notify(c, syscall.SIGUSR1)
	go func() {
		for {
			s := <-c
			fmt.Println("Got signal:", s)
			buf := make([]byte, 1<<16)
			runtime.Stack(buf, true)
			fmt.Println(string(buf))
			fmt.Print("Number of goroutines:", runtime.NumGoroutine())
			m := new(runtime.MemStats)
			runtime.GC()
			runtime.ReadMemStats(m)
			fmt.Printf(", Memory allocated: %+v\n", m.Alloc)
		}
	}()
}

// Kite defines a single process that enables distributed service messaging
// amongst the peers it is connected. A Kite process acts as a Client and as a
// Server. That means it can receive request, process them, but it also can
// make request to other kites. A Kite can be anything. It can be simple Image
// processing kite (which would process data), it could be a Chat kite that
// enables peer-to-peer chat. For examples we have FileSystem kite that expose
// the file system to a client, which in order build the filetree.
type Kite struct {
	protocol.Kite

	// KodingKey is used for authenticate to Kontrol.
	KodingKey string

	// Is this Kite Public or Private? Default is Private.
	Visibility protocol.Visibility

	// Points to the Kontrol instance if enabled
	Kontrol *Kontrol

	// Wheter we want to connect to Kontrol on startup, true by default.
	KontrolEnabled bool

	// Wheter we want to register our Kite to Kontrol, true by default.
	RegisterToKontrol bool

	// method map for exported methods
	handlers map[string]HandlerFunc

	// Dnode rpc server
	server *rpc.Server

	listener net.Listener

	// Handlers to call when a Kite opens a connection to this Kite.
	onConnectHandlers []func(*RemoteKite)

	// Handlers to call when a client has disconnected.
	onDisconnectHandlers []func(*RemoteKite)

	// Contains different functions for authenticating user from request.
	// Keys are the authentication types (options.authentication.type).
	Authenticators map[string]func(*Request) error

	// Used to signal if the kite is ready to start and make calls to
	// other kites.
	ready chan bool
	end   chan bool

	// Prints logging messages to stderr and syslog.
	Log *logging.Logger
}

// New creates, initialize and then returns a new Kite instance. It accepts
// a single options argument that is a config struct that needs to be filled
// with several informations like Name, Port, IP and so on.
func New(options *Options) *Kite {
	var err error
	if options == nil {
		options, err = ReadKiteOptions("manifest.json")
		if err != nil {
			log.Fatal("error: could not read config file", err)
		}
	}

	options.validate() // exits if validating fails

	hostname, _ := os.Hostname()
	kiteID := utils.GenerateUUID()
	kodingKey, err := utils.GetKodingKey()
	if err != nil {
		log.Fatal("Couldn't find koding.key. Please run 'kd register'.")
	}

	k := &Kite{
		Kite: protocol.Kite{
			Name:        options.Kitename,
			Username:    options.Username,
			ID:          kiteID,
			Version:     options.Version,
			Hostname:    hostname,
			Port:        options.Port,
			Environment: options.Environment,
			Region:      options.Region,
			Visibility:  options.Visibility,

			// PublicIP will be set by Kontrol after registering if it is not set.
			PublicIP: options.PublicIP,
		},
		KodingKey:         kodingKey,
		server:            rpc.NewServer(),
		KontrolEnabled:    true,
		RegisterToKontrol: true,
		Authenticators:    make(map[string]func(*Request) error),
		handlers:          make(map[string]HandlerFunc),
		ready:             make(chan bool),
		end:               make(chan bool, 1),
	}

	k.Log = newLogger(k.Name, k.hasDebugFlag())
	k.Kontrol = k.NewKontrol(options.KontrolAddr)

	// Call registered handlers when a client has disconnected.
	k.server.OnDisconnect(func(c *rpc.Client) {
		if r, ok := c.Properties()["remoteKite"]; ok {
			// Run OnDisconnect handlers.
			k.notifyRemoteKiteDisconnected(r.(*RemoteKite))
		}
	})

	// Every kite should be able to authenticate the user from token.
	k.Authenticators["token"] = k.AuthenticateFromToken
	// A kite accepts requests from Kontrol.
	k.Authenticators["kodingKey"] = k.AuthenticateFromKodingKey

	// Register our internal methods
	k.HandleFunc("systemInfo", new(Status).Info)
	k.HandleFunc("heartbeat", k.handleHeartbeat)
	k.HandleFunc("log", k.handleLog)

	return k
}

// Run is a blocking method. It runs the kite server and then accepts requests
// asynchronously.
func (k *Kite) Run() {
	k.Start()
	<-k.end
	k.Log.Notice("Kite server is closed.")
}

// Start is like Run(), but does not wait for it to complete. It's nonblocking.
func (k *Kite) Start() {
	k.parseVersionFlag()

	go func() {
		err := k.listenAndServe()
		if err != nil {
			k.Log.Fatal(err)
		}
	}()

	<-k.ready // wait until we are ready
}

// Close stops the server.
func (k *Kite) Close() {
	k.Log.Notice("Closing server...")
	k.listener.Close()
}

func (k *Kite) handleHeartbeat(r *Request) (interface{}, error) {
	args := r.Args.MustSliceOfLength(2)
	seconds := args[0].MustFloat64()
	ping := args[1].MustFunction()

	go func() {
		for {
			time.Sleep(time.Duration(seconds) * time.Second)
			if ping() != nil {
				return
			}
		}
	}()

	return nil, nil
}

// handleLog prints a log message to stdout.
func (k *Kite) handleLog(r *Request) (interface{}, error) {
	msg := r.Args.MustString()
	k.Log.Info(fmt.Sprintf("%s: %s", r.RemoteKite.Name, msg))
	return nil, nil
}

func init() {
	// These logging related stuff needs to be called once because stupid
	// logging library uses global variables and resets the backends every time.
	logging.SetFormatter(logging.MustStringFormatter("%{level:-8s} ▶ %{message}"))
	stderrBackend := logging.NewLogBackend(os.Stderr, "", log.LstdFlags)
	stderrBackend.Color = true
	syslogBackend, _ := logging.NewSyslogBackend("")
	logging.SetBackend(stderrBackend, syslogBackend)
}

// newLogger returns a new logger object for desired name and level.
func newLogger(name string, debug bool) *logging.Logger {
	logger := logging.MustGetLogger(name)

	level := logging.INFO
	if debug {
		level = logging.DEBUG
	}

	logging.SetLevel(level, name)
	return logger
}

// If the user wants to call flag.Parse() the flag must be defined in advance.
var _ = flag.Bool("version", false, "show version")
var _ = flag.Bool("debug", false, "print debug logs")

// parseVersionFlag prints the version number of the kite and exits with 0
// if "-version" flag is enabled.
// We did not use the "flag" package because it causes trouble if the user
// also calls "flag.Parse()" in his code. flag.Parse() can be called only once.
func (k *Kite) parseVersionFlag() {
	for _, flag := range os.Args {
		if flag == "-version" {
			fmt.Println(k.Version)
			os.Exit(0)
		}
	}
}

// hasDebugFlag returns true if -debug flag is present in os.Args.
func (k *Kite) hasDebugFlag() bool {
	for _, flag := range os.Args {
		if flag == "-debug" {
			return true
		}
	}

	// We can't use flags when running "go test" command.
	// This is another way to print debug logs.
	if os.Getenv("DEBUG") != "" {
		return true
	}

	return false
}

// listenAndServe starts our rpc server with the given addr.
func (k *Kite) listenAndServe() (err error) {
	// An error string equivalent to net.errClosing for using with http.Serve()
	// during a graceful exit.
	// I had to put it here because it is not exported by "net" package.
	const errClosing = "use of closed network connection"

	k.listener, err = net.Listen("tcp4", ":"+k.Port)
	if err != nil {
		return err
	}
	k.Log.Info("Listening: %s", k.listener.Addr().String())

	// Port is known here if "0" is used as port number
	_, k.Port, _ = net.SplitHostPort(k.listener.Addr().String())

	// We must connect to Kontrol after starting to listen on port
	if k.KontrolEnabled {
		if k.RegisterToKontrol {
			k.Kontrol.OnConnect(k.registerToKontrol)
		}

		k.Kontrol.DialForever()
	}

	k.ready <- true // listener is ready, means we are ready too

	err = http.Serve(k.listener, k.server)
	if strings.Contains(err.Error(), errClosing) {
		// The server is closed by Close() method
		err = nil
	}

	k.end <- true // Serving is finished.

	return err
}

func (k *Kite) registerToKontrol() {
	err := k.Kontrol.Register()
	if err != nil {
		k.Log.Fatalf("Cannot register to Kontrol: %s", err)
	}
}

// OnConnect registers a function to run when a Kite connects to this Kite.
func (k *Kite) OnConnect(handler func(*RemoteKite)) {
	k.onConnectHandlers = append(k.onConnectHandlers, handler)
}

// OnDisconnect registers a function to run when a connected Kite is disconnected.
func (k *Kite) OnDisconnect(handler func(*RemoteKite)) {
	k.onDisconnectHandlers = append(k.onDisconnectHandlers, handler)
}

// notifyRemoteKiteConnected runs the registered handlers with OnConnect().
func (k *Kite) notifyRemoteKiteConnected(r *RemoteKite) {
	k.Log.Info("Client is connected to us: [%s %s]", r.Name, r.Addr())

	for _, handler := range k.onConnectHandlers {
		go handler(r)
	}
}

func (k *Kite) notifyRemoteKiteDisconnected(r *RemoteKite) {
	k.Log.Info("Client has disconnected: [%s %s]", r.Name, r.Addr())

	for _, handler := range k.onDisconnectHandlers {
		go handler(r)
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package oglematchers_test

import (
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////

type IdenticalToTest struct {
}

func init() { RegisterTestSuite(&IdenticalToTest{}) }

////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////

func (t *IdenticalToTest) TypesNotIdentical() {
	var m Matcher
	var err error

	type intAlias int

	// Type alias expected value
	m = IdenticalTo(intAlias(17))
	err = m.Matches(int(17))
	ExpectThat(err, Error(Equals("which is of type int")))

	// Type alias candidate
	m = IdenticalTo(int(17))
	err = m.Matches(intAlias(17))
	ExpectThat(err, Error(Equals("which is of type intAlias")))

	// int and uint
	m = IdenticalTo(int(17))
	err = m.Matches(uint(17))
	ExpectThat(err, Error(Equals("which is of type uint")))
}

func (t *IdenticalToTest) InvalidTypeExpectedValue() {
	f := func() { IdenticalTo(nil) }
	ExpectThat(f, Panics(AllOf(HasSubstr("IdenticalTo"), HasSubstr("invalid"))))
}

func (t *IdenticalToTest) InvalidTypeCandidate() {
	var m Matcher
	var err error

	// Nil chan expected value
	m = IdenticalTo((chan int)(nil))
	err = m.Matches(nil)
	ExpectThat(err, Error(Equals("which is of type <nil>")))

	// Non-nil chan expected value
	m = IdenticalTo(make(chan int))
	err = m.Matches(nil)
	ExpectThat(err, Error(Equals("which is of type <nil>")))
}

func (t *IdenticalToTest) Slices() {
}

func (t *IdenticalToTest) Maps() {
}

func (t *IdenticalToTest) Functions() {
}

func (t *IdenticalToTest) Channels() {
}

func (t *IdenticalToTest) Bools() {
}

func (t *IdenticalToTest) Ints() {
}

func (t *IdenticalToTest) Int8s() {
}

func (t *IdenticalToTest) Int16s() {
}

func (t *IdenticalToTest) Int32s() {
}

func (t *IdenticalToTest) Int64s() {
}

func (t *IdenticalToTest) Uints() {
}

func (t *IdenticalToTest) Uint8s() {
}

func (t *IdenticalToTest) Uint16s() {
}

func (t *IdenticalToTest) Uint32s() {
}

func (t *IdenticalToTest) Uint64s() {
}

func (t *IdenticalToTest) Float32s() {
}

func (t *IdenticalToTest) Float64s() {
}

func (t *IdenticalToTest) Complex64s() {
}

func (t *IdenticalToTest) Complex128s() {
}

func (t *IdenticalToTest) ComparableArrays() {
}

func (t *IdenticalToTest) NonComparableArrays() {
}

func (t *IdenticalToTest) ComparableInterfaces() {
}

func (t *IdenticalToTest) NonComparableInterfaces() {
}

func (t *IdenticalToTest) Pointers() {
}

func (t *IdenticalToTest) Strings() {
}

func (t *IdenticalToTest) ComparableStructs() {
}

func (t *IdenticalToTest) NonComparableStructs() {
}

func (t *IdenticalToTest) UnsafePointers() {
}

func (t *IdenticalToTest) IntAlias() {
}
<commit_msg>IdenticalToTest.Slices<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package oglematchers_test

import (
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////

type IdenticalToTest struct {
}

func init() { RegisterTestSuite(&IdenticalToTest{}) }

////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////

func (t *IdenticalToTest) TypesNotIdentical() {
	var m Matcher
	var err error

	type intAlias int

	// Type alias expected value
	m = IdenticalTo(intAlias(17))
	err = m.Matches(int(17))
	ExpectThat(err, Error(Equals("which is of type int")))

	// Type alias candidate
	m = IdenticalTo(int(17))
	err = m.Matches(intAlias(17))
	ExpectThat(err, Error(Equals("which is of type intAlias")))

	// int and uint
	m = IdenticalTo(int(17))
	err = m.Matches(uint(17))
	ExpectThat(err, Error(Equals("which is of type uint")))
}

func (t *IdenticalToTest) InvalidTypeExpectedValue() {
	f := func() { IdenticalTo(nil) }
	ExpectThat(f, Panics(AllOf(HasSubstr("IdenticalTo"), HasSubstr("invalid"))))
}

func (t *IdenticalToTest) InvalidTypeCandidate() {
	var m Matcher
	var err error

	// Nil chan expected value
	m = IdenticalTo((chan int)(nil))
	err = m.Matches(nil)
	ExpectThat(err, Error(Equals("which is of type <nil>")))

	// Non-nil chan expected value
	m = IdenticalTo(make(chan int))
	err = m.Matches(nil)
	ExpectThat(err, Error(Equals("which is of type <nil>")))
}

func (t *IdenticalToTest) Slices() {
	var m Matcher
	var err error

	// Nil expected value
	m = IdenticalTo(([]int)(nil))

	err = m.Matches(([]int)(nil))
	ExpectEq(nil, err)

	err = m.Matches([]int{})
	ExpectThat(err, Equals("which is not an identical reference"))

	// Non-nil expected value
	s1 := []int{}
	s2 := []int{}
	m = IdenticalTo(s1)

	err = m.Matches(s1)
	ExpectEq(nil, err)

	err = m.Matches(s2)
	ExpectThat(err, Equals("which is not an identical reference"))
}

func (t *IdenticalToTest) Maps() {
}

func (t *IdenticalToTest) Functions() {
}

func (t *IdenticalToTest) Channels() {
}

func (t *IdenticalToTest) Bools() {
}

func (t *IdenticalToTest) Ints() {
}

func (t *IdenticalToTest) Int8s() {
}

func (t *IdenticalToTest) Int16s() {
}

func (t *IdenticalToTest) Int32s() {
}

func (t *IdenticalToTest) Int64s() {
}

func (t *IdenticalToTest) Uints() {
}

func (t *IdenticalToTest) Uint8s() {
}

func (t *IdenticalToTest) Uint16s() {
}

func (t *IdenticalToTest) Uint32s() {
}

func (t *IdenticalToTest) Uint64s() {
}

func (t *IdenticalToTest) Float32s() {
}

func (t *IdenticalToTest) Float64s() {
}

func (t *IdenticalToTest) Complex64s() {
}

func (t *IdenticalToTest) Complex128s() {
}

func (t *IdenticalToTest) ComparableArrays() {
}

func (t *IdenticalToTest) NonComparableArrays() {
}

func (t *IdenticalToTest) ComparableInterfaces() {
}

func (t *IdenticalToTest) NonComparableInterfaces() {
}

func (t *IdenticalToTest) Pointers() {
}

func (t *IdenticalToTest) Strings() {
}

func (t *IdenticalToTest) ComparableStructs() {
}

func (t *IdenticalToTest) NonComparableStructs() {
}

func (t *IdenticalToTest) UnsafePointers() {
}

func (t *IdenticalToTest) IntAlias() {
}
<|endoftext|>


<commit_before>package handler

import (
	"context"

	"github.com/micro/go-micro/v2/broker"
	pb "github.com/micro/go-micro/v2/broker/service/proto"
	"github.com/micro/go-micro/v2/errors"
	log "github.com/micro/go-micro/v2/logger"
	"github.com/micro/micro/v2/internal/namespace"
)

type Broker struct {
	Broker broker.Broker
}

func (b *Broker) Publish(ctx context.Context, req *pb.PublishRequest, rsp *pb.Empty) error {
	ns := namespace.FromContext(ctx)

	log.Debugf("Publishing message to %s topic in the %v namespace", req.Topic, ns)
	err := b.Broker.Publish(ns+"."+req.Topic, &broker.Message{
		Header: req.Message.Header,
		Body:   req.Message.Body,
	})
	log.Debugf("Published message to %s topic in the %v namespace", req.Topic, ns)
	if err != nil {
		return errors.InternalServerError("go.micro.broker", err.Error())
	}
	return nil
}

func (b *Broker) Subscribe(ctx context.Context, req *pb.SubscribeRequest, stream pb.Broker_SubscribeStream) error {
	ns := namespace.FromContext(ctx)
	errChan := make(chan error, 1)

	// message handler to stream back messages from broker
	handler := func(p broker.Event) error {
		if err := stream.Send(&pb.Message{
			Header: p.Message().Header,
			Body:   p.Message().Body,
		}); err != nil {
			select {
			case errChan <- err:
				return err
			default:
				return err
			}
		}
		return nil
	}

	log.Debugf("Subscribing to %s topic in namespace %v", req.Topic, ns)
	sub, err := b.Broker.Subscribe(ns+"."+req.Topic, handler, broker.Queue(ns+"."+req.Queue))
	if err != nil {
		return errors.InternalServerError("go.micro.broker", err.Error())
	}
	defer func() {
		log.Debugf("Unsubscribing from topic %s in namespace %v", req.Topic, ns)
		sub.Unsubscribe()
	}()

	select {
	case <-ctx.Done():
		log.Debugf("Context done for subscription to topic %s", req.Topic)
		return nil
	case err := <-errChan:
		log.Debugf("Subscription error for topic %s: %v", req.Topic, err)
		return err
	}
}
<commit_msg>service/broker: use namespace.Authorize<commit_after>package handler

import (
	"context"

	"github.com/micro/go-micro/v2/broker"
	pb "github.com/micro/go-micro/v2/broker/service/proto"
	"github.com/micro/go-micro/v2/errors"
	log "github.com/micro/go-micro/v2/logger"
	"github.com/micro/micro/v2/internal/namespace"
)

type Broker struct {
	Broker broker.Broker
}

func (b *Broker) Publish(ctx context.Context, req *pb.PublishRequest, rsp *pb.Empty) error {
	ns := namespace.FromContext(ctx)

	// authorize the request
	if err := namespace.Authorize(ctx, ns); err == namespace.ErrForbidden {
		return errors.Forbidden("go.micro.broker", err.Error())
	} else if err == namespace.ErrUnauthorized {
		return errors.Unauthorized("go.micro.broker", err.Error())
	} else if err != nil {
		return errors.InternalServerError("go.micro.broker", err.Error())
	}

	log.Debugf("Publishing message to %s topic in the %v namespace", req.Topic, ns)
	err := b.Broker.Publish(ns+"."+req.Topic, &broker.Message{
		Header: req.Message.Header,
		Body:   req.Message.Body,
	})
	log.Debugf("Published message to %s topic in the %v namespace", req.Topic, ns)
	if err != nil {
		return errors.InternalServerError("go.micro.broker", err.Error())
	}
	return nil
}

func (b *Broker) Subscribe(ctx context.Context, req *pb.SubscribeRequest, stream pb.Broker_SubscribeStream) error {
	ns := namespace.FromContext(ctx)
	errChan := make(chan error, 1)

	// authorize the request
	if err := namespace.Authorize(ctx, ns); err == namespace.ErrForbidden {
		return errors.Forbidden("go.micro.broker", err.Error())
	} else if err == namespace.ErrUnauthorized {
		return errors.Unauthorized("go.micro.broker", err.Error())
	} else if err != nil {
		return errors.InternalServerError("go.micro.broker", err.Error())
	}

	// message handler to stream back messages from broker
	handler := func(p broker.Event) error {
		if err := stream.Send(&pb.Message{
			Header: p.Message().Header,
			Body:   p.Message().Body,
		}); err != nil {
			select {
			case errChan <- err:
				return err
			default:
				return err
			}
		}
		return nil
	}

	log.Debugf("Subscribing to %s topic in namespace %v", req.Topic, ns)
	sub, err := b.Broker.Subscribe(ns+"."+req.Topic, handler, broker.Queue(ns+"."+req.Queue))
	if err != nil {
		return errors.InternalServerError("go.micro.broker", err.Error())
	}
	defer func() {
		log.Debugf("Unsubscribing from topic %s in namespace %v", req.Topic, ns)
		sub.Unsubscribe()
	}()

	select {
	case <-ctx.Done():
		log.Debugf("Context done for subscription to topic %s", req.Topic)
		return nil
	case err := <-errChan:
		log.Debugf("Subscription error for topic %s: %v", req.Topic, err)
		return err
	}
}
<|endoftext|>


<commit_before>// Copyright © 2016 NAME HERE <EMAIL ADDRESS>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"github.com/tylerharter/open-lambda/lambda-generator/experimental/frontends"
	"github.com/tylerharter/open-lambda/lambda-generator/experimental/frontends/effe"
)

var (
	cfgFile     string
	frontendStr string
	fe          frontends.FrontEnd
)

// This represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   fmt.Sprintf("%s", os.Args[0]),
	Short: "Helps to template, organize, build and deploy OpenLambda Lambdas",
	Long:  "Helps to template, organize, build and deploy OpenLambda Lambdas",
	// Uncomment the following line if your bare application
	// has an action associated with it:
	//	Run: func(cmd *cobra.Command, args []string) { },
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	// Here you will define your flags and configuration settings.
	// Cobra supports Persistent Flags, which, if defined here,
	// will be global for your application.

	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.experimental.yaml)")
	RootCmd.PersistentFlags().StringVar(&frontendStr, "frontend", "effe", "OpenLambda frontend framework (default is effe)")
	// Cobra also supports local flags, which will only run
	// when this action is called directly.
	RootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")

	// find the .openlambda folder or warn user if not found
	olDir := findOlDir()
	if olDir == "" {
		fmt.Printf("WARNING: no .openlambda directory found (Have you called %s init yet?)\n\n", os.Args[0])
		return
	}

	// Here we select the frontend, based on user configs found from above
	switch frontendStr {
	case "effe":
		fe = effe.NewFrontEnd(olDir)
	default:
		fmt.Println("frontend %s is unsupported\n")
		os.Exit(1)
	}
}

func findOlDir() string {
	wd, err := os.Getwd()
	if err != nil {
		fmt.Printf("failed to get wd with err %v\n", err)
		os.Exit(1)
	}

	curr, err := filepath.Abs(wd)
	if err != nil {
		fmt.Printf("failed to get create abs path with err %v\n", err)
		os.Exit(1)
	}

	// Walk one dir up each loop
	// TODO "/" is ommitted. Do we want this?
	for ; curr != "/"; curr = filepath.Dir(curr) {
		dir := filepath.Join(curr, ".openlambda")
		info, err := os.Stat(dir)
		if err != nil {
			if !os.IsNotExist(err) {
				fmt.Printf("failed to get info on %s with err %v\n", dir, err)
				os.Exit(1)
			}
			continue
		}
		if !info.IsDir() {
			fmt.Printf("warning: non-directory .openlambda at %s\n", dir)
			continue
		}
		// found dir
		return dir
	}
	// caller will log
	return ""
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	if cfgFile != "" { // enable ability to specify config file via flag
		viper.SetConfigFile(cfgFile)
	}

	viper.SetConfigName(".experimental") // name of config file (without extension)
	viper.AddConfigPath("$HOME")         // adding home directory as first search path
	viper.AutomaticEnv()                 // read in environment variables that match

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		fmt.Println("Using config file:", viper.ConfigFileUsed())
	}
}
<commit_msg>remove duplicate default log<commit_after>// Copyright © 2016 NAME HERE <EMAIL ADDRESS>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"github.com/tylerharter/open-lambda/lambda-generator/experimental/frontends"
	"github.com/tylerharter/open-lambda/lambda-generator/experimental/frontends/effe"
)

var (
	cfgFile     string
	frontendStr string
	fe          frontends.FrontEnd
)

// This represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   fmt.Sprintf("%s", os.Args[0]),
	Short: "Helps to template, organize, build and deploy OpenLambda Lambdas",
	Long:  "Helps to template, organize, build and deploy OpenLambda Lambdas",
	// Uncomment the following line if your bare application
	// has an action associated with it:
	//	Run: func(cmd *cobra.Command, args []string) { },
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	// Here you will define your flags and configuration settings.
	// Cobra supports Persistent Flags, which, if defined here,
	// will be global for your application.

	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.experimental.yaml)")
	RootCmd.PersistentFlags().StringVar(&frontendStr, "frontend", "effe", "OpenLambda frontend framework")
	// Cobra also supports local flags, which will only run
	// when this action is called directly.
	RootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")

	// find the .openlambda folder or warn user if not found
	olDir := findOlDir()
	if olDir == "" {
		fmt.Printf("WARNING: no .openlambda directory found (Have you called %s init yet?)\n\n", os.Args[0])
		return
	}

	// Here we select the frontend, based on user configs found from above
	switch frontendStr {
	case "effe":
		fe = effe.NewFrontEnd(olDir)
	default:
		fmt.Println("frontend %s is unsupported\n")
		os.Exit(1)
	}
}

func findOlDir() string {
	wd, err := os.Getwd()
	if err != nil {
		fmt.Printf("failed to get wd with err %v\n", err)
		os.Exit(1)
	}

	curr, err := filepath.Abs(wd)
	if err != nil {
		fmt.Printf("failed to get create abs path with err %v\n", err)
		os.Exit(1)
	}

	// Walk one dir up each loop
	// TODO "/" is ommitted. Do we want this?
	for ; curr != "/"; curr = filepath.Dir(curr) {
		dir := filepath.Join(curr, ".openlambda")
		info, err := os.Stat(dir)
		if err != nil {
			if !os.IsNotExist(err) {
				fmt.Printf("failed to get info on %s with err %v\n", dir, err)
				os.Exit(1)
			}
			continue
		}
		if !info.IsDir() {
			fmt.Printf("warning: non-directory .openlambda at %s\n", dir)
			continue
		}
		// found dir
		return dir
	}
	// caller will log
	return ""
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	if cfgFile != "" { // enable ability to specify config file via flag
		viper.SetConfigFile(cfgFile)
	}

	viper.SetConfigName(".experimental") // name of config file (without extension)
	viper.AddConfigPath("$HOME")         // adding home directory as first search path
	viper.AutomaticEnv()                 // read in environment variables that match

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		fmt.Println("Using config file:", viper.ConfigFileUsed())
	}
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package avalanche

import (
	"fmt"

	"github.com/ava-labs/gecko/cache"
	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/snow/choices"
	"github.com/ava-labs/gecko/snow/consensus/avalanche"
	"github.com/ava-labs/gecko/snow/engine/common"
	"github.com/ava-labs/gecko/snow/engine/common/queue"
	"github.com/ava-labs/gecko/utils/formatting"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	// We cache processed vertices where height = c * stripeDistance for c = {1,2,3...}
	// This forms a "stripe" of cached DAG vertices at height stripeDistance, 2*stripeDistance, etc.
	// This helps to limit the number of repeated DAG traversals performed
	stripeDistance = 2000
	stripeWidth    = 5
	cacheSize      = 100000
)

// BootstrapConfig ...
type BootstrapConfig struct {
	common.Config

	// VtxBlocked tracks operations that are blocked on vertices
	// TxBlocked tracks operations that are blocked on transactions
	VtxBlocked, TxBlocked *queue.Jobs

	State State
	VM    DAGVM
}

type bootstrapper struct {
	BootstrapConfig
	metrics
	common.Bootstrapper

	// number of vertices fetched so far
	numFetched uint32

	// tracks which validators were asked for which containers in which requests
	outstandingRequests common.Requests

	// IDs of vertices that we will send a GetAncestors request for once we are
	// not at the max number of outstanding requests
	// Invariant: The intersection of needToFetch and outstandingRequests is
	//            empty
	needToFetch ids.Set

	// Contains IDs of vertices that have recently been processed
	processedCache *cache.LRU

	// true if bootstrapping is done
	finished bool

	// Called when bootstrapping is done
	onFinished func() error
}

// Initialize this engine.
func (b *bootstrapper) Initialize(config BootstrapConfig) error {
	b.BootstrapConfig = config
	b.processedCache = &cache.LRU{Size: cacheSize}

	b.VtxBlocked.SetParser(&vtxParser{
		log:         config.Context.Log,
		numAccepted: b.numBSVtx,
		numDropped:  b.numBSDroppedVtx,
		state:       b.State,
	})

	b.TxBlocked.SetParser(&txParser{
		log:         config.Context.Log,
		numAccepted: b.numBSTx,
		numDropped:  b.numBSDroppedTx,
		vm:          b.VM,
	})

	config.Bootstrapable = b
	b.Bootstrapper.Initialize(config.Config)
	return nil
}

// CurrentAcceptedFrontier ...
func (b *bootstrapper) CurrentAcceptedFrontier() ids.Set {
	acceptedFrontier := ids.Set{}
	acceptedFrontier.Add(b.State.Edge()...)
	return acceptedFrontier
}

// FilterAccepted ...
func (b *bootstrapper) FilterAccepted(containerIDs ids.Set) ids.Set {
	acceptedVtxIDs := ids.Set{}
	for _, vtxID := range containerIDs.List() {
		if vtx, err := b.State.GetVertex(vtxID); err == nil && vtx.Status() == choices.Accepted {
			acceptedVtxIDs.Add(vtxID)
		}
	}
	return acceptedVtxIDs
}

// Fetch vertices and their ancestors from the set of vertices that are needed
// to be fetched.
func (b *bootstrapper) fetch(vtxIDs ...ids.ID) error {
	b.needToFetch.Add(vtxIDs...)
	for b.needToFetch.Len() > 0 && b.outstandingRequests.Len() < common.MaxOutstandingRequests {
		vtxID := b.needToFetch.CappedList(1)[0]
		b.needToFetch.Remove(vtxID)

		// Make sure we haven't already requested this vertex
		if b.outstandingRequests.Contains(vtxID) {
			continue
		}

		// Make sure we don't already have this vertex
		if _, err := b.State.GetVertex(vtxID); err == nil {
			continue
		}

		validators := b.BootstrapConfig.Validators.Sample(1) // validator to send request to
		if len(validators) == 0 {
			return fmt.Errorf("Dropping request for %s as there are no validators", vtxID)
		}
		validatorID := validators[0].ID()
		b.RequestID++

		b.outstandingRequests.Add(validatorID, b.RequestID, vtxID)
		b.needToFetch.Remove(vtxID)                                            // maintains invariant that intersection with outstandingRequests is empty
		b.BootstrapConfig.Sender.GetAncestors(validatorID, b.RequestID, vtxID) // request vertex and ancestors
	}
	return b.finish()
}

// Process vertices
func (b *bootstrapper) process(vtxs ...avalanche.Vertex) error {
	toProcess := newMaxVertexHeap()
	for _, vtx := range vtxs {
		if _, ok := b.processedCache.Get(vtx.ID()); !ok { // only process if we haven't already
			toProcess.Push(vtx)
		}
	}

	for toProcess.Len() > 0 {
		vtx := toProcess.Pop()
		vtxID := vtx.ID()

		switch vtx.Status() {
		case choices.Unknown:
			b.fetch(vtxID)
		case choices.Rejected:
			b.needToFetch.Remove(vtxID)
			return fmt.Errorf("tried to accept %s even though it was previously rejected", vtx.ID())
		case choices.Processing:
			b.needToFetch.Remove(vtxID)

			if err := b.VtxBlocked.Push(&vertexJob{
				log:         b.BootstrapConfig.Context.Log,
				numAccepted: b.numBSVtx,
				numDropped:  b.numBSDroppedVtx,
				vtx:         vtx,
			}); err == nil {
				b.numBSBlockedVtx.Inc()
				b.numFetched++ // Progress tracker
				if b.numFetched%common.StatusUpdateFrequency == 0 {
					b.BootstrapConfig.Context.Log.Info("fetched %d vertices", b.numFetched)
				}
			} else {
				b.BootstrapConfig.Context.Log.Verbo("couldn't push to vtxBlocked: %s", err)
			}
			for _, tx := range vtx.Txs() {
				if err := b.TxBlocked.Push(&txJob{
					log:         b.BootstrapConfig.Context.Log,
					numAccepted: b.numBSTx,
					numDropped:  b.numBSDroppedTx,
					tx:          tx,
				}); err == nil {
					b.numBSBlockedTx.Inc()
				} else {
					b.BootstrapConfig.Context.Log.Verbo("couldn't push to txBlocked: %s", err)
				}
			}
			for _, parent := range vtx.Parents() {
				if _, ok := b.processedCache.Get(parent.ID()); !ok { // already processed this
					toProcess.Push(parent)
				}
			}
			if vtx.Height()%stripeDistance < stripeWidth {
				b.processedCache.Put(vtx.ID(), nil)
			}
		}
	}

	if err := b.VtxBlocked.Commit(); err != nil {
		return err
	}
	if err := b.TxBlocked.Commit(); err != nil {
		return err
	}

	return b.fetch()
}

// MultiPut handles the receipt of multiple containers. Should be received in response to a GetAncestors message to [vdr]
// with request ID [requestID]. Expects vtxs[0] to be the vertex requested in the corresponding GetAncestors.
func (b *bootstrapper) MultiPut(vdr ids.ShortID, requestID uint32, vtxs [][]byte) error {
	if lenVtxs := len(vtxs); lenVtxs > common.MaxContainersPerMultiPut {
		b.BootstrapConfig.Context.Log.Debug("MultiPut(%s, %d) contains more than maximum number of vertices", vdr, requestID)
		return b.GetAncestorsFailed(vdr, requestID)
	} else if lenVtxs == 0 {
		b.BootstrapConfig.Context.Log.Debug("MultiPut(%s, %d) contains no vertices", vdr, requestID)
		return b.GetAncestorsFailed(vdr, requestID)
	}

	// Make sure this is in response to a request we made
	neededVtxID, needed := b.outstandingRequests.Remove(vdr, requestID)
	if !needed { // this message isn't in response to a request we made
		b.BootstrapConfig.Context.Log.Debug("received unexpected MultiPut from %s with ID %d", vdr, requestID)
		return nil
	}

	neededVtx, err := b.State.ParseVertex(vtxs[0]) // the vertex we requested
	if err != nil {
		b.BootstrapConfig.Context.Log.Debug("Failed to parse requested vertex %s: %w", neededVtxID, err)
		b.BootstrapConfig.Context.Log.Verbo("vertex: %s", formatting.DumpBytes{Bytes: vtxs[0]})
		return b.fetch(neededVtxID)
	} else if actualID := neededVtx.ID(); !actualID.Equals(neededVtxID) {
		b.BootstrapConfig.Context.Log.Debug("expected the first block to be the requested block, %s, but is %s", neededVtxID, actualID)
		return b.fetch(neededVtxID)
	}

	processVertices := make([]avalanche.Vertex, 1, len(vtxs))
	processVertices[0] = neededVtx

	for _, vtxBytes := range vtxs[1:] { // Parse/persist all the vertices
		if vtx, err := b.State.ParseVertex(vtxBytes); err != nil { // Persists the vtx
			b.BootstrapConfig.Context.Log.Debug("Failed to parse vertex: %w", err)
			b.BootstrapConfig.Context.Log.Verbo("vertex: %s", formatting.DumpBytes{Bytes: vtxBytes})
		} else {
			processVertices = append(processVertices, vtx)
			b.needToFetch.Remove(vtx.ID()) // No need to fetch this vertex since we have it now
		}
	}

	return b.process(processVertices...)
}

// GetAncestorsFailed is called when a GetAncestors message we sent fails
func (b *bootstrapper) GetAncestorsFailed(vdr ids.ShortID, requestID uint32) error {
	vtxID, ok := b.outstandingRequests.Remove(vdr, requestID)
	if !ok {
		b.BootstrapConfig.Context.Log.Debug("GetAncestorsFailed(%s, %d) called but there was no outstanding request to this validator with this ID", vdr, requestID)
		return nil
	}
	// Send another request for the vertex
	return b.fetch(vtxID)
}

// ForceAccepted ...
func (b *bootstrapper) ForceAccepted(acceptedContainerIDs ids.Set) error {
	if err := b.VM.Bootstrapping(); err != nil {
		return fmt.Errorf("failed to notify VM that bootstrapping has started: %w",
			err)
	}

	storedVtxs := make([]avalanche.Vertex, 0, acceptedContainerIDs.Len())
	for _, vtxID := range acceptedContainerIDs.List() {
		if vtx, err := b.State.GetVertex(vtxID); err == nil {
			storedVtxs = append(storedVtxs, vtx)
		} else {
			b.needToFetch.Add(vtxID)
		}
	}
	if err := b.process(storedVtxs...); err != nil {
		return err
	}
	return b.fetch()
}

// Finish bootstrapping
func (b *bootstrapper) finish() error {
	if b.finished || b.outstandingRequests.Len() > 0 || b.needToFetch.Len() > 0 {
		return nil
	}
	b.BootstrapConfig.Context.Log.Info("finished fetching vertices. executing transaction state transitions...")

	if err := b.executeAll(b.TxBlocked, b.numBSBlockedTx); err != nil {
		return err
	}

	b.BootstrapConfig.Context.Log.Info("executing vertex state transitions...")

	if err := b.executeAll(b.VtxBlocked, b.numBSBlockedVtx); err != nil {
		return err
	}

	if err := b.VM.Bootstrapped(); err != nil {
		return fmt.Errorf("failed to notify VM that bootstrapping has finished: %w",
			err)
	}

	// Start consensus
	if err := b.onFinished(); err != nil {
		return err
	}
	b.finished = true
	return nil
}

func (b *bootstrapper) executeAll(jobs *queue.Jobs, numBlocked prometheus.Gauge) error {
	numExecuted := 0
	for job, err := jobs.Pop(); err == nil; job, err = jobs.Pop() {
		numBlocked.Dec()
		b.BootstrapConfig.Context.Log.Debug("Executing: %s", job.ID())
		if err := jobs.Execute(job); err != nil {
			b.BootstrapConfig.Context.Log.Error("Error executing: %s", err)
			return err
		}
		if err := jobs.Commit(); err != nil {
			return err
		}
		numExecuted++
		if numExecuted%common.StatusUpdateFrequency == 0 { // Periodically print progress
			b.BootstrapConfig.Context.Log.Info("executed %d operations", numExecuted)
		}
	}
	return nil
}
<commit_msg>Removed no longer upheld invariant<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package avalanche

import (
	"fmt"

	"github.com/ava-labs/gecko/cache"
	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/snow/choices"
	"github.com/ava-labs/gecko/snow/consensus/avalanche"
	"github.com/ava-labs/gecko/snow/engine/common"
	"github.com/ava-labs/gecko/snow/engine/common/queue"
	"github.com/ava-labs/gecko/utils/formatting"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	// We cache processed vertices where height = c * stripeDistance for c = {1,2,3...}
	// This forms a "stripe" of cached DAG vertices at height stripeDistance, 2*stripeDistance, etc.
	// This helps to limit the number of repeated DAG traversals performed
	stripeDistance = 2000
	stripeWidth    = 5
	cacheSize      = 100000
)

// BootstrapConfig ...
type BootstrapConfig struct {
	common.Config

	// VtxBlocked tracks operations that are blocked on vertices
	// TxBlocked tracks operations that are blocked on transactions
	VtxBlocked, TxBlocked *queue.Jobs

	State State
	VM    DAGVM
}

type bootstrapper struct {
	BootstrapConfig
	metrics
	common.Bootstrapper

	// number of vertices fetched so far
	numFetched uint32

	// tracks which validators were asked for which containers in which requests
	outstandingRequests common.Requests

	// IDs of vertices that we will send a GetAncestors request for once we are
	// not at the max number of outstanding requests
	needToFetch ids.Set

	// Contains IDs of vertices that have recently been processed
	processedCache *cache.LRU

	// true if bootstrapping is done
	finished bool

	// Called when bootstrapping is done
	onFinished func() error
}

// Initialize this engine.
func (b *bootstrapper) Initialize(config BootstrapConfig) error {
	b.BootstrapConfig = config
	b.processedCache = &cache.LRU{Size: cacheSize}

	b.VtxBlocked.SetParser(&vtxParser{
		log:         config.Context.Log,
		numAccepted: b.numBSVtx,
		numDropped:  b.numBSDroppedVtx,
		state:       b.State,
	})

	b.TxBlocked.SetParser(&txParser{
		log:         config.Context.Log,
		numAccepted: b.numBSTx,
		numDropped:  b.numBSDroppedTx,
		vm:          b.VM,
	})

	config.Bootstrapable = b
	b.Bootstrapper.Initialize(config.Config)
	return nil
}

// CurrentAcceptedFrontier ...
func (b *bootstrapper) CurrentAcceptedFrontier() ids.Set {
	acceptedFrontier := ids.Set{}
	acceptedFrontier.Add(b.State.Edge()...)
	return acceptedFrontier
}

// FilterAccepted ...
func (b *bootstrapper) FilterAccepted(containerIDs ids.Set) ids.Set {
	acceptedVtxIDs := ids.Set{}
	for _, vtxID := range containerIDs.List() {
		if vtx, err := b.State.GetVertex(vtxID); err == nil && vtx.Status() == choices.Accepted {
			acceptedVtxIDs.Add(vtxID)
		}
	}
	return acceptedVtxIDs
}

// Fetch vertices and their ancestors from the set of vertices that are needed
// to be fetched.
func (b *bootstrapper) fetch(vtxIDs ...ids.ID) error {
	b.needToFetch.Add(vtxIDs...)
	for b.needToFetch.Len() > 0 && b.outstandingRequests.Len() < common.MaxOutstandingRequests {
		vtxID := b.needToFetch.CappedList(1)[0]
		b.needToFetch.Remove(vtxID)

		// Make sure we haven't already requested this vertex
		if b.outstandingRequests.Contains(vtxID) {
			continue
		}

		// Make sure we don't already have this vertex
		if _, err := b.State.GetVertex(vtxID); err == nil {
			continue
		}

		validators := b.BootstrapConfig.Validators.Sample(1) // validator to send request to
		if len(validators) == 0 {
			return fmt.Errorf("Dropping request for %s as there are no validators", vtxID)
		}
		validatorID := validators[0].ID()
		b.RequestID++

		b.outstandingRequests.Add(validatorID, b.RequestID, vtxID)
		b.needToFetch.Remove(vtxID)                                            // maintains invariant that intersection with outstandingRequests is empty
		b.BootstrapConfig.Sender.GetAncestors(validatorID, b.RequestID, vtxID) // request vertex and ancestors
	}
	return b.finish()
}

// Process vertices
func (b *bootstrapper) process(vtxs ...avalanche.Vertex) error {
	toProcess := newMaxVertexHeap()
	for _, vtx := range vtxs {
		if _, ok := b.processedCache.Get(vtx.ID()); !ok { // only process if we haven't already
			toProcess.Push(vtx)
		}
	}

	for toProcess.Len() > 0 {
		vtx := toProcess.Pop()
		vtxID := vtx.ID()

		switch vtx.Status() {
		case choices.Unknown:
			b.fetch(vtxID)
		case choices.Rejected:
			b.needToFetch.Remove(vtxID)
			return fmt.Errorf("tried to accept %s even though it was previously rejected", vtx.ID())
		case choices.Processing:
			b.needToFetch.Remove(vtxID)

			if err := b.VtxBlocked.Push(&vertexJob{
				log:         b.BootstrapConfig.Context.Log,
				numAccepted: b.numBSVtx,
				numDropped:  b.numBSDroppedVtx,
				vtx:         vtx,
			}); err == nil {
				b.numBSBlockedVtx.Inc()
				b.numFetched++ // Progress tracker
				if b.numFetched%common.StatusUpdateFrequency == 0 {
					b.BootstrapConfig.Context.Log.Info("fetched %d vertices", b.numFetched)
				}
			} else {
				b.BootstrapConfig.Context.Log.Verbo("couldn't push to vtxBlocked: %s", err)
			}
			for _, tx := range vtx.Txs() {
				if err := b.TxBlocked.Push(&txJob{
					log:         b.BootstrapConfig.Context.Log,
					numAccepted: b.numBSTx,
					numDropped:  b.numBSDroppedTx,
					tx:          tx,
				}); err == nil {
					b.numBSBlockedTx.Inc()
				} else {
					b.BootstrapConfig.Context.Log.Verbo("couldn't push to txBlocked: %s", err)
				}
			}
			for _, parent := range vtx.Parents() {
				if _, ok := b.processedCache.Get(parent.ID()); !ok { // already processed this
					toProcess.Push(parent)
				}
			}
			if vtx.Height()%stripeDistance < stripeWidth {
				b.processedCache.Put(vtx.ID(), nil)
			}
		}
	}

	if err := b.VtxBlocked.Commit(); err != nil {
		return err
	}
	if err := b.TxBlocked.Commit(); err != nil {
		return err
	}

	return b.fetch()
}

// MultiPut handles the receipt of multiple containers. Should be received in response to a GetAncestors message to [vdr]
// with request ID [requestID]. Expects vtxs[0] to be the vertex requested in the corresponding GetAncestors.
func (b *bootstrapper) MultiPut(vdr ids.ShortID, requestID uint32, vtxs [][]byte) error {
	if lenVtxs := len(vtxs); lenVtxs > common.MaxContainersPerMultiPut {
		b.BootstrapConfig.Context.Log.Debug("MultiPut(%s, %d) contains more than maximum number of vertices", vdr, requestID)
		return b.GetAncestorsFailed(vdr, requestID)
	} else if lenVtxs == 0 {
		b.BootstrapConfig.Context.Log.Debug("MultiPut(%s, %d) contains no vertices", vdr, requestID)
		return b.GetAncestorsFailed(vdr, requestID)
	}

	// Make sure this is in response to a request we made
	neededVtxID, needed := b.outstandingRequests.Remove(vdr, requestID)
	if !needed { // this message isn't in response to a request we made
		b.BootstrapConfig.Context.Log.Debug("received unexpected MultiPut from %s with ID %d", vdr, requestID)
		return nil
	}

	neededVtx, err := b.State.ParseVertex(vtxs[0]) // the vertex we requested
	if err != nil {
		b.BootstrapConfig.Context.Log.Debug("Failed to parse requested vertex %s: %w", neededVtxID, err)
		b.BootstrapConfig.Context.Log.Verbo("vertex: %s", formatting.DumpBytes{Bytes: vtxs[0]})
		return b.fetch(neededVtxID)
	} else if actualID := neededVtx.ID(); !actualID.Equals(neededVtxID) {
		b.BootstrapConfig.Context.Log.Debug("expected the first block to be the requested block, %s, but is %s", neededVtxID, actualID)
		return b.fetch(neededVtxID)
	}

	processVertices := make([]avalanche.Vertex, 1, len(vtxs))
	processVertices[0] = neededVtx

	for _, vtxBytes := range vtxs[1:] { // Parse/persist all the vertices
		if vtx, err := b.State.ParseVertex(vtxBytes); err != nil { // Persists the vtx
			b.BootstrapConfig.Context.Log.Debug("Failed to parse vertex: %w", err)
			b.BootstrapConfig.Context.Log.Verbo("vertex: %s", formatting.DumpBytes{Bytes: vtxBytes})
		} else {
			processVertices = append(processVertices, vtx)
			b.needToFetch.Remove(vtx.ID()) // No need to fetch this vertex since we have it now
		}
	}

	return b.process(processVertices...)
}

// GetAncestorsFailed is called when a GetAncestors message we sent fails
func (b *bootstrapper) GetAncestorsFailed(vdr ids.ShortID, requestID uint32) error {
	vtxID, ok := b.outstandingRequests.Remove(vdr, requestID)
	if !ok {
		b.BootstrapConfig.Context.Log.Debug("GetAncestorsFailed(%s, %d) called but there was no outstanding request to this validator with this ID", vdr, requestID)
		return nil
	}
	// Send another request for the vertex
	return b.fetch(vtxID)
}

// ForceAccepted ...
func (b *bootstrapper) ForceAccepted(acceptedContainerIDs ids.Set) error {
	if err := b.VM.Bootstrapping(); err != nil {
		return fmt.Errorf("failed to notify VM that bootstrapping has started: %w",
			err)
	}

	storedVtxs := make([]avalanche.Vertex, 0, acceptedContainerIDs.Len())
	for _, vtxID := range acceptedContainerIDs.List() {
		if vtx, err := b.State.GetVertex(vtxID); err == nil {
			storedVtxs = append(storedVtxs, vtx)
		} else {
			b.needToFetch.Add(vtxID)
		}
	}
	if err := b.process(storedVtxs...); err != nil {
		return err
	}
	return b.fetch()
}

// Finish bootstrapping
func (b *bootstrapper) finish() error {
	if b.finished || b.outstandingRequests.Len() > 0 || b.needToFetch.Len() > 0 {
		return nil
	}
	b.BootstrapConfig.Context.Log.Info("finished fetching vertices. executing transaction state transitions...")

	if err := b.executeAll(b.TxBlocked, b.numBSBlockedTx); err != nil {
		return err
	}

	b.BootstrapConfig.Context.Log.Info("executing vertex state transitions...")

	if err := b.executeAll(b.VtxBlocked, b.numBSBlockedVtx); err != nil {
		return err
	}

	if err := b.VM.Bootstrapped(); err != nil {
		return fmt.Errorf("failed to notify VM that bootstrapping has finished: %w",
			err)
	}

	// Start consensus
	if err := b.onFinished(); err != nil {
		return err
	}
	b.finished = true
	return nil
}

func (b *bootstrapper) executeAll(jobs *queue.Jobs, numBlocked prometheus.Gauge) error {
	numExecuted := 0
	for job, err := jobs.Pop(); err == nil; job, err = jobs.Pop() {
		numBlocked.Dec()
		b.BootstrapConfig.Context.Log.Debug("Executing: %s", job.ID())
		if err := jobs.Execute(job); err != nil {
			b.BootstrapConfig.Context.Log.Error("Error executing: %s", err)
			return err
		}
		if err := jobs.Commit(); err != nil {
			return err
		}
		numExecuted++
		if numExecuted%common.StatusUpdateFrequency == 0 { // Periodically print progress
			b.BootstrapConfig.Context.Log.Info("executed %d operations", numExecuted)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package command

import (
	"os"
)

type MountOptions struct {
	filer                       *string
	filerMountRootPath          *string
	dir                         *string
	dirListCacheLimit           *int64
	collection                  *string
	replication                 *string
	ttlSec                      *int
	chunkSizeLimitMB            *int
	cacheDir                    *string
	cacheSizeMB                 *int64
	dataCenter                  *string
	allowOthers                 *bool
	umaskString                 *string
	nonempty                    *bool
	outsideContainerClusterMode *bool
}

var (
	mountOptions    MountOptions
	mountCpuProfile *string
	mountMemProfile *string
)

func init() {
	cmdMount.Run = runMount // break init cycle
	mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location")
	mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server")
	mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory")
	mountOptions.dirListCacheLimit = cmdMount.Flag.Int64("dirListCacheLimit", 1000000, "limit cache size to speed up directory long format listing")
	mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files")
	mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.")
	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 4, "local write buffer size, also chunk large files")
	mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks")
	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local cache capacity in MB")
	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
	mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")
	mountOptions.nonempty = cmdMount.Flag.Bool("nonempty", false, "allows the mounting over a non-empty directory")
	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
	mountOptions.outsideContainerClusterMode = cmdMount.Flag.Bool("outsideContainerClusterMode", false, "allows other users to access the file system")
}

var cmdMount = &Command{
	UsageLine: "mount -filer=localhost:8888 -dir=/some/dir",
	Short:     "mount weed filer to a directory as file system in userspace(FUSE)",
	Long: `mount weed filer to userspace.

  Pre-requisites:
  1) have SeaweedFS master and volume servers running
  2) have a "weed filer" running
  These 2 requirements can be achieved with one command "weed server -filer=true"

  This uses github.com/seaweedfs/fuse, which enables writing FUSE file systems on
  Linux, and OS X.

  On OS X, it requires OSXFUSE (http://osxfuse.github.com/).

  If the SeaweedFS system runs in a container cluster, e.g. managed by kubernetes or docker compose,
  the volume servers are not accessible by their own ip addresses.
  In "outsideContainerClusterMode", the mount will use the filer ip address instead, assuming:
    * All volume server containers are accessible through the same hostname or IP address as the filer.
    * All volume server container ports are open external to the cluster.

  `,
}
<commit_msg>set default chunk size to 16<commit_after>package command

import (
	"os"
)

type MountOptions struct {
	filer                       *string
	filerMountRootPath          *string
	dir                         *string
	dirListCacheLimit           *int64
	collection                  *string
	replication                 *string
	ttlSec                      *int
	chunkSizeLimitMB            *int
	cacheDir                    *string
	cacheSizeMB                 *int64
	dataCenter                  *string
	allowOthers                 *bool
	umaskString                 *string
	nonempty                    *bool
	outsideContainerClusterMode *bool
}

var (
	mountOptions    MountOptions
	mountCpuProfile *string
	mountMemProfile *string
)

func init() {
	cmdMount.Run = runMount // break init cycle
	mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location")
	mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server")
	mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory")
	mountOptions.dirListCacheLimit = cmdMount.Flag.Int64("dirListCacheLimit", 1000000, "limit cache size to speed up directory long format listing")
	mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files")
	mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.")
	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 16, "local write buffer size, also chunk large files")
	mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks")
	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local cache capacity in MB")
	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
	mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")
	mountOptions.nonempty = cmdMount.Flag.Bool("nonempty", false, "allows the mounting over a non-empty directory")
	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
	mountOptions.outsideContainerClusterMode = cmdMount.Flag.Bool("outsideContainerClusterMode", false, "allows other users to access the file system")
}

var cmdMount = &Command{
	UsageLine: "mount -filer=localhost:8888 -dir=/some/dir",
	Short:     "mount weed filer to a directory as file system in userspace(FUSE)",
	Long: `mount weed filer to userspace.

  Pre-requisites:
  1) have SeaweedFS master and volume servers running
  2) have a "weed filer" running
  These 2 requirements can be achieved with one command "weed server -filer=true"

  This uses github.com/seaweedfs/fuse, which enables writing FUSE file systems on
  Linux, and OS X.

  On OS X, it requires OSXFUSE (http://osxfuse.github.com/).

  If the SeaweedFS system runs in a container cluster, e.g. managed by kubernetes or docker compose,
  the volume servers are not accessible by their own ip addresses.
  In "outsideContainerClusterMode", the mount will use the filer ip address instead, assuming:
    * All volume server containers are accessible through the same hostname or IP address as the filer.
    * All volume server container ports are open external to the cluster.

  `,
}
<|endoftext|>


<commit_before>package vsphere

import (
	"log"

	"github.com/vmware/govmomi"
	"github.com/vmware/govmomi/find"
	"github.com/vmware/govmomi/object"
	"github.com/vmware/govmomi/vim25/mo"
	"github.com/vmware/govmomi/vim25/types"
	"golang.org/x/net/context"
)

const (
	defaultTimeZone = "Etc/UTC"
	defaultDomain   = "vsphere.local"
)

type networkInterface struct {
	deviceName string
	label      string
	ipAddress  string
	subnetMask string
}

type additionalHardDisk struct {
	size int64
	iops int64
}

type virtualMachine struct {
	name                string
	datacenter          string
	cluster             string
	resourcePool        string
	datastore           string
	vcpu                int
	memoryMb            int64
	template            string
	networkInterfaces   []networkInterface
	additionalHardDisks []additionalHardDisk
	gateway             string
	domain              string
	timeZone            string
	dnsSuffixes         []string
	dnsServers          []string
}

func (vm *virtualMachine) deployVirtualMachine(c *govmomi.Client) error {
	if len(vm.dnsServers) == 0 {
		vm.dnsServers = []string{
			"8.8.8.8",
			"8.8.4.4",
		}
	}

	if len(vm.dnsSuffixes) == 0 {
		vm.dnsSuffixes = []string{
			defaultDomain,
		}
	}

	if vm.domain == "" {
		vm.domain = defaultDomain
	}

	if vm.timeZone == "" {
		vm.timeZone = defaultTimeZone
	}

	finder := find.NewFinder(c.Client, true)
	dc, err := findDatacenter(finder, vm.datacenter)
	if err != nil {
		return err
	}
	finder = finder.SetDatacenter(dc)

	template, err := finder.VirtualMachine(context.TODO(), vm.template)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] template: %#v", template)

	resourcePool, err := findResourcePool(finder, vm.resourcePool, vm.cluster)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] resource pool: %#v", resourcePool)

	dcFolders, err := dc.Folders(context.TODO())
	if err != nil {
		return err
	}
	datastore, err := findDatastore(c, finder, dcFolders, template, resourcePool, vm.datastore)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] datastore: %#v", datastore)

	relocateSpec, err := getVMRelocateSpec(resourcePool, datastore, template)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] relocate spec: %v", relocateSpec)

	// network
	networkDevices := []types.BaseVirtualDeviceConfigSpec{}
	networkConfigs := []types.CustomizationAdapterMapping{}
	for _, network := range vm.networkInterfaces {
		// network device
		device, err := networkDevice(finder, network.label)
		if err != nil {
			return err
		}
		networkDevices = append(networkDevices, device)

		var ipSetting types.CustomizationIPSettings
		if network.ipAddress == "" {
			ipSetting = types.CustomizationIPSettings{
				Ip: &types.CustomizationDhcpIpGenerator{},
			}
		} else {
			ipSetting = types.CustomizationIPSettings{
				Gateway: []string{
					vm.gateway,
				},
				Ip: &types.CustomizationFixedIp{
					IpAddress: network.ipAddress,
				},
				SubnetMask: network.subnetMask,
			}
		}

		// network config
		config := types.CustomizationAdapterMapping{
			Adapter: ipSetting,
		}
		networkConfigs = append(networkConfigs, config)
	}
	log.Printf("[DEBUG] network configs: %v", networkConfigs[0].Adapter)

	// make config spec
	configSpec := types.VirtualMachineConfigSpec{
		NumCPUs:           vm.vcpu,
		NumCoresPerSocket: 1,
		MemoryMB:          vm.memoryMb,
		DeviceChange:      networkDevices,
	}
	log.Printf("[DEBUG] virtual machine config spec: %v", configSpec)

	// make custom spec
	customSpec := createCustomizationSpec(vm.name, vm.domain, vm.timeZone, vm.dnsSuffixes, vm.dnsServers, networkConfigs)
	log.Printf("[DEBUG] custom spec: %v", customSpec)

	// make vm clone spec
	cloneSpec := types.VirtualMachineCloneSpec{
		Location:      relocateSpec,
		Template:      false,
		Config:        &configSpec,
		Customization: &customSpec,
		PowerOn:       true,
	}
	log.Printf("[DEBUG] clone spec: %v", cloneSpec)

	task, err := template.Clone(context.TODO(), dcFolders.VmFolder, vm.name, cloneSpec)
	if err != nil {
		return err
	}

	err = task.Wait(context.TODO())
	if err != nil {
		return err
	}

	newVM, err := finder.VirtualMachine(context.TODO(), vm.name)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] new vm: %v", newVM)

	ip, err := newVM.WaitForIP(context.TODO())
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] ip address: %v", ip)

	for _, hd := range vm.additionalHardDisks {
		err = addHardDisk(newVM, hd.size, hd.iops)
		if err != nil {
			return err
		}
	}

	return nil
}

func findDatastoreForClone(c *govmomi.Client, storagePod *object.Folder, template *object.VirtualMachine, vmFolder *object.Folder, resourcePool *object.ResourcePool) (*object.Datastore, error) {

	templateRef := template.Reference()
	vmFolderRef := vmFolder.Reference()
	resourcePoolRef := resourcePool.Reference()
	storagePodRef := storagePod.Reference()

	var o mo.VirtualMachine
	err := template.Properties(context.TODO(), templateRef, []string{"datastore"}, &o)
	if err != nil {
		return nil, err
	}
	templateDatastore := object.NewDatastore(c.Client, o.Datastore[0])
	log.Printf("[DEBUG] %#v\n", templateDatastore)

	devices, err := template.Device(context.TODO())
	if err != nil {
		return nil, err
	}

	var key int
	for _, d := range devices.SelectByType((*types.VirtualDisk)(nil)) {
		key = d.GetVirtualDevice().Key
		log.Printf("[DEBUG] %#v\n", d.GetVirtualDevice())
	}

	sps := types.StoragePlacementSpec{
		Type: "clone",
		Vm:   &templateRef,
		PodSelectionSpec: types.StorageDrsPodSelectionSpec{
			StoragePod: &storagePodRef,
		},
		CloneSpec: &types.VirtualMachineCloneSpec{
			Location: types.VirtualMachineRelocateSpec{
				Disk: []types.VirtualMachineRelocateSpecDiskLocator{
					types.VirtualMachineRelocateSpecDiskLocator{
						Datastore:       templateDatastore.Reference(),
						DiskBackingInfo: &types.VirtualDiskFlatVer2BackingInfo{},
						DiskId:          key,
					},
				},
				Pool: &resourcePoolRef,
			},
			PowerOn:  false,
			Template: false,
		},
		CloneName: "dummy",
		Folder:    &vmFolderRef,
	}
	log.Printf("[DEBUG] findDatastoreForClone: StoragePlacementSpec: %v", sps)

	srm := object.NewStorageResourceManager(c.Client)
	result, err := srm.RecommendDatastores(context.TODO(), sps)
	if err != nil {
		return nil, err
	}
	log.Printf("[DEBUG] findDatastoreForClone: result: %v", result)
	spa := result.Recommendations[0].Action[0].(*types.StoragePlacementAction)
	datastore := object.NewDatastore(c.Client, spa.Destination)

	return datastore, nil
}

// findDatastore finds Datastore object.
func findDatastore(c *govmomi.Client, finder *find.Finder, f *object.DatacenterFolders, template *object.VirtualMachine, resourcePool *object.ResourcePool, name string) (*object.Datastore, error) {
	if name == "" {
		datastore, err := finder.DefaultDatastore(context.TODO())
		if err != nil {
			return nil, err
		}
		log.Printf("[DEBUG] findDatastore: datastore: %#v", datastore)
		return datastore, nil
	} else {
		var datastore *object.Datastore
		s := object.NewSearchIndex(c.Client)
		ref, err := s.FindChild(context.TODO(), f.DatastoreFolder, name)
		if err != nil {
			return nil, err
		}
		log.Printf("[DEBUG] findDatastore: reference: %#v", ref)

		mor := ref.Reference()
		if mor.Type == "StoragePod" {
			s := object.NewFolder(c.Client, mor)
			datastore, err = findDatastoreForClone(c, s, template, f.VmFolder, resourcePool)
			if err != nil {
				return nil, err
			}
		} else {
			datastore = object.NewDatastore(c.Client, mor)
		}
		log.Printf("[DEBUG] findDatastore: datastore: %#v", datastore)
		return datastore, nil
	}
}

func networkDevice(f *find.Finder, label string) (*types.VirtualDeviceConfigSpec, error) {
	network, err := f.Network(context.TODO(), "*"+label)
	if err != nil {
		return nil, err
	}
	backing, err := network.EthernetCardBackingInfo(context.TODO())
	if err != nil {
		return nil, err
	}

	d := types.VirtualDeviceConfigSpec{
		Operation: types.VirtualDeviceConfigSpecOperationAdd,
		Device: &types.VirtualVmxnet3{
			types.VirtualVmxnet{
				types.VirtualEthernetCard{
					VirtualDevice: types.VirtualDevice{
						Key:     -1,
						Backing: backing,
					},
					AddressType: string(types.VirtualEthernetCardMacTypeGenerated),
				},
			},
		},
	}
	return &d, nil
}

// findDatacenter finds Datacenter object.
func findDatacenter(f *find.Finder, name string) (*object.Datacenter, error) {
	if name != "" {
		dc, err := f.Datacenter(context.TODO(), name)
		if err != nil {
			return nil, err
		}
		return dc, nil
	} else {
		dc, err := f.DefaultDatacenter(context.TODO())
		if err != nil {
			return nil, err
		}
		return dc, nil
	}
}

// findResourcePool finds ResourcePool object
func findResourcePool(f *find.Finder, name, cluster string) (*object.ResourcePool, error) {
	if name == "" {
		if cluster == "" {
			resourcePool, err := f.DefaultResourcePool(context.TODO())
			if err != nil {
				return nil, err
			}
			return resourcePool, nil
		} else {
			resourcePool, err := f.ResourcePool(context.TODO(), "*"+cluster+"/Resources")
			if err != nil {
				return nil, err
			}
			return resourcePool, nil
		}
	} else {
		resourcePool, err := f.ResourcePool(context.TODO(), name)
		if err != nil {
			return nil, err
		}
		return resourcePool, nil
	}
}

func getVMRelocateSpec(rp *object.ResourcePool, ds *object.Datastore, vm *object.VirtualMachine) (types.VirtualMachineRelocateSpec, error) {
	var key int

	devices, err := vm.Device(context.TODO())
	if err != nil {
		return types.VirtualMachineRelocateSpec{}, err
	}
	for _, d := range devices {
		if devices.Type(d) == "disk" {
			key = d.GetVirtualDevice().Key
		}
	}

	rpr := rp.Reference()
	dsr := ds.Reference()
	return types.VirtualMachineRelocateSpec{
		Datastore: &dsr,
		Pool:      &rpr,
		Disk: []types.VirtualMachineRelocateSpecDiskLocator{
			types.VirtualMachineRelocateSpecDiskLocator{
				Datastore: dsr,
				DiskBackingInfo: &types.VirtualDiskFlatVer2BackingInfo{
					DiskMode:        "persistent",
					ThinProvisioned: types.NewBool(false),
					EagerlyScrub:    types.NewBool(true),
				},
				DiskId: key,
			},
		},
	}, nil
}

// createCustomizationSpec creates the CustomizationSpec object.
func createCustomizationSpec(name, domain, tz string, suffixes, servers []string, nics []types.CustomizationAdapterMapping) types.CustomizationSpec {
	return types.CustomizationSpec{
		Identity: &types.CustomizationLinuxPrep{
			HostName: &types.CustomizationFixedName{
				Name: name,
			},
			Domain:     domain,
			TimeZone:   tz,
			HwClockUTC: types.NewBool(true),
		},
		GlobalIPSettings: types.CustomizationGlobalIPSettings{
			DnsSuffixList: suffixes,
			DnsServerList: servers,
		},
		NicSettingMap: nics,
	}
}

func addHardDisk(vm *object.VirtualMachine, size, iops int64) error {
	devices, err := vm.Device(context.TODO())
	if err != nil {
		return err
	}

	controller, err := devices.FindDiskController("scsi")
	if err != nil {
		log.Printf("[ERROR] %s", err)
	}
	log.Printf("[DEBUG] %#v\n", controller)

	disk := devices.CreateDisk(controller, "")

	existing := devices.SelectByBackingInfo(disk.Backing)
	log.Printf("[DEBUG] %#v\n", existing)

	if len(existing) == 0 {
		disk.CapacityInKB = int64(size * 1024 * 1024)
		disk.StorageIOAllocation = &types.StorageIOAllocationInfo{
			Limit: iops,
		}
	} else {
		log.Printf("Disk already present\n")
	}

	backing := disk.Backing.(*types.VirtualDiskFlatVer2BackingInfo)

	// eager zeroed thick virtual disk
	backing.ThinProvisioned = types.NewBool(false)
	backing.EagerlyScrub = types.NewBool(true)

	return vm.AddDevice(context.TODO(), disk)
}
<commit_msg>Add if statement for empty of iops value<commit_after>package vsphere

import (
	"log"

	"github.com/vmware/govmomi"
	"github.com/vmware/govmomi/find"
	"github.com/vmware/govmomi/object"
	"github.com/vmware/govmomi/vim25/mo"
	"github.com/vmware/govmomi/vim25/types"
	"golang.org/x/net/context"
)

const (
	defaultTimeZone = "Etc/UTC"
	defaultDomain   = "vsphere.local"
)

type networkInterface struct {
	deviceName string
	label      string
	ipAddress  string
	subnetMask string
}

type additionalHardDisk struct {
	size int64
	iops int64
}

type virtualMachine struct {
	name                string
	datacenter          string
	cluster             string
	resourcePool        string
	datastore           string
	vcpu                int
	memoryMb            int64
	template            string
	networkInterfaces   []networkInterface
	additionalHardDisks []additionalHardDisk
	gateway             string
	domain              string
	timeZone            string
	dnsSuffixes         []string
	dnsServers          []string
}

func (vm *virtualMachine) deployVirtualMachine(c *govmomi.Client) error {
	if len(vm.dnsServers) == 0 {
		vm.dnsServers = []string{
			"8.8.8.8",
			"8.8.4.4",
		}
	}

	if len(vm.dnsSuffixes) == 0 {
		vm.dnsSuffixes = []string{
			defaultDomain,
		}
	}

	if vm.domain == "" {
		vm.domain = defaultDomain
	}

	if vm.timeZone == "" {
		vm.timeZone = defaultTimeZone
	}

	finder := find.NewFinder(c.Client, true)
	dc, err := findDatacenter(finder, vm.datacenter)
	if err != nil {
		return err
	}
	finder = finder.SetDatacenter(dc)

	template, err := finder.VirtualMachine(context.TODO(), vm.template)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] template: %#v", template)

	resourcePool, err := findResourcePool(finder, vm.resourcePool, vm.cluster)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] resource pool: %#v", resourcePool)

	dcFolders, err := dc.Folders(context.TODO())
	if err != nil {
		return err
	}
	datastore, err := findDatastore(c, finder, dcFolders, template, resourcePool, vm.datastore)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] datastore: %#v", datastore)

	relocateSpec, err := getVMRelocateSpec(resourcePool, datastore, template)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] relocate spec: %v", relocateSpec)

	// network
	networkDevices := []types.BaseVirtualDeviceConfigSpec{}
	networkConfigs := []types.CustomizationAdapterMapping{}
	for _, network := range vm.networkInterfaces {
		// network device
		device, err := networkDevice(finder, network.label)
		if err != nil {
			return err
		}
		networkDevices = append(networkDevices, device)

		var ipSetting types.CustomizationIPSettings
		if network.ipAddress == "" {
			ipSetting = types.CustomizationIPSettings{
				Ip: &types.CustomizationDhcpIpGenerator{},
			}
		} else {
			ipSetting = types.CustomizationIPSettings{
				Gateway: []string{
					vm.gateway,
				},
				Ip: &types.CustomizationFixedIp{
					IpAddress: network.ipAddress,
				},
				SubnetMask: network.subnetMask,
			}
		}

		// network config
		config := types.CustomizationAdapterMapping{
			Adapter: ipSetting,
		}
		networkConfigs = append(networkConfigs, config)
	}
	log.Printf("[DEBUG] network configs: %v", networkConfigs[0].Adapter)

	// make config spec
	configSpec := types.VirtualMachineConfigSpec{
		NumCPUs:           vm.vcpu,
		NumCoresPerSocket: 1,
		MemoryMB:          vm.memoryMb,
		DeviceChange:      networkDevices,
	}
	log.Printf("[DEBUG] virtual machine config spec: %v", configSpec)

	// make custom spec
	customSpec := createCustomizationSpec(vm.name, vm.domain, vm.timeZone, vm.dnsSuffixes, vm.dnsServers, networkConfigs)
	log.Printf("[DEBUG] custom spec: %v", customSpec)

	// make vm clone spec
	cloneSpec := types.VirtualMachineCloneSpec{
		Location:      relocateSpec,
		Template:      false,
		Config:        &configSpec,
		Customization: &customSpec,
		PowerOn:       true,
	}
	log.Printf("[DEBUG] clone spec: %v", cloneSpec)

	task, err := template.Clone(context.TODO(), dcFolders.VmFolder, vm.name, cloneSpec)
	if err != nil {
		return err
	}

	err = task.Wait(context.TODO())
	if err != nil {
		return err
	}

	newVM, err := finder.VirtualMachine(context.TODO(), vm.name)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] new vm: %v", newVM)

	ip, err := newVM.WaitForIP(context.TODO())
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] ip address: %v", ip)

	for _, hd := range vm.additionalHardDisks {
		err = addHardDisk(newVM, hd.size, hd.iops)
		if err != nil {
			return err
		}
	}

	return nil
}

func findDatastoreForClone(c *govmomi.Client, storagePod *object.Folder, template *object.VirtualMachine, vmFolder *object.Folder, resourcePool *object.ResourcePool) (*object.Datastore, error) {

	templateRef := template.Reference()
	vmFolderRef := vmFolder.Reference()
	resourcePoolRef := resourcePool.Reference()
	storagePodRef := storagePod.Reference()

	var o mo.VirtualMachine
	err := template.Properties(context.TODO(), templateRef, []string{"datastore"}, &o)
	if err != nil {
		return nil, err
	}
	templateDatastore := object.NewDatastore(c.Client, o.Datastore[0])
	log.Printf("[DEBUG] %#v\n", templateDatastore)

	devices, err := template.Device(context.TODO())
	if err != nil {
		return nil, err
	}

	var key int
	for _, d := range devices.SelectByType((*types.VirtualDisk)(nil)) {
		key = d.GetVirtualDevice().Key
		log.Printf("[DEBUG] %#v\n", d.GetVirtualDevice())
	}

	sps := types.StoragePlacementSpec{
		Type: "clone",
		Vm:   &templateRef,
		PodSelectionSpec: types.StorageDrsPodSelectionSpec{
			StoragePod: &storagePodRef,
		},
		CloneSpec: &types.VirtualMachineCloneSpec{
			Location: types.VirtualMachineRelocateSpec{
				Disk: []types.VirtualMachineRelocateSpecDiskLocator{
					types.VirtualMachineRelocateSpecDiskLocator{
						Datastore:       templateDatastore.Reference(),
						DiskBackingInfo: &types.VirtualDiskFlatVer2BackingInfo{},
						DiskId:          key,
					},
				},
				Pool: &resourcePoolRef,
			},
			PowerOn:  false,
			Template: false,
		},
		CloneName: "dummy",
		Folder:    &vmFolderRef,
	}
	log.Printf("[DEBUG] findDatastoreForClone: StoragePlacementSpec: %v", sps)

	srm := object.NewStorageResourceManager(c.Client)
	result, err := srm.RecommendDatastores(context.TODO(), sps)
	if err != nil {
		return nil, err
	}
	log.Printf("[DEBUG] findDatastoreForClone: result: %v", result)
	spa := result.Recommendations[0].Action[0].(*types.StoragePlacementAction)
	datastore := object.NewDatastore(c.Client, spa.Destination)

	return datastore, nil
}

// findDatastore finds Datastore object.
func findDatastore(c *govmomi.Client, finder *find.Finder, f *object.DatacenterFolders, template *object.VirtualMachine, resourcePool *object.ResourcePool, name string) (*object.Datastore, error) {
	if name == "" {
		datastore, err := finder.DefaultDatastore(context.TODO())
		if err != nil {
			return nil, err
		}
		log.Printf("[DEBUG] findDatastore: datastore: %#v", datastore)
		return datastore, nil
	} else {
		var datastore *object.Datastore
		s := object.NewSearchIndex(c.Client)
		ref, err := s.FindChild(context.TODO(), f.DatastoreFolder, name)
		if err != nil {
			return nil, err
		}
		log.Printf("[DEBUG] findDatastore: reference: %#v", ref)

		mor := ref.Reference()
		if mor.Type == "StoragePod" {
			s := object.NewFolder(c.Client, mor)
			datastore, err = findDatastoreForClone(c, s, template, f.VmFolder, resourcePool)
			if err != nil {
				return nil, err
			}
		} else {
			datastore = object.NewDatastore(c.Client, mor)
		}
		log.Printf("[DEBUG] findDatastore: datastore: %#v", datastore)
		return datastore, nil
	}
}

func networkDevice(f *find.Finder, label string) (*types.VirtualDeviceConfigSpec, error) {
	network, err := f.Network(context.TODO(), "*"+label)
	if err != nil {
		return nil, err
	}
	backing, err := network.EthernetCardBackingInfo(context.TODO())
	if err != nil {
		return nil, err
	}

	d := types.VirtualDeviceConfigSpec{
		Operation: types.VirtualDeviceConfigSpecOperationAdd,
		Device: &types.VirtualVmxnet3{
			types.VirtualVmxnet{
				types.VirtualEthernetCard{
					VirtualDevice: types.VirtualDevice{
						Key:     -1,
						Backing: backing,
					},
					AddressType: string(types.VirtualEthernetCardMacTypeGenerated),
				},
			},
		},
	}
	return &d, nil
}

// findDatacenter finds Datacenter object.
func findDatacenter(f *find.Finder, name string) (*object.Datacenter, error) {
	if name != "" {
		dc, err := f.Datacenter(context.TODO(), name)
		if err != nil {
			return nil, err
		}
		return dc, nil
	} else {
		dc, err := f.DefaultDatacenter(context.TODO())
		if err != nil {
			return nil, err
		}
		return dc, nil
	}
}

// findResourcePool finds ResourcePool object
func findResourcePool(f *find.Finder, name, cluster string) (*object.ResourcePool, error) {
	if name == "" {
		if cluster == "" {
			resourcePool, err := f.DefaultResourcePool(context.TODO())
			if err != nil {
				return nil, err
			}
			return resourcePool, nil
		} else {
			resourcePool, err := f.ResourcePool(context.TODO(), "*"+cluster+"/Resources")
			if err != nil {
				return nil, err
			}
			return resourcePool, nil
		}
	} else {
		resourcePool, err := f.ResourcePool(context.TODO(), name)
		if err != nil {
			return nil, err
		}
		return resourcePool, nil
	}
}

func getVMRelocateSpec(rp *object.ResourcePool, ds *object.Datastore, vm *object.VirtualMachine) (types.VirtualMachineRelocateSpec, error) {
	var key int

	devices, err := vm.Device(context.TODO())
	if err != nil {
		return types.VirtualMachineRelocateSpec{}, err
	}
	for _, d := range devices {
		if devices.Type(d) == "disk" {
			key = d.GetVirtualDevice().Key
		}
	}

	rpr := rp.Reference()
	dsr := ds.Reference()
	return types.VirtualMachineRelocateSpec{
		Datastore: &dsr,
		Pool:      &rpr,
		Disk: []types.VirtualMachineRelocateSpecDiskLocator{
			types.VirtualMachineRelocateSpecDiskLocator{
				Datastore: dsr,
				DiskBackingInfo: &types.VirtualDiskFlatVer2BackingInfo{
					DiskMode:        "persistent",
					ThinProvisioned: types.NewBool(false),
					EagerlyScrub:    types.NewBool(true),
				},
				DiskId: key,
			},
		},
	}, nil
}

// createCustomizationSpec creates the CustomizationSpec object.
func createCustomizationSpec(name, domain, tz string, suffixes, servers []string, nics []types.CustomizationAdapterMapping) types.CustomizationSpec {
	return types.CustomizationSpec{
		Identity: &types.CustomizationLinuxPrep{
			HostName: &types.CustomizationFixedName{
				Name: name,
			},
			Domain:     domain,
			TimeZone:   tz,
			HwClockUTC: types.NewBool(true),
		},
		GlobalIPSettings: types.CustomizationGlobalIPSettings{
			DnsSuffixList: suffixes,
			DnsServerList: servers,
		},
		NicSettingMap: nics,
	}
}

func addHardDisk(vm *object.VirtualMachine, size, iops int64) error {
	devices, err := vm.Device(context.TODO())
	if err != nil {
		return err
	}

	controller, err := devices.FindDiskController("scsi")
	if err != nil {
		return err
	}

	disk := devices.CreateDisk(controller, "")
	existing := devices.SelectByBackingInfo(disk.Backing)

	if len(existing) == 0 {
		disk.CapacityInKB = int64(size * 1024 * 1024)
		if iops != 0 {
			disk.StorageIOAllocation = &types.StorageIOAllocationInfo{
				Limit: iops,
			}
		}
		backing := disk.Backing.(*types.VirtualDiskFlatVer2BackingInfo)

		// eager zeroed thick virtual disk
		backing.ThinProvisioned = types.NewBool(false)
		backing.EagerlyScrub = types.NewBool(true)

		log.Printf("[DEBUG] addHardDisk: %#v\n", disk)

		return vm.AddDevice(context.TODO(), disk)
	} else {
		log.Printf("[DEBUG] addHardDisk: Disk already present.\n")

		return nil
	}
}
<|endoftext|>


<commit_before>/*
 *   Copyright 2015 Benoit LETONDOR
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */

package main

// Rename this file conf.go and replace with actual values

//const USER_NAME = "twitterName"
//const ACTIONS_INTERVAL = "@every 30m"
//const WAKE_UP_HOUR int = 16
//const GO_TO_BED_HOUR int = 8

//const CONSUMER_KEY string = ""
//const CONSUMER_SECRET string = ""
//const TOKEN string = ""
//const TOKEN_SECRET string = ""

//const WIT_ACCESS_TOKEN string = ""

//const ACTION_FOLLOW_WEIGHT int = 1
//const ACTION_UNFOLLOW_WEIGHT int = 1
//const ACTION_FAVORITE_WEIGHT int = 1
//const ACTION_UNFAVORITE_WEIGHT int = 1
//const ACTION_TWEET_WEIGHT int = 1
//const ACTION_REPLY_WEIGHT int = 1
//const ACTION_NIGHTLY_FOLLOW_WEIGHT int = 1
//const ACTION_NIGHTLY_UNFOLLOW_WEIGHT int = 1
//const ACTION_NIGHTLY_FAVORITE_WEIGHT int = 1
//const ACTION_NIGHTLY_UNFAVORITE_WEIGHT int = 1
//const ACTION_NIGHTLY_TWEET_WEIGHT int = 1
//const ACTION_NIGHTLY_REPLY_WEIGHT int = 1
//const FAV_LIMIT_IN_A_ROW int = 5
//const UNFOLLOW_LIMIT_IN_A_ROW int = 5
//const UNFAVORITE_LIMIT_IN_A_ROW int = 1
//const MAX_TWEET_IN_A_DAY int = 5

//const T_CO_URL_LENGTH = 25

//const MYSQL_USER string = ""
//const MYSQL_PASSWORD string = ""
//const MYSQL_SCHEMA string = ""

//var KIMONO_DATA_SOURCES = []string{"https://www.kimonolabs.com/api", "https://www.kimonolabs.com/api"}
//var KEYWORDS = []string{"key1", "key2}
//var HASHTAGS = []string{"hash1", "hash2"}

//const ACCEPTED_LANGUAGE string = "en"

//var BANNED_KEYWORDS = []string{"porn"}
<commit_msg>Change conf example according to time.Duration format.<commit_after>/*
 *   Copyright 2015 Benoit LETONDOR
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */

package main

// Rename this file conf.go and replace with actual values

//const USER_NAME = "twitterName"
//const ACTIONS_INTERVAL = "30m"
//const WAKE_UP_HOUR int = 16
//const GO_TO_BED_HOUR int = 8

//const CONSUMER_KEY string = ""
//const CONSUMER_SECRET string = ""
//const TOKEN string = ""
//const TOKEN_SECRET string = ""

//const WIT_ACCESS_TOKEN string = ""

//const ACTION_FOLLOW_WEIGHT int = 1
//const ACTION_UNFOLLOW_WEIGHT int = 1
//const ACTION_FAVORITE_WEIGHT int = 1
//const ACTION_UNFAVORITE_WEIGHT int = 1
//const ACTION_TWEET_WEIGHT int = 1
//const ACTION_REPLY_WEIGHT int = 1
//const ACTION_NIGHTLY_FOLLOW_WEIGHT int = 1
//const ACTION_NIGHTLY_UNFOLLOW_WEIGHT int = 1
//const ACTION_NIGHTLY_FAVORITE_WEIGHT int = 1
//const ACTION_NIGHTLY_UNFAVORITE_WEIGHT int = 1
//const ACTION_NIGHTLY_TWEET_WEIGHT int = 1
//const ACTION_NIGHTLY_REPLY_WEIGHT int = 1
//const FAV_LIMIT_IN_A_ROW int = 5
//const UNFOLLOW_LIMIT_IN_A_ROW int = 5
//const UNFAVORITE_LIMIT_IN_A_ROW int = 1
//const MAX_TWEET_IN_A_DAY int = 5

//const T_CO_URL_LENGTH = 25

//const MYSQL_USER string = ""
//const MYSQL_PASSWORD string = ""
//const MYSQL_SCHEMA string = ""

//var KIMONO_DATA_SOURCES = []string{"https://www.kimonolabs.com/api", "https://www.kimonolabs.com/api"}
//var KEYWORDS = []string{"key1", "key2}
//var HASHTAGS = []string{"hash1", "hash2"}

//const ACCEPTED_LANGUAGE string = "en"

//var BANNED_KEYWORDS = []string{"porn"}
<|endoftext|>