<commit_before>// Package flatfs is a Datastore implementation that stores all
// objects in a two-level directory structure in the local file
// system, regardless of the hierarchy of the keys.
package flatfs

import (
	"encoding/hex"
	"errors"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/ipfs/go-datastore"
	"github.com/ipfs/go-datastore/query"
	"github.com/jbenet/go-os-rename"

	logging "github.com/ipfs/go-log"
)

var log = logging.Logger("flatfs")

const (
	extension    = ".data"
	maxPrefixLen = 16
)

var (
	ErrBadPrefixLen = errors.New("bad prefix length")
)

type Datastore struct {
	path string
	// length of the dir splay prefix, in bytes of hex digits
	hexPrefixLen int

	// sychronize all writes and directory changes for added safety
	sync bool
}

var _ datastore.Datastore = (*Datastore)(nil)

func New(path string, prefixLen int, sync bool) (*Datastore, error) {
	if prefixLen <= 0 || prefixLen > maxPrefixLen {
		return nil, ErrBadPrefixLen
	}
	fs := &Datastore{
		path: path,
		// convert from binary bytes to bytes of hex encoding
		hexPrefixLen: prefixLen * hex.EncodedLen(1),
		sync:         sync,
	}
	return fs, nil
}

var padding = strings.Repeat("_", maxPrefixLen*hex.EncodedLen(1))

func (fs *Datastore) encode(key datastore.Key) (dir, file string) {
	safe := hex.EncodeToString(key.Bytes()[1:])
	prefix := (safe + padding)[:fs.hexPrefixLen]
	dir = path.Join(fs.path, prefix)
	file = path.Join(dir, safe+extension)
	return dir, file
}

func (fs *Datastore) decode(file string) (key datastore.Key, ok bool) {
	if path.Ext(file) != extension {
		return datastore.Key{}, false
	}
	name := file[:len(file)-len(extension)]
	k, err := hex.DecodeString(name)
	if err != nil {
		return datastore.Key{}, false
	}
	return datastore.NewKey(string(k)), true
}

func (fs *Datastore) makePrefixDir(dir string) error {
	if err := fs.makePrefixDirNoSync(dir); err != nil {
		return err
	}

	// In theory, if we create a new prefix dir and add a file to
	// it, the creation of the prefix dir itself might not be
	// durable yet. Sync the root dir after a successful mkdir of
	// a prefix dir, just to be paranoid.
	if fs.sync {
		if err := syncDir(fs.path); err != nil {
			return err
		}
	}
	return nil
}

func (fs *Datastore) makePrefixDirNoSync(dir string) error {
	if err := os.Mkdir(dir, 0777); err != nil {
		// EEXIST is safe to ignore here, that just means the prefix
		// directory already existed.
		if !os.IsExist(err) {
			return err
		}
	}
	return nil
}

var putMaxRetries = 3

func (fs *Datastore) Put(key datastore.Key, value interface{}) error {
	val, ok := value.([]byte)
	if !ok {
		return datastore.ErrInvalidType
	}

	var err error
	for i := 0; i < putMaxRetries; i++ {
		err = fs.doPut(key, val)
		if err == nil {
			return nil
		}

		if !strings.Contains(err.Error(), "too many open files") {
			return err
		}

		log.Error("too many open files, retrying in %dms", 100*i)
		time.Sleep(time.Millisecond * 100 * time.Duration(i))
	}
	return err
}

func (fs *Datastore) doPut(key datastore.Key, val []byte) error {
	dir, path := fs.encode(key)
	if err := fs.makePrefixDir(dir); err != nil {
		return err
	}

	tmp, err := ioutil.TempFile(dir, "put-")
	if err != nil {
		return err
	}
	closed := false
	removed := false
	defer func() {
		if !closed {
			// silence errcheck
			_ = tmp.Close()
		}
		if !removed {
			// silence errcheck
			_ = os.Remove(tmp.Name())
		}
	}()

	if _, err := tmp.Write(val); err != nil {
		return err
	}
	if fs.sync {
		if err := tmp.Sync(); err != nil {
			return err
		}
	}
	if err := tmp.Close(); err != nil {
		return err
	}
	closed = true

	err = osrename.Rename(tmp.Name(), path)
	if err != nil {
		return err
	}
	removed = true

	if fs.sync {
		if err := syncDir(dir); err != nil {
			return err
		}
	}
	return nil
}

func (fs *Datastore) putMany(data map[datastore.Key]interface{}) error {
	var dirsToSync []string
	files := make(map[*os.File]string)

	for key, value := range data {
		val, ok := value.([]byte)
		if !ok {
			return datastore.ErrInvalidType
		}
		dir, path := fs.encode(key)
		if err := fs.makePrefixDirNoSync(dir); err != nil {
			return err
		}
		dirsToSync = append(dirsToSync, dir)

		tmp, err := ioutil.TempFile(dir, "put-")
		if err != nil {
			return err
		}

		if _, err := tmp.Write(val); err != nil {
			return err
		}

		files[tmp] = path
	}

	ops := make(map[*os.File]int)

	defer func() {
		for fi, _ := range files {
			val, _ := ops[fi]
			switch val {
			case 0:
				_ = fi.Close()
				fallthrough
			case 1:
				_ = os.Remove(fi.Name())
			}
		}
	}()

	// Now we sync everything
	// sync and close files
	for fi, _ := range files {
		if fs.sync {
			if err := fi.Sync(); err != nil {
				return err
			}
		}

		if err := fi.Close(); err != nil {
			return err
		}

		// signify closed
		ops[fi] = 1
	}

	// move files to their proper places
	for fi, path := range files {
		if err := osrename.Rename(fi.Name(), path); err != nil {
			return err
		}

		// signify removed
		ops[fi] = 2
	}

	// now sync the dirs for those files
	if fs.sync {
		for _, dir := range dirsToSync {
			if err := syncDir(dir); err != nil {
				return err
			}
		}

		// sync top flatfs dir
		if err := syncDir(fs.path); err != nil {
			return err
		}
	}

	return nil
}

func (fs *Datastore) Get(key datastore.Key) (value interface{}, err error) {
	_, path := fs.encode(key)
	data, err := ioutil.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, datastore.ErrNotFound
		}
		// no specific error to return, so just pass it through
		return nil, err
	}
	return data, nil
}

func (fs *Datastore) Has(key datastore.Key) (exists bool, err error) {
	_, path := fs.encode(key)
	switch _, err := os.Stat(path); {
	case err == nil:
		return true, nil
	case os.IsNotExist(err):
		return false, nil
	default:
		return false, err
	}
}

func (fs *Datastore) Delete(key datastore.Key) error {
	_, path := fs.encode(key)
	switch err := os.Remove(path); {
	case err == nil:
		return nil
	case os.IsNotExist(err):
		return datastore.ErrNotFound
	default:
		return err
	}
}

func (fs *Datastore) Query(q query.Query) (query.Results, error) {
	if (q.Prefix != "" && q.Prefix != "/") ||
		len(q.Filters) > 0 ||
		len(q.Orders) > 0 ||
		q.Limit > 0 ||
		q.Offset > 0 ||
		!q.KeysOnly {
		// TODO this is overly simplistic, but the only caller is
		// `ipfs refs local` for now, and this gets us moving.
		return nil, errors.New("flatfs only supports listing all keys in random order")
	}

	reschan := make(chan query.Result)
	go func() {
		defer close(reschan)
		err := filepath.Walk(fs.path, func(path string, info os.FileInfo, err error) error {

			if !info.Mode().IsRegular() || strings.HasPrefix(info.Name(), ".") {
				return nil
			}

			key, ok := fs.decode(info.Name())
			if !ok {
				log.Warning("failed to decode entry in flatfs")
				return nil
			}

			reschan <- query.Result{
				Entry: query.Entry{
					Key: key.String(),
				},
			}
			return nil
		})
		if err != nil {
			log.Warning("walk failed: ", err)
		}
	}()
	return query.ResultsWithChan(q, reschan), nil
}

func (fs *Datastore) Close() error {
	return nil
}

type flatfsBatch struct {
	puts    map[datastore.Key]interface{}
	deletes map[datastore.Key]struct{}

	ds *Datastore
}

func (fs *Datastore) Batch() (datastore.Batch, error) {
	return &flatfsBatch{
		puts:    make(map[datastore.Key]interface{}),
		deletes: make(map[datastore.Key]struct{}),
		ds:      fs,
	}, nil
}

func (bt *flatfsBatch) Put(key datastore.Key, val interface{}) error {
	bt.puts[key] = val
	return nil
}

func (bt *flatfsBatch) Delete(key datastore.Key) error {
	bt.deletes[key] = struct{}{}
	return nil
}

func (bt *flatfsBatch) Commit() error {
	if err := bt.ds.putMany(bt.puts); err != nil {
		return err
	}

	for k, _ := range bt.deletes {
		if err := bt.ds.Delete(k); err != nil {
			return err
		}
	}

	return nil
}

var _ datastore.ThreadSafeDatastore = (*Datastore)(nil)

func (*Datastore) IsThreadSafe() {}
<commit_msg>Return and log errors in Walk function<commit_after>// Package flatfs is a Datastore implementation that stores all
// objects in a two-level directory structure in the local file
// system, regardless of the hierarchy of the keys.
package flatfs

import (
	"encoding/hex"
	"errors"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/ipfs/go-datastore"
	"github.com/ipfs/go-datastore/query"
	"github.com/jbenet/go-os-rename"

	logging "github.com/ipfs/go-log"
)

var log = logging.Logger("flatfs")

const (
	extension    = ".data"
	maxPrefixLen = 16
)

var (
	ErrBadPrefixLen = errors.New("bad prefix length")
)

type Datastore struct {
	path string
	// length of the dir splay prefix, in bytes of hex digits
	hexPrefixLen int

	// sychronize all writes and directory changes for added safety
	sync bool
}

var _ datastore.Datastore = (*Datastore)(nil)

func New(path string, prefixLen int, sync bool) (*Datastore, error) {
	if prefixLen <= 0 || prefixLen > maxPrefixLen {
		return nil, ErrBadPrefixLen
	}
	fs := &Datastore{
		path: path,
		// convert from binary bytes to bytes of hex encoding
		hexPrefixLen: prefixLen * hex.EncodedLen(1),
		sync:         sync,
	}
	return fs, nil
}

var padding = strings.Repeat("_", maxPrefixLen*hex.EncodedLen(1))

func (fs *Datastore) encode(key datastore.Key) (dir, file string) {
	safe := hex.EncodeToString(key.Bytes()[1:])
	prefix := (safe + padding)[:fs.hexPrefixLen]
	dir = path.Join(fs.path, prefix)
	file = path.Join(dir, safe+extension)
	return dir, file
}

func (fs *Datastore) decode(file string) (key datastore.Key, ok bool) {
	if path.Ext(file) != extension {
		return datastore.Key{}, false
	}
	name := file[:len(file)-len(extension)]
	k, err := hex.DecodeString(name)
	if err != nil {
		return datastore.Key{}, false
	}
	return datastore.NewKey(string(k)), true
}

func (fs *Datastore) makePrefixDir(dir string) error {
	if err := fs.makePrefixDirNoSync(dir); err != nil {
		return err
	}

	// In theory, if we create a new prefix dir and add a file to
	// it, the creation of the prefix dir itself might not be
	// durable yet. Sync the root dir after a successful mkdir of
	// a prefix dir, just to be paranoid.
	if fs.sync {
		if err := syncDir(fs.path); err != nil {
			return err
		}
	}
	return nil
}

func (fs *Datastore) makePrefixDirNoSync(dir string) error {
	if err := os.Mkdir(dir, 0777); err != nil {
		// EEXIST is safe to ignore here, that just means the prefix
		// directory already existed.
		if !os.IsExist(err) {
			return err
		}
	}
	return nil
}

var putMaxRetries = 3

func (fs *Datastore) Put(key datastore.Key, value interface{}) error {
	val, ok := value.([]byte)
	if !ok {
		return datastore.ErrInvalidType
	}

	var err error
	for i := 0; i < putMaxRetries; i++ {
		err = fs.doPut(key, val)
		if err == nil {
			return nil
		}

		if !strings.Contains(err.Error(), "too many open files") {
			return err
		}

		log.Error("too many open files, retrying in %dms", 100*i)
		time.Sleep(time.Millisecond * 100 * time.Duration(i))
	}
	return err
}

func (fs *Datastore) doPut(key datastore.Key, val []byte) error {
	dir, path := fs.encode(key)
	if err := fs.makePrefixDir(dir); err != nil {
		return err
	}

	tmp, err := ioutil.TempFile(dir, "put-")
	if err != nil {
		return err
	}
	closed := false
	removed := false
	defer func() {
		if !closed {
			// silence errcheck
			_ = tmp.Close()
		}
		if !removed {
			// silence errcheck
			_ = os.Remove(tmp.Name())
		}
	}()

	if _, err := tmp.Write(val); err != nil {
		return err
	}
	if fs.sync {
		if err := tmp.Sync(); err != nil {
			return err
		}
	}
	if err := tmp.Close(); err != nil {
		return err
	}
	closed = true

	err = osrename.Rename(tmp.Name(), path)
	if err != nil {
		return err
	}
	removed = true

	if fs.sync {
		if err := syncDir(dir); err != nil {
			return err
		}
	}
	return nil
}

func (fs *Datastore) putMany(data map[datastore.Key]interface{}) error {
	var dirsToSync []string
	files := make(map[*os.File]string)

	for key, value := range data {
		val, ok := value.([]byte)
		if !ok {
			return datastore.ErrInvalidType
		}
		dir, path := fs.encode(key)
		if err := fs.makePrefixDirNoSync(dir); err != nil {
			return err
		}
		dirsToSync = append(dirsToSync, dir)

		tmp, err := ioutil.TempFile(dir, "put-")
		if err != nil {
			return err
		}

		if _, err := tmp.Write(val); err != nil {
			return err
		}

		files[tmp] = path
	}

	ops := make(map[*os.File]int)

	defer func() {
		for fi, _ := range files {
			val, _ := ops[fi]
			switch val {
			case 0:
				_ = fi.Close()
				fallthrough
			case 1:
				_ = os.Remove(fi.Name())
			}
		}
	}()

	// Now we sync everything
	// sync and close files
	for fi, _ := range files {
		if fs.sync {
			if err := fi.Sync(); err != nil {
				return err
			}
		}

		if err := fi.Close(); err != nil {
			return err
		}

		// signify closed
		ops[fi] = 1
	}

	// move files to their proper places
	for fi, path := range files {
		if err := osrename.Rename(fi.Name(), path); err != nil {
			return err
		}

		// signify removed
		ops[fi] = 2
	}

	// now sync the dirs for those files
	if fs.sync {
		for _, dir := range dirsToSync {
			if err := syncDir(dir); err != nil {
				return err
			}
		}

		// sync top flatfs dir
		if err := syncDir(fs.path); err != nil {
			return err
		}
	}

	return nil
}

func (fs *Datastore) Get(key datastore.Key) (value interface{}, err error) {
	_, path := fs.encode(key)
	data, err := ioutil.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, datastore.ErrNotFound
		}
		// no specific error to return, so just pass it through
		return nil, err
	}
	return data, nil
}

func (fs *Datastore) Has(key datastore.Key) (exists bool, err error) {
	_, path := fs.encode(key)
	switch _, err := os.Stat(path); {
	case err == nil:
		return true, nil
	case os.IsNotExist(err):
		return false, nil
	default:
		return false, err
	}
}

func (fs *Datastore) Delete(key datastore.Key) error {
	_, path := fs.encode(key)
	switch err := os.Remove(path); {
	case err == nil:
		return nil
	case os.IsNotExist(err):
		return datastore.ErrNotFound
	default:
		return err
	}
}

func (fs *Datastore) Query(q query.Query) (query.Results, error) {
	if (q.Prefix != "" && q.Prefix != "/") ||
		len(q.Filters) > 0 ||
		len(q.Orders) > 0 ||
		q.Limit > 0 ||
		q.Offset > 0 ||
		!q.KeysOnly {
		// TODO this is overly simplistic, but the only caller is
		// `ipfs refs local` for now, and this gets us moving.
		return nil, errors.New("flatfs only supports listing all keys in random order")
	}

	reschan := make(chan query.Result)
	go func() {
		defer close(reschan)
		err := filepath.Walk(fs.path, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				log.Errorf("Walk func in Query got error: %v", err)
				return err
			}

			if !info.Mode().IsRegular() || strings.HasPrefix(info.Name(), ".") {
				return nil
			}

			key, ok := fs.decode(info.Name())
			if !ok {
				log.Warning("failed to decode entry in flatfs")
				return nil
			}

			reschan <- query.Result{
				Entry: query.Entry{
					Key: key.String(),
				},
			}
			return nil
		})
		if err != nil {
			log.Warning("walk failed: ", err)
		}
	}()
	return query.ResultsWithChan(q, reschan), nil
}

func (fs *Datastore) Close() error {
	return nil
}

type flatfsBatch struct {
	puts    map[datastore.Key]interface{}
	deletes map[datastore.Key]struct{}

	ds *Datastore
}

func (fs *Datastore) Batch() (datastore.Batch, error) {
	return &flatfsBatch{
		puts:    make(map[datastore.Key]interface{}),
		deletes: make(map[datastore.Key]struct{}),
		ds:      fs,
	}, nil
}

func (bt *flatfsBatch) Put(key datastore.Key, val interface{}) error {
	bt.puts[key] = val
	return nil
}

func (bt *flatfsBatch) Delete(key datastore.Key) error {
	bt.deletes[key] = struct{}{}
	return nil
}

func (bt *flatfsBatch) Commit() error {
	if err := bt.ds.putMany(bt.puts); err != nil {
		return err
	}

	for k, _ := range bt.deletes {
		if err := bt.ds.Delete(k); err != nil {
			return err
		}
	}

	return nil
}

var _ datastore.ThreadSafeDatastore = (*Datastore)(nil)

func (*Datastore) IsThreadSafe() {}
<|endoftext|>


<commit_before>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"syscall"

	"github.com/coreos/fleet/Godeps/_workspace/src/github.com/rakyll/globalconf"

	"github.com/coreos/fleet/agent"
	"github.com/coreos/fleet/config"
	"github.com/coreos/fleet/log"
	"github.com/coreos/fleet/pkg"
	"github.com/coreos/fleet/registry"
	"github.com/coreos/fleet/server"
	"github.com/coreos/fleet/version"
)

const (
	DefaultConfigFile = "/etc/fleet/fleet.conf"
)

func main() {
	userset := flag.NewFlagSet("fleet", flag.ExitOnError)
	printVersion := userset.Bool("version", false, "Print the version and exit")
	cfgPath := userset.String("config", "", fmt.Sprintf("Path to config file. Fleet will look for a config at %s by default.", DefaultConfigFile))

	err := userset.Parse(os.Args[1:])
	if err == flag.ErrHelp {
		userset.Usage()
		os.Exit(1)
	}

	args := userset.Args()
	if len(args) == 1 && args[0] == "version" {
		*printVersion = true
	} else if len(args) != 0 {
		userset.Usage()
		os.Exit(1)
	}

	if *printVersion {
		fmt.Println("fleetd version", version.Version)
		os.Exit(0)
	}

	log.Infof("Starting fleetd version %v", version.Version)

	cfgset := flag.NewFlagSet("fleet", flag.ExitOnError)
	cfgset.Int("verbosity", 0, "Logging level")
	cfgset.Var(&pkg.StringSlice{"http://127.0.0.1:2379", "http://127.0.0.1:4001"}, "etcd_servers", "List of etcd endpoints")
	cfgset.String("etcd_keyfile", "", "SSL key file used to secure etcd communication")
	cfgset.String("etcd_certfile", "", "SSL certification file used to secure etcd communication")
	cfgset.String("etcd_cafile", "", "SSL Certificate Authority file used to secure etcd communication")
	cfgset.String("etcd_key_prefix", registry.DefaultKeyPrefix, "Keyspace for fleet data in etcd")
	cfgset.Float64("etcd_request_timeout", 1.0, "Amount of time in seconds to allow a single etcd request before considering it failed.")
	cfgset.Float64("engine_reconcile_interval", 2.0, "Interval at which the engine should reconcile the cluster schedule in etcd.")
	cfgset.String("public_ip", "", "IP address that fleet machine should publish")
	cfgset.String("metadata", "", "List of key-value metadata to assign to the fleet machine")
	cfgset.String("agent_ttl", agent.DefaultTTL, "TTL in seconds of fleet machine state in etcd")
	cfgset.Int("token_limit", 100, "Maximum number of entries per page returned from API requests")
	cfgset.Bool("disable_engine", false, "Disable the engine entirely, use with care")
	cfgset.Bool("verify_units", false, "DEPRECATED - This option is ignored")
	cfgset.String("authorized_keys_file", "", "DEPRECATED - This option is ignored")

	globalconf.Register("", cfgset)
	cfg, err := getConfig(cfgset, *cfgPath)
	if err != nil {
		log.Fatalf(err.Error())
	}

	log.Debugf("Creating Server")
	srv, err := server.New(*cfg)
	if err != nil {
		log.Fatalf("Failed creating Server: %v", err.Error())
	}
	srv.Run()

	reconfigure := func() {
		log.Infof("Reloading configuration from %s", *cfgPath)

		cfg, err := getConfig(cfgset, *cfgPath)
		if err != nil {
			log.Fatalf(err.Error())
		}

		log.Infof("Restarting server components")
		srv.Stop()

		srv, err = server.New(*cfg)
		if err != nil {
			log.Fatalf(err.Error())
		}
		srv.Run()
	}

	shutdown := func() {
		log.Infof("Gracefully shutting down")
		srv.Stop()
		srv.Purge()
		os.Exit(0)
	}

	writeState := func() {
		log.Infof("Dumping server state")

		encoded, err := json.Marshal(srv)
		if err != nil {
			log.Errorf("Failed to dump server state: %v", err)
			return
		}

		if _, err := os.Stdout.Write(encoded); err != nil {
			log.Errorf("Failed to dump server state: %v", err)
			return
		}

		os.Stdout.Write([]byte("\n"))

		log.Debugf("Finished dumping server state")
	}

	signals := map[os.Signal]func(){
		syscall.SIGHUP:  reconfigure,
		syscall.SIGTERM: shutdown,
		syscall.SIGINT:  shutdown,
		syscall.SIGUSR1: writeState,
	}

	listenForSignals(signals)
}

func getConfig(flagset *flag.FlagSet, userCfgFile string) (*config.Config, error) {
	opts := globalconf.Options{EnvPrefix: "FLEET_"}

	if userCfgFile != "" {
		// Fail hard if a user-provided config is not usable
		fi, err := os.Stat(userCfgFile)
		if err != nil {
			log.Fatalf("Unable to use config file %s: %v", userCfgFile, err)
		}
		if fi.IsDir() {
			log.Fatalf("Provided config %s is a directory, not a file", userCfgFile)
		}

		log.Infof("Using provided config file %s", userCfgFile)
		opts.Filename = userCfgFile

	} else if _, err := os.Stat(DefaultConfigFile); err == nil {
		log.Infof("Using default config file %s", DefaultConfigFile)
		opts.Filename = DefaultConfigFile
	} else {
		log.Infof("No provided or default config file found - proceeding without")
	}

	gconf, err := globalconf.NewWithOptions(&opts)
	if err != nil {
		return nil, err
	}

	gconf.ParseSet("", flagset)

	cfg := config.Config{
		Verbosity:               (*flagset.Lookup("verbosity")).Value.(flag.Getter).Get().(int),
		EtcdServers:             (*flagset.Lookup("etcd_servers")).Value.(flag.Getter).Get().(pkg.StringSlice),
		EtcdKeyPrefix:           (*flagset.Lookup("etcd_key_prefix")).Value.(flag.Getter).Get().(string),
		EtcdKeyFile:             (*flagset.Lookup("etcd_keyfile")).Value.(flag.Getter).Get().(string),
		EtcdCertFile:            (*flagset.Lookup("etcd_certfile")).Value.(flag.Getter).Get().(string),
		EtcdCAFile:              (*flagset.Lookup("etcd_cafile")).Value.(flag.Getter).Get().(string),
		EtcdRequestTimeout:      (*flagset.Lookup("etcd_request_timeout")).Value.(flag.Getter).Get().(float64),
		EngineReconcileInterval: (*flagset.Lookup("engine_reconcile_interval")).Value.(flag.Getter).Get().(float64),
		PublicIP:                (*flagset.Lookup("public_ip")).Value.(flag.Getter).Get().(string),
		RawMetadata:             (*flagset.Lookup("metadata")).Value.(flag.Getter).Get().(string),
		AgentTTL:                (*flagset.Lookup("agent_ttl")).Value.(flag.Getter).Get().(string),
		DisableEngine:           (*flagset.Lookup("disable_engine")).Value.(flag.Getter).Get().(bool),
		VerifyUnits:             (*flagset.Lookup("verify_units")).Value.(flag.Getter).Get().(bool),
		TokenLimit:              (*flagset.Lookup("token_limit")).Value.(flag.Getter).Get().(int),
		AuthorizedKeysFile:      (*flagset.Lookup("authorized_keys_file")).Value.(flag.Getter).Get().(string),
	}

	if cfg.VerifyUnits {
		log.Error("Config option verify_units is no longer supported - ignoring")
	}
	if len(cfg.AuthorizedKeysFile) > 0 {
		log.Error("Config option authorized_keys_file is no longer supported - ignoring")
	}

	if cfg.Verbosity > 0 {
		log.EnableDebug()
	}

	return &cfg, nil
}

func listenForSignals(sigmap map[os.Signal]func()) {
	sigchan := make(chan os.Signal, 1)

	for k := range sigmap {
		signal.Notify(sigchan, k)
	}

	for true {
		sig := <-sigchan
		handler, ok := sigmap[sig]
		if ok {
			handler()
		}
	}
}
<commit_msg>fleetd: fix argument handling<commit_after>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"syscall"

	"github.com/coreos/fleet/Godeps/_workspace/src/github.com/rakyll/globalconf"

	"github.com/coreos/fleet/agent"
	"github.com/coreos/fleet/config"
	"github.com/coreos/fleet/log"
	"github.com/coreos/fleet/pkg"
	"github.com/coreos/fleet/registry"
	"github.com/coreos/fleet/server"
	"github.com/coreos/fleet/version"
)

const (
	DefaultConfigFile = "/etc/fleet/fleet.conf"
)

func main() {
	userset := flag.NewFlagSet("fleet", flag.ExitOnError)
	printVersion := userset.Bool("version", false, "Print the version and exit")
	cfgPath := userset.String("config", "", fmt.Sprintf("Path to config file. Fleet will look for a config at %s by default.", DefaultConfigFile))

	userset.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		userset.PrintDefaults()
	}

	err := userset.Parse(os.Args[1:])
	if err != nil {
		userset.Usage()
		os.Exit(1)
	}

	args := userset.Args()
	if len(args) == 1 && args[0] == "version" {
		*printVersion = true
	} else if len(args) != 0 {
		userset.Usage()
		os.Exit(1)
	}

	if *printVersion {
		fmt.Println("fleetd version", version.Version)
		os.Exit(0)
	}

	log.Infof("Starting fleetd version %v", version.Version)

	cfgset := flag.NewFlagSet("fleet", flag.ExitOnError)
	cfgset.Int("verbosity", 0, "Logging level")
	cfgset.Var(&pkg.StringSlice{"http://127.0.0.1:2379", "http://127.0.0.1:4001"}, "etcd_servers", "List of etcd endpoints")
	cfgset.String("etcd_keyfile", "", "SSL key file used to secure etcd communication")
	cfgset.String("etcd_certfile", "", "SSL certification file used to secure etcd communication")
	cfgset.String("etcd_cafile", "", "SSL Certificate Authority file used to secure etcd communication")
	cfgset.String("etcd_key_prefix", registry.DefaultKeyPrefix, "Keyspace for fleet data in etcd")
	cfgset.Float64("etcd_request_timeout", 1.0, "Amount of time in seconds to allow a single etcd request before considering it failed.")
	cfgset.Float64("engine_reconcile_interval", 2.0, "Interval at which the engine should reconcile the cluster schedule in etcd.")
	cfgset.String("public_ip", "", "IP address that fleet machine should publish")
	cfgset.String("metadata", "", "List of key-value metadata to assign to the fleet machine")
	cfgset.String("agent_ttl", agent.DefaultTTL, "TTL in seconds of fleet machine state in etcd")
	cfgset.Int("token_limit", 100, "Maximum number of entries per page returned from API requests")
	cfgset.Bool("disable_engine", false, "Disable the engine entirely, use with care")
	cfgset.Bool("verify_units", false, "DEPRECATED - This option is ignored")
	cfgset.String("authorized_keys_file", "", "DEPRECATED - This option is ignored")

	globalconf.Register("", cfgset)
	cfg, err := getConfig(cfgset, *cfgPath)
	if err != nil {
		log.Fatalf(err.Error())
	}

	log.Debugf("Creating Server")
	srv, err := server.New(*cfg)
	if err != nil {
		log.Fatalf("Failed creating Server: %v", err.Error())
	}
	srv.Run()

	reconfigure := func() {
		log.Infof("Reloading configuration from %s", *cfgPath)

		cfg, err := getConfig(cfgset, *cfgPath)
		if err != nil {
			log.Fatalf(err.Error())
		}

		log.Infof("Restarting server components")
		srv.Stop()

		srv, err = server.New(*cfg)
		if err != nil {
			log.Fatalf(err.Error())
		}
		srv.Run()
	}

	shutdown := func() {
		log.Infof("Gracefully shutting down")
		srv.Stop()
		srv.Purge()
		os.Exit(0)
	}

	writeState := func() {
		log.Infof("Dumping server state")

		encoded, err := json.Marshal(srv)
		if err != nil {
			log.Errorf("Failed to dump server state: %v", err)
			return
		}

		if _, err := os.Stdout.Write(encoded); err != nil {
			log.Errorf("Failed to dump server state: %v", err)
			return
		}

		os.Stdout.Write([]byte("\n"))

		log.Debugf("Finished dumping server state")
	}

	signals := map[os.Signal]func(){
		syscall.SIGHUP:  reconfigure,
		syscall.SIGTERM: shutdown,
		syscall.SIGINT:  shutdown,
		syscall.SIGUSR1: writeState,
	}

	listenForSignals(signals)
}

func getConfig(flagset *flag.FlagSet, userCfgFile string) (*config.Config, error) {
	opts := globalconf.Options{EnvPrefix: "FLEET_"}

	if userCfgFile != "" {
		// Fail hard if a user-provided config is not usable
		fi, err := os.Stat(userCfgFile)
		if err != nil {
			log.Fatalf("Unable to use config file %s: %v", userCfgFile, err)
		}
		if fi.IsDir() {
			log.Fatalf("Provided config %s is a directory, not a file", userCfgFile)
		}

		log.Infof("Using provided config file %s", userCfgFile)
		opts.Filename = userCfgFile

	} else if _, err := os.Stat(DefaultConfigFile); err == nil {
		log.Infof("Using default config file %s", DefaultConfigFile)
		opts.Filename = DefaultConfigFile
	} else {
		log.Infof("No provided or default config file found - proceeding without")
	}

	gconf, err := globalconf.NewWithOptions(&opts)
	if err != nil {
		return nil, err
	}

	gconf.ParseSet("", flagset)

	cfg := config.Config{
		Verbosity:               (*flagset.Lookup("verbosity")).Value.(flag.Getter).Get().(int),
		EtcdServers:             (*flagset.Lookup("etcd_servers")).Value.(flag.Getter).Get().(pkg.StringSlice),
		EtcdKeyPrefix:           (*flagset.Lookup("etcd_key_prefix")).Value.(flag.Getter).Get().(string),
		EtcdKeyFile:             (*flagset.Lookup("etcd_keyfile")).Value.(flag.Getter).Get().(string),
		EtcdCertFile:            (*flagset.Lookup("etcd_certfile")).Value.(flag.Getter).Get().(string),
		EtcdCAFile:              (*flagset.Lookup("etcd_cafile")).Value.(flag.Getter).Get().(string),
		EtcdRequestTimeout:      (*flagset.Lookup("etcd_request_timeout")).Value.(flag.Getter).Get().(float64),
		EngineReconcileInterval: (*flagset.Lookup("engine_reconcile_interval")).Value.(flag.Getter).Get().(float64),
		PublicIP:                (*flagset.Lookup("public_ip")).Value.(flag.Getter).Get().(string),
		RawMetadata:             (*flagset.Lookup("metadata")).Value.(flag.Getter).Get().(string),
		AgentTTL:                (*flagset.Lookup("agent_ttl")).Value.(flag.Getter).Get().(string),
		DisableEngine:           (*flagset.Lookup("disable_engine")).Value.(flag.Getter).Get().(bool),
		VerifyUnits:             (*flagset.Lookup("verify_units")).Value.(flag.Getter).Get().(bool),
		TokenLimit:              (*flagset.Lookup("token_limit")).Value.(flag.Getter).Get().(int),
		AuthorizedKeysFile:      (*flagset.Lookup("authorized_keys_file")).Value.(flag.Getter).Get().(string),
	}

	if cfg.VerifyUnits {
		log.Error("Config option verify_units is no longer supported - ignoring")
	}
	if len(cfg.AuthorizedKeysFile) > 0 {
		log.Error("Config option authorized_keys_file is no longer supported - ignoring")
	}

	if cfg.Verbosity > 0 {
		log.EnableDebug()
	}

	return &cfg, nil
}

func listenForSignals(sigmap map[os.Signal]func()) {
	sigchan := make(chan os.Signal, 1)

	for k := range sigmap {
		signal.Notify(sigchan, k)
	}

	for true {
		sig := <-sigchan
		handler, ok := sigmap[sig]
		if ok {
			handler()
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/thehowl/go-osuapi"
	"github.com/bwmarrin/discordgo"
	"fmt"
	"strings"
	"strconv"
)

func checkBeatmapLink(s *discordgo.Session, m *discordgo.MessageCreate) {

	//check if message contains only one beatmap link
	BeatmapSet := strings.Count(m.Content, "https://osu.ppy.sh/s/")
	Beatmap := strings.Count(m.Content, "https://osu.ppy.sh/b/")
	if BeatmapSet + Beatmap != 1 {
		return
	}

	id := 0

	//check if message contains beatmap set
	if BeatmapSet == 1 {

		//get beatmap id in the message
		tmp_id, err := strconv.Atoi(strings.Split(string(m.Content[strings.Index(m.Content,"https://osu.ppy.sh/s/")+21:])," ")[0])
		if err != nil {
			fmt.Println("An error ocurred while getting beatmap id, ",err)
			return
		}
		id = tmp_id
	}

	//check if message contains specific beatmap difficulty
	if Beatmap == 1 {

		//get beatmap id in the message
		tmp_id, err := strconv.Atoi(strings.Split(strings.Split(string(m.Content[strings.Index(m.Content,"https://osu.ppy.sh/b/")+21:])," ")[0],"?")[0])
		if err != nil {
			fmt.Println("An error ocurred while getting beatmap id, ",err)
			return
		}
		id = tmp_id
	}

	//return if there's no beatmap link
	if id == 0 {
		return
	}

	var opts osuapi.GetBeatmapsOpts

	if BeatmapSet == 1 {
		opts = osuapi.GetBeatmapsOpts{BeatmapSetID: id}
	} else {
		opts = osuapi.GetBeatmapsOpts{BeatmapID: id}
	}

	//get beatmap info
	beatmaps, err := osu_client.GetBeatmaps(opts)
	if err != nil {
		fmt.Println("An error ocurred while fecthing beatmap, ",err)
		return
	}

	//check for empty list in case of no beatmaps found
	if len(beatmaps) == 0 {
		return
	}

	//get the highest difficulty in mapset
	beatmap := beatmaps[0]
	for i := 1; i < len(beatmaps); i++ {
		if beatmaps[i].DifficultyRating > beatmap.DifficultyRating {
			beatmap = beatmaps[i]
		}
	}

	//create the embed message to send
	message := discordgo.MessageEmbed{
		URL: "https://osu.ppy.sh/s/"+strconv.Itoa(beatmap.BeatmapSetID),
		Title: beatmap.Artist+" - "+beatmap.Title+" ["+beatmap.DiffName+"]",
		Description: "**Mode:** "+beatmap.Mode.String()+" | **Length:** "+parseTime(beatmap.TotalLength)+"\n**Star rating:** "+strconv.FormatFloat(beatmap.DifficultyRating,'f',2,64)+" | **BPM:** "+strconv.FormatFloat(beatmap.BPM,'f',2,64)+"\n**OD:** "+strconv.FormatFloat(beatmap.OverallDifficulty,'f',2,64)+" | **CS:** "+strconv.FormatFloat(beatmap.CircleSize,'f',2,64)+"\n**AR:** "+strconv.FormatFloat(beatmap.ApproachRate,'f',2,64)+" | **HP:** "+strconv.FormatFloat(beatmap.HPDrain,'f',2,64),
		Thumbnail: &discordgo.MessageEmbedThumbnail{
			URL: "https://b.ppy.sh/thumb/"+strconv.Itoa(beatmap.BeatmapSetID)+"l.jpg",
		},
		Color: 16763135, //this should be pink
		Footer: &discordgo.MessageEmbedFooter{
			Text: "Mapped by "+beatmap.Creator+" | Status: "+beatmap.Approved.String(),
		},
	}

	//send the message. finally.
	_, err = s.ChannelMessageSendEmbed(m.ChannelID, &message)
	if err != nil {
		fmt.Println("An error ocurred while sending embed message, ",err)
	}
}

//get time in format MM:SS
func parseTime(s int) string {

	if s < 10 {
		return "00:0"+strconv.Itoa(s)
	}
	if s < 61 {
		return "00:"+strconv.Itoa(s)
	}
	m := int(s/60)
	s = s - m * 60
	if s < 10 {
		return strconv.Itoa(m)+":0"+strconv.Itoa(s)
	}
	return strconv.Itoa(m)+":"+strconv.Itoa(s)
}
<commit_msg>Revert "Fix on beatmap linker length"<commit_after>package main

import (
	"github.com/thehowl/go-osuapi"
	"github.com/bwmarrin/discordgo"
	"fmt"
	"strings"
	"strconv"
)

func checkBeatmapLink(s *discordgo.Session, m *discordgo.MessageCreate) {

	//check if message contains only one beatmap link
	BeatmapSet := strings.Count(m.Content, "https://osu.ppy.sh/s/")
	Beatmap := strings.Count(m.Content, "https://osu.ppy.sh/b/")
	if BeatmapSet + Beatmap != 1 {
		return
	}

	id := 0

	//check if message contains beatmap set
	if BeatmapSet == 1 {

		//get beatmap id in the message
		tmp_id, err := strconv.Atoi(strings.Split(string(m.Content[strings.Index(m.Content,"https://osu.ppy.sh/s/")+21:])," ")[0])
		if err != nil {
			fmt.Println("An error ocurred while getting beatmap id, ",err)
			return
		}
		id = tmp_id
	}

	//check if message contains specific beatmap difficulty
	if Beatmap == 1 {

		//get beatmap id in the message
		tmp_id, err := strconv.Atoi(strings.Split(strings.Split(string(m.Content[strings.Index(m.Content,"https://osu.ppy.sh/b/")+21:])," ")[0],"?")[0])
		if err != nil {
			fmt.Println("An error ocurred while getting beatmap id, ",err)
			return
		}
		id = tmp_id
	}

	//return if there's no beatmap link
	if id == 0 {
		return
	}

	var opts osuapi.GetBeatmapsOpts

	if BeatmapSet == 1 {
		opts = osuapi.GetBeatmapsOpts{BeatmapSetID: id}
	} else {
		opts = osuapi.GetBeatmapsOpts{BeatmapID: id}
	}

	//get beatmap info
	beatmaps, err := osu_client.GetBeatmaps(opts)
	if err != nil {
		fmt.Println("An error ocurred while fecthing beatmap, ",err)
		return
	}

	//check for empty list in case of no beatmaps found
	if len(beatmaps) == 0 {
		return
	}

	//get the highest difficulty in mapset
	beatmap := beatmaps[0]
	for i := 1; i < len(beatmaps); i++ {
		if beatmaps[i].DifficultyRating > beatmap.DifficultyRating {
			beatmap = beatmaps[i]
		}
	}

	//create the embed message to send
	message := discordgo.MessageEmbed{
		URL: "https://osu.ppy.sh/s/"+strconv.Itoa(beatmap.BeatmapSetID),
		Title: beatmap.Artist+" - "+beatmap.Title+" ["+beatmap.DiffName+"]",
		Description: "**Mode:** "+beatmap.Mode.String()+" | **Length:** "+parseTime(beatmap.TotalLength)+"\n**Star rating:** "+strconv.FormatFloat(beatmap.DifficultyRating,'f',2,64)+" | **BPM:** "+strconv.FormatFloat(beatmap.BPM,'f',2,64)+"\n**OD:** "+strconv.FormatFloat(beatmap.OverallDifficulty,'f',2,64)+" | **CS:** "+strconv.FormatFloat(beatmap.CircleSize,'f',2,64)+"\n**AR:** "+strconv.FormatFloat(beatmap.ApproachRate,'f',2,64)+" | **HP:** "+strconv.FormatFloat(beatmap.HPDrain,'f',2,64),
		Thumbnail: &discordgo.MessageEmbedThumbnail{
			URL: "https://b.ppy.sh/thumb/"+strconv.Itoa(beatmap.BeatmapSetID)+"l.jpg",
		},
		Color: 16763135, //this should be pink
		Footer: &discordgo.MessageEmbedFooter{
			Text: "Mapped by "+beatmap.Creator+" | Status: "+beatmap.Approved.String(),
		},
	}

	//send the message. finally.
	_, err = s.ChannelMessageSendEmbed(m.ChannelID, &message)
	if err != nil {
		fmt.Println("An error ocurred while sending embed message, ",err)
	}
}

//get time in format MM:SS
func parseTime(s int) string {

	if s < 61 {
		return "00:"+strconv.Itoa(s)
	}
	m := int(s/60)
	s = s - m * 60
	return strconv.Itoa(m)+":"+strconv.Itoa(s)
}<|endoftext|>


<commit_before>package client

import (
	"syscall"
	"unsafe"
)

type Termios struct {
	Iflag  uintptr
	Oflag  uintptr
	Cflag  uintptr
	Lflag  uintptr
	Cc     [20]byte
	Ispeed uintptr
	Ospeed uintptr
}

const (
	// Input flags
	inpck  = 0x010
	istrip = 0x020
	icrnl  = 0x100
	ixon   = 0x200

	// Output flags
	opost = 0x1

	// Control flags
	cs8 = 0x300

	// Local flags
	icanon = 0x100
	iexten = 0x400
)

const (
        HUPCL                             = 0x4000
        ICANON                            = 0x100
        ICRNL                             = 0x100
        IEXTEN                            = 0x400
        BRKINT                            = 0x2
        CFLUSH                            = 0xf
        CLOCAL                            = 0x8000
        CREAD                             = 0x800
        CS5                               = 0x0
        CS6                               = 0x100
        CS7                               = 0x200
        CS8                               = 0x300
        CSIZE                             = 0x300
        CSTART                            = 0x11
        CSTATUS                           = 0x14
        CSTOP                             = 0x13
        CSTOPB                            = 0x400
        CSUSP                             = 0x1a
        IGNBRK                            = 0x1
        IGNCR                             = 0x80
        IGNPAR                            = 0x4
        IMAXBEL                           = 0x2000
        INLCR                             = 0x40
        INPCK                             = 0x10
        ISIG                              = 0x80
        ISTRIP                            = 0x20
        IUTF8                             = 0x4000
        IXANY                             = 0x800
        IXOFF                             = 0x400
        IXON                              = 0x200
        NOFLSH                            = 0x80000000
        OCRNL                             = 0x10
        OFDEL                             = 0x20000
        OFILL                             = 0x80
        ONLCR                             = 0x2
        ONLRET                            = 0x40
        ONOCR                             = 0x20
        ONOEOT                            = 0x8
        OPOST                             = 0x1
RENB                            = 0x1000
        PARMRK                            = 0x8
        PARODD                            = 0x2000

        TOSTOP                            = 0x400000
        VDISCARD                          = 0xf
        VDSUSP                            = 0xb
        VEOF                              = 0x0
        VEOL                              = 0x1
        VEOL2                             = 0x2
        VERASE                            = 0x3
        VINTR                             = 0x8
        VKILL                             = 0x5
        VLNEXT                            = 0xe
        VMIN                              = 0x10
        VQUIT                             = 0x9
        VREPRINT                          = 0x6
        VSTART                            = 0xc
        VSTATUS                           = 0x12
        VSTOP                             = 0xd
        VSUSP                             = 0xa
        VT0                               = 0x0
        VT1                               = 0x10000
        VTDLY                             = 0x10000
        VTIME                             = 0x11
	ECHO				  = 0x00000008

        PENDIN                            = 0x20000000
)

type State struct {
       termios Termios
}

// IsTerminal returns true if the given file descriptor is a terminal.
func IsTerminal(fd int) bool {
        var termios Termios
        _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(getTermios), uintptr(unsafe.Pointer(&termios)), 0, 0, 0)
        return err == 0
}

// MakeRaw put the terminal connected to the given file descriptor into raw
// mode and returns the previous state of the terminal so that it can be
// restored.
func MakeRaw(fd int) (*State, error) {
        var oldState State
        if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(getTermios), uintptr(unsafe.Pointer(&oldState.termios)), 0, 0, 0); err != 0 {
                return nil, err
        }

        newState := oldState.termios
        newState.Iflag &^= istrip | INLCR | ICRNL | IGNCR | IXON | IXOFF
        newState.Lflag &^= ECHO | ICANON | ISIG
        if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(setTermios), uintptr(unsafe.Pointer(&newState)), 0, 0, 0); err != 0 {
                return nil, err
        }

        return &oldState, nil
}

// Restore restores the terminal connected to the given file descriptor to a
// previous state.
func Restore(fd int, state *State) error {
        _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(setTermios), uintptr(unsafe.Pointer(&state.termios)), 0, 0, 0)
        return err
}

<commit_msg>make sure the standard "replace input carriage returns with line feeds" and "replace output line feeds with carriage return and line feed" flags are set, even on raw (I dont think it should actually be raw)<commit_after>package client

import (
	"syscall"
	"unsafe"
)

type Termios struct {
	Iflag  uintptr
	Oflag  uintptr
	Cflag  uintptr
	Lflag  uintptr
	Cc     [20]byte
	Ispeed uintptr
	Ospeed uintptr
}

const (
	// Input flags
	inpck  = 0x010
	istrip = 0x020
	icrnl  = 0x100
	ixon   = 0x200

	// Output flags
	opost = 0x1

	// Control flags
	cs8 = 0x300

	// Local flags
	icanon = 0x100
	iexten = 0x400
)

const (
        HUPCL                             = 0x4000
        ICANON                            = 0x100
        ICRNL                             = 0x100
        IEXTEN                            = 0x400
        BRKINT                            = 0x2
        CFLUSH                            = 0xf
        CLOCAL                            = 0x8000
        CREAD                             = 0x800
        CS5                               = 0x0
        CS6                               = 0x100
        CS7                               = 0x200
        CS8                               = 0x300
        CSIZE                             = 0x300
        CSTART                            = 0x11
        CSTATUS                           = 0x14
        CSTOP                             = 0x13
        CSTOPB                            = 0x400
        CSUSP                             = 0x1a
        IGNBRK                            = 0x1
        IGNCR                             = 0x80
        IGNPAR                            = 0x4
        IMAXBEL                           = 0x2000
        INLCR                             = 0x40
        INPCK                             = 0x10
        ISIG                              = 0x80
        ISTRIP                            = 0x20
        IUTF8                             = 0x4000
        IXANY                             = 0x800
        IXOFF                             = 0x400
        IXON                              = 0x200
        NOFLSH                            = 0x80000000
        OCRNL                             = 0x10
        OFDEL                             = 0x20000
        OFILL                             = 0x80
        ONLCR                             = 0x2
        ONLRET                            = 0x40
        ONOCR                             = 0x20
        ONOEOT                            = 0x8
        OPOST                             = 0x1
RENB                            = 0x1000
        PARMRK                            = 0x8
        PARODD                            = 0x2000

        TOSTOP                            = 0x400000
        VDISCARD                          = 0xf
        VDSUSP                            = 0xb
        VEOF                              = 0x0
        VEOL                              = 0x1
        VEOL2                             = 0x2
        VERASE                            = 0x3
        VINTR                             = 0x8
        VKILL                             = 0x5
        VLNEXT                            = 0xe
        VMIN                              = 0x10
        VQUIT                             = 0x9
        VREPRINT                          = 0x6
        VSTART                            = 0xc
        VSTATUS                           = 0x12
        VSTOP                             = 0xd
        VSUSP                             = 0xa
        VT0                               = 0x0
        VT1                               = 0x10000
        VTDLY                             = 0x10000
        VTIME                             = 0x11
	ECHO				  = 0x00000008

        PENDIN                            = 0x20000000
)

type State struct {
       termios Termios
}

// IsTerminal returns true if the given file descriptor is a terminal.
func IsTerminal(fd int) bool {
        var termios Termios
        _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(getTermios), uintptr(unsafe.Pointer(&termios)), 0, 0, 0)
        return err == 0
}

// MakeRaw put the terminal connected to the given file descriptor into raw
// mode and returns the previous state of the terminal so that it can be
// restored.
func MakeRaw(fd int) (*State, error) {
        var oldState State
        if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(getTermios), uintptr(unsafe.Pointer(&oldState.termios)), 0, 0, 0); err != 0 {
                return nil, err
        }

        newState := oldState.termios
        newState.Iflag &^= ISTRIP | INLCR | IGNCR | IXON | IXOFF
	newState.Iflag |= ICRNL
	newState.Oflag |= ONLCR
        newState.Lflag &^= ECHO | ICANON | ISIG
        if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(setTermios), uintptr(unsafe.Pointer(&newState)), 0, 0, 0); err != 0 {
                return nil, err
        }

        return &oldState, nil
}

// Restore restores the terminal connected to the given file descriptor to a
// previous state.
func Restore(fd int, state *State) error {
        _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(setTermios), uintptr(unsafe.Pointer(&state.termios)), 0, 0, 0)
        return err
}

<|endoftext|>


<commit_before>package vkapi

import (
	"log"
	"net/http"
	"net/url"
)

func ExampleNewClientFromToken() {
	client, _ := NewClientFromToken("<access_token>")
	if err := client.InitLongPoll(0, 2); err != nil {
		log.Panic(err)
	}
	client.Log(true)
	updates, _, err := client.GetLPUpdatesChan(100, LPConfig{25, LPModeAttachments})
	if err != nil {
		log.Panic(err)
	}

	for update := range updates {
		if update.Message == nil {
			continue
		}

		log.Printf("%s", update.Message.String())
		if update.IsNewMessage() && update.Message.Text == "/start" {
			client.SendMessage(NewMessage(NewDstFromUserID(update.Message.FromID), "Hello!"))
		}

	}
}

func ExampleNewClientFromAPIClient() {
	apiClient := NewApiClient()
	apiClient.SetHTTPClient(http.DefaultClient)
	apiClient.SetAccessToken("<access token>")
	client, _ := NewClientFromAPIClient(apiClient)
	if err := client.InitMyProfile(); err != nil {
		log.Panic(err.Error())
	}

	log.Printf("My name is %s", client.VKUser.Me.FirstName)
}

func ExampleNewClientFromApplication() {
	client, err := NewClientFromApplication(Application{
		Username:     "<username>",
		Password:     "<password>",
		GrantType:    "password",
		ClientID:     "<client_id>",
		ClientSecret: "<client_secret>",
	})
	if err != nil {
		log.Panic(err)
	}

	if err := client.InitMyProfile(); err != nil {
		log.Panic(err.Error())
	}

	log.Printf("My name is %s", client.VKUser.Me.FirstName)
}

func ExampleClient_Do() {
	client, _ := NewClientFromToken("<access token>")
	values := url.Values{}
	values.Set("user_id", "1")
	values.Set("count", "10")

	res, err := client.Do(NewRequest("groups.get", "", values))
	if err != nil {
		panic(err.Error())
	}

	println(res.Response.String())
}<commit_msg>[ FIX ] Refactoring<commit_after>package vkapi

import (
	"log"
	"net/http"
	"net/url"
)

func ExampleNewClientFromToken() {
	client, _ := NewClientFromToken("<access_token>")
	if err := client.InitLongPoll(0, 2); err != nil {
		log.Panic(err)
	}
	client.Log(true)
	updates, _, err := client.GetLPUpdatesChan(100, LPConfig{25, LPModeAttachments})
	if err != nil {
		log.Panic(err)
	}

	for update := range updates {
		if update.Message == nil {
			continue
		}

		log.Printf("%s", update.Message.String())
		if update.IsNewMessage() && update.Message.Text == "/start" {
			client.SendMessage(NewMessage(NewDstFromUserID(update.Message.FromID), "Hello!"))
		}

	}
}

func ExampleNewClientFromAPIClient() {
	apiClient := NewApiClient()
	apiClient.SetHTTPClient(http.DefaultClient)
	apiClient.SetAccessToken("<access token>")
	client, _ := NewClientFromAPIClient(apiClient)
	if err := client.InitMyProfile(); err != nil {
		log.Panic(err.Error())
	}

	log.Printf("My name is %s", client.VKUser.Me.FirstName)
}

func ExampleNewClientFromApplication() {
	client, err := NewClientFromApplication(Application{
		Username:     "<username>",
		Password:     "<password>",
		GrantType:    "password",
		ClientID:     "<client_id>",
		ClientSecret: "<client_secret>",
	})
	if err != nil {
		log.Panic(err)
	}

	if err := client.InitMyProfile(); err != nil {
		log.Panic(err.Error())
	}

	log.Printf("My name is %s", client.VKUser.Me.FirstName)
}

func ExampleClient_Do() {
	client, _ := NewClientFromToken("<access token>")
	values := url.Values{}
	values.Set("user_id", "1")
	values.Set("count", "10")

	res, err := client.Do(NewRequest("groups.get", "", values))
	if err != nil {
		panic(err.Error())
	}

	println(res.Response.String())
}
<|endoftext|>


<commit_before>package dockertest

import (
	"context"
	"fmt"
	"os"
	"time"

	. "gopkg.in/check.v1"
)

type ClientTest struct{}

var _ = Suite(&ClientTest{})

func (s *ClientTest) TestNewClient(c *C) {
	_, err := NewClient()
	c.Assert(err, IsNil)

	dockerhost := os.Getenv("DOCKER_HOST")
	if dockerhost != "" {
		defer os.Setenv("DOCKER_HOST", dockerhost)
	} else {
		defer os.Unsetenv("DOCKER_HOST")
	}

	os.Setenv("DOCKER_HOST", "/////")
	_, err = NewClient()
	c.Assert(err, ErrorMatches, "unable to parse docker host `/////`")
}

func (s *ClientTest) TestRunAndRemoveContainer(c *C) {
	dc, err := NewClient()
	c.Assert(err, IsNil)
	input := NewClientInput("rabbitmq:3")

	info, err := dc.RunContainer(context.Background(), input)
	c.Assert(err, IsNil)
	c.Assert(info.Refresh(), IsNil)
	defer dc.RemoveContainer(context.Background(), info.Data.ID)
}

func (s *ClientTest) TestRunContainerAttemptsToRetrieveImage(c *C) {
	dc, err := NewClient()
	c.Assert(err, IsNil)

	// Some random image so it will force the client to try to pull the
	// image down.
	input := NewClientInput("abcdefgzyn")
	_, err = dc.RunContainer(context.Background(), input)
	c.Assert(err, ErrorMatches,
		"Error response from daemon: repository abcdefgzyn not " +
			"found: does not exist or no pull access")

	//c.Assert(err, IsNil)
	//c.Assert(info.Refresh(), IsNil)
	//defer dc.RemoveContainer(context.Background(), info.Data.ID)
}

func (s *ClientTest) TestRemoveContainer(c *C) {
	dc, err := NewClient()
	c.Assert(err, IsNil)
	input := NewClientInput("rabbitmq:3")
	info, err := dc.RunContainer(context.Background(), input)
	c.Assert(err, IsNil)
	c.Assert(dc.RemoveContainer(context.Background(), info.Data.ID), IsNil)
	c.Assert(dc.RemoveContainer(context.Background(), info.Data.ID), IsNil)
}

func (s *ClientTest) TestListContainers(c *C) {
	dc, err := NewClient()
	c.Assert(err, IsNil)

	label := fmt.Sprintf("%d", time.Now().Nanosecond())
	infos := map[string]*ContainerInfo{}
	for i := 0; i < 4; i++ {
		input := NewClientInput("rabbitmq:3")
		input.SetLabel("time", label)
		info, err := dc.RunContainer(context.Background(), input)
		c.Assert(err, IsNil)
		infos[info.Data.ID] = info
		defer dc.RemoveContainer(context.Background(), info.Data.ID)
	}

	input := NewClientInput("rabbitmq:3")
	input.SetLabel("time", label)

	containers, err := dc.ListContainers(context.Background(), input)
	c.Assert(err, IsNil)
	for _, entry := range containers {
		_, ok := infos[entry.Data.ID]
		c.Assert(ok, Equals, true)

	}
}
<commit_msg>be sure to close the connection, add test for Service()<commit_after>package dockertest

import (
	"context"
	"fmt"
	"os"
	"time"

	. "gopkg.in/check.v1"
)

type ClientTest struct{}

var _ = Suite(&ClientTest{})

func (s *ClientTest) TestNewClient(c *C) {
	dc, err := NewClient()
	defer dc.Client.Close()
	c.Assert(err, IsNil)

	dockerhost := os.Getenv("DOCKER_HOST")
	if dockerhost != "" {
		defer os.Setenv("DOCKER_HOST", dockerhost)
	} else {
		defer os.Unsetenv("DOCKER_HOST")
	}

	os.Setenv("DOCKER_HOST", "/////")
	_, err = NewClient()
	c.Assert(err, ErrorMatches, "unable to parse docker host `/////`")
}

func (s *ClientTest) TestRunAndRemoveContainer(c *C) {
	dc, err := NewClient()
	defer dc.Client.Close()
	c.Assert(err, IsNil)
	input := NewClientInput("rabbitmq:3")

	info, err := dc.RunContainer(context.Background(), input)
	c.Assert(err, IsNil)
	c.Assert(info.Refresh(), IsNil)
	defer dc.RemoveContainer(context.Background(), info.Data.ID)
}

func (s *ClientTest) TestRunContainerAttemptsToRetrieveImage(c *C) {
	dc, err := NewClient()
	c.Assert(err, IsNil)

	// Some random image so it will force the client to try to pull the
	// image down.
	input := NewClientInput("abcdefgzyn")
	_, err = dc.RunContainer(context.Background(), input)
	c.Assert(err, ErrorMatches,
		"Error response from daemon: repository abcdefgzyn not "+
			"found: does not exist or no pull access")
}

func (s *ClientTest) TestRemoveContainer(c *C) {
	dc, err := NewClient()
	defer dc.Client.Close()
	c.Assert(err, IsNil)
	input := NewClientInput("rabbitmq:3")
	info, err := dc.RunContainer(context.Background(), input)
	c.Assert(err, IsNil)
	c.Assert(dc.RemoveContainer(context.Background(), info.Data.ID), IsNil)
	c.Assert(dc.RemoveContainer(context.Background(), info.Data.ID), IsNil)
}

func (s *ClientTest) TestListContainers(c *C) {
	dc, err := NewClient()
	c.Assert(err, IsNil)
	defer dc.Client.Close()

	label := fmt.Sprintf("%d", time.Now().Nanosecond())
	infos := map[string]*ContainerInfo{}
	for i := 0; i < 4; i++ {
		input := NewClientInput("rabbitmq:3")
		input.SetLabel("time", label)
		info, err := dc.RunContainer(context.Background(), input)
		c.Assert(err, IsNil)
		infos[info.Data.ID] = info
		defer dc.RemoveContainer(context.Background(), info.Data.ID)
	}

	input := NewClientInput("rabbitmq:3")
	input.SetLabel("time", label)

	containers, err := dc.ListContainers(context.Background(), input)
	c.Assert(err, IsNil)
	for _, entry := range containers {
		_, ok := infos[entry.Data.ID]
		c.Assert(ok, Equals, true)

	}
}

func (s *ClientTest) TestService(c *C) {
	dc, err := NewClient()
	c.Assert(err, IsNil)
	defer dc.Client.Close()
	svc := dc.Service("foobar")
	c.Assert(svc.Image, Equals, "foobar")
}
<|endoftext|>


<commit_before>package ovn

import (
	"fmt"
	"net"
	"os/exec"
	"runtime"
	"strconv"
	"strings"

	"github.com/openvswitch/ovn-kubernetes/go-controller/pkg/config"
	"github.com/openvswitch/ovn-kubernetes/go-controller/pkg/util"
	"github.com/sirupsen/logrus"
)

const (
	windowsOS = "windows"
)

func configureManagementPortWindows(clusterSubnet, clusterServicesSubnet,
	routerIP, interfaceName, interfaceIP string) error {
	// Up the interface.
	_, _, err := util.RunPowershell("Enable-NetAdapter", interfaceName)
	if err != nil {
		return err
	}

	//check if interface already exists
	ifAlias := fmt.Sprintf("-InterfaceAlias %s", interfaceName)
	_, _, err = util.RunPowershell("Get-NetIPAddress", ifAlias)
	if err == nil {
		//The interface already exists, we should delete the routes and IP
		logrus.Debugf("Interface %s exists, removing.", interfaceName)
		_, _, err = util.RunPowershell("Remove-NetIPAddress", ifAlias, "-Confirm:$false")
		if err != nil {
			return err
		}
	}

	// Assign IP address to the internal interface.
	portIP, interfaceIPNet, err := net.ParseCIDR(interfaceIP)
	if err != nil {
		return fmt.Errorf("Failed to parse interfaceIP %v : %v", interfaceIP, err)
	}
	portPrefix, _ := interfaceIPNet.Mask.Size()
	_, _, err = util.RunPowershell("New-NetIPAddress",
		fmt.Sprintf("-IPAddress %s", portIP),
		fmt.Sprintf("-PrefixLength %d", portPrefix),
		ifAlias)
	if err != nil {
		return err
	}

	// Set MTU for the interface
	_, _, err = util.RunNetsh("interface", "ipv4", "set", "subinterface",
		interfaceName, fmt.Sprintf("mtu=%d", config.Default.MTU),
		"store=persistent")
	if err != nil {
		return err
	}

	// Retrieve the interface index
	stdout, stderr, err := util.RunPowershell("$(Get-NetAdapter", "|", "Where",
		"{", "$_.Name", "-Match", fmt.Sprintf("\"%s\"", interfaceName), "}).ifIndex")
	if err != nil {
		logrus.Errorf("Failed to fetch interface index, stderr: %q, error: %v", stderr, err)
		return err
	}
	if _, err := strconv.Atoi(stdout); err != nil {
		logrus.Errorf("Failed to parse interface index %q: %v", stdout, err)
		return err
	}
	interfaceIndex := stdout

	clusterIP, clusterIPNet, err := net.ParseCIDR(clusterSubnet)
	if err != nil {
		return fmt.Errorf("Failed to parse clusterSubnet %v : %v", clusterSubnet, err)
	}
	// Checking if the route already exists, in which case it will not be created again
	stdout, stderr, err = util.RunRoute("print", "-4", clusterIP.String())
	if err != nil {
		logrus.Debugf("Failed to run route print, stderr: %q, error: %v", stderr, err)
	}

	if strings.Contains(stdout, clusterIP.String()) {
		logrus.Debugf("Route was found, skipping route add")
	} else {
		// Windows route command requires the mask to be specified in the IP format
		clusterMask := net.IP(clusterIPNet.Mask).String()
		// Create a route for the entire subnet.
		_, stderr, err = util.RunRoute("-p", "add",
			clusterIP.String(), "mask", clusterMask,
			routerIP, "METRIC", "2", "IF", interfaceIndex)
		if err != nil {
			logrus.Errorf("failed to run route add, stderr: %q, error: %v", stderr, err)
			return err
		}
	}

	if clusterServicesSubnet != "" {
		clusterServiceIP, clusterServiceIPNet, err := net.ParseCIDR(clusterServicesSubnet)
		if err != nil {
			return fmt.Errorf("Failed to parse clusterServicesSubnet %v : %v", clusterServicesSubnet, err)
		}
		// Checking if the route already exists, in which case it will not be created again
		stdout, stderr, err = util.RunRoute("print", "-4", clusterServiceIP.String())
		if err != nil {
			logrus.Debugf("Failed to run route print, stderr: %q, error: %v", stderr, err)
		}

		if strings.Contains(stdout, clusterServiceIP.String()) {
			logrus.Debugf("Route was found, skipping route add")
		} else {
			// Windows route command requires the mask to be specified in the IP format
			clusterServiceMask := net.IP(clusterServiceIPNet.Mask).String()
			// Create a route for the entire subnet.
			_, stderr, err = util.RunRoute("-p", "add",
				clusterServiceIP.String(), "mask", clusterServiceMask,
				routerIP, "METRIC", "2", "IF", interfaceIndex)
			if err != nil {
				logrus.Errorf("failed to run route add, stderr: %q, error: %v", stderr, err)
				return err
			}
		}
	}

	return nil
}

func configureManagementPort(clusterSubnet, clusterServicesSubnet,
	routerIP, interfaceName, interfaceIP string) error {
	if runtime.GOOS == windowsOS {
		// Return here for Windows, the commands for enabling the interface, setting the IP and adding the
		// route will be done in the above function
		return configureManagementPortWindows(clusterSubnet, clusterServicesSubnet,
			routerIP, interfaceName, interfaceIP)
	}

	// Up the interface.
	_, _, err := util.RunIP("link", "set", interfaceName, "up")
	if err != nil {
		return err
	}

	// The interface may already exist, in which case delete the routes and IP.
	_, _, err = util.RunIP("addr", "flush", "dev", interfaceName)
	if err != nil {
		return err
	}

	// Assign IP address to the internal interface.
	_, _, err = util.RunIP("addr", "add", interfaceIP, "dev", interfaceName)
	if err != nil {
		return err
	}

	// Flush the route for the entire subnet (in case it was added before).
	_, _, err = util.RunIP("route", "flush", clusterSubnet)
	if err != nil {
		return err
	}

	// Create a route for the entire subnet.
	_, _, err = util.RunIP("route", "add", clusterSubnet, "via", routerIP)
	if err != nil {
		return err
	}

	if clusterServicesSubnet != "" {
		// Flush the route for the services subnet (in case it was added before).
		_, _, err = util.RunIP("route", "flush", clusterServicesSubnet)
		if err != nil {
			return err
		}

		// Create a route for the services subnet.
		_, _, err = util.RunIP("route", "add", clusterServicesSubnet,
			"via", routerIP)
		if err != nil {
			return err
		}
	}

	return nil
}

// CreateManagementPort creates a logical switch for the node and connect it to the distributed router. This switch will start with one logical port (A OVS internal interface).
// 1. This logical port is via which a node can access all other nodes and the containers running inside them using the private IP addresses.
// 2. When this port is created on the master node, the K8s daemons become reachable from the containers without any NAT.
// 3. The nodes can health-check the pod IP addresses.
func CreateManagementPort(nodeName, localSubnet, clusterSubnet,
	clusterServicesSubnet string) error {
	// Create a router port and provide it the first address in the 'local_subnet'.
	ip, localSubnetNet, err := net.ParseCIDR(localSubnet)
	if err != nil {
		return fmt.Errorf("Failed to parse local subnet %v : %v", localSubnetNet, err)
	}
	ip = util.NextIP(ip)
	n, _ := localSubnetNet.Mask.Size()
	routerIPMask := fmt.Sprintf("%s/%d", ip.String(), n)
	routerIP := ip.String()
	// Kubernetes emits events when pods are created. The event will contain
	// only lowercase letters of the hostname even though the kubelet is
	// started with a hostname that contains lowercase and uppercase letters.
	// When the kubelet is started with a hostname containing lowercase and
	// uppercase letters, this causes a mismatch between what the watcher
	// will try to fetch and what kubernetes provides, thus failing to
	// create the port on the logical switch.
	// Until the above is changed, switch to a lowercase hostname for
	// initMinion.
	nodeName = strings.ToLower(nodeName)

	routerMac, stderr, err := util.RunOVNNbctl("--if-exist", "get", "logical_router_port", "rtos-"+nodeName, "mac")
	if err != nil {
		logrus.Errorf("Failed to get logical router port,stderr: %q, error: %v", stderr, err)
		return err
	}

	var clusterRouter string
	if routerMac == "" {
		routerMac = util.GenerateMac()
	}

	clusterRouter, err = util.GetK8sClusterRouter()
	if err != nil {
		return err
	}

	_, stderr, err = util.RunOVNNbctl("--may-exist", "lrp-add", clusterRouter, "rtos-"+nodeName, routerMac, routerIPMask)
	if err != nil {
		logrus.Errorf("Failed to add logical port to router, stderr: %q, error: %v", stderr, err)
		return err
	}

	// Create a logical switch and set its subnet.
	stdout, stderr, err := util.RunOVNNbctl("--", "--may-exist", "ls-add", nodeName, "--", "set", "logical_switch", nodeName, "other-config:subnet="+localSubnet, "external-ids:gateway_ip="+routerIPMask)
	if err != nil {
		logrus.Errorf("Failed to create a logical switch %v, stdout: %q, stderr: %q, error: %v", nodeName, stdout, stderr, err)
		return err
	}

	// Connect the switch to the router.
	stdout, stderr, err = util.RunOVNNbctl("--", "--may-exist", "lsp-add", nodeName, "stor-"+nodeName, "--", "set", "logical_switch_port", "stor-"+nodeName, "type=router", "options:router-port=rtos-"+nodeName, "addresses="+"\""+routerMac+"\"")
	if err != nil {
		logrus.Errorf("Failed to add logical port to switch, stdout: %q, stderr: %q, error: %v", stdout, stderr, err)
		return err
	}

	// Make sure br-int is created.
	stdout, stderr, err = util.RunOVSVsctl("--", "--may-exist", "add-br", "br-int")
	if err != nil {
		logrus.Errorf("Failed to create br-int, stdout: %q, stderr: %q, error: %v", stdout, stderr, err)
		return err
	}

	// Create a OVS internal interface.
	var interfaceName string
	if len(nodeName) > 11 {
		interfaceName = "k8s-" + (nodeName[:11])
	} else {
		interfaceName = "k8s-" + nodeName
	}

	stdout, stderr, err = util.RunOVSVsctl("--", "--may-exist", "add-port",
		"br-int", interfaceName, "--", "set", "interface", interfaceName,
		"type=internal", "mtu_request="+fmt.Sprintf("%d", config.Default.MTU),
		"external-ids:iface-id=k8s-"+nodeName)
	if err != nil {
		logrus.Errorf("Failed to add port to br-int, stdout: %q, stderr: %q, error: %v", stdout, stderr, err)
		return err
	}
	macAddress, stderr, err := util.RunOVSVsctl("--if-exists", "get", "interface", interfaceName, "mac_in_use")
	if err != nil {
		logrus.Errorf("Failed to get mac address of %v, stderr: %q, error: %v", interfaceName, stderr, err)
		return err
	}
	if macAddress == "[]" {
		return fmt.Errorf("Failed to get mac address of %v", interfaceName)
	}

	if runtime.GOOS == windowsOS && macAddress == "00:00:00:00:00:00" {
		var stdoutStderr []byte
		stdoutStderr, err = exec.Command("powershell", "$(Get-NetAdapter", "|", "Where", "{", "$_.Name",
			"-Match", fmt.Sprintf("\"%s\"", interfaceName), "}).MacAddress").CombinedOutput()
		if err != nil {
			logrus.Errorf("Failed to get mac address of ovn-k8s-master, stderr: %q, error: %v", fmt.Sprintf("%s", stdoutStderr), err)
			return err
		}
		// Windows returns it in 00-00-00-00-00-00 format, we want ':' instead of '-'
		macAddress = strings.Replace(strings.TrimSpace(fmt.Sprintf("%s", stdoutStderr)), "-", ":", -1)
	}

	// Create the OVN logical port.
	ip = util.NextIP(ip)
	portIP := ip.String()
	portIPMask := fmt.Sprintf("%s/%d", portIP, n)
	stdout, stderr, err = util.RunOVNNbctl("--", "--may-exist", "lsp-add", nodeName, "k8s-"+nodeName, "--", "lsp-set-addresses", "k8s-"+nodeName, macAddress+" "+portIP)
	if err != nil {
		logrus.Errorf("Failed to add logical port to switch, stdout: %q, stderr: %q, error: %v", stdout, stderr, err)
		return err
	}
	err = configureManagementPort(clusterSubnet, clusterServicesSubnet,
		routerIP, interfaceName, portIPMask)
	if err != nil {
		return err
	}

	// Add the load_balancer to the switch.
	k8sClusterLbTCP, stderr, err := util.RunOVNNbctl("--data=bare", "--no-heading", "--columns=_uuid", "find", "load_balancer", "external_ids:k8s-cluster-lb-tcp=yes")
	if err != nil {
		logrus.Errorf("Failed to get k8sClusterLbTCP, stderr: %q, error: %v", stderr, err)
		return err
	}
	if k8sClusterLbTCP == "" {
		return fmt.Errorf("Failed to get k8sClusterLbTCP")
	}

	stdout, stderr, err = util.RunOVNNbctl("set", "logical_switch", nodeName, "load_balancer="+k8sClusterLbTCP)
	if err != nil {
		logrus.Errorf("Failed to set logical switch %v's loadbalancer, stdout: %q, stderr: %q, error: %v", nodeName, stdout, stderr, err)
		return err
	}

	k8sClusterLbUDP, stderr, err := util.RunOVNNbctl("--data=bare", "--no-heading", "--columns=_uuid", "find", "load_balancer", "external_ids:k8s-cluster-lb-udp=yes")
	if err != nil {
		logrus.Errorf("Failed to get k8sClusterLbUDP, stderr: %q, error: %v", stderr, err)
		return err
	}
	if k8sClusterLbUDP == "" {
		return fmt.Errorf("Failed to get k8sClusterLbUDP")
	}

	stdout, stderr, err = util.RunOVNNbctl("add", "logical_switch", nodeName, "load_balancer", k8sClusterLbUDP)
	if err != nil {
		logrus.Errorf("Failed to add logical switch %v's loadbalancer, stdout: %q, stderr: %q, error: %v", nodeName, stdout, stderr, err)
		return err
	}

	return nil
}
<commit_msg>windows, go-controller: Include hidden adapters on management port<commit_after>package ovn

import (
	"fmt"
	"net"
	"os/exec"
	"runtime"
	"strconv"
	"strings"

	"github.com/openvswitch/ovn-kubernetes/go-controller/pkg/config"
	"github.com/openvswitch/ovn-kubernetes/go-controller/pkg/util"
	"github.com/sirupsen/logrus"
)

const (
	windowsOS = "windows"
)

func configureManagementPortWindows(clusterSubnet, clusterServicesSubnet,
	routerIP, interfaceName, interfaceIP string) error {
	// Up the interface.
	_, _, err := util.RunPowershell("Enable-NetAdapter", "-IncludeHidden", interfaceName)
	if err != nil {
		return err
	}

	//check if interface already exists
	ifAlias := fmt.Sprintf("-InterfaceAlias %s", interfaceName)
	_, _, err = util.RunPowershell("Get-NetIPAddress", ifAlias)
	if err == nil {
		//The interface already exists, we should delete the routes and IP
		logrus.Debugf("Interface %s exists, removing.", interfaceName)
		_, _, err = util.RunPowershell("Remove-NetIPAddress", ifAlias, "-Confirm:$false")
		if err != nil {
			return err
		}
	}

	// Assign IP address to the internal interface.
	portIP, interfaceIPNet, err := net.ParseCIDR(interfaceIP)
	if err != nil {
		return fmt.Errorf("Failed to parse interfaceIP %v : %v", interfaceIP, err)
	}
	portPrefix, _ := interfaceIPNet.Mask.Size()
	_, _, err = util.RunPowershell("New-NetIPAddress",
		fmt.Sprintf("-IPAddress %s", portIP),
		fmt.Sprintf("-PrefixLength %d", portPrefix),
		ifAlias)
	if err != nil {
		return err
	}

	// Set MTU for the interface
	_, _, err = util.RunNetsh("interface", "ipv4", "set", "subinterface",
		interfaceName, fmt.Sprintf("mtu=%d", config.Default.MTU),
		"store=persistent")
	if err != nil {
		return err
	}

	// Retrieve the interface index
	stdout, stderr, err := util.RunPowershell("$(Get-NetAdapter", "-IncludeHidden", "|", "Where",
		"{", "$_.Name", "-Match", fmt.Sprintf("\"%s\"", interfaceName), "}).ifIndex")
	if err != nil {
		logrus.Errorf("Failed to fetch interface index, stderr: %q, error: %v", stderr, err)
		return err
	}
	if _, err := strconv.Atoi(stdout); err != nil {
		logrus.Errorf("Failed to parse interface index %q: %v", stdout, err)
		return err
	}
	interfaceIndex := stdout

	clusterIP, clusterIPNet, err := net.ParseCIDR(clusterSubnet)
	if err != nil {
		return fmt.Errorf("Failed to parse clusterSubnet %v : %v", clusterSubnet, err)
	}
	// Checking if the route already exists, in which case it will not be created again
	stdout, stderr, err = util.RunRoute("print", "-4", clusterIP.String())
	if err != nil {
		logrus.Debugf("Failed to run route print, stderr: %q, error: %v", stderr, err)
	}

	if strings.Contains(stdout, clusterIP.String()) {
		logrus.Debugf("Route was found, skipping route add")
	} else {
		// Windows route command requires the mask to be specified in the IP format
		clusterMask := net.IP(clusterIPNet.Mask).String()
		// Create a route for the entire subnet.
		_, stderr, err = util.RunRoute("-p", "add",
			clusterIP.String(), "mask", clusterMask,
			routerIP, "METRIC", "2", "IF", interfaceIndex)
		if err != nil {
			logrus.Errorf("failed to run route add, stderr: %q, error: %v", stderr, err)
			return err
		}
	}

	if clusterServicesSubnet != "" {
		clusterServiceIP, clusterServiceIPNet, err := net.ParseCIDR(clusterServicesSubnet)
		if err != nil {
			return fmt.Errorf("Failed to parse clusterServicesSubnet %v : %v", clusterServicesSubnet, err)
		}
		// Checking if the route already exists, in which case it will not be created again
		stdout, stderr, err = util.RunRoute("print", "-4", clusterServiceIP.String())
		if err != nil {
			logrus.Debugf("Failed to run route print, stderr: %q, error: %v", stderr, err)
		}

		if strings.Contains(stdout, clusterServiceIP.String()) {
			logrus.Debugf("Route was found, skipping route add")
		} else {
			// Windows route command requires the mask to be specified in the IP format
			clusterServiceMask := net.IP(clusterServiceIPNet.Mask).String()
			// Create a route for the entire subnet.
			_, stderr, err = util.RunRoute("-p", "add",
				clusterServiceIP.String(), "mask", clusterServiceMask,
				routerIP, "METRIC", "2", "IF", interfaceIndex)
			if err != nil {
				logrus.Errorf("failed to run route add, stderr: %q, error: %v", stderr, err)
				return err
			}
		}
	}

	return nil
}

func configureManagementPort(clusterSubnet, clusterServicesSubnet,
	routerIP, interfaceName, interfaceIP string) error {
	if runtime.GOOS == windowsOS {
		// Return here for Windows, the commands for enabling the interface, setting the IP and adding the
		// route will be done in the above function
		return configureManagementPortWindows(clusterSubnet, clusterServicesSubnet,
			routerIP, interfaceName, interfaceIP)
	}

	// Up the interface.
	_, _, err := util.RunIP("link", "set", interfaceName, "up")
	if err != nil {
		return err
	}

	// The interface may already exist, in which case delete the routes and IP.
	_, _, err = util.RunIP("addr", "flush", "dev", interfaceName)
	if err != nil {
		return err
	}

	// Assign IP address to the internal interface.
	_, _, err = util.RunIP("addr", "add", interfaceIP, "dev", interfaceName)
	if err != nil {
		return err
	}

	// Flush the route for the entire subnet (in case it was added before).
	_, _, err = util.RunIP("route", "flush", clusterSubnet)
	if err != nil {
		return err
	}

	// Create a route for the entire subnet.
	_, _, err = util.RunIP("route", "add", clusterSubnet, "via", routerIP)
	if err != nil {
		return err
	}

	if clusterServicesSubnet != "" {
		// Flush the route for the services subnet (in case it was added before).
		_, _, err = util.RunIP("route", "flush", clusterServicesSubnet)
		if err != nil {
			return err
		}

		// Create a route for the services subnet.
		_, _, err = util.RunIP("route", "add", clusterServicesSubnet,
			"via", routerIP)
		if err != nil {
			return err
		}
	}

	return nil
}

// CreateManagementPort creates a logical switch for the node and connect it to the distributed router. This switch will start with one logical port (A OVS internal interface).
// 1. This logical port is via which a node can access all other nodes and the containers running inside them using the private IP addresses.
// 2. When this port is created on the master node, the K8s daemons become reachable from the containers without any NAT.
// 3. The nodes can health-check the pod IP addresses.
func CreateManagementPort(nodeName, localSubnet, clusterSubnet,
	clusterServicesSubnet string) error {
	// Create a router port and provide it the first address in the 'local_subnet'.
	ip, localSubnetNet, err := net.ParseCIDR(localSubnet)
	if err != nil {
		return fmt.Errorf("Failed to parse local subnet %v : %v", localSubnetNet, err)
	}
	ip = util.NextIP(ip)
	n, _ := localSubnetNet.Mask.Size()
	routerIPMask := fmt.Sprintf("%s/%d", ip.String(), n)
	routerIP := ip.String()
	// Kubernetes emits events when pods are created. The event will contain
	// only lowercase letters of the hostname even though the kubelet is
	// started with a hostname that contains lowercase and uppercase letters.
	// When the kubelet is started with a hostname containing lowercase and
	// uppercase letters, this causes a mismatch between what the watcher
	// will try to fetch and what kubernetes provides, thus failing to
	// create the port on the logical switch.
	// Until the above is changed, switch to a lowercase hostname for
	// initMinion.
	nodeName = strings.ToLower(nodeName)

	routerMac, stderr, err := util.RunOVNNbctl("--if-exist", "get", "logical_router_port", "rtos-"+nodeName, "mac")
	if err != nil {
		logrus.Errorf("Failed to get logical router port,stderr: %q, error: %v", stderr, err)
		return err
	}

	var clusterRouter string
	if routerMac == "" {
		routerMac = util.GenerateMac()
	}

	clusterRouter, err = util.GetK8sClusterRouter()
	if err != nil {
		return err
	}

	_, stderr, err = util.RunOVNNbctl("--may-exist", "lrp-add", clusterRouter, "rtos-"+nodeName, routerMac, routerIPMask)
	if err != nil {
		logrus.Errorf("Failed to add logical port to router, stderr: %q, error: %v", stderr, err)
		return err
	}

	// Create a logical switch and set its subnet.
	stdout, stderr, err := util.RunOVNNbctl("--", "--may-exist", "ls-add", nodeName, "--", "set", "logical_switch", nodeName, "other-config:subnet="+localSubnet, "external-ids:gateway_ip="+routerIPMask)
	if err != nil {
		logrus.Errorf("Failed to create a logical switch %v, stdout: %q, stderr: %q, error: %v", nodeName, stdout, stderr, err)
		return err
	}

	// Connect the switch to the router.
	stdout, stderr, err = util.RunOVNNbctl("--", "--may-exist", "lsp-add", nodeName, "stor-"+nodeName, "--", "set", "logical_switch_port", "stor-"+nodeName, "type=router", "options:router-port=rtos-"+nodeName, "addresses="+"\""+routerMac+"\"")
	if err != nil {
		logrus.Errorf("Failed to add logical port to switch, stdout: %q, stderr: %q, error: %v", stdout, stderr, err)
		return err
	}

	// Make sure br-int is created.
	stdout, stderr, err = util.RunOVSVsctl("--", "--may-exist", "add-br", "br-int")
	if err != nil {
		logrus.Errorf("Failed to create br-int, stdout: %q, stderr: %q, error: %v", stdout, stderr, err)
		return err
	}

	// Create a OVS internal interface.
	var interfaceName string
	if len(nodeName) > 11 {
		interfaceName = "k8s-" + (nodeName[:11])
	} else {
		interfaceName = "k8s-" + nodeName
	}

	stdout, stderr, err = util.RunOVSVsctl("--", "--may-exist", "add-port",
		"br-int", interfaceName, "--", "set", "interface", interfaceName,
		"type=internal", "mtu_request="+fmt.Sprintf("%d", config.Default.MTU),
		"external-ids:iface-id=k8s-"+nodeName)
	if err != nil {
		logrus.Errorf("Failed to add port to br-int, stdout: %q, stderr: %q, error: %v", stdout, stderr, err)
		return err
	}
	macAddress, stderr, err := util.RunOVSVsctl("--if-exists", "get", "interface", interfaceName, "mac_in_use")
	if err != nil {
		logrus.Errorf("Failed to get mac address of %v, stderr: %q, error: %v", interfaceName, stderr, err)
		return err
	}
	if macAddress == "[]" {
		return fmt.Errorf("Failed to get mac address of %v", interfaceName)
	}

	if runtime.GOOS == windowsOS && macAddress == "00:00:00:00:00:00" {
		var stdoutStderr []byte
		stdoutStderr, err = exec.Command("powershell", "$(Get-NetAdapter", "-IncludeHidden", "|", "Where", "{", "$_.Name",
			"-Match", fmt.Sprintf("\"%s\"", interfaceName), "}).MacAddress").CombinedOutput()
		if err != nil {
			logrus.Errorf("Failed to get mac address of ovn-k8s-master, stderr: %q, error: %v", fmt.Sprintf("%s", stdoutStderr), err)
			return err
		}
		// Windows returns it in 00-00-00-00-00-00 format, we want ':' instead of '-'
		macAddress = strings.Replace(strings.TrimSpace(fmt.Sprintf("%s", stdoutStderr)), "-", ":", -1)
	}

	// Create the OVN logical port.
	ip = util.NextIP(ip)
	portIP := ip.String()
	portIPMask := fmt.Sprintf("%s/%d", portIP, n)
	stdout, stderr, err = util.RunOVNNbctl("--", "--may-exist", "lsp-add", nodeName, "k8s-"+nodeName, "--", "lsp-set-addresses", "k8s-"+nodeName, macAddress+" "+portIP)
	if err != nil {
		logrus.Errorf("Failed to add logical port to switch, stdout: %q, stderr: %q, error: %v", stdout, stderr, err)
		return err
	}
	err = configureManagementPort(clusterSubnet, clusterServicesSubnet,
		routerIP, interfaceName, portIPMask)
	if err != nil {
		return err
	}

	// Add the load_balancer to the switch.
	k8sClusterLbTCP, stderr, err := util.RunOVNNbctl("--data=bare", "--no-heading", "--columns=_uuid", "find", "load_balancer", "external_ids:k8s-cluster-lb-tcp=yes")
	if err != nil {
		logrus.Errorf("Failed to get k8sClusterLbTCP, stderr: %q, error: %v", stderr, err)
		return err
	}
	if k8sClusterLbTCP == "" {
		return fmt.Errorf("Failed to get k8sClusterLbTCP")
	}

	stdout, stderr, err = util.RunOVNNbctl("set", "logical_switch", nodeName, "load_balancer="+k8sClusterLbTCP)
	if err != nil {
		logrus.Errorf("Failed to set logical switch %v's loadbalancer, stdout: %q, stderr: %q, error: %v", nodeName, stdout, stderr, err)
		return err
	}

	k8sClusterLbUDP, stderr, err := util.RunOVNNbctl("--data=bare", "--no-heading", "--columns=_uuid", "find", "load_balancer", "external_ids:k8s-cluster-lb-udp=yes")
	if err != nil {
		logrus.Errorf("Failed to get k8sClusterLbUDP, stderr: %q, error: %v", stderr, err)
		return err
	}
	if k8sClusterLbUDP == "" {
		return fmt.Errorf("Failed to get k8sClusterLbUDP")
	}

	stdout, stderr, err = util.RunOVNNbctl("add", "logical_switch", nodeName, "load_balancer", k8sClusterLbUDP)
	if err != nil {
		logrus.Errorf("Failed to add logical switch %v's loadbalancer, stdout: %q, stderr: %q, error: %v", nodeName, stdout, stderr, err)
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"
)

var (
	ping = make(chan int)
	quit = make(chan struct{})
)

func loopA() {
	for {
		select {
		case i := <-ping:
			fmt.Printf("loopA ping= %d\n", i)
			i++
		case <-quit:
			fmt.Println("after return will block")
			return
		}
	}
}

func loopB() {
	for _ = range time.After(3 * time.Second) {
		close(quit)
	}
}

func main() {
	go loopA()
	go loopB()
	for i := 0; i <= 10; i++ {
		ping <- i
		time.Sleep(1 * time.Second)
	}
}
<commit_msg>fixed example<commit_after>package main

import (
	"fmt"
	"time"
)

var (
	ping = make(chan int)
	pong = make(chan int)
	quit = make(chan struct{})
	done = make(chan struct{})
)

func loopA() {
	for {
		select {
		case i := <-ping:
			fmt.Printf("loopA ping= %d\n", i)
		case <-quit:
			fmt.Println("after return will block")
			close(done)
			return
		}
	}
}

func loopB() {
	for i := 0; i <= 100000000000; i++ {
		ping <- i
	}
}

func exit() {
	for _ = range time.After(3 * time.Second) {
		close(quit)
	}
}

func main() {
	go loopA()
	go loopB()
	go exit()
	ping <- 0
	for _ = range done {
		fmt.Println("waiting")
	}
}
<|endoftext|>


<commit_before>package kloud

import (
	"encoding/json"
	"errors"
	"fmt"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/terraformer"

	"golang.org/x/net/context"

	"github.com/koding/kite"
)

func (k *Kloud) Apply(r *kite.Request) (interface{}, error) {
	if r.Args == nil {
		return nil, NewError(ErrNoArguments)
	}

	var args *TerraformKloudRequest
	if err := r.Args.One().Unmarshal(&args); err != nil {
		return nil, err
	}

	if args.TerraformContext == "" {
		return nil, NewError(ErrTerraformContextIsMissing)
	}

	if len(args.PublicKeys) == 0 {
		return nil, errors.New("publicKeys are not passed")
	}

	if len(args.MachineIds) == 0 {
		return nil, errors.New("machine ids are not passed")
	}

	if len(args.MachineIds) != len(args.PublicKeys) {
		return nil, errors.New("machineIds and publicKeys do not match")
	}

	ctx := k.ContextCreator(context.Background())

	sess, ok := session.FromContext(ctx)
	if !ok {
		return nil, errors.New("session context is not passed")
	}

	creds, err := fetchCredentials(r.Username, sess.DB, args.PublicKeys)
	if err != nil {
		return nil, err
	}

	tfKite, err := terraformer.Connect(sess.Kite)
	if err != nil {
		return nil, err
	}
	defer tfKite.Close()

	args.TerraformContext = appendVariables(args.TerraformContext, creds)
	state, err := tfKite.Apply(args.TerraformContext)
	if err != nil {
		return nil, err
	}

	region, err := regionFromHCL(args.TerraformContext)
	if err != nil {
		return nil, err
	}

	machines, err := machinesFromState(state)
	if err != nil {
		return nil, err
	}

	machines.AppendRegion(region)

	if err := updateMachines(ctx, machines, args.MachineIds...); err != nil {
		return nil, err
	}

	d, err := json.MarshalIndent(machines, "", " ")
	if err != nil {
		return nil, err
	}

	fmt.Printf(string(d))

	return nil, errors.New("not implemented yet")
}

func updateMachines(ctx context.Context, data *Machines, ids ...string) error {
	_, ok := session.FromContext(ctx)
	if !ok {
		return errors.New("session context is not passed")
	}

	return nil
}
<commit_msg>kloud/apply: fetch jMachines to be validated and updated<commit_after>package kloud

import (
	"encoding/json"
	"errors"
	"fmt"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/provider/generic"
	"koding/kites/kloud/terraformer"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	"golang.org/x/net/context"

	"github.com/koding/kite"
)

func (k *Kloud) Apply(r *kite.Request) (interface{}, error) {
	if r.Args == nil {
		return nil, NewError(ErrNoArguments)
	}

	var args *TerraformKloudRequest
	if err := r.Args.One().Unmarshal(&args); err != nil {
		return nil, err
	}

	if args.TerraformContext == "" {
		return nil, NewError(ErrTerraformContextIsMissing)
	}

	if len(args.PublicKeys) == 0 {
		return nil, errors.New("publicKeys are not passed")
	}

	if len(args.MachineIds) == 0 {
		return nil, errors.New("machine ids are not passed")
	}

	if len(args.MachineIds) != len(args.PublicKeys) {
		return nil, errors.New("machineIds and publicKeys do not match")
	}

	ctx := k.ContextCreator(context.Background())

	sess, ok := session.FromContext(ctx)
	if !ok {
		return nil, errors.New("session context is not passed")
	}

	_, err := fetchMachines(ctx, args.MachineIds...)
	if err != nil {
		return nil, err
	}

	creds, err := fetchCredentials(r.Username, sess.DB, args.PublicKeys)
	if err != nil {
		return nil, err
	}

	tfKite, err := terraformer.Connect(sess.Kite)
	if err != nil {
		return nil, err
	}
	defer tfKite.Close()

	args.TerraformContext = appendVariables(args.TerraformContext, creds)
	state, err := tfKite.Apply(args.TerraformContext)
	if err != nil {
		return nil, err
	}

	region, err := regionFromHCL(args.TerraformContext)
	if err != nil {
		return nil, err
	}

	machines, err := machinesFromState(state)
	if err != nil {
		return nil, err
	}

	machines.AppendRegion(region)

	if err := updateMachines(ctx, machines, args.MachineIds...); err != nil {
		return nil, err
	}

	d, err := json.MarshalIndent(machines, "", " ")
	if err != nil {
		return nil, err
	}

	fmt.Printf(string(d))

	return nil, errors.New("not implemented yet")
}

func fetchMachines(ctx context.Context, ids ...string) ([]*generic.Machine, error) {
	sess, ok := session.FromContext(ctx)
	if !ok {
		return nil, errors.New("session context is not passed")
	}

	mongodbIds := make([]bson.ObjectId, len(ids))
	for i, id := range ids {
		mongodbIds[i] = bson.ObjectIdHex(id)
	}

	machines := make([]*generic.Machine, 0)
	if err := sess.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.Find(bson.M{"_id": bson.M{"$in": mongodbIds}}).All(&machines)
	}); err != nil {
		return nil, err
	}

	for i, m := range machines {
		fmt.Printf("[%d] %v\n", i, m)
	}

	return nil, nil
}

func updateMachines(ctx context.Context, data *Machines, ids ...string) error {
	sess, ok := session.FromContext(ctx)
	if !ok {
		return errors.New("session context is not passed")
	}

	for _, id := range ids {
		bsonId := bson.ObjectIdHex(id)

		sess.DB.Run("jMachines", func(c *mgo.Collection) error {
			return c.UpdateId(
				bsonId,
				bson.M{"$set": bson.M{
					"meta.instanceId": "",
					"queryString":     "",
					"meta.region":     "us-east-1",
				}},
			)
		})

	}

	return nil
}
<|endoftext|>