<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

/*
This file implements reverse buildlets. These are buildlets that are not
started by the coordinator. They dial the coordinator and then accept
instructions. This feature is used for machines that cannot be started by
an API, for example real OS X machines with iOS and Android devices attached.

You can test this setup locally. In one terminal start a coordinator.
It will default to dev mode, using a dummy TLS cert and not talking to GCE.

	$ coordinator

In another terminal, start a reverse buildlet:

	$ buildlet -reverse "darwin-amd64"

It will dial and register itself with the coordinator. To confirm the
coordinator can see the buildlet, check the logs output or visit its
diagnostics page: https://localhost:8119. To send the buildlet some
work, go to:

	https://localhost:8119/dosomework
*/

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"sort"
	"sync"
	"time"

	"golang.org/x/build/buildlet"
)

const minBuildletVersion = 1

var reversePool = &reverseBuildletPool{
	buildletReturned: make(chan token, 1),
}

type token struct{}

type reverseBuildletPool struct {
	buildletReturned chan token // best-effort tickle when any buildlet becomes free

	mu        sync.Mutex // guards buildlets and their fields
	buildlets []*reverseBuildlet
}

var errInUse = errors.New("all buildlets are in use")

func (p *reverseBuildletPool) tryToGrab(machineType string) (*buildlet.Client, error) {
	p.mu.Lock()
	defer p.mu.Unlock()
	usableCount := 0
	for _, b := range p.buildlets {
		usable := false
		for _, m := range b.modes {
			if m == machineType {
				usable = true
				usableCount++
				break
			}
		}
		if usable && b.inUseAs == "" {
			// Found an unused match.
			b.inUseAs = machineType
			b.client.SetCloseFunc(func() error {
				p.mu.Lock()
				b.inUseAs = ""
				p.mu.Unlock()
				select {
				case p.buildletReturned <- token{}:
				default:
				}
				return nil
			})
			return b.client, nil
		}
	}
	if usableCount == 0 {
		return nil, fmt.Errorf("no buildlets registered for machine type %q", machineType)
	}
	return nil, errInUse
}

func (p *reverseBuildletPool) GetBuildlet(machineType, rev string, el eventTimeLogger) (*buildlet.Client, error) {
	start := time.Now()
	for {
		el.logEventTime("try_to_grab=" + machineType + ", wait=" + time.Now().Sub(start).String())
		b, err := p.tryToGrab(machineType)
		if err == errInUse {
			select {
			case <-p.buildletReturned:
			// As multiple goroutines can be listening for the
			// buildletReturned signal, it must be treated as
			// a best effort signal. So periodically try to grab
			// a buildlet again.
			case <-time.After(30 * time.Second):
			}
		} else if err != nil {
			return nil, err
		} else {
			return b, nil
		}
	}
}

func (p *reverseBuildletPool) String() string {
	buf := bytes.NewBufferString("Reverse Buildlet Pool:")
	p.mu.Lock()
	defer p.mu.Unlock()
	inUse := 0
	for _, b := range p.buildlets {
		if b.inUseAs != "" {
			inUse++
		}
	}
	fmt.Fprintf(buf, " %d buildlets registered, %d in use", len(p.buildlets), inUse)
	for _, b := range p.buildlets {
		fmt.Fprintf(buf, "\n\t%v", b.modes)
		if b.inUseAs != "" {
			fmt.Fprintf(buf, ", in use as: %v", b.inUseAs)
		}
	}
	buf.WriteByte('\n')
	return buf.String()
}

// Modes returns the a deduplicated list of buildlet modes curently supported
// by the pool. Buildlet modes are described on reverseBuildlet comments.
func (p *reverseBuildletPool) Modes() (modes []string) {
	mm := make(map[string]bool)
	p.mu.Lock()
	for _, b := range p.buildlets {
		for _, mode := range b.modes {
			mm[mode] = true
		}
	}
	p.mu.Unlock()

	for mode := range mm {
		modes = append(modes, mode)
	}
	sort.Strings(modes)
	return modes
}

// CanBuild reports whether the pool has a machine capable of building mode.
// The machine may be in use, so you may have to wait.
func (p *reverseBuildletPool) CanBuild(mode string) bool {
	p.mu.Lock()
	defer p.mu.Unlock()
	for _, b := range p.buildlets {
		for _, m := range b.modes {
			if m == mode {
				return true
			}
		}
	}
	return false
}

// reverseBuildlet is a registered reverse buildlet.
// Its immediate fields are guarded by the reverseBuildletPool mutex.
type reverseBuildlet struct {
	client *buildlet.Client

	// modes is the set of valid modes for this buildlet.
	//
	// A mode is the equivalent of a builder name, for example
	// "darwin-amd64", "android-arm", or "linux-amd64-race".
	//
	// Each buildlet may potentially have several modes. For example a
	// Mac OS X machine with an attached iOS device may be registered
	// as both "darwin-amd64", "darwin-arm64".
	modes []string

	// inUseAs signifies that the buildlet is in use as the named mode.
	// guarded by mutex on reverseBuildletPool.
	inUseAs string
}

func handleReverse(w http.ResponseWriter, r *http.Request) {
	if r.TLS == nil {
		http.Error(w, "buildlet registration requires SSL", http.StatusInternalServerError)
		return
	}
	// Check build keys.
	modes := r.Header["X-Go-Builder-Type"]
	gobuildkeys := r.Header["X-Go-Builder-Key"]
	if len(modes) == 0 || len(modes) != len(gobuildkeys) {
		http.Error(w, fmt.Sprintf("need at least one mode and matching key, got %d/%d", len(modes), len(gobuildkeys)), http.StatusPreconditionFailed)
		return
	}
	for i, m := range modes {
		if gobuildkeys[i] != builderKey(m) {
			http.Error(w, fmt.Sprintf("bad key for mode %q", m), http.StatusPreconditionFailed)
			return
		}
	}

	conn, bufrw, err := w.(http.Hijacker).Hijack()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	log.Printf("Registering reverse buildlet %s", r.RemoteAddr)

	// The server becomes a (very simple) http client.
	(&http.Response{StatusCode: 200, Proto: "HTTP/1.1"}).Write(conn)

	client := buildlet.NewClient("none", buildlet.NoKeyPair)
	client.SetHTTPClient(&http.Client{
		Transport: newRoundTripper(bufrw),
	})
	status, err := client.Status()
	if err != nil {
		log.Printf("Reverse connection did not answer status: %v", err)
		conn.Close()
		return
	}
	if status.Version < minBuildletVersion {
		log.Printf("Buildlet too old: %s, %+v", r.RemoteAddr, status)
		conn.Close()
		return
	}
	log.Printf("Buildlet %s: %+v for %s", r.RemoteAddr, status, modes)

	// TODO(crawshaw): unregister buildlet when it disconnects. Maybe just
	// periodically request Status, and if there's no response unregister.
	reversePool.mu.Lock()
	defer reversePool.mu.Unlock()
	b := &reverseBuildlet{
		modes:  modes,
		client: client,
	}
	reversePool.buildlets = append(reversePool.buildlets, b)
	registerBuildlet(*b)
}

var registerBuildlet = func(b reverseBuildlet) {} // test hook

func newRoundTripper(bufrw *bufio.ReadWriter) *reverseRoundTripper {
	return &reverseRoundTripper{
		bufrw: bufrw,
		sema:  make(chan bool, 1),
	}
}

// reverseRoundTripper is an http client that serializes all requests
// over a *bufio.ReadWriter.
//
// Attempts at concurrent requests return an error.
type reverseRoundTripper struct {
	bufrw *bufio.ReadWriter
	sema  chan bool
}

func (c *reverseRoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	// Serialize trips. It is up to callers to avoid deadlocking.
	c.sema <- true
	if err := req.Write(c.bufrw); err != nil {
		return nil, err
	}
	if err := c.bufrw.Flush(); err != nil {
		return nil, err
	}
	resp, err = http.ReadResponse(c.bufrw.Reader, req)
	if err != nil {
		return nil, err
	}
	resp.Body = &reverseLockedBody{resp.Body, c.sema}
	return resp, err
}

type reverseLockedBody struct {
	body io.ReadCloser
	sema chan bool
}

func (b *reverseLockedBody) Read(p []byte) (n int, err error) {
	return b.body.Read(p)
}

func (b *reverseLockedBody) Close() error {
	err := b.body.Close()
	<-b.sema
	b.body = nil // prevent double close
	return err
}
<commit_msg>cmd/coordinator: very simple health checks<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

/*
This file implements reverse buildlets. These are buildlets that are not
started by the coordinator. They dial the coordinator and then accept
instructions. This feature is used for machines that cannot be started by
an API, for example real OS X machines with iOS and Android devices attached.

You can test this setup locally. In one terminal start a coordinator.
It will default to dev mode, using a dummy TLS cert and not talking to GCE.

	$ coordinator

In another terminal, start a reverse buildlet:

	$ buildlet -reverse "darwin-amd64"

It will dial and register itself with the coordinator. To confirm the
coordinator can see the buildlet, check the logs output or visit its
diagnostics page: https://localhost:8119. To send the buildlet some
work, go to:

	https://localhost:8119/dosomework
*/

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"sort"
	"sync"
	"time"

	"golang.org/x/build/buildlet"
)

const minBuildletVersion = 1

var reversePool = &reverseBuildletPool{
	buildletReturned: make(chan token, 1),
}

func init() {
	go func() {
		for {
			time.Sleep(30 * time.Second)
			reversePool.reverseHealthCheck()
		}
	}()
}

type token struct{}

type reverseBuildletPool struct {
	buildletReturned chan token // best-effort tickle when any buildlet becomes free

	mu        sync.Mutex // guards buildlets and their fields
	buildlets []*reverseBuildlet
}

var errInUse = errors.New("all buildlets are in use")

func (p *reverseBuildletPool) tryToGrab(machineType string) (*buildlet.Client, error) {
	p.mu.Lock()
	defer p.mu.Unlock()
	usableCount := 0
	for _, b := range p.buildlets {
		usable := false
		for _, m := range b.modes {
			if m == machineType {
				usable = true
				usableCount++
				break
			}
		}
		if usable && b.inUseAs == "" {
			// Found an unused match.
			b.inUseAs = machineType
			b.client.SetCloseFunc(func() error {
				p.mu.Lock()
				b.inUseAs = ""
				p.mu.Unlock()
				select {
				case p.buildletReturned <- token{}:
				default:
				}
				return nil
			})
			return b.client, nil
		}
	}
	if usableCount == 0 {
		return nil, fmt.Errorf("no buildlets registered for machine type %q", machineType)
	}
	return nil, errInUse
}

// reverseHealthCheck requests the status page of each idle buildlet.
// If the buildlet fails to respond promptly, it is removed from the pool.
func (p *reverseBuildletPool) reverseHealthCheck() {
	p.mu.Lock()
	responses := make(map[*reverseBuildlet]chan error)
	for _, b := range p.buildlets {
		if b.inUseAs == "health" { // sanity check
			panic("previous health check still running")
		}
		if b.inUseAs != "" {
			continue // skip busy buildlets
		}
		b.inUseAs = "health"
		res := make(chan error, 1)
		responses[b] = res
		client := b.client
		go func() {
			_, err := client.Status()
			res <- err
		}()
	}
	p.mu.Unlock()
	time.Sleep(5 * time.Second) // give buildlets time to respond
	p.mu.Lock()

	var buildlets []*reverseBuildlet
	for _, b := range p.buildlets {
		res := responses[b]
		if b.inUseAs != "health" || res == nil {
			// buildlet skipped or registered after health check
			buildlets = append(buildlets, b)
			continue
		}
		b.inUseAs = ""
		err, done := <-res
		if !done {
			err = errors.New("health check timeout")
		}
		if err == nil {
			buildlets = append(buildlets, b)
			continue
		}
		// remove bad buildlet
		log.Printf("Reverse buildlet %s %v not responding, removing from pool", b.client, b.modes)
		b.client.Close()
	}
	p.buildlets = buildlets
	p.mu.Unlock()
}

func (p *reverseBuildletPool) GetBuildlet(machineType, rev string, el eventTimeLogger) (*buildlet.Client, error) {
	start := time.Now()
	for {
		el.logEventTime("try_to_grab=" + machineType + ", wait=" + time.Now().Sub(start).String())
		b, err := p.tryToGrab(machineType)
		if err == errInUse {
			select {
			case <-p.buildletReturned:
			// As multiple goroutines can be listening for the
			// buildletReturned signal, it must be treated as
			// a best effort signal. So periodically try to grab
			// a buildlet again.
			case <-time.After(30 * time.Second):
			}
		} else if err != nil {
			return nil, err
		} else {
			return b, nil
		}
	}
}

func (p *reverseBuildletPool) String() string {
	buf := bytes.NewBufferString("Reverse Buildlet Pool:")
	p.mu.Lock()
	defer p.mu.Unlock()
	inUse := 0
	for _, b := range p.buildlets {
		if b.inUseAs != "" {
			inUse++
		}
	}
	fmt.Fprintf(buf, " %d buildlets registered, %d in use", len(p.buildlets), inUse)
	for _, b := range p.buildlets {
		fmt.Fprintf(buf, "\n\t%v", b.modes)
		if b.inUseAs != "" {
			fmt.Fprintf(buf, ", in use as: %v", b.inUseAs)
		}
	}
	buf.WriteByte('\n')
	return buf.String()
}

// Modes returns the a deduplicated list of buildlet modes curently supported
// by the pool. Buildlet modes are described on reverseBuildlet comments.
func (p *reverseBuildletPool) Modes() (modes []string) {
	mm := make(map[string]bool)
	p.mu.Lock()
	for _, b := range p.buildlets {
		for _, mode := range b.modes {
			mm[mode] = true
		}
	}
	p.mu.Unlock()

	for mode := range mm {
		modes = append(modes, mode)
	}
	sort.Strings(modes)
	return modes
}

// CanBuild reports whether the pool has a machine capable of building mode.
// The machine may be in use, so you may have to wait.
func (p *reverseBuildletPool) CanBuild(mode string) bool {
	p.mu.Lock()
	defer p.mu.Unlock()
	for _, b := range p.buildlets {
		for _, m := range b.modes {
			if m == mode {
				return true
			}
		}
	}
	return false
}

// reverseBuildlet is a registered reverse buildlet.
// Its immediate fields are guarded by the reverseBuildletPool mutex.
type reverseBuildlet struct {
	client *buildlet.Client

	// modes is the set of valid modes for this buildlet.
	//
	// A mode is the equivalent of a builder name, for example
	// "darwin-amd64", "android-arm", or "linux-amd64-race".
	//
	// Each buildlet may potentially have several modes. For example a
	// Mac OS X machine with an attached iOS device may be registered
	// as both "darwin-amd64", "darwin-arm64".
	modes []string

	// inUseAs signifies that the buildlet is in use as the named mode.
	// guarded by mutex on reverseBuildletPool.
	inUseAs string
}

func handleReverse(w http.ResponseWriter, r *http.Request) {
	if r.TLS == nil {
		http.Error(w, "buildlet registration requires SSL", http.StatusInternalServerError)
		return
	}
	// Check build keys.
	modes := r.Header["X-Go-Builder-Type"]
	gobuildkeys := r.Header["X-Go-Builder-Key"]
	if len(modes) == 0 || len(modes) != len(gobuildkeys) {
		http.Error(w, fmt.Sprintf("need at least one mode and matching key, got %d/%d", len(modes), len(gobuildkeys)), http.StatusPreconditionFailed)
		return
	}
	for i, m := range modes {
		if gobuildkeys[i] != builderKey(m) {
			http.Error(w, fmt.Sprintf("bad key for mode %q", m), http.StatusPreconditionFailed)
			return
		}
	}

	conn, bufrw, err := w.(http.Hijacker).Hijack()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	log.Printf("Registering reverse buildlet %s", r.RemoteAddr)

	// The server becomes a (very simple) http client.
	(&http.Response{StatusCode: 200, Proto: "HTTP/1.1"}).Write(conn)

	client := buildlet.NewClient("none", buildlet.NoKeyPair)
	client.SetHTTPClient(&http.Client{
		Transport: newRoundTripper(bufrw),
	})
	status, err := client.Status()
	if err != nil {
		log.Printf("Reverse connection did not answer status: %v", err)
		conn.Close()
		return
	}
	if status.Version < minBuildletVersion {
		log.Printf("Buildlet too old: %s, %+v", r.RemoteAddr, status)
		conn.Close()
		return
	}
	log.Printf("Buildlet %s: %+v for %s", r.RemoteAddr, status, modes)

	// TODO(crawshaw): unregister buildlet when it disconnects. Maybe just
	// periodically request Status, and if there's no response unregister.
	reversePool.mu.Lock()
	defer reversePool.mu.Unlock()
	b := &reverseBuildlet{
		modes:  modes,
		client: client,
	}
	reversePool.buildlets = append(reversePool.buildlets, b)
	registerBuildlet(*b)
}

var registerBuildlet = func(b reverseBuildlet) {} // test hook

func newRoundTripper(bufrw *bufio.ReadWriter) *reverseRoundTripper {
	return &reverseRoundTripper{
		bufrw: bufrw,
		sema:  make(chan bool, 1),
	}
}

// reverseRoundTripper is an http client that serializes all requests
// over a *bufio.ReadWriter.
//
// Attempts at concurrent requests return an error.
type reverseRoundTripper struct {
	bufrw *bufio.ReadWriter
	sema  chan bool
}

func (c *reverseRoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	// Serialize trips. It is up to callers to avoid deadlocking.
	c.sema <- true
	if err := req.Write(c.bufrw); err != nil {
		return nil, err
	}
	if err := c.bufrw.Flush(); err != nil {
		return nil, err
	}
	resp, err = http.ReadResponse(c.bufrw.Reader, req)
	if err != nil {
		return nil, err
	}
	resp.Body = &reverseLockedBody{resp.Body, c.sema}
	return resp, err
}

type reverseLockedBody struct {
	body io.ReadCloser
	sema chan bool
}

func (b *reverseLockedBody) Read(p []byte) (n int, err error) {
	return b.body.Read(p)
}

func (b *reverseLockedBody) Close() error {
	err := b.body.Close()
	<-b.sema
	b.body = nil // prevent double close
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/lifeforms/httpcheck/httpcheck"
	"io/ioutil"
	"os"
)

func parseArgs() (manifestfile string, server string, rt uint, st uint, verbose bool) {
	flagi := flag.String("i", "manifest.yaml", "Input file with YAML manifest")
	flags := flag.String("s", "", "Only check this server, default check all servers in manifest")
	flagr := flag.Uint("r", 5, "Timeout for each HTTP request in seconds, 0 for no timeout")
	flagt := flag.Uint("t", 120, "Timeout for each server in seconds, 0 for no timeout")
	flagv := flag.Bool("v", false, "Verbose, prints the result of each test")
	flag.Parse()

	manifestfile = *flagi
	server = *flags
	rt = *flagr
	st = *flagt
	verbose = *flagv
	return
}

func readManifest(manifestfile string) (manifest httpcheck.Manifest, err error) {
	y, err := ioutil.ReadFile(manifestfile)
	if err == nil {
		manifest, err = httpcheck.FromYAML(y)
	}
	return
}

func filterManifest(in httpcheck.Manifest, server string) (out httpcheck.Manifest, err error) {
	if server == "" {
		return in, nil
	}

	for _, s := range in {
		if s.Name == server {
			out = append(out, s)
		}
	}
	if len(out) == 0 {
		return nil, errors.New("Server not in manifest: " + server)
	}
	return
}

func main() {
	manifestfile, server, rt, st, verbose := parseArgs()

	httpcheck.RequestTimeout = rt
	httpcheck.ServerTimeout = st
	httpcheck.Verbose = verbose
	manifest, err := readManifest(manifestfile)

	if err == nil {
		manifest, err = filterManifest(manifest, server)
	}

	if err == nil {
		err = manifest.Test()
	}

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	} else {
		if verbose {
			fmt.Println("OK")
		}
	}
}
<commit_msg>main: if -u parameter given, set the base URL on the manifest<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/lifeforms/httpcheck/httpcheck"
	"io/ioutil"
	"os"
)

func parseArgs() (manifestfile string, server string, baseurl string, rt uint, st uint, verbose bool) {
	flagi := flag.String("i", "manifest.yaml", "Input file with YAML manifest")
	flags := flag.String("s", "", "Only check this server, default check all servers in manifest")
	flagu := flag.String("u", "", "Prefix any tests not having a scheme with this base URL")
	flagr := flag.Uint("r", 5, "Timeout for each HTTP request in seconds, 0 for no timeout")
	flagt := flag.Uint("t", 120, "Timeout for each server in seconds, 0 for no timeout")
	flagv := flag.Bool("v", false, "Verbose, prints the result of each test")
	flag.Parse()

	manifestfile = *flagi
	server = *flags
	baseurl = *flagu
	rt = *flagr
	st = *flagt
	verbose = *flagv
	return
}

func readManifest(manifestfile string) (manifest httpcheck.Manifest, err error) {
	y, err := ioutil.ReadFile(manifestfile)
	if err == nil {
		manifest, err = httpcheck.FromYAML(y)
	}
	return
}

func filterManifest(in httpcheck.Manifest, server string) (out httpcheck.Manifest, err error) {
	if server == "" {
		return in, nil
	}

	for _, s := range in {
		if s.Name == server {
			out = append(out, s)
		}
	}
	if len(out) == 0 {
		return nil, errors.New("Server not in manifest: " + server)
	}
	return
}

func main() {
	manifestfile, server, baseurl, rt, st, verbose := parseArgs()

	httpcheck.RequestTimeout = rt
	httpcheck.ServerTimeout = st
	httpcheck.Verbose = verbose
	manifest, err := readManifest(manifestfile)

	if err == nil {
		manifest, err = filterManifest(manifest, server)
	}

	if err == nil {
		if baseurl != "" {
			(&manifest).SetBaseURL(baseurl)
		}
	}

	if err == nil {
		err = manifest.Test()
	}

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	} else {
		if verbose {
			fmt.Println("OK")
		}
	}
}
<|endoftext|>


<commit_before>// Command juggler-server implements a juggler server that listens for
// connections and serves the requests.
package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/PuerkitoBio/exp/juggler"
	"github.com/PuerkitoBio/exp/juggler/broker/redisbroker"
	"github.com/garyburd/redigo/redis"
	"github.com/gorilla/websocket"
)

var (
	redisAddrFlag       = flag.String("redis", ":6379", "Redis `address`.")
	allowEmptyProtoFlag = flag.Bool("allow-empty-subprotocol", false, "Allow empty subprotocol during handshake.")
	portFlag            = flag.Int("port", 9000, "Server `port`.")
	configFlag          = flag.String("config", "", "Path of the configuration `file`.")
	helpFlag            = flag.Bool("help", false, "Show help.")
)

// Redis defines the redis-specific configuration options.
type Redis struct {
	Addr        string        `yaml:"addr"`
	MaxActive   int           `yaml:"max_active"`
	MaxIdle     int           `yaml:"max_idle"`
	IdleTimeout time.Duration `yaml:"idle_timeout"`
}

// Config defines the configuration options of the server.
type Config struct {
	Redis *Redis `yaml:"redis"`
}

func getConfigFromReader(r io.Reader) (*Config, error) {
	// set default values
	conf := &Config{
		Redis: &Redis{
			Addr:      *redisAddrFlag,
			MaxActive: 100,
			MaxIdle:   10,
		},
	}

	if r != nil {
		b, err := ioutil.ReadAll(r)
		if err != nil {
			return nil, err
		}
		if err := yaml.Unmarshal(b, conf); err != nil {
			return nil, err
		}
	}
	return conf, nil
}

func getConfigFromFile(file string) (*Config, error) {
	var r io.Reader
	if file != "" {
		f, err := os.Open(file)
		if err != nil {
			return nil, err
		}
		defer f.Close()

		r = f
	}
	return getConfigFromReader(r)
}

func main() {
	flag.Parse()
	if *helpFlag {
		flag.Usage()
		return
	}

	conf, err := getConfigFromFile(*configFlag)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to load configuration file: %v\n", err)
		flag.Usage()
		os.Exit(1)
	}

	// wrap LogMsg and ProcessMsg in a PanicRecover handler
	h := juggler.PanicRecover(
		juggler.Chain(
			juggler.HandlerFunc(juggler.LogMsg),
			juggler.HandlerFunc(juggler.ProcessMsg),
		), true, true)

	pool := newRedisPool(conf.Redis)
	broker := &redisbroker.Broker{
		Pool:      pool,
		Dial:      pool.Dial,
		CallCap:   100,
		ResultCap: 100,
	}
	// test the connection so that it fails fast if redis is not available
	c := pool.Get()
	if _, err := c.Do("PING"); err != nil {
		log.Fatalf("redis PING failed: %v", err)
	}

	if *allowEmptyProtoFlag {
		juggler.Subprotocols = append(juggler.Subprotocols, "")
	}
	upg := &websocket.Upgrader{Subprotocols: juggler.Subprotocols}
	srv := &juggler.Server{
		ReadLimit:               4096,
		WriteLimit:              4096,
		AcquireWriteLockTimeout: 200 * time.Millisecond,
		ConnState:               juggler.LogConn,
		Handler:                 h,
		PubSubBroker:            broker,
		CallerBroker:            broker,
	}
	http.Handle("/ws", juggler.Upgrade(upg, srv))

	log.Printf("listening on port %d", *portFlag)
	if err := http.ListenAndServe(fmt.Sprintf(":%d", *portFlag), nil); err != nil {
		log.Fatalf("ListenAndServe failed: %v", err)
	}
}

func newRedisPool(conf *Redis) *redis.Pool {
	addr := conf.Addr
	return &redis.Pool{
		MaxIdle:     conf.MaxIdle,
		MaxActive:   conf.MaxActive,
		IdleTimeout: conf.IdleTimeout,
		Dial: func() (redis.Conn, error) {
			c, err := redis.Dial("tcp", addr)
			if err != nil {
				return nil, err
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}
<commit_msg>juggler/cmd: support configuration file for juggler-server<commit_after>// Command juggler-server implements a juggler server that listens for
// connections and serves the requests.
package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/PuerkitoBio/exp/juggler"
	"github.com/PuerkitoBio/exp/juggler/broker"
	"github.com/PuerkitoBio/exp/juggler/broker/redisbroker"
	"github.com/garyburd/redigo/redis"
	"github.com/gorilla/websocket"
)

var (
	redisAddrFlag       = flag.String("redis", ":6379", "Redis `address`.")
	allowEmptyProtoFlag = flag.Bool("allow-empty-subprotocol", false, "Allow empty subprotocol during handshake.")
	portFlag            = flag.Int("port", 9000, "Server `port`.")
	configFlag          = flag.String("config", "", "Path of the configuration `file`.")
	helpFlag            = flag.Bool("help", false, "Show help.")
)

// Redis defines the redis-specific configuration options.
type Redis struct {
	Addr        string        `yaml:"addr"`
	MaxActive   int           `yaml:"max_active"`
	MaxIdle     int           `yaml:"max_idle"`
	IdleTimeout time.Duration `yaml:"idle_timeout"`
	PubSub      *Redis        `yaml:"pubsub"`
	Caller      *Redis        `yaml:"caller"`
}

// CallerBroker defines the configuration options for the caller broker.
type CallerBroker struct {
	BlockingTimeout time.Duration `yaml:"blocking_timeout"`
	CallCap         int           `yaml:"call_cap"`
}

// Server defines the juggler server configuration options.
type Server struct {
	// HTTP server configuration for the websocket handshake/upgrade
	Addr               string        `yaml:"addr"`
	Paths              []string      `yaml:"paths"`
	MaxHeaderBytes     int           `yaml:"max_header_bytes"`
	ReadBufferSize     int           `yaml:"read_buffer_size"`
	WriteBufferSize    int           `yaml:"write_buffer_size"`
	HandshakeTimeout   time.Duration `yaml:"handshake_timeout"`
	WhitelistedOrigins []string      `yaml:"whitelisted_origins"`

	// websocket/juggler configuration
	ReadLimit               int64         `yaml:"read_limit"`
	ReadTimeout             time.Duration `yaml:"read_timeout"`
	WriteLimit              int64         `yaml:"write_limit"`
	WriteTimeout            time.Duration `yaml:"write_timeout"`
	AcquireWriteLockTimeout time.Duration `yaml:"acquire_write_lock_timeout"`
	AllowEmptySubprotocol   bool          `yaml:"allow_empty_subprotocol"`
}

// Config defines the configuration options of the server.
type Config struct {
	Redis        *Redis        `yaml:"redis"`
	CallerBroker *CallerBroker `yaml:"caller_broker"`
	Server       *Server       `yaml:"server"`
}

func getConfigFromReader(r io.Reader) (*Config, error) {
	// set default values
	conf := &Config{
		Redis: &Redis{
			Addr:        *redisAddrFlag,
			MaxActive:   0,
			MaxIdle:     0,
			IdleTimeout: 0,
		},
		CallerBroker: &CallerBroker{
			BlockingTimeout: 0,
			CallCap:         0,
		},
		Server: &Server{
			Addr:                    ":" + strconv.Itoa(*portFlag),
			Paths:                   []string{"/ws"},
			ReadLimit:               0,
			ReadTimeout:             0,
			WriteLimit:              0,
			WriteTimeout:            0,
			AcquireWriteLockTimeout: 0,
			AllowEmptySubprotocol:   *allowEmptyProtoFlag,
		},
	}

	if r != nil {
		b, err := ioutil.ReadAll(r)
		if err != nil {
			return nil, err
		}
		if err := yaml.Unmarshal(b, conf); err != nil {
			return nil, err
		}
	}
	return conf, nil
}

func getConfigFromFile(file string) (*Config, error) {
	var r io.Reader
	if file != "" {
		f, err := os.Open(file)
		if err != nil {
			return nil, err
		}
		defer f.Close()

		r = f
	}
	return getConfigFromReader(r)
}

var zeroRedis = Redis{}

func isZeroRedis(rc *Redis) bool {
	// nil the pubsub and caller
	copy := *rc
	copy.PubSub = nil
	copy.Caller = nil
	return copy == zeroRedis
}

// TODO : check redis configuration: use Config.Redis to use the same pool
// for pubsub and caller, or use Config.Redis.PubSub and Config.Redis.Caller.
// No other combination is accepted.
func checkRedisConfig(conf *Redis) error {
	return nil
}

func main() {
	flag.Parse()
	if *helpFlag {
		flag.Usage()
		return
	}

	conf, err := getConfigFromFile(*configFlag)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to load configuration file: %v\n", err)
		flag.Usage()
		os.Exit(1)
	}

	if err := checkRedisConfig(conf.Redis); err != nil {
		fmt.Fprintf(os.Stderr, "invalid redis configuration: %v\n", err)
		flag.Usage()
		os.Exit(3)
	}

	// create pool, brokers, server, upgrader, HTTP server
	pool := newRedisPool(conf.Redis)
	psb := newPubSubBroker(pool)
	cb := newCallerBroker(conf.CallerBroker, pool)
	srv := newServer(conf.Server, psb, cb)
	srv.Handler = newHandler()
	upg := newUpgrader(conf.Server) // must be after newServer, for Subprotocols

	upgh := juggler.Upgrade(upg, srv)
	for _, p := range conf.Server.Paths {
		http.Handle(p, upgh)
	}

	httpSrv := newHTTPServer(conf.Server)

	log.Printf("listening on %s", conf.Server.Addr)
	if err := httpSrv.ListenAndServe(); err != nil {
		log.Fatalf("ListenAndServe failed: %v", err)
	}
}

func newHandler() juggler.Handler {
	return juggler.PanicRecover(
		juggler.Chain(
			juggler.HandlerFunc(juggler.LogMsg),
			juggler.HandlerFunc(juggler.ProcessMsg),
		), true, true)
}

func newPubSubBroker(pool *redis.Pool) broker.PubSubBroker {
	return &redisbroker.Broker{
		Pool: pool,
		Dial: pool.Dial,
	}
}

func newCallerBroker(conf *CallerBroker, pool *redis.Pool) broker.CallerBroker {
	return &redisbroker.Broker{
		Pool:            pool,
		Dial:            pool.Dial,
		BlockingTimeout: conf.BlockingTimeout,
		CallCap:         conf.CallCap,
	}
}

func isIn(list []string, v string) bool {
	for _, vv := range list {
		if v == vv {
			return true
		}
	}
	return false
}

func newUpgrader(conf *Server) *websocket.Upgrader {
	upg := &websocket.Upgrader{
		HandshakeTimeout: conf.HandshakeTimeout,
		ReadBufferSize:   conf.ReadBufferSize,
		WriteBufferSize:  conf.WriteBufferSize,
		Subprotocols:     juggler.Subprotocols,
	}

	if len(conf.WhitelistedOrigins) > 0 {
		oris := conf.WhitelistedOrigins
		upg.CheckOrigin = func(r *http.Request) bool {
			o := r.Header.Get("Origin")
			return isIn(oris, o)
		}
	}
	return upg
}

func newHTTPServer(conf *Server) *http.Server {
	return &http.Server{
		Addr:           conf.Addr,
		ReadTimeout:    conf.ReadTimeout,
		WriteTimeout:   conf.WriteTimeout,
		MaxHeaderBytes: conf.MaxHeaderBytes,
	}
}

func newServer(conf *Server, pubSub broker.PubSubBroker, caller broker.CallerBroker) *juggler.Server {
	if conf.AllowEmptySubprotocol {
		juggler.Subprotocols = append(juggler.Subprotocols, "")
	}

	return &juggler.Server{
		ReadLimit:               conf.ReadLimit,
		ReadTimeout:             conf.ReadTimeout,
		WriteLimit:              conf.WriteLimit,
		WriteTimeout:            conf.WriteTimeout,
		AcquireWriteLockTimeout: conf.AcquireWriteLockTimeout,
		ConnState:               juggler.LogConn,
		PubSubBroker:            pubSub,
		CallerBroker:            caller,
	}
}

func newRedisPool(conf *Redis) *redis.Pool {
	addr := conf.Addr
	p := &redis.Pool{
		MaxIdle:     conf.MaxIdle,
		MaxActive:   conf.MaxActive,
		IdleTimeout: conf.IdleTimeout,
		Dial: func() (redis.Conn, error) {
			c, err := redis.Dial("tcp", addr)
			if err != nil {
				return nil, err
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}

	// test the connection so that it fails fast if redis is not available
	c := p.Get()
	defer c.Close()

	if _, err := c.Do("PING"); err != nil {
		log.Fatalf("redis PING failed: %v", err)
	}

	return p
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"fmt"
	"io"
	"os"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"launchpad.net/gnuflag"

	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/state/backups"
)

const uploadDoc = `
"upload" sends a backup archive file to remote storage.
`

// UploadCommand is the sub-command for uploading a backup archive.
type UploadCommand struct {
	CommandBase
	// Filename is where to find the archive to upload.
	Filename string
	// ShowMeta indicates that the uploaded metadata should be printed.
	ShowMeta bool
	// Quiet indicates that the new backup ID should not be printed.
	Quiet bool
}

// SetFlags implements Command.SetFlags.
func (c *UploadCommand) SetFlags(f *gnuflag.FlagSet) {
	f.BoolVar(&c.ShowMeta, "verbose", false, "show the uploaded metadata")
	f.BoolVar(&c.Quiet, "quiet", false, "do not print the new backup ID")
}

// Info implements Command.Info.
func (c *UploadCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "upload",
		Args:    "<filename>",
		Purpose: "store a backup archive file remotely in juju",
		Doc:     uploadDoc,
	}
}

// Init implements Command.Init.
func (c *UploadCommand) Init(args []string) error {
	if len(args) == 0 {
		return errors.New("missing filename")
	}
	filename, args := args[0], args[1:]
	if err := cmd.CheckEmpty(args); err != nil {
		return errors.Trace(err)
	}
	c.Filename = filename
	return nil
}

// Run implements Command.Run.
func (c *UploadCommand) Run(ctx *cmd.Context) error {
	client, err := c.NewAPIClient()
	if err != nil {
		return errors.Trace(err)
	}
	defer client.Close()

	archive, meta, err := c.getArchive(c.Filename)
	if err != nil {
		return errors.Trace(err)
	}
	defer archive.Close()

	if c.ShowMeta {
		fmt.Fprintln(ctx.Stdout, "Uploaded metadata:")
		c.dumpMetadata(ctx, meta)
		fmt.Fprintln(ctx.Stdout)
	}

	// Upload the archive.
	id, err := client.Upload(archive, *meta)
	if err != nil {
		return errors.Trace(err)
	}

	if c.Quiet {
		fmt.Fprintln(ctx.Stdout, id)
		return nil
	}

	// Pull the stored metadata.
	stored, err := client.Info(id)
	if err != nil {
		return errors.Trace(err)
	}

	c.dumpMetadata(ctx, stored)
	return nil
}

func (c *UploadCommand) getArchive(filename string) (io.ReadCloser, *params.BackupsMetadataResult, error) {

	archive, err := os.Open(filename)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}

	// Extract the metadata.
	ad, err := backups.NewArchiveDataReader(archive)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	archive.Seek(0, os.SEEK_SET)
	meta, err := ad.Metadata()
	if err != nil {
		if !errors.IsNotFound(err) {
			return nil, nil, errors.Trace(err)
		}
		meta, err = backups.BuildMetadata(archive)
		if err != nil {
			return nil, nil, errors.Trace(err)
		}
	} else {
		// Make sure the file info is set.
		fileMeta, err := backups.BuildMetadata(archive)
		if err != nil {
			return nil, nil, errors.Trace(err)
		}
		if meta.Size() == int64(0) {
			if err := meta.SetFileInfo(fileMeta.Size(), "", ""); err != nil {
				return nil, nil, errors.Trace(err)
			}
		}
		if meta.Checksum() == "" {
			err := meta.SetFileInfo(0, fileMeta.Checksum(), fileMeta.ChecksumFormat())
			if err != nil {
				return nil, nil, errors.Trace(err)
			}
		}
		if meta.Finished == nil || meta.Finished.IsZero() {
			meta.Finished = fileMeta.Finished
		}
	}
	archive.Seek(0, os.SEEK_SET)

	// Pack the metadata into a result.
	var metaResult params.BackupsMetadataResult
	metaResult.UpdateFromMetadata(meta)

	return archive, &metaResult, nil
}
<commit_msg>Clarify an error message.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"fmt"
	"io"
	"os"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"launchpad.net/gnuflag"

	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/state/backups"
)

const uploadDoc = `
"upload" sends a backup archive file to remote storage.
`

// UploadCommand is the sub-command for uploading a backup archive.
type UploadCommand struct {
	CommandBase
	// Filename is where to find the archive to upload.
	Filename string
	// ShowMeta indicates that the uploaded metadata should be printed.
	ShowMeta bool
	// Quiet indicates that the new backup ID should not be printed.
	Quiet bool
}

// SetFlags implements Command.SetFlags.
func (c *UploadCommand) SetFlags(f *gnuflag.FlagSet) {
	f.BoolVar(&c.ShowMeta, "verbose", false, "show the uploaded metadata")
	f.BoolVar(&c.Quiet, "quiet", false, "do not print the new backup ID")
}

// Info implements Command.Info.
func (c *UploadCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "upload",
		Args:    "<filename>",
		Purpose: "store a backup archive file remotely in juju",
		Doc:     uploadDoc,
	}
}

// Init implements Command.Init.
func (c *UploadCommand) Init(args []string) error {
	if len(args) == 0 {
		return errors.New("backup filename not specified")
	}
	filename, args := args[0], args[1:]
	if err := cmd.CheckEmpty(args); err != nil {
		return errors.Trace(err)
	}
	c.Filename = filename
	return nil
}

// Run implements Command.Run.
func (c *UploadCommand) Run(ctx *cmd.Context) error {
	client, err := c.NewAPIClient()
	if err != nil {
		return errors.Trace(err)
	}
	defer client.Close()

	archive, meta, err := c.getArchive(c.Filename)
	if err != nil {
		return errors.Trace(err)
	}
	defer archive.Close()

	if c.ShowMeta {
		fmt.Fprintln(ctx.Stdout, "Uploaded metadata:")
		c.dumpMetadata(ctx, meta)
		fmt.Fprintln(ctx.Stdout)
	}

	// Upload the archive.
	id, err := client.Upload(archive, *meta)
	if err != nil {
		return errors.Trace(err)
	}

	if c.Quiet {
		fmt.Fprintln(ctx.Stdout, id)
		return nil
	}

	// Pull the stored metadata.
	stored, err := client.Info(id)
	if err != nil {
		return errors.Trace(err)
	}

	c.dumpMetadata(ctx, stored)
	return nil
}

func (c *UploadCommand) getArchive(filename string) (io.ReadCloser, *params.BackupsMetadataResult, error) {

	archive, err := os.Open(filename)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}

	// Extract the metadata.
	ad, err := backups.NewArchiveDataReader(archive)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	archive.Seek(0, os.SEEK_SET)
	meta, err := ad.Metadata()
	if err != nil {
		if !errors.IsNotFound(err) {
			return nil, nil, errors.Trace(err)
		}
		meta, err = backups.BuildMetadata(archive)
		if err != nil {
			return nil, nil, errors.Trace(err)
		}
	} else {
		// Make sure the file info is set.
		fileMeta, err := backups.BuildMetadata(archive)
		if err != nil {
			return nil, nil, errors.Trace(err)
		}
		if meta.Size() == int64(0) {
			if err := meta.SetFileInfo(fileMeta.Size(), "", ""); err != nil {
				return nil, nil, errors.Trace(err)
			}
		}
		if meta.Checksum() == "" {
			err := meta.SetFileInfo(0, fileMeta.Checksum(), fileMeta.ChecksumFormat())
			if err != nil {
				return nil, nil, errors.Trace(err)
			}
		}
		if meta.Finished == nil || meta.Finished.IsZero() {
			meta.Finished = fileMeta.Finished
		}
	}
	archive.Seek(0, os.SEEK_SET)

	// Pack the metadata into a result.
	var metaResult params.BackupsMetadataResult
	metaResult.UpdateFromMetadata(meta)

	return archive, &metaResult, nil
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: MIT
//
// Copyright © 2018 Kent Gibson <warthog618@gmail.com>.

// modeminfo collects and displays information related to the modem and its
// current configuration.
//
// This serves as an example of how interact with a modem, as well as
// providing information which may be useful for debugging.
package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"github.com/warthog618/modem/at"
	"github.com/warthog618/modem/serial"
	"github.com/warthog618/modem/trace"
)

var version = "undefined"

func main() {
	dev := flag.String("d", "/dev/ttyUSB0", "path to modem device")
	baud := flag.Int("b", 115200, "baud rate")
	timeout := flag.Duration("t", 400*time.Millisecond, "command timeout period")
	verbose := flag.Bool("v", false, "log modem interactions")
	vsn := flag.Bool("version", false, "report version and exit")
	flag.Parse()
	if *vsn {
		fmt.Printf("%s %s\n", os.Args[0], version)
		os.Exit(0)
	}
	m, err := serial.New(serial.WithPort(*dev), serial.WithBaud(*baud))
	if err != nil {
		log.Println(err)
		return
	}
	defer m.Close()
	var mio io.ReadWriter = m
	if *verbose {
		mio = trace.New(m)
	}
	a := at.New(mio, at.WithTimeout(*timeout))
	err = a.Init()
	if err != nil {
		log.Println(err)
		return
	}
	cmds := []string{
		"I",
		"+GCAP",
		"+CMEE=2",
		"+CGMI",
		"+CGMM",
		"+CGMR",
		"+CGSN",
		"+CSQ",
		"+CIMI",
		"+CREG?",
		"+CNUM",
		"+CPIN?",
		"+CEER",
		"+CSCA?",
		"+CSMS?",
		"+CSMS=?",
		"+CPMS=?",
		"+CNMI?",
		"+CNMI=?",
		"+CNMA=?",
		"+CMGF?",
		"+CMGF=?",
		"+CUSD?",
		"+CUSD=?",
		"^USSDMODE?",
		"^USSDMODE=?",
	}
	for _, cmd := range cmds {
		info, err := a.Command(cmd)
		fmt.Println("AT" + cmd)
		if err != nil {
			fmt.Printf(" %s\n", err)
			continue
		}
		for _, l := range info {
			fmt.Printf(" %s\n", l)
		}
	}
}
<commit_msg>add getting ICCID in modeminfo<commit_after>// SPDX-License-Identifier: MIT
//
// Copyright © 2018 Kent Gibson <warthog618@gmail.com>.

// modeminfo collects and displays information related to the modem and its
// current configuration.
//
// This serves as an example of how interact with a modem, as well as
// providing information which may be useful for debugging.
package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"github.com/warthog618/modem/at"
	"github.com/warthog618/modem/serial"
	"github.com/warthog618/modem/trace"
)

var version = "undefined"

func main() {
	dev := flag.String("d", "/dev/ttyUSB0", "path to modem device")
	baud := flag.Int("b", 115200, "baud rate")
	timeout := flag.Duration("t", 400*time.Millisecond, "command timeout period")
	verbose := flag.Bool("v", false, "log modem interactions")
	vsn := flag.Bool("version", false, "report version and exit")
	flag.Parse()
	if *vsn {
		fmt.Printf("%s %s\n", os.Args[0], version)
		os.Exit(0)
	}
	m, err := serial.New(serial.WithPort(*dev), serial.WithBaud(*baud))
	if err != nil {
		log.Println(err)
		return
	}
	defer m.Close()
	var mio io.ReadWriter = m
	if *verbose {
		mio = trace.New(m)
	}
	a := at.New(mio, at.WithTimeout(*timeout))
	err = a.Init()
	if err != nil {
		log.Println(err)
		return
	}
	cmds := []string{
		"I",
		"+GCAP",
		"+CMEE=2",
		"+CGMI",
		"+CGMM",
		"+CGMR",
		"+CGSN",
		"+CSQ",
		"+CIMI",
		"+CREG?",
		"+CNUM",
		"+CPIN?",
		"+CEER",
		"+CSCA?",
		"+CSMS?",
		"+CSMS=?",
		"+CPMS=?",
		"+CCID?",
		"+CCID=?",
		"^ICCID?",
		"+CNMI?",
		"+CNMI=?",
		"+CNMA=?",
		"+CMGF?",
		"+CMGF=?",
		"+CUSD?",
		"+CUSD=?",
		"^USSDMODE?",
		"^USSDMODE=?",
	}
	for _, cmd := range cmds {
		info, err := a.Command(cmd)
		fmt.Println("AT" + cmd)
		if err != nil {
			fmt.Printf(" %s\n", err)
			continue
		}
		for _, l := range info {
			fmt.Printf(" %s\n", l)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016-2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"math/rand"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"syscall"
	"time"

	"github.com/golang/glog"

	"github.com/Mirantis/virtlet/pkg/tapmanager"
)

const (
	fdSocketPath    = "/var/lib/virtlet/tapfdserver.sock"
	defaultEmulator = "/usr/bin/qemu-system-x86_64" // FIXME
	emulatorVar     = "VIRTLET_EMULATOR"
	netKeyEnvVar    = "VIRTLET_NET_KEY"
)

// Spawn child process and wait for its termination.
// If process receives EOF on pipe from parent or SIGTERM
// it forwards SIGKILL or SIGTERM to child accordingly.
func runCommand(name string, args []string, exitEOF chan bool, sigTERM chan os.Signal, tapFile *os.File) (error, bool) {
	var err error
	cmd := exec.Command(name, args...)
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout

	if len(args) >= 1 && args[0] == "-child" && tapFile != nil {
		glog.Error("Unexpected combination of input args: '-child' and tapFile are set simultaneously")
		return errors.New("Unexpected combination of input args: '-child' and tapFile are set simultaneously"), false
	}

	if tapFile != nil {
		cmd.ExtraFiles = []*os.File{
			tapFile,
		}
		defer tapFile.Close()
	}

	if len(args) >= 1 && args[0] == "-child" {
		pipeR, _, err := os.Pipe()
		if err != nil {
			glog.Errorf("Failed to create pipe: %v", err)
		}
		cmd.ExtraFiles = []*os.File{
			pipeR,
		}
		defer pipeR.Close()
	}

	if err = cmd.Start(); err != nil {
		glog.Errorf("Failed to start %q: %v", name, err)
		return err, false
	}

	done := make(chan bool, 1)
	go func() {
		if err = cmd.Wait(); err != nil {
			glog.Errorf("Failed to run %q: %v", name, err)
		}
		done <- true
	}()
	procTerminated := false

L:
	for {
		select {
		case <-done:
			break L
		case ret := <-exitEOF:
			if ret {
				glog.Info("Received EOF on pipe from parent process. Forwarding to child SIGKILL.")
			} else {
				glog.Info("Unexpected error on read pipe from parent process. Forwarding to child SIGKILL.")
			}
			if err := cmd.Process.Signal(syscall.SIGKILL); err != nil {
				glog.Errorf("Error forwarding SIGKILL to child %v.", err)
			}
			procTerminated = true
		case <-sigTERM:
			glog.Infof("Received SIGTERM. Forwarding to child and wait the process to stop.")
			if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
				glog.Errorf("Error forwarding SIGTERM to child %v.", err)
			}
			procTerminated = true
		}
	}
	return err, procTerminated
}

func runCommandIfExists(path string, args []string, exitEOF chan bool, sigTERM chan os.Signal, tapFile *os.File) (error, bool) {
	_, err := os.Stat(path)
	switch {
	case err == nil:
		return runCommand(path, args, exitEOF, sigTERM, tapFile)
	case os.IsNotExist(err):
		return nil, false
	default:
		return fmt.Errorf("os.Stat(): %v", err), false
	}
}

func catchParentKilled(exitEOF chan bool) {
	// Child process is connected to read endpoint of pipe, on 3rd file descriptor.
	// In loop, try to read something from pipe until it's closed by second side.
	pipeR := os.NewFile(uintptr(3), "pipe")

	data := make([]byte, 1)
	for {
		_, err := pipeR.Read(data)
		if err == io.EOF {
			break
		} else if err != nil {
			glog.Errorf("Read from pipe error: %v", err)
			exitEOF <- false
			break
		}
		time.Sleep(100 * time.Millisecond)
	}
	exitEOF <- true
}

// The purpose of this parent process is to be able to do cleanup
// in case of libvirt terminates process with SIGKILL.
// It is done using pipe with read end passed to child,
// which watches for EOF on the pipe.

func parent(exitEOF chan bool, sigTERM chan os.Signal) {
	emulator := os.Getenv(emulatorVar)
	emulatorArgs := os.Args[1:]

	if emulator == "" {
		// this happens during qemu -help invocation by virtlet
		// (capability check)
		// TODO: only do this when *all* above vars are empty
		// TODO: use per-emulator symlinks to vmwrapper to determine
		// the necessary binary instead of using fixed defaultEmulator
		// here
		// Exec'ing as cleanup is not need in this case
		emulatorArgs := append([]string{defaultEmulator}, emulatorArgs...)
		env := os.Environ()
		if err := syscall.Exec(defaultEmulator, emulatorArgs, env); err != nil {
			glog.Errorf("Emulator %q failed: %v", defaultEmulator, err)
			os.Exit(1)
		}
		// Wrapper process ends here and is replaced by qemu process
	}

	childArgs := append([]string{"-child"}, emulatorArgs...)
	runCommandIfExists("/vmwrapper", childArgs, exitEOF, sigTERM, nil)
}

func child(exitEOF chan bool, sigTERM chan os.Signal) {
	go catchParentKilled(exitEOF)

	emulatorArgs := os.Args[2:]

	// FIXME
	os.Args = []string{os.Args[0], "-v=3", "-alsologtostderr=true"}
	flag.Parse()

	emulator := os.Getenv(emulatorVar)
	netFdKey := os.Getenv(netKeyEnvVar)

	// TODO: use cleaner way to do this
	runHook := func(name string) {
		err, procTerminated := runCommandIfExists(name, append([]string{emulator, netFdKey}, emulatorArgs...), exitEOF, sigTERM, nil)
		if err != nil {
			glog.Errorf("Hook %q: %v", name, err)
			if procTerminated {
				os.Exit(1)
			}
		}
		if procTerminated {
			glog.Errorf("Hook %q was terminated either forcibly or gracefully. Terminate execution.", name)
			os.Exit(0)
		}
	}

	runHook("/vmwrapper-entry.sh")

	c := tapmanager.NewFDClient(fdSocketPath)
	if err := c.Connect(); err != nil {
		glog.Errorf("Can't connect to fd server: %v", err)
		os.Exit(1)
	}
	tapFd, hwAddr, err := c.GetFD(netFdKey)
	if err != nil {
		glog.Errorf("Failed to obtain tap fd for key %q: %v", netFdKey, err)
		os.Exit(1)
	}

	tapFile := os.NewFile(uintptr(tapFd), "acquired-fd")
	defer tapFile.Close()

	netArgs := []string{
		"-netdev",
		// Qemu process is started with 3d FD set to already opened TAP device
		"tap,id=tap0,fd=3",
		"-device",
		"virtio-net-pci,netdev=tap0,id=net0,mac=" + net.HardwareAddr(hwAddr).String(),
	}

	// On domain's shutdown/destroy libvirt terminates corresponding qemu process using SIGTERM or SIGKILL.
	// SIGKILL is detected by catching EOF on a pipe from parent process.
	// On catching SIGTERM just forward it to qemu process and wait for it to exit.

	err, _ = runCommand(emulator, append(emulatorArgs, netArgs...), exitEOF, sigTERM, tapFile)
	runHook("/vmwrapper-after-qemu.sh")
	if err != nil {
		glog.Errorf("Error occurred while starting subprocess in Virtlet base network namespace: %v", err)
		os.Exit(1)
	}
}

func main() {
	rand.Seed(time.Now().UnixNano())

	exitEOF := make(chan bool, 1)
	sigTERM := make(chan os.Signal)
	signal.Notify(sigTERM, syscall.SIGTERM)

	if len(os.Args) > 1 && os.Args[1] == "-child" {
		child(exitEOF, sigTERM)
	} else {
		parent(exitEOF, sigTERM)
	}
}

// TODO: convert to syscall.Exec
<commit_msg>Use syscall.Exec to start emulator from vmwrapper<commit_after>/*
Copyright 2016-2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"net"
	"os"
	"syscall"

	"github.com/golang/glog"

	"github.com/Mirantis/virtlet/pkg/tapmanager"
)

const (
	fdSocketPath    = "/var/lib/virtlet/tapfdserver.sock"
	defaultEmulator = "/usr/bin/qemu-system-x86_64" // FIXME
	emulatorVar     = "VIRTLET_EMULATOR"
	netKeyEnvVar    = "VIRTLET_NET_KEY"
)

func main() {
	// configure glog (apparently no better way to do it ...)
	flag.CommandLine.Parse([]string{"-v=3", "-alsologtostderr=true"})

	emulator := os.Getenv(emulatorVar)
	emulatorArgs := os.Args[1:]
	var netArgs []string
	if emulator == "" {
		// this happens during 'qemu -help' invocation by libvirt
		// (capability check)
		emulator = defaultEmulator
	} else {
		netFdKey := os.Getenv(netKeyEnvVar)

		c := tapmanager.NewFDClient(fdSocketPath)
		if err := c.Connect(); err != nil {
			glog.Errorf("Can't connect to fd server: %v", err)
			os.Exit(1)
		}
		tapFd, hwAddr, err := c.GetFD(netFdKey)
		if err != nil {
			glog.Errorf("Failed to obtain tap fd for key %q: %v", netFdKey, err)
			os.Exit(1)
		}

		netArgs = []string{
			"-netdev",
			fmt.Sprintf("tap,id=tap0,fd=%d", tapFd),
			"-device",
			"virtio-net-pci,netdev=tap0,id=net0,mac=" + net.HardwareAddr(hwAddr).String(),
		}
	}

	glog.V(0).Infof("executing emulator %q: args %#v", emulator, emulatorArgs)
	args := append([]string{emulator}, emulatorArgs...)
	if err := syscall.Exec(emulator, append(args, netArgs...), os.Environ()); err != nil {
		glog.Errorf("can't exec emulator %q: %v", emulator, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016-2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net"
	"os"
	"strings"
	"syscall"

	"github.com/golang/glog"

	"github.com/Mirantis/virtlet/pkg/tapmanager"
	"github.com/Mirantis/virtlet/pkg/utils"
)

// Here we use cgo constructor trick to avoid threading-related problems
// (not being able to enter the mount namespace)
// when working with process uids/gids and namespaces
// https://github.com/golang/go/issues/8676#issuecomment-66098496

/*
#define _GNU_SOURCE

#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <sched.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <linux/limits.h>

static void vmwrapper_perr(const char* msg) {
	perror(msg);
	exit(1);
}

static void vmwrapper_setns(int my_pid, int target_pid, int nstype, const char* nsname) {
	int my_ns_inode, fd;
        struct stat st;
	char my_ns_path[PATH_MAX], target_ns_path[PATH_MAX];
	snprintf(my_ns_path, sizeof(my_ns_path), "/proc/%u/ns/%s", my_pid, nsname);
	snprintf(target_ns_path, sizeof(target_ns_path), "/proc/%u/ns/%s", target_pid, nsname);
	if (stat(my_ns_path, &st) < 0) {
		vmwrapper_perr("stat() my ns");
	}
	my_ns_inode = st.st_ino;
	if (stat(target_ns_path, &st) < 0) {
		vmwrapper_perr("stat() target ns");
	}

	// Check if that's the same namespace
	// (actually only critical for CLONE_NEWUSER)
	if (my_ns_inode == st.st_ino)
		return;

	if ((fd = open(target_ns_path, O_RDONLY)) < 0) {
		vmwrapper_perr("open() target ns");
	}

	if (setns(fd, nstype) < 0) {
		vmwrapper_perr("setns()");
	}
}

// This function is a high-priority constructor that will be invoked
// before any Go code starts.
__attribute__((constructor (200))) void vmwrapper_handle_reexec(void) {
	int my_pid, target_pid;
	char* pid_str;
	if ((pid_str = getenv("VMWRAPPER_NS_PID")) == NULL)
		return;

	my_pid = getpid();
        target_pid = atoi(pid_str);

        // Other namespaces:
        // cgroup, user - not touching
        // pid - host pid namespace is used by virtlet
        // net - host network is used by virtlet
	fprintf(stderr, "vmwrapper reexec: entering vms container namespaces\n");
	vmwrapper_setns(my_pid, target_pid, CLONE_NEWNS, "mnt");
	vmwrapper_setns(my_pid, target_pid, CLONE_NEWUTS, "uts");
	vmwrapper_setns(my_pid, target_pid, CLONE_NEWIPC, "ipc");

	// permanently drop privs
	fprintf(stderr, "vmwrapper reexec: dropping privs\n");
	if (setgid(getgid()) < 0)
		vmwrapper_perr("setgid()");
	if (setuid(getuid()) < 0)
		vmwrapper_perr("setuid()");
}
*/
import "C"

const (
	fdSocketPath    = "/var/lib/virtlet/tapfdserver.sock"
	defaultEmulator = "/usr/bin/qemu-system-x86_64" // FIXME
	emulatorVar     = "VIRTLET_EMULATOR"
	netKeyEnvVar    = "VIRTLET_NET_KEY"
	vmsProcFile     = "/var/lib/virtlet/vms.procfile"
)

func main() {
	// configure glog (apparently no better way to do it ...)
	flag.CommandLine.Parse([]string{"-v=3", "-alsologtostderr=true"})

	if os.Getenv("VMWRAPPER_NS_PID") != "" {
		if err := syscall.Exec(os.Args[1], os.Args[1:], os.Environ()); err != nil {
			glog.Errorf("Can't exec emulator: %v", err)
			os.Exit(1)
		}
	}

	runInAnotherContainer := os.Getuid() != 0

	var pid int
	var err error
	if runInAnotherContainer {
		glog.V(0).Infof("Obtaining PID of the VM container process...")
		pid, err = utils.WaitForProcess(vmsProcFile)
		if err != nil {
			glog.Errorf("Can't obtain PID of the VM container process")
			os.Exit(1)
		}
	}

	emulator := os.Getenv(emulatorVar)
	emulatorArgs := os.Args[1:]
	var netArgs []string
	if emulator == "" {
		// this happens during 'qemu -help' invocation by libvirt
		// (capability check)
		emulator = defaultEmulator
	} else {
		netFdKey := os.Getenv(netKeyEnvVar)

		if netFdKey != "" {
			c := tapmanager.NewFDClient(fdSocketPath)
			if err := c.Connect(); err != nil {
				glog.Errorf("Can't connect to fd server: %v", err)
				os.Exit(1)
			}
			tapFds, marshaledHwAddrs, err := c.GetFDs(netFdKey)
			if err != nil {
				glog.Errorf("Failed to obtain tap fd for key %q: %v", netFdKey, err)
				os.Exit(1)
			}

			var hwAddrs []net.HardwareAddr
			if err := json.Unmarshal(marshaledHwAddrs, &hwAddrs); err != nil {
				glog.Errorf("Failed to unmarshal info about hardware addressess for network interfaces: %v", err)
				os.Exit(1)
			}

			netArgs = []string{
				"-netdev",
				fmt.Sprintf("tap,id=tap0,fd=%d", tapFds[0]),
				"-device",
				"virtio-net-pci,netdev=tap0,id=net0,mac=" + hwAddrs[0].String(),
			}
		}
	}

	args := append([]string{emulator}, emulatorArgs...)
	args = append(args, netArgs...)
	env := os.Environ()
	if runInAnotherContainer {
		// re-execute itself because entering mount namespace
		// is impossible after Go runtime spawns some threads
		env = append(env, fmt.Sprintf("VMWRAPPER_NS_PID=%d", pid))
		args = append([]string{os.Args[0]}, args...)
	}

	glog.V(0).Infof("Executing emulator: %s", strings.Join(args, " "))
	if err := syscall.Exec(args[0], args, env); err != nil {
		glog.Errorf("Can't exec emulator: %v", err)
		os.Exit(1)
	}
}
<commit_msg>Support multiple tap interfaces in vmwrapper<commit_after>/*
Copyright 2016-2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net"
	"os"
	"strings"
	"syscall"

	"github.com/golang/glog"

	"github.com/Mirantis/virtlet/pkg/tapmanager"
	"github.com/Mirantis/virtlet/pkg/utils"
)

// Here we use cgo constructor trick to avoid threading-related problems
// (not being able to enter the mount namespace)
// when working with process uids/gids and namespaces
// https://github.com/golang/go/issues/8676#issuecomment-66098496

/*
#define _GNU_SOURCE

#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <sched.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <linux/limits.h>

static void vmwrapper_perr(const char* msg) {
	perror(msg);
	exit(1);
}

static void vmwrapper_setns(int my_pid, int target_pid, int nstype, const char* nsname) {
	int my_ns_inode, fd;
        struct stat st;
	char my_ns_path[PATH_MAX], target_ns_path[PATH_MAX];
	snprintf(my_ns_path, sizeof(my_ns_path), "/proc/%u/ns/%s", my_pid, nsname);
	snprintf(target_ns_path, sizeof(target_ns_path), "/proc/%u/ns/%s", target_pid, nsname);
	if (stat(my_ns_path, &st) < 0) {
		vmwrapper_perr("stat() my ns");
	}
	my_ns_inode = st.st_ino;
	if (stat(target_ns_path, &st) < 0) {
		vmwrapper_perr("stat() target ns");
	}

	// Check if that's the same namespace
	// (actually only critical for CLONE_NEWUSER)
	if (my_ns_inode == st.st_ino)
		return;

	if ((fd = open(target_ns_path, O_RDONLY)) < 0) {
		vmwrapper_perr("open() target ns");
	}

	if (setns(fd, nstype) < 0) {
		vmwrapper_perr("setns()");
	}
}

// This function is a high-priority constructor that will be invoked
// before any Go code starts.
__attribute__((constructor (200))) void vmwrapper_handle_reexec(void) {
	int my_pid, target_pid;
	char* pid_str;
	if ((pid_str = getenv("VMWRAPPER_NS_PID")) == NULL)
		return;

	my_pid = getpid();
        target_pid = atoi(pid_str);

        // Other namespaces:
        // cgroup, user - not touching
        // pid - host pid namespace is used by virtlet
        // net - host network is used by virtlet
	fprintf(stderr, "vmwrapper reexec: entering vms container namespaces\n");
	vmwrapper_setns(my_pid, target_pid, CLONE_NEWNS, "mnt");
	vmwrapper_setns(my_pid, target_pid, CLONE_NEWUTS, "uts");
	vmwrapper_setns(my_pid, target_pid, CLONE_NEWIPC, "ipc");

	// permanently drop privs
	fprintf(stderr, "vmwrapper reexec: dropping privs\n");
	if (setgid(getgid()) < 0)
		vmwrapper_perr("setgid()");
	if (setuid(getuid()) < 0)
		vmwrapper_perr("setuid()");
}
*/
import "C"

const (
	fdSocketPath    = "/var/lib/virtlet/tapfdserver.sock"
	defaultEmulator = "/usr/bin/qemu-system-x86_64" // FIXME
	emulatorVar     = "VIRTLET_EMULATOR"
	netKeyEnvVar    = "VIRTLET_NET_KEY"
	vmsProcFile     = "/var/lib/virtlet/vms.procfile"
)

func main() {
	// configure glog (apparently no better way to do it ...)
	flag.CommandLine.Parse([]string{"-v=3", "-alsologtostderr=true"})

	if os.Getenv("VMWRAPPER_NS_PID") != "" {
		if err := syscall.Exec(os.Args[1], os.Args[1:], os.Environ()); err != nil {
			glog.Errorf("Can't exec emulator: %v", err)
			os.Exit(1)
		}
	}

	runInAnotherContainer := os.Getuid() != 0

	var pid int
	var err error
	if runInAnotherContainer {
		glog.V(0).Infof("Obtaining PID of the VM container process...")
		pid, err = utils.WaitForProcess(vmsProcFile)
		if err != nil {
			glog.Errorf("Can't obtain PID of the VM container process")
			os.Exit(1)
		}
	}

	emulator := os.Getenv(emulatorVar)
	emulatorArgs := os.Args[1:]
	var netArgs []string
	if emulator == "" {
		// this happens during 'qemu -help' invocation by libvirt
		// (capability check)
		emulator = defaultEmulator
	} else {
		netFdKey := os.Getenv(netKeyEnvVar)

		if netFdKey != "" {
			c := tapmanager.NewFDClient(fdSocketPath)
			if err := c.Connect(); err != nil {
				glog.Errorf("Can't connect to fd server: %v", err)
				os.Exit(1)
			}
			tapFds, marshaledHwAddrs, err := c.GetFDs(netFdKey)
			if err != nil {
				glog.Errorf("Failed to obtain tap fd for key %q: %v", netFdKey, err)
				os.Exit(1)
			}

			var hwAddrs []net.HardwareAddr
			if err := json.Unmarshal(marshaledHwAddrs, &hwAddrs); err != nil {
				glog.Errorf("Failed to unmarshal info about hardware addressess for network interfaces: %v", err)
				os.Exit(1)
			}
			if len(tapFds) != len(hwAddrs) {
				glog.Error("The number of open tap fds is different from the number of hardware addresses")
				os.Exit(1)
			}

			for i, addr := range hwAddrs {
				netArgs = append(netArgs,
					"-netdev",
					fmt.Sprintf("tap,id=tap%d,fd=%d", i, tapFds[i]),
					"-device",
					fmt.Sprintf("virtio-net-pci,netdev=tap%d,id=net%d,mac=%s", i, i, addr),
				)
			}
		}
	}

	args := append([]string{emulator}, emulatorArgs...)
	args = append(args, netArgs...)
	env := os.Environ()
	if runInAnotherContainer {
		// re-execute itself because entering mount namespace
		// is impossible after Go runtime spawns some threads
		env = append(env, fmt.Sprintf("VMWRAPPER_NS_PID=%d", pid))
		args = append([]string{os.Args[0]}, args...)
	}

	glog.V(0).Infof("Executing emulator: %s", strings.Join(args, " "))
	if err := syscall.Exec(args[0], args, env); err != nil {
		glog.Errorf("Can't exec emulator: %v", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package infrastructure

import (
	"encoding/json"
	"io/ioutil"
	"net/http"

	"github.com/marcusolsson/goddd/domain/cargo"
	"github.com/marcusolsson/goddd/domain/location"
	"github.com/marcusolsson/goddd/domain/routing"
	"github.com/marcusolsson/goddd/domain/voyage"
)

type routeDTO struct {
	Edges []edgeDTO `json:"edges"`
}

type edgeDTO struct {
	Voyage      string `json:"voyage"`
	Origin      string `json:"origin"`
	Destination string `json:"destination"`
	Departure   string `json:"departure"`
	Arrival     string `json:"arrival"`
}

type externalRoutingService struct {
	locationRepository location.Repository
}

func (s *externalRoutingService) FetchRoutesForSpecification(routeSpecification cargo.RouteSpecification) []cargo.Itinerary {

	query := "from=" + string(routeSpecification.Origin) + "&to=" + string(routeSpecification.Destination)
	resp, err := http.Get("http://ddd-pathfinder.herokuapp.com/paths?" + query)

	if err != nil {
		return []cargo.Itinerary{}
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)

	var routes []routeDTO
	err = json.Unmarshal(body, &routes)

	if err != nil {
		return []cargo.Itinerary{}
	}

	var itineraries []cargo.Itinerary
	for _, r := range routes {
		var legs []cargo.Leg
		for _, e := range r.Edges {
			legs = append(legs, cargo.Leg{
				VoyageNumber:   voyage.Number(e.Voyage),
				LoadLocation:   location.UNLocode(e.Origin),
				UnloadLocation: location.UNLocode(e.Destination),
			})
		}

		itineraries = append(itineraries, cargo.Itinerary{Legs: legs})
	}

	return itineraries
}

func NewExternalRoutingService(locationRepository location.Repository) routing.RoutingService {
	return &externalRoutingService{locationRepository: locationRepository}
}
<commit_msg>Add time to edge DTO.<commit_after>package infrastructure

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"time"

	"github.com/marcusolsson/goddd/domain/cargo"
	"github.com/marcusolsson/goddd/domain/location"
	"github.com/marcusolsson/goddd/domain/routing"
	"github.com/marcusolsson/goddd/domain/voyage"
)

type routeDTO struct {
	Edges []edgeDTO `json:"edges"`
}

type edgeDTO struct {
	Voyage      string    `json:"voyage"`
	Origin      string    `json:"origin"`
	Destination string    `json:"destination"`
	Departure   time.Time `json:"departure"`
	Arrival     time.Time `json:"arrival"`
}

type externalRoutingService struct {
	locationRepository location.Repository
}

func (s *externalRoutingService) FetchRoutesForSpecification(routeSpecification cargo.RouteSpecification) []cargo.Itinerary {

	query := "from=" + string(routeSpecification.Origin) + "&to=" + string(routeSpecification.Destination)
	resp, err := http.Get("http://ddd-pathfinder.herokuapp.com/paths?" + query)

	if err != nil {
		return []cargo.Itinerary{}
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)

	var routes []routeDTO
	err = json.Unmarshal(body, &routes)

	if err != nil {
		return []cargo.Itinerary{}
	}

	var itineraries []cargo.Itinerary
	for _, r := range routes {
		var legs []cargo.Leg
		for _, e := range r.Edges {
			legs = append(legs, cargo.Leg{
				VoyageNumber:   voyage.Number(e.Voyage),
				LoadLocation:   location.UNLocode(e.Origin),
				UnloadLocation: location.UNLocode(e.Destination),
				LoadTime:       e.Departure,
				UnloadTime:     e.Arrival,
			})
		}

		itineraries = append(itineraries, cargo.Itinerary{Legs: legs})
	}

	return itineraries
}

func NewExternalRoutingService(locationRepository location.Repository) routing.RoutingService {
	return &externalRoutingService{locationRepository: locationRepository}
}
<|endoftext|>


<commit_before>package daemon

import (
	"encoding/binary"
	"fmt"
	"time"

	"github.com/codahale/hdrhistogram"
)

type subscriber struct {
	peer
	id          int
	numMessages int
	messageSize int64
	test        test
	hasStarted  bool
	started     int64
	stopped     int64
	counter     int
	results     chan *result
}

type latencyResults struct {
	Min    int64   `json:"min"`
	Q1     int64   `json:"q1"`
	Q2     int64   `json:"q2"`
	Q3     int64   `json:"q3"`
	Max    int64   `json:"max"`
	Mean   float64 `json:"mean"`
	StdDev float64 `json:"std_dev"`
}

func (s *subscriber) start() {
	switch s.test {
	case throughput:
		s.testThroughput()
	case latency:
		s.testLatency()
	default:
		panic(fmt.Sprintf("Invalid test: %s", s.test))
	}
}

func (s *subscriber) testThroughput() {
	for {
		s.Recv()

		if !s.hasStarted {
			s.hasStarted = true
			s.started = time.Now().UnixNano()
		}

		s.counter++
		if s.counter == s.numMessages {
			s.stopped = time.Now().UnixNano()
			ms := float32(s.stopped-s.started) / 1000000.0
			s.results <- &result{
				Duration:   ms,
				Throughput: 1000 * float32(s.numMessages) / ms,
			}
			return
		}
	}
}

func (s *subscriber) testLatency() {
	latencies := hdrhistogram.New(0, 60000, 5)
	for {
		message := s.Recv()
		now := time.Now().UnixNano()
		then, _ := binary.Varint(message)

		latencies.RecordValue((now - then) / 1000000)

		s.counter++
		if s.counter == s.numMessages {
			s.results <- &result{
				Latency: &latencyResults{
					Min:    latencies.Min(),
					Q1:     latencies.ValueAtQuantile(25),
					Q2:     latencies.ValueAtQuantile(50),
					Q3:     latencies.ValueAtQuantile(75),
					Max:    latencies.Max(),
					Mean:   latencies.Mean(),
					StdDev: latencies.StdDev(),
				},
			}
			return
		}
	}
}
<commit_msg>Fix histogram resolution<commit_after>package daemon

import (
	"encoding/binary"
	"fmt"
	"time"

	"github.com/codahale/hdrhistogram"
)

type subscriber struct {
	peer
	id          int
	numMessages int
	messageSize int64
	test        test
	hasStarted  bool
	started     int64
	stopped     int64
	counter     int
	results     chan *result
}

type latencyResults struct {
	Min    int64   `json:"min"`
	Q1     int64   `json:"q1"`
	Q2     int64   `json:"q2"`
	Q3     int64   `json:"q3"`
	Max    int64   `json:"max"`
	Mean   float64 `json:"mean"`
	StdDev float64 `json:"std_dev"`
}

func (s *subscriber) start() {
	switch s.test {
	case throughput:
		s.testThroughput()
	case latency:
		s.testLatency()
	default:
		panic(fmt.Sprintf("Invalid test: %s", s.test))
	}
}

func (s *subscriber) testThroughput() {
	for {
		s.Recv()

		if !s.hasStarted {
			s.hasStarted = true
			s.started = time.Now().UnixNano()
		}

		s.counter++
		if s.counter == s.numMessages {
			s.stopped = time.Now().UnixNano()
			ms := float32(s.stopped-s.started) / 1000000.0
			s.results <- &result{
				Duration:   ms,
				Throughput: 1000 * float32(s.numMessages) / ms,
			}
			return
		}
	}
}

func (s *subscriber) testLatency() {
	latencies := hdrhistogram.New(0, 3600000, 5)
	for {
		message := s.Recv()
		now := time.Now().UnixNano()
		then, _ := binary.Varint(message)

		latencies.RecordValue((now - then) / 1000000)

		s.counter++
		if s.counter == s.numMessages {
			s.results <- &result{
				Latency: &latencyResults{
					Min:    latencies.Min(),
					Q1:     latencies.ValueAtQuantile(25),
					Q2:     latencies.ValueAtQuantile(50),
					Q3:     latencies.ValueAtQuantile(75),
					Max:    latencies.Max(),
					Mean:   latencies.Mean(),
					StdDev: latencies.StdDev(),
				},
			}
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"koding/kite-handler/command"
	"koding/kite-handler/fs"
	"koding/kite-handler/terminal"
	"log"
	"net/url"
	"os"
	"strconv"

	"github.com/koding/kite"
	"github.com/koding/kite/config"
)

const (
	VERSION = "0.0.1"
	NAME    = "klient"
)

var (
	flagIP          = flag.String("ip", "", "Change public ip")
	flagPort        = flag.Int("port", 3000, "Change running port")
	flagVersion     = flag.Bool("version", false, "Show version and exit")
	flagEnvironment = flag.String("environment", "public-host", "Change environment")
)

func main() {
	flag.Parse()
	if *flagVersion {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	k := kite.New(NAME, VERSION)
	k.Config = config.MustGet()
	k.Config.Port = *flagPort
	k.Config.Environment = *flagEnvironment

	u, _ := url.Parse("wss://kontrol.koding.com")
	k.Config.KontrolURL = u

	k.HandleFunc("fs.readDirectory", fs.ReadDirectory)
	k.HandleFunc("fs.glob", fs.Glob)
	k.HandleFunc("fs.readFile", fs.ReadFile)
	k.HandleFunc("fs.writeFile", fs.WriteFile)
	k.HandleFunc("fs.uniquePath", fs.UniquePath)
	k.HandleFunc("fs.getInfo", fs.GetInfo)
	k.HandleFunc("fs.setPermissions", fs.SetPermissions)
	k.HandleFunc("fs.remove", fs.Remove)
	k.HandleFunc("fs.rename", fs.Rename)
	k.HandleFunc("fs.createDirectory", fs.CreateDirectory)
	k.HandleFunc("fs.move", fs.Move)
	k.HandleFunc("fs.copy", fs.Copy)

	k.HandleFunc("webterm.getSessions", terminal.GetSessions)
	k.HandleFunc("webterm.connect", terminal.Connect)
	k.HandleFunc("webterm.killSession", terminal.KillSession)

	k.HandleFunc("exec", command.Exec)

	if err := k.RegisterForever(registerURL()); err != nil {
		log.Fatal(err)
	}

	k.Run()
}

func registerURL() *url.URL {
	l := &localhost{}
	p, err := l.PublicIp()
	if err != nil {
		return nil
	}

	return &url.URL{
		Scheme: "ws",
		Host:   p.String() + ":" + strconv.Itoa(*flagPort),
		Path:   "/" + NAME + "-" + VERSION,
	}
}
<commit_msg>klient: kontrolURL is already inside the token<commit_after>package main

import (
	"flag"
	"fmt"
	"koding/kite-handler/command"
	"koding/kite-handler/fs"
	"koding/kite-handler/terminal"
	"log"
	"net/url"
	"os"
	"strconv"

	"github.com/koding/kite"
	"github.com/koding/kite/config"
)

const (
	VERSION = "0.0.1"
	NAME    = "klient"
)

var (
	flagIP          = flag.String("ip", "", "Change public ip")
	flagPort        = flag.Int("port", 3000, "Change running port")
	flagVersion     = flag.Bool("version", false, "Show version and exit")
	flagEnvironment = flag.String("environment", "public-host", "Change environment")
)

func main() {
	flag.Parse()
	if *flagVersion {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	k := kite.New(NAME, VERSION)
	k.Config = config.MustGet()
	k.Config.Port = *flagPort
	k.Config.Environment = *flagEnvironment

	k.HandleFunc("fs.readDirectory", fs.ReadDirectory)
	k.HandleFunc("fs.glob", fs.Glob)
	k.HandleFunc("fs.readFile", fs.ReadFile)
	k.HandleFunc("fs.writeFile", fs.WriteFile)
	k.HandleFunc("fs.uniquePath", fs.UniquePath)
	k.HandleFunc("fs.getInfo", fs.GetInfo)
	k.HandleFunc("fs.setPermissions", fs.SetPermissions)
	k.HandleFunc("fs.remove", fs.Remove)
	k.HandleFunc("fs.rename", fs.Rename)
	k.HandleFunc("fs.createDirectory", fs.CreateDirectory)
	k.HandleFunc("fs.move", fs.Move)
	k.HandleFunc("fs.copy", fs.Copy)

	k.HandleFunc("webterm.getSessions", terminal.GetSessions)
	k.HandleFunc("webterm.connect", terminal.Connect)
	k.HandleFunc("webterm.killSession", terminal.KillSession)

	k.HandleFunc("exec", command.Exec)

	if err := k.RegisterForever(registerURL()); err != nil {
		log.Fatal(err)
	}

	k.Run()
}

func registerURL() *url.URL {
	l := &localhost{}
	p, err := l.PublicIp()
	if err != nil {
		return nil
	}

	return &url.URL{
		Scheme: "ws",
		Host:   p.String() + ":" + strconv.Itoa(*flagPort),
		Path:   "/" + NAME + "-" + VERSION,
	}
}
<|endoftext|>


<commit_before>package clusters

import (
	"math"
	"math/rand"
	"sync"
	"time"

	"gonum.org/v1/gonum/floats"
)

const (
	CHANGES_THRESHOLD = 2
)

type kmeansClusterer struct {
	iterations int
	number     int
	dimension  int

	// Variables keeping count of changes of points' membership every iteration. User as a stopping condition.
	changes, oldchanges, counter, threshold int

	// For online learning only
	alpha float64

	distance DistanceFunc

	// Mapping from training set points to clusters' numbers.
	a map[int]int

	// Mapping from clusters' numbers to set of points they contain.
	b [][]int

	// Mapping from clusters' numbers to their means
	m [][]float64

	// Training set
	d [][]float64

	// Computed clusters. Access is synchronized.
	mu sync.RWMutex
	c  []HardCluster
}

func KmeansClusterer(iterations, clusters int, distance DistanceFunc) (HardClusterer, error) {
	if iterations < 1 {
		return nil, ErrZeroIterations
	}

	if clusters < 2 {
		return nil, ErrOneCluster
	}

	var d DistanceFunc
	{
		if distance != nil {
			d = distance
		} else {
			d = EuclideanDistance
		}
	}

	return &kmeansClusterer{
		iterations: iterations,
		number:     clusters,
		distance:   d,
	}, nil
}

func (c *kmeansClusterer) WithOnline(o Online) HardClusterer {
	c.alpha = o.Alpha
	c.dimension = o.Dimension

	c.d = make([][]float64, 0, 100)

	c.initializeMeans()

	return c
}

func (c *kmeansClusterer) Learn(data [][]float64) error {
	if len(data) == 0 {
		return ErrEmptySet
	}

	c.mu.Lock()

	c.d = data

	c.a = make(map[int]int, len(data))
	c.b = make([][]int, c.number)

	c.counter = 0
	c.threshold = CHANGES_THRESHOLD
	c.changes = 0
	c.oldchanges = 0

	c.initializeMeansWithData()

	for i := 0; i < c.iterations && c.notConverged(); i++ {
		c.run()
	}

	var wg sync.WaitGroup
	{
		wg.Add(c.number)
	}

	for j := 0; j < c.number; j++ {
		go func(n int) {
			defer wg.Done()

			c.c[n] = make([][]float64, len(c.b[n]))

			for k := 0; k < len(c.b[n]); k++ {
				c.c[n][k] = c.d[c.b[n][k]]
			}
		}(j)
	}

	wg.Wait()

	c.mu.Unlock()

	c.a = nil
	c.b = nil

	return nil
}

func (c *kmeansClusterer) Guesses() []HardCluster {
	c.mu.RLock()
	defer c.mu.RUnlock()

	return c.c
}

func (c *kmeansClusterer) Predict(p []float64) HardCluster {
	var (
		l HardCluster
		d float64
		m float64 = math.MaxFloat64
	)

	for i := 0; i < len(c.c); i++ {
		if d = c.distance(p, c.m[i]); d < m {
			m = d
			l = c.c[i]
		}
	}

	return l
}

func (c *kmeansClusterer) Online(observations chan []float64, done chan struct{}, callback func([]float64, int)) {
	c.mu.Lock()

	var (
		l, f int     = len(c.m), len(c.m[0])
		h    float64 = 1 - c.alpha
	)

	/* The first step of online learning is adjusting the centroids by finding the one closes to new data point
	 * and modifying it's location using given alpha. Once the client quits sending new data, the actual clusters
	 * are computed and the mutex is unlocked. */

	go func() {
		for {
			select {
			case o := <-observations:
				var (
					k int
					n float64
					m float64 = c.distance(o, c.m[0])
				)

				for i := 1; i < l; i++ {
					if n = c.distance(o, c.m[i]); n < m {
						m = n
						k = i
					}
				}

				go callback(o, k)

				for i := 0; i < f; i++ {
					c.m[k][i] = c.alpha*o[i] + h*c.m[k][i]
				}

				c.d = append(c.d, o)
			case <-done:
				go func() {
					var (
						n    int
						l    int = len(c.d) / c.number
						d, m float64
					)

					for i := 0; i < c.number; i++ {
						c.c[n] = make([][]float64, 0, l)
					}

					for i := 0; i < len(c.d); i++ {
						m = c.distance(c.d[i], c.m[0])
						n = 0

						for j := 1; j < c.number; j++ {
							if d = c.distance(c.d[i], c.m[j]); d < m {
								m = d
								n = j
							}
						}

						c.c[n] = append(c.c[n], c.d[i])
					}

					c.mu.Unlock()
				}()

				return
			}
		}
	}()
}

// private
func (c *kmeansClusterer) initializeMeansWithData() {
	c.m = make([][]float64, c.number)
	c.c = make([]HardCluster, c.number)

	rand.Seed(time.Now().UTC().Unix())

	var (
		k          int
		s, t, l, f float64
		d          []float64 = make([]float64, len(c.d))
	)

	c.m[0] = c.d[rand.Intn(len(c.d)-1)]

	for i := 1; i < c.number; i++ {
		s = 0
		t = 0
		for j := 0; j < len(c.d); j++ {

			l = c.distance(c.m[0], c.d[j])
			for g := 1; g < i; g++ {
				if f = c.distance(c.m[g], c.d[j]); f < l {
					l = f
				}
			}

			d[j] = math.Pow(l, 2)
			s += d[j]
		}

		t = rand.Float64() * s
		k = 0
		for s = d[0]; s < t; s += d[k] {
			k++
		}

		c.m[i] = c.d[k]
	}

}

func (c *kmeansClusterer) initializeMeans() {
	c.m = make([][]float64, c.number)
	c.c = make([]HardCluster, c.number)

	rand.Seed(time.Now().UTC().Unix())

	for i := 0; i < c.number; i++ {
		c.m[i] = make([]float64, c.dimension)
		for j := 0; j < c.dimension; j++ {
			c.m[i][j] = 10 * (rand.Float64() - 0.5)
		}
	}
}

func (c *kmeansClusterer) run() {
	var (
		l, n int
		d, m float64
	)

	for i := 0; i < len(c.c); i++ {
		if l = len(c.b[i]); l == 0 {
			continue
		}

		c.m[i] = make([]float64, len(c.d[0]))

		for j := 0; j < l; j++ {
			floats.Add(c.m[i], c.d[c.b[i][j]])
		}

		floats.Scale(1/float64(l), c.m[i])

		c.b[i] = c.b[i][:0]
	}

	for i := 0; i < len(c.d); i++ {
		m = c.distance(c.d[i], c.m[0])
		n = 0

		for j := 1; j < len(c.c); j++ {
			if d = c.distance(c.d[i], c.m[j]); d < m {
				m = d
				n = j
			}
		}

		if v, ok := c.a[i]; ok && v != n {
			c.changes++
		}

		c.a[i] = n
		c.b[n] = append(c.b[n], i)
	}
}

func (c *kmeansClusterer) notConverged() bool {
	if c.counter == c.threshold {
		return false
	}

	if c.changes == c.oldchanges {
		c.counter++
	}

	c.oldchanges = c.changes

	return true
}
<commit_msg>Cleaned up<commit_after>package clusters

import (
	"math"
	"math/rand"
	"sync"
	"time"

	"gonum.org/v1/gonum/floats"
)

const (
	CHANGES_THRESHOLD = 2
)

type kmeansClusterer struct {
	iterations int
	number     int
	dimension  int

	// Variables keeping count of changes of points' membership every iteration. User as a stopping condition.
	changes, oldchanges, counter, threshold int

	// For online learning only
	alpha float64

	distance DistanceFunc

	// Mapping from training set points to clusters' numbers.
	a map[int]int

	// Mapping from clusters' numbers to set of points they contain.
	b [][]int

	// Mapping from clusters' numbers to their means
	m [][]float64

	// Training set
	d [][]float64

	// Computed clusters. Access is synchronized.
	mu sync.RWMutex
	c  []HardCluster
}

func KmeansClusterer(iterations, clusters int, distance DistanceFunc) (HardClusterer, error) {
	if iterations < 1 {
		return nil, ErrZeroIterations
	}

	if clusters < 2 {
		return nil, ErrOneCluster
	}

	var d DistanceFunc
	{
		if distance != nil {
			d = distance
		} else {
			d = EuclideanDistance
		}
	}

	return &kmeansClusterer{
		iterations: iterations,
		number:     clusters,
		distance:   d,
	}, nil
}

func (c *kmeansClusterer) WithOnline(o Online) HardClusterer {
	c.alpha = o.Alpha
	c.dimension = o.Dimension

	c.d = make([][]float64, 0, 100)

	c.initializeMeans()

	return c
}

func (c *kmeansClusterer) Learn(data [][]float64) error {
	if len(data) == 0 {
		return ErrEmptySet
	}

	c.mu.Lock()

	c.d = data

	c.a = make(map[int]int, len(data))
	c.b = make([][]int, c.number)

	c.counter = 0
	c.threshold = CHANGES_THRESHOLD
	c.changes = 0
	c.oldchanges = 0

	c.initializeMeansWithData()

	for i := 0; i < c.iterations && c.notConverged(); i++ {
		c.run()
	}

	var wg sync.WaitGroup
	{
		wg.Add(c.number)
	}

	for j := 0; j < c.number; j++ {
		go func(n int) {
			defer wg.Done()

			c.c[n] = make([][]float64, len(c.b[n]))

			for k := 0; k < len(c.b[n]); k++ {
				c.c[n][k] = c.d[c.b[n][k]]
			}
		}(j)
	}

	wg.Wait()

	c.mu.Unlock()

	c.a = nil
	c.b = nil

	return nil
}

func (c *kmeansClusterer) Guesses() []HardCluster {
	c.mu.RLock()
	defer c.mu.RUnlock()

	return c.c
}

func (c *kmeansClusterer) Predict(p []float64) HardCluster {
	var (
		l HardCluster
		d float64
		m float64 = math.MaxFloat64
	)

	for i := 0; i < len(c.c); i++ {
		if d = c.distance(p, c.m[i]); d < m {
			m = d
			l = c.c[i]
		}
	}

	return l
}

func (c *kmeansClusterer) Online(observations chan []float64, done chan struct{}, callback func([]float64, int)) {
	c.mu.Lock()

	var (
		l, f int     = len(c.m), len(c.m[0])
		h    float64 = 1 - c.alpha
	)

	/* The first step of online learning is adjusting the centroids by finding the one closes to new data point
	 * and modifying it's location using given alpha. Once the client quits sending new data, the actual clusters
	 * are computed and the mutex is unlocked. */

	go func() {
		for {
			select {
			case o := <-observations:
				var (
					k int
					n float64
					m float64 = math.pow(c.distance(o, c.m[0]), 2)
				)

				for i := 1; i < l; i++ {
					if n = math.Pow(c.distance(o, c.m[i]), 2); n < m {
						m = n
						k = i
					}
				}

				go callback(o, k)

				for i := 0; i < f; i++ {
					c.m[k][i] = c.alpha*o[i] + h*c.m[k][i]
				}

				c.d = append(c.d, o)
			case <-done:
				go func() {
					var (
						n    int
						l    int = len(c.d) / c.number
						d, m float64
					)

					for i := 0; i < c.number; i++ {
						c.c[n] = make([][]float64, 0, l)
					}

					for i := 0; i < len(c.d); i++ {
						m = c.distance(c.d[i], c.m[0])
						n = 0

						for j := 1; j < c.number; j++ {
							if d = c.distance(c.d[i], c.m[j]); d < m {
								m = d
								n = j
							}
						}

						c.c[n] = append(c.c[n], c.d[i])
					}

					c.mu.Unlock()
				}()

				return
			}
		}
	}()
}

// private
func (c *kmeansClusterer) initializeMeansWithData() {
	c.m = make([][]float64, c.number)
	c.c = make([]HardCluster, c.number)

	rand.Seed(time.Now().UTC().Unix())

	var (
		k          int
		s, t, l, f float64
		d          []float64 = make([]float64, len(c.d))
	)

	c.m[0] = c.d[rand.Intn(len(c.d)-1)]

	for i := 1; i < c.number; i++ {
		s = 0
		t = 0
		for j := 0; j < len(c.d); j++ {

			l = c.distance(c.m[0], c.d[j])
			for g := 1; g < i; g++ {
				if f = c.distance(c.m[g], c.d[j]); f < l {
					l = f
				}
			}

			d[j] = math.Pow(l, 2)
			s += d[j]
		}

		t = rand.Float64() * s
		k = 0
		for s = d[0]; s < t; s += d[k] {
			k++
		}

		c.m[i] = c.d[k]
	}

}

func (c *kmeansClusterer) initializeMeans() {
	c.m = make([][]float64, c.number)
	c.c = make([]HardCluster, c.number)

	rand.Seed(time.Now().UTC().Unix())

	for i := 0; i < c.number; i++ {
		c.m[i] = make([]float64, c.dimension)
		for j := 0; j < c.dimension; j++ {
			c.m[i][j] = 10 * (rand.Float64() - 0.5)
		}
	}
}

func (c *kmeansClusterer) run() {
	var (
		l, n int
		d, m float64
	)

	for i := 0; i < len(c.c); i++ {
		if l = len(c.b[i]); l == 0 {
			continue
		}

		c.m[i] = make([]float64, len(c.d[0]))

		for j := 0; j < l; j++ {
			floats.Add(c.m[i], c.d[c.b[i][j]])
		}

		floats.Scale(1/float64(l), c.m[i])

		c.b[i] = c.b[i][:0]
	}

	for i := 0; i < len(c.d); i++ {
		m = c.distance(c.d[i], c.m[0])
		n = 0

		for j := 1; j < len(c.c); j++ {
			if d = c.distance(c.d[i], c.m[j]); d < m {
				m = d
				n = j
			}
		}

		if v, ok := c.a[i]; ok && v != n {
			c.changes++
		}

		c.a[i] = n
		c.b[n] = append(c.b[n], i)
	}
}

func (c *kmeansClusterer) notConverged() bool {
	if c.counter == c.threshold {
		return false
	}

	if c.changes == c.oldchanges {
		c.counter++
	}

	c.oldchanges = c.changes

	return true
}
<|endoftext|>


<commit_before>package app

import (
	"errors"
	"github.com/timeredbull/openstack/keystone"
	"github.com/timeredbull/openstack/nova"
	"github.com/timeredbull/tsuru/config"
	"github.com/timeredbull/tsuru/log"
)

var (
	Client     keystone.Client
	authUrl    string
	authUser   string
	authPass   string
	authTenant string
)

type keystoneEnv struct {
	TenantId  string
	UserId    string
	AccessKey string
	secretKey string
	novaApi   nova.NetworkDisassociator
}

func (k *keystoneEnv) disassociate() error {
	err := k.disassociator().DisassociateNetwork(k.TenantId)
	if err == nova.ErrNoNetwork {
		return nil
	}
	return err
}

func (k *keystoneEnv) disassociator() nova.NetworkDisassociator {
	if k.novaApi == nil {
		k.novaApi = &nova.Client{KeystoneClient: &Client}
	}
	return k.novaApi
}

// getAuth retrieves information about openstack nova authentication. Uses the
// following confs:
//
//   - nova:
//     auth-url
//     user
//     password
//     tenant
//
// Returns error in case of failure obtaining any of the previous confs.
func getAuth() (err error) {
	if authUrl == "" {
		authUrl, err = config.GetString("nova:auth-url")
		if err != nil {
			log.Printf("ERROR: %s", err.Error())
			return
		}
	}
	if authUser == "" {
		authUser, err = config.GetString("nova:user")
		if err != nil {
			log.Printf("ERROR: %s", err.Error())
			return
		}
	}
	if authPass == "" {
		authPass, err = config.GetString("nova:password")
		if err != nil {
			log.Printf("ERROR: %s", err.Error())
			return
		}
	}
	if authTenant == "" {
		authTenant, err = config.GetString("nova:tenant")
		if err != nil {
			log.Printf("ERROR: %s", err.Error())
			return
		}
	}
	return
}

// getClient fills global Client variable with the returned value from
// keystone.NewClient.
//
// Uses the conf variables filled by getAuth function.
func getClient() (err error) {
	if Client.Token != "" {
		return
	}
	err = getAuth()
	if err != nil {
		return
	}
	c, err := keystone.NewClient(authUser, authPass, authTenant, authUrl)
	if err != nil {
		log.Printf("ERROR: a problem occurred while trying to obtain keystone's client: %s", err.Error())
		return
	}
	Client = *c
	return
}

func newKeystoneEnv(name string) (keystoneEnv, error) {
	err := getClient()
	if err != nil {
		return keystoneEnv{}, err
	}
	desc := "Tenant for " + name
	log.Printf("DEBUG: attempting to create tenant %s via keystone api...", name)
	tenant, err := Client.NewTenant(name, desc, true)
	if err != nil {
		log.Printf("ERROR: %s", err)
		return keystoneEnv{}, err
	}
	// var memberRole string
	// memberRole, err = config.GetString("nova:member-role")
	// if err != nil {
	// 	return keystoneEnv{}, err
	// }
	userId, err := config.GetString("nova:user-id")
	if err != nil {
		return keystoneEnv{}, err
	}
	creds, err := Client.NewEc2(userId, tenant.Id)
	if err != nil {
		log.Printf("ERROR: %s", err)
		return keystoneEnv{}, err
	}
	env := keystoneEnv{
		TenantId:  tenant.Id,
		UserId:    userId,
		AccessKey: creds.Access,
		secretKey: creds.Secret,
	}
	return env, nil
}

func destroyKeystoneEnv(env *keystoneEnv) error {
	if env.AccessKey == "" {
		return errors.New("Missing EC2 credentials.")
	}
	if env.UserId == "" {
		return errors.New("Missing user.")
	}
	if env.TenantId == "" {
		return errors.New("Missing tenant.")
	}
	var memberRole string
	memberRole, err := config.GetString("nova:member-role")
	if err != nil {
		return err
	}
	err = getClient()
	if err != nil {
		return err
	}
	err = Client.RemoveEc2(env.UserId, env.AccessKey)
	if err != nil {
		return err
	}
	err = Client.RemoveUser(env.UserId, env.TenantId, memberRole)
	if err != nil {
		return err
	}
	return Client.RemoveTenant(env.TenantId)
}
<commit_msg>Expliciting the return of some methods on keystone.go<commit_after>package app

import (
	"errors"
	"github.com/timeredbull/openstack/keystone"
	"github.com/timeredbull/openstack/nova"
	"github.com/timeredbull/tsuru/config"
	"github.com/timeredbull/tsuru/log"
)

var (
	Client     keystone.Client
	authUrl    string
	authUser   string
	authPass   string
	authTenant string
)

type keystoneEnv struct {
	TenantId  string
	UserId    string
	AccessKey string
	secretKey string
	novaApi   nova.NetworkDisassociator
}

func (k *keystoneEnv) disassociate() error {
	err := k.disassociator().DisassociateNetwork(k.TenantId)
	if err == nova.ErrNoNetwork {
		return nil
	}
	return err
}

func (k *keystoneEnv) disassociator() nova.NetworkDisassociator {
	if k.novaApi == nil {
		k.novaApi = &nova.Client{KeystoneClient: &Client}
	}
	return k.novaApi
}

// getAuth retrieves information about openstack nova authentication. Uses the
// following confs:
//
//   - nova:
//     auth-url
//     user
//     password
//     tenant
//
// Returns error in case of failure obtaining any of the previous confs.
func getAuth() error {
	var err error
	if authUrl == "" {
		authUrl, err = config.GetString("nova:auth-url")
		if err != nil {
			log.Printf("ERROR: %s", err.Error())
			return err
		}
	}
	if authUser == "" {
		authUser, err = config.GetString("nova:user")
		if err != nil {
			log.Printf("ERROR: %s", err.Error())
			return err
		}
	}
	if authPass == "" {
		authPass, err = config.GetString("nova:password")
		if err != nil {
			log.Printf("ERROR: %s", err.Error())
			return err
		}
	}
	if authTenant == "" {
		authTenant, err = config.GetString("nova:tenant")
		if err != nil {
			log.Printf("ERROR: %s", err.Error())
			return err
		}
	}
	return nil
}

// getClient fills global Client variable with the returned value from
// keystone.NewClient.
//
// Uses the conf variables filled by getAuth function.
func getClient() error {
	if Client.Token != "" {
		return nil
	}
	err := getAuth()
	if err != nil {
		return err
	}
	c, err := keystone.NewClient(authUser, authPass, authTenant, authUrl)
	if err != nil {
		log.Printf("ERROR: a problem occurred while trying to obtain keystone's client: %s", err.Error())
		return err
	}
	Client = *c
	return nil
}

func newKeystoneEnv(name string) (keystoneEnv, error) {
	err := getClient()
	if err != nil {
		return keystoneEnv{}, err
	}
	desc := "Tenant for " + name
	log.Printf("DEBUG: attempting to create tenant %s via keystone api...", name)
	tenant, err := Client.NewTenant(name, desc, true)
	if err != nil {
		log.Printf("ERROR: %s", err)
		return keystoneEnv{}, err
	}
	// var memberRole string
	// memberRole, err = config.GetString("nova:member-role")
	// if err != nil {
	// 	return keystoneEnv{}, err
	// }
	userId, err := config.GetString("nova:user-id")
	if err != nil {
		return keystoneEnv{}, err
	}
	creds, err := Client.NewEc2(userId, tenant.Id)
	if err != nil {
		log.Printf("ERROR: %s", err)
		return keystoneEnv{}, err
	}
	env := keystoneEnv{
		TenantId:  tenant.Id,
		UserId:    userId,
		AccessKey: creds.Access,
		secretKey: creds.Secret,
	}
	return env, nil
}

func destroyKeystoneEnv(env *keystoneEnv) error {
	if env.AccessKey == "" {
		return errors.New("Missing EC2 credentials.")
	}
	if env.UserId == "" {
		return errors.New("Missing user.")
	}
	if env.TenantId == "" {
		return errors.New("Missing tenant.")
	}
	var memberRole string
	memberRole, err := config.GetString("nova:member-role")
	if err != nil {
		return err
	}
	err = getClient()
	if err != nil {
		return err
	}
	err = Client.RemoveEc2(env.UserId, env.AccessKey)
	if err != nil {
		return err
	}
	err = Client.RemoveUser(env.UserId, env.TenantId, memberRole)
	if err != nil {
		return err
	}
	return Client.RemoveTenant(env.TenantId)
}
<|endoftext|>