<commit_before>package addressupdater

import (
	"fmt"
	"time"

	"launchpad.net/loggo"

	"launchpad.net/juju-core/errors"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/watcher"
)

var logger = loggo.GetLogger("juju.worker.addressupdater")

var (
	longPoll  = 10 * time.Second
	shortPoll = 500 * time.Millisecond
)

//func NewAddressPublisher() worker.Worker {
//	p := &updater{
//		st:
//	}
//	// wait for environment
//	go func() {
//		defer p.tomb.Done()
//		p.tomb.Kill(p.loop())
//	}()
//}

//type updater struct {
//	st   *state.State
//	tomb tomb.Tomb
//
//	mu      sync.Mutex
//	environ environs.Environ
//}

type machine interface {
	Id() string
	Addresses() []instance.Address
	InstanceId() (instance.Id, error)
	SetAddresses([]instance.Address) error
	Jobs() []state.MachineJob
	String() string
	Refresh() error
	Life() state.Life
}

type machineContext interface {
	killAll(err error)
	addresses(id instance.Id) ([]instance.Address, error)
	dying() <-chan struct{}
}

type machineAddress struct {
	machine   machine
	addresses []instance.Address
}

var _ machine = (*state.Machine)(nil)

type machinesWatcher interface {
	Changes() <-chan []string
	Err() error
	Stop() error
}

type updaterContext interface {
	newMachineContext() machineContext
	getMachine(id string) (machine, error)
	dying() <-chan struct{}
}

type updater struct {
	context     updaterContext
	machines    map[string]chan struct{}
	machineDead chan machine
}

// watchMachinesLoop watches for changes provided by the given
// machinesWatcher and starts machine goroutines to deal
// with them, using the provided newMachineContext
// function to create the appropriate context for each new machine id.
func watchMachinesLoop(context updaterContext, w machinesWatcher) (err error) {
	p := &updater{
		context:     context,
		machines:    make(map[string]chan struct{}),
		machineDead: make(chan machine),
	}
	defer func() {
		if stopErr := w.Stop(); stopErr != nil {
			if err == nil {
				err = fmt.Errorf("error stopping watcher: %v", stopErr)
			} else {
				logger.Warningf("ignoring error when stopping watcher: %v", stopErr)
			}
		}
		for len(p.machines) > 0 {
			delete(p.machines, (<-p.machineDead).Id())
		}
	}()
	for {
		select {
		case ids, ok := <-w.Changes():
			if !ok {
				return watcher.MustErr(w)
			}
			if err := p.startMachines(ids); err != nil {
				return err
			}
		case m := <-p.machineDead:
			delete(p.machines, m.Id())
		case <-p.context.dying():
			return nil
		}
	}
}

func (p *updater) startMachines(ids []string) error {
	for _, id := range ids {
		if c := p.machines[id]; c == nil {
			// We don't know about the machine - start
			// a goroutine to deal with it.
			m, err := p.context.getMachine(id)
			if errors.IsNotFoundError(err) {
				logger.Warningf("watcher gave notification of non-existent machine %q", id)
				continue
			}
			if err != nil {
				return err
			}
			c = make(chan struct{})
			p.machines[id] = c
			go runMachine(p.context.newMachineContext(), m, c, p.machineDead)
		} else {
			c <- struct{}{}
		}
	}
	return nil
}

// runMachine processes the address publishing for a given machine.
// We assume that the machine is alive when this is first called.
func runMachine(context machineContext, m machine, changed <-chan struct{}, died chan<- machine) {
	defer func() {
		// We can't just send on the died channel because the
		// central loop might be trying to write to us on the
		// changed channel.
		for {
			select {
			case died <- m:
				return
			case <-changed:
			}
		}
	}()
	if err := machineLoop(context, m, changed); err != nil {
		context.killAll(err)
	}
}

func machineLoop(context machineContext, m machine, changed <-chan struct{}) error {
	// Use a short poll interval when initially waiting for
	// a machine's address, and a long one when it already
	// has an address.
	pollInterval := longPoll
	if len(m.Addresses()) == 0 {
		pollInterval = shortPoll
	}
	for {
		instId, err := m.InstanceId()
		if err != nil {
			return fmt.Errorf("cannot get machine's instance id: %v", err)
		}
		newAddrs, err := context.addresses(instId)
		if err != nil {
			logger.Warningf("cannot get addresses for instance %q: %v", instId, err)
		} else if !addressesEqual(m.Addresses(), newAddrs) {
			if err := m.SetAddresses(newAddrs); err != nil {
				return fmt.Errorf("cannot set addresses on %q: %v", m, err)
			}
			pollInterval = longPoll
		}
		select {
		case <-time.After(pollInterval):
		case <-context.dying():
			return nil
		case <-changed:
			if err := m.Refresh(); err != nil {
				return err
			}
			// In practice the only event that will trigger
			// a change is the life state changing to dying or dead,
			// in which case we return. The logic will still work
			// if a change is triggered for some other reason,
			// but we don't mind an extra address check in that case,
			// seeing as it's unlikely.
			if m.Life() == state.Dead {
				return nil
			}
		}
	}
}

func addressesEqual(a0, a1 []instance.Address) bool {
	if len(a0) != len(a1) {
		return false
	}
	for i := range a0 {
		if a0[i] != a1[i] {
			return false
		}
	}
	return true
}
<commit_msg>worker/updater: don't fetch addresses unnecessarily<commit_after>package addressupdater

import (
	"fmt"
	"time"

	"launchpad.net/loggo"

	"launchpad.net/juju-core/errors"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/watcher"
)

var logger = loggo.GetLogger("juju.worker.addressupdater")

var (
	longPoll  = 10 * time.Second
	shortPoll = 500 * time.Millisecond
)

//func NewAddressPublisher() worker.Worker {
//	p := &updater{
//		st:
//	}
//	// wait for environment
//	go func() {
//		defer p.tomb.Done()
//		p.tomb.Kill(p.loop())
//	}()
//}

//type updater struct {
//	st   *state.State
//	tomb tomb.Tomb
//
//	mu      sync.Mutex
//	environ environs.Environ
//}

type machine interface {
	Id() string
	Addresses() []instance.Address
	InstanceId() (instance.Id, error)
	SetAddresses([]instance.Address) error
	Jobs() []state.MachineJob
	String() string
	Refresh() error
	Life() state.Life
}

type machineContext interface {
	killAll(err error)
	addresses(id instance.Id) ([]instance.Address, error)
	dying() <-chan struct{}
}

type machineAddress struct {
	machine   machine
	addresses []instance.Address
}

var _ machine = (*state.Machine)(nil)

type machinesWatcher interface {
	Changes() <-chan []string
	Err() error
	Stop() error
}

type updaterContext interface {
	newMachineContext() machineContext
	getMachine(id string) (machine, error)
	dying() <-chan struct{}
}

type updater struct {
	context     updaterContext
	machines    map[string]chan struct{}
	machineDead chan machine
}

// watchMachinesLoop watches for changes provided by the given
// machinesWatcher and starts machine goroutines to deal
// with them, using the provided newMachineContext
// function to create the appropriate context for each new machine id.
func watchMachinesLoop(context updaterContext, w machinesWatcher) (err error) {
	p := &updater{
		context:     context,
		machines:    make(map[string]chan struct{}),
		machineDead: make(chan machine),
	}
	defer func() {
		if stopErr := w.Stop(); stopErr != nil {
			if err == nil {
				err = fmt.Errorf("error stopping watcher: %v", stopErr)
			} else {
				logger.Warningf("ignoring error when stopping watcher: %v", stopErr)
			}
		}
		for len(p.machines) > 0 {
			delete(p.machines, (<-p.machineDead).Id())
		}
	}()
	for {
		select {
		case ids, ok := <-w.Changes():
			if !ok {
				return watcher.MustErr(w)
			}
			if err := p.startMachines(ids); err != nil {
				return err
			}
		case m := <-p.machineDead:
			delete(p.machines, m.Id())
		case <-p.context.dying():
			return nil
		}
	}
}

func (p *updater) startMachines(ids []string) error {
	for _, id := range ids {
		if c := p.machines[id]; c == nil {
			// We don't know about the machine - start
			// a goroutine to deal with it.
			m, err := p.context.getMachine(id)
			if errors.IsNotFoundError(err) {
				logger.Warningf("watcher gave notification of non-existent machine %q", id)
				continue
			}
			if err != nil {
				return err
			}
			c = make(chan struct{})
			p.machines[id] = c
			go runMachine(p.context.newMachineContext(), m, c, p.machineDead)
		} else {
			c <- struct{}{}
		}
	}
	return nil
}

// runMachine processes the address publishing for a given machine.
// We assume that the machine is alive when this is first called.
func runMachine(context machineContext, m machine, changed <-chan struct{}, died chan<- machine) {
	defer func() {
		// We can't just send on the died channel because the
		// central loop might be trying to write to us on the
		// changed channel.
		for {
			select {
			case died <- m:
				return
			case <-changed:
			}
		}
	}()
	if err := machineLoop(context, m, changed); err != nil {
		context.killAll(err)
	}
}

func machineLoop(context machineContext, m machine, changed <-chan struct{}) error {
	// Use a short poll interval when initially waiting for
	// a machine's address, and a long one when it already
	// has an address.
	pollInterval := shortPoll
	checkAddress := true
	for {
		if checkAddress {
			if err := checkMachineAddresses(context, m); err != nil {
				return err
			}
			if len(m.Addresses()) > 0 {
				pollInterval = longPoll
			}
			checkAddress = false
		}
		select {
		case <-time.After(pollInterval):
			checkAddress = true
		case <-context.dying():
			return nil
		case <-changed:
			if err := m.Refresh(); err != nil {
				return err
			}
			if m.Life() == state.Dead {
				return nil
			}
		}
	}
}

// checkMachineAddresses checks the current provider addresses
// for the given machine's instance, and sets them
// on the machine if they've changed.
func checkMachineAddresses(context machineContext, m machine) error {
	instId, err := m.InstanceId()
	if err != nil {
		return fmt.Errorf("cannot get machine's instance id: %v", err)
	}
	newAddrs, err := context.addresses(instId)
	if err != nil {
		logger.Warningf("cannot get addresses for instance %q: %v", instId, err)
		return nil
	}
	if addressesEqual(m.Addresses(), newAddrs) {
		return nil
	}
	if err := m.SetAddresses(newAddrs); err != nil {
		return fmt.Errorf("cannot set addresses on %q: %v", m, err)
	}
	return nil
}

func addressesEqual(a0, a1 []instance.Address) bool {
	if len(a0) != len(a1) {
		return false
	}
	for i := range a0 {
		if a0[i] != a1[i] {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package xfer

import (
	"compress/gzip"
	"encoding/gob"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"net/url"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/gorilla/handlers"
	"github.com/weaveworks/scope/report"
	"github.com/weaveworks/scope/test"
)

type publisherFunc func(io.Reader) error

func (p publisherFunc) Publish(r io.Reader) error {
	return p(r)
}

func (publisherFunc) Stop() {}

func TestAppClientPublish(t *testing.T) {
	var (
		token = "abcdefg"
		id    = "1234567"
		rpt   = report.MakeReport()
		done  = make(chan struct{})
	)

	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if want, have := fmt.Sprintf("Scope-Probe token=%s", token), r.Header.Get("Authorization"); want != have {
			t.Errorf("want %q, have %q", want, have)
		}

		if want, have := id, r.Header.Get(ScopeProbeIDHeader); want != have {
			t.Errorf("want %q, have %q", want, have)
		}

		var have report.Report

		reader := r.Body
		var err error
		if strings.Contains(r.Header.Get("Content-Encoding"), "gzip") {
			reader, err = gzip.NewReader(r.Body)
			if err != nil {
				http.Error(w, err.Error(), http.StatusBadRequest)
				return
			}
			defer reader.Close()
		}

		if err := gob.NewDecoder(reader).Decode(&have); err != nil {
			t.Error(err)
			return
		}
		if want := rpt; !reflect.DeepEqual(want, have) {
			t.Error(test.Diff(want, have))
			return
		}
		w.WriteHeader(http.StatusOK)
		close(done)
	})

	s := httptest.NewServer(handlers.CompressHandler(handler))
	defer s.Close()

	u, err := url.Parse(s.URL)
	if err != nil {
		t.Fatal(err)
	}

	pc := ProbeConfig{
		Token:    token,
		ProbeID:  id,
		Insecure: false,
	}

	p, err := NewAppClient(pc, u.Host, s.URL)
	if err != nil {
		t.Fatal(err)
	}
	defer p.Stop()
	rp := NewReportPublisher(publisherFunc(p.(*appClient).publish))
	if err := rp.Publish(rpt); err != nil {
		t.Error(err)
	}

	select {
	case <-done:
	case <-time.After(100 * time.Millisecond):
		t.Error("timeout")
	}
}
<commit_msg>Extend the testing of the AppClient.<commit_after>package xfer

import (
	"compress/gzip"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"net/url"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/gorilla/handlers"
	"github.com/weaveworks/scope/report"
	"github.com/weaveworks/scope/test"
)

type publisherFunc func(io.Reader) error

func (p publisherFunc) Publish(r io.Reader) error {
	return p(r)
}

func (publisherFunc) Stop() {}

func dummyServer(t *testing.T, expectedToken, expectedID string, expectedReport report.Report, done chan struct{}) *httptest.Server {
	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if have := r.Header.Get("Authorization"); fmt.Sprintf("Scope-Probe token=%s", expectedToken) != have {
			t.Errorf("want %q, have %q", expectedToken, have)
		}

		if have := r.Header.Get(ScopeProbeIDHeader); expectedID != have {
			t.Errorf("want %q, have %q", expectedID, have)
		}

		var have report.Report

		reader := r.Body
		var err error
		if strings.Contains(r.Header.Get("Content-Encoding"), "gzip") {
			reader, err = gzip.NewReader(r.Body)
			if err != nil {
				http.Error(w, err.Error(), http.StatusBadRequest)
				return
			}
			defer reader.Close()
		}

		if err := gob.NewDecoder(reader).Decode(&have); err != nil {
			t.Error(err)
			return
		}
		if !reflect.DeepEqual(expectedReport, have) {
			t.Error(test.Diff(expectedReport, have))
			return
		}
		w.WriteHeader(http.StatusOK)
		close(done)
	})

	return httptest.NewServer(handlers.CompressHandler(handler))
}

func TestAppClientPublishInternal(t *testing.T) {
	var (
		token = "abcdefg"
		id    = "1234567"
		rpt   = report.MakeReport()
		done  = make(chan struct{})
	)

	s := dummyServer(t, token, id, rpt, done)
	defer s.Close()

	u, err := url.Parse(s.URL)
	if err != nil {
		t.Fatal(err)
	}

	pc := ProbeConfig{
		Token:    token,
		ProbeID:  id,
		Insecure: false,
	}

	p, err := NewAppClient(pc, u.Host, s.URL)
	if err != nil {
		t.Fatal(err)
	}
	defer p.Stop()
	rp := NewReportPublisher(publisherFunc(p.(*appClient).publish))
	if err := rp.Publish(rpt); err != nil {
		t.Error(err)
	}

	select {
	case <-done:
	case <-time.After(100 * time.Millisecond):
		t.Error("timeout")
	}
}

func TestAppClientDetails(t *testing.T) {
	var (
		token = "abcdefg"
		id    = "1234567"
		rpt   = report.MakeReport()
		done  = make(chan struct{})
	)

	s := dummyServer(t, token, id, rpt, done)
	defer s.Close()

	u, err := url.Parse(s.URL)
	if err != nil {
		t.Fatal(err)
	}

	pc := ProbeConfig{
		Token:    token,
		ProbeID:  id,
		Insecure: false,
	}

	p, err := NewAppClient(pc, u.Host, s.URL)
	if err != nil {
		t.Fatal(err)
	}
	defer p.Stop()

	// First few reports might be dropped as the client is spinning up.
	rp := NewReportPublisher(p)
	for i := 0; i < 3; i++ {
		if err := rp.Publish(rpt); err != nil {
			t.Error(err)
		}
	}

	select {
	case <-done:
	case <-time.After(100 * time.Millisecond):
		t.Error("timeout")
	}
}

func TestAppClientPublish(t *testing.T) {
	var (
		id      = "foobarbaz"
		version = "imalittleteapot"
		want    = Details{ID: id, Version: version}
	)

	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if err := json.NewEncoder(w).Encode(want); err != nil {
			t.Fatal(err)
		}
	})

	s := httptest.NewServer(handlers.CompressHandler(handler))
	defer s.Close()

	u, err := url.Parse(s.URL)
	if err != nil {
		t.Fatal(err)
	}

	pc := ProbeConfig{
		Token:    "",
		ProbeID:  "",
		Insecure: false,
	}
	p, err := NewAppClient(pc, u.Host, s.URL)
	if err != nil {
		t.Fatal(err)
	}
	defer p.Stop()

	have, err := p.Details()
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(want, have) {
		t.Error(test.Diff(want, have))
		return
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package safehttp

import (
	"net/http"
)

// ResponseWriter TODO
type ResponseWriter struct {
	d  Dispatcher
	rw http.ResponseWriter

	// Having this field unexported is essential for security. Otherwise one can
	// easily overwrite the struct bypassing all our safety guarantees.
	header       Header
	muxInterceps map[string]Interceptor
	written      bool
}

// NewResponseWriter creates a ResponseWriter from a safehttp.Dispatcher, an
// http.ResponseWriter and a list of interceptors associated with a ServeMux.
func NewResponseWriter(d Dispatcher, rw http.ResponseWriter, muxInterceps map[string]Interceptor) *ResponseWriter {
	header := newHeader(rw.Header())
	return &ResponseWriter{
		d:            d,
		rw:           rw,
		header:       header,
		muxInterceps: muxInterceps,
	}
}

// Interceptor returns the interceptor associated with the given key.
func (w *ResponseWriter) Interceptor(key string) Interceptor {
	mp, ok := w.muxInterceps[key]
	if !ok {
		return nil
	}
	return mp
}

// Result TODO
type Result struct{}

// Write TODO
func (w *ResponseWriter) Write(resp Response) Result {
	w.markWritten()
	if err := w.d.Write(w.rw, resp); err != nil {
		panic("error")
	}
	return Result{}
}

// WriteTemplate TODO
func (w *ResponseWriter) WriteTemplate(t Template, data interface{}) Result {
	w.markWritten()
	if err := w.d.ExecuteTemplate(w.rw, t, data); err != nil {
		panic("error")
	}
	return Result{}
}

// ClientError TODO
func (w *ResponseWriter) ClientError(code StatusCode) Result {
	w.markWritten()
	if code < 400 || code >= 500 {
		// TODO(@mihalimara22): Replace this when we decide how to handle this case
		panic("wrong method called")
	}
	http.Error(w.rw, http.StatusText(int(code)), int(code))
	return Result{}
}

// ServerError TODO
func (w *ResponseWriter) ServerError(code StatusCode) Result {
	w.markWritten()
	if code < 500 || code >= 600 {
		// TODO(@mattiasgrenfeldt, @mihalimara22, @kele, @empijei): Decide how it should
		// be communicated to the user of the framework that they've called the wrong
		// method.
		panic("wrong method called")
		return Result{}
	}
	http.Error(w.rw, http.StatusText(int(code)), int(code))
	return Result{}
}

// Redirect responds with a redirect to a given url, using code as the status code.
func (w *ResponseWriter) Redirect(r *IncomingRequest, url string, code StatusCode) Result {
	w.markWritten()
	if code < 300 || code >= 400 {
		panic("wrong method called")
	}
	http.Redirect(w.rw, r.req, url, int(code))
	return Result{}
}

// markWritten sets written to true. If written was already true, it panics.
func (w *ResponseWriter) markWritten() {
	if w.written {
		panic("ResponseWriter already written to")
	}
	w.written = true
}

// Header returns the collection of headers that will be set
// on the response. Headers must be set before writing a
// response (e.g. Write, WriteTemplate).
func (w ResponseWriter) Header() Header {
	return w.header
}

// SetCookie adds a Set-Cookie header to the provided ResponseWriter's headers.
// The provided cookie must have a valid Name. Otherwise an error will be
// returned.
func (w *ResponseWriter) SetCookie(c *Cookie) error {
	return w.header.addCookie(c)
}

// Dispatcher TODO
type Dispatcher interface {
	Write(rw http.ResponseWriter, resp Response) error
	ExecuteTemplate(rw http.ResponseWriter, t Template, data interface{}) error
}
<commit_msg>Removed unreachable return statement<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package safehttp

import (
	"net/http"
)

// ResponseWriter TODO
type ResponseWriter struct {
	d  Dispatcher
	rw http.ResponseWriter

	// Having this field unexported is essential for security. Otherwise one can
	// easily overwrite the struct bypassing all our safety guarantees.
	header       Header
	muxInterceps map[string]Interceptor
	written      bool
}

// NewResponseWriter creates a ResponseWriter from a safehttp.Dispatcher, an
// http.ResponseWriter and a list of interceptors associated with a ServeMux.
func NewResponseWriter(d Dispatcher, rw http.ResponseWriter, muxInterceps map[string]Interceptor) *ResponseWriter {
	header := newHeader(rw.Header())
	return &ResponseWriter{
		d:            d,
		rw:           rw,
		header:       header,
		muxInterceps: muxInterceps,
	}
}

// Interceptor returns the interceptor associated with the given key.
func (w *ResponseWriter) Interceptor(key string) Interceptor {
	mp, ok := w.muxInterceps[key]
	if !ok {
		return nil
	}
	return mp
}

// Result TODO
type Result struct{}

// Write TODO
func (w *ResponseWriter) Write(resp Response) Result {
	w.markWritten()
	if err := w.d.Write(w.rw, resp); err != nil {
		panic("error")
	}
	return Result{}
}

// WriteTemplate TODO
func (w *ResponseWriter) WriteTemplate(t Template, data interface{}) Result {
	w.markWritten()
	if err := w.d.ExecuteTemplate(w.rw, t, data); err != nil {
		panic("error")
	}
	return Result{}
}

// ClientError TODO
func (w *ResponseWriter) ClientError(code StatusCode) Result {
	w.markWritten()
	if code < 400 || code >= 500 {
		// TODO(@mihalimara22): Replace this when we decide how to handle this case
		panic("wrong method called")
	}
	http.Error(w.rw, http.StatusText(int(code)), int(code))
	return Result{}
}

// ServerError TODO
func (w *ResponseWriter) ServerError(code StatusCode) Result {
	w.markWritten()
	if code < 500 || code >= 600 {
		// TODO(@mattiasgrenfeldt, @mihalimara22, @kele, @empijei): Decide how it should
		// be communicated to the user of the framework that they've called the wrong
		// method.
		panic("wrong method called")
	}
	http.Error(w.rw, http.StatusText(int(code)), int(code))
	return Result{}
}

// Redirect responds with a redirect to a given url, using code as the status code.
func (w *ResponseWriter) Redirect(r *IncomingRequest, url string, code StatusCode) Result {
	w.markWritten()
	if code < 300 || code >= 400 {
		panic("wrong method called")
	}
	http.Redirect(w.rw, r.req, url, int(code))
	return Result{}
}

// markWritten sets written to true. If written was already true, it panics.
func (w *ResponseWriter) markWritten() {
	if w.written {
		panic("ResponseWriter already written to")
	}
	w.written = true
}

// Header returns the collection of headers that will be set
// on the response. Headers must be set before writing a
// response (e.g. Write, WriteTemplate).
func (w ResponseWriter) Header() Header {
	return w.header
}

// SetCookie adds a Set-Cookie header to the provided ResponseWriter's headers.
// The provided cookie must have a valid Name. Otherwise an error will be
// returned.
func (w *ResponseWriter) SetCookie(c *Cookie) error {
	return w.header.addCookie(c)
}

// Dispatcher TODO
type Dispatcher interface {
	Write(rw http.ResponseWriter, resp Response) error
	ExecuteTemplate(rw http.ResponseWriter, t Template, data interface{}) error
}
<|endoftext|>


<commit_before>package dynamics

import (
	"fmt"
	"math"

	"github.com/gonum/floats"
)

const (
	eps = 1e-3
)

func floatEqual(a, b float64) (bool, error) {
	if !floats.EqualWithinRel(a, b, eps) {
		return false, fmt.Errorf("difference of %3.10f", math.Abs(a-b))
	}
	return true, nil
}

func vectorsEqual(a, b []float64) bool {
	if len(a) != len(b) {
		return false
	}
	for i := len(a) - 1; i >= 0; i-- {
		if !floats.EqualWithinRel(a[i], b[i], eps) {
			return false
		}
	}
	return true
}

//anglesEqual returns whether two angles in Radians are equal.
func anglesEqual(a, b float64) (bool, error) {
	diff := math.Mod(math.Abs(a-b), 2*math.Pi)
	if diff < angleε {
		return true, nil
	}
	return false, fmt.Errorf("difference of %3.10f degrees", math.Abs(Rad2deg(diff)))
}
<commit_msg>Replaced floatsEqual with floats.EqualWithinAbs<commit_after>package dynamics

import (
	"fmt"
	"math"

	"github.com/gonum/floats"
)

const (
	eps = 1e-3
)

func vectorsEqual(a, b []float64) bool {
	if len(a) != len(b) {
		return false
	}
	for i := len(a) - 1; i >= 0; i-- {
		if !floats.EqualWithinRel(a[i], b[i], eps) {
			return false
		}
	}
	return true
}

//anglesEqual returns whether two angles in Radians are equal.
func anglesEqual(a, b float64) (bool, error) {
	diff := math.Mod(math.Abs(a-b), 2*math.Pi)
	if diff < angleε {
		return true, nil
	}
	return false, fmt.Errorf("difference of %3.10f degrees", math.Abs(Rad2deg(diff)))
}
<|endoftext|>


<commit_before>package console

import (
	"fmt"
	"os"
	"time"
)

var logFile *os.File

func Open(logfile string) (err error) {
	logFile, err = os.Create(logfile)
	if err != nil {
		return
	}
	return
}

func Close() {
	logFile.Close()
}

func Log(format string, args ...interface{}) {
	if logFile == nil {
		return
	}
	text := fmt.Sprintf(format, args...)
	text = fmt.Sprintf("%s %s\n", time.Now().String(), text)
	logFile.WriteString(text)
}
<commit_msg>Use numeric time in debug log, kernel style<commit_after>package console

import (
	"fmt"
	"os"
	"time"
)

var logFile *os.File

var start = time.Now()

func Open(logfile string) (err error) {
	logFile, err = os.Create(logfile)
	if err != nil {
		return
	}
	return
}

func Close() {
	logFile.Close()
}

func Log(format string, args ...interface{}) {
	if logFile == nil {
		return
	}
	since := time.Since(start)
	text := fmt.Sprintf(format, args...)
	text = fmt.Sprintf("[%.5f] %s\n", since.Seconds(), text)
	logFile.WriteString(text)
}
<|endoftext|>


<commit_before>package consul

import (
	"strconv"
	"strings"

	"github.com/doubledutch/lager"
	"github.com/doubledutch/quantum"
	"github.com/doubledutch/quantum/client"
	"github.com/miekg/dns"
)

type resolveResult struct {
	address string
}

// NewClientResolver creates a consul client resolver
func NewClientResolver(config quantum.ClientResolverConfig) quantum.ClientResolver {
	if config.Config == nil {
		config.Config = quantum.DefaultConfig()
	}
	return &ClientResolver{
		lgr:    config.Config.Lager,
		server: config.Server,
	}
}

// ClientResolver is a client resolver that leverages Consul's service discovery.
type ClientResolver struct {
	config *quantum.ConnConfig
	lgr    lager.Lager
	server string
}

// Resolve resolves a ClientConn using a ResolveRequest
func (cr *ClientResolver) Resolve(request quantum.ResolveRequest) (quantum.ClientConn, error) {
	// Get ResolveResults
	results, err := cr.resolveResults(request)
	if err != nil {
		return nil, err
	}

	if len(results) == 0 {
		return nil, quantum.NoAgentsFromRequest(request)
	}

	// Ping each one, return the first one to respond
	return cr.resolveClient(results)
}

// ResolveConfigs resolves client configs given the specified arguments
func (cr *ClientResolver) resolveResults(rr quantum.ResolveRequest) (results []resolveResult, err error) {
	m := new(dns.Msg)
	srv := rr.Type + ".service.consul."
	m.SetQuestion(srv, dns.TypeSRV)

	c := &dns.Client{Net: "tcp"}
	in, _, err := c.Exchange(m, cr.server)
	if err != nil {
		cr.lgr.Errorf("DNS Exchange failed: %s\n", err)
		return nil, err
	}

	return newResolveResults(in, rr), nil
}

func newResolveResults(in *dns.Msg, rr quantum.ResolveRequest) (results []resolveResult) {
	for i, a := range in.Answer {
		srv := a.(*dns.SRV)
		if rr.Agent != "" {
			// We were given an agent name, match it to the hostname of the SRV record
			targetSplit := strings.Split(srv.Target, ".")
			if len(targetSplit) < 5 {
				// We expect name.node.dc1.consul. (5)
				continue
			}
			// Drop .node.dc1.consul.
			hostnameSplit := targetSplit[:len(targetSplit)-4]
			hostname := strings.Join(hostnameSplit, ".")
			if hostname != rr.Agent {
				continue
			}
		}
		a := in.Extra[i].(*dns.A)
		sPort := ":" + strconv.Itoa(int(srv.Port))
		results = append(results, resolveResult{
			address: a.A.String() + sPort,
		})
	}
	return
}

func (cr *ClientResolver) resolveClient(results []resolveResult) (conn quantum.ClientConn, err error) {
	// TODO: Do this concurrently, first one to respond wins
	for _, result := range results {
		client := client.New(cr.config)
		conn, err = client.Dial(result.address)
		if err == nil {
			break
		}
	}
	return
}
<commit_msg>resolve requests with agent and type with API<commit_after>package consul

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/doubledutch/lager"
	"github.com/doubledutch/quantum"
	"github.com/doubledutch/quantum/client"
	"github.com/hashicorp/consul/api"
	"github.com/miekg/dns"
)

type resolveResult struct {
	address string
}

// NewClientResolver creates a consul client resolver
func NewClientResolver(config quantum.ClientResolverConfig) quantum.ClientResolver {
	if config.Config == nil {
		config.Config = quantum.DefaultConfig()
	}
	return &ClientResolver{
		lgr:    config.Config.Lager,
		server: config.Server,
	}
}

// ClientResolver is a client resolver that leverages Consul's service discovery.
type ClientResolver struct {
	config *quantum.ConnConfig
	lgr    lager.Lager
	server string
}

// Resolve resolves a ClientConn using a ResolveRequest
func (cr *ClientResolver) Resolve(request quantum.ResolveRequest) (quantum.ClientConn, error) {
	// Get ResolveResults
	results, err := cr.resolveResults(request)
	if err != nil {
		return nil, err
	}

	if len(results) == 0 {
		return nil, quantum.NoAgentsFromRequest(request)
	}

	// Ping each one, return the first one to respond
	return cr.resolveClient(results)
}

// ResolveConfigs resolves client configs given the specified arguments
func (cr *ClientResolver) resolveResults(rr quantum.ResolveRequest) (results []resolveResult, err error) {
	if rr.Agent == "" {
		return cr.resolveWithDNS(rr)
	}

	return cr.resolveWithAPI(rr)
}

func (cr *ClientResolver) resolveWithDNS(rr quantum.ResolveRequest) (results []resolveResult, err error) {
	m := new(dns.Msg)
	srv := rr.Type + ".service.consul."
	m.SetQuestion(srv, dns.TypeSRV)

	c := &dns.Client{Net: "tcp"}
	in, _, err := c.Exchange(m, cr.server)
	if err != nil {
		cr.lgr.Errorf("DNS Exchange failed: %s\n", err)
		return nil, err
	}

	return newResolveResults(in, rr), nil
}

func newResolveResults(in *dns.Msg, rr quantum.ResolveRequest) (results []resolveResult) {
	for i, a := range in.Answer {
		srv := a.(*dns.SRV)
		if rr.Agent != "" {
			// We were given an agent name, match it to the hostname of the SRV record
			targetSplit := strings.Split(srv.Target, ".")
			if len(targetSplit) < 5 {
				// We expect name.node.dc1.consul. (5)
				continue
			}
			// Drop .node.dc1.consul.
			hostnameSplit := targetSplit[:len(targetSplit)-4]
			hostname := strings.Join(hostnameSplit, ".")
			if hostname != rr.Agent {
				continue
			}
		}
		a := in.Extra[i].(*dns.A)
		sPort := ":" + strconv.Itoa(int(srv.Port))
		results = append(results, resolveResult{
			address: a.A.String() + sPort,
		})
	}
	return
}

func (cr *ClientResolver) resolveWithAPI(rr quantum.ResolveRequest) (results []resolveResult, err error) {
	client, err := api.NewClient(&api.Config{
		Address: cr.server,
	})
	if err != nil {
		return nil, err
	}

	catalog := client.Catalog()
	node, _, err := catalog.Node(rr.Agent, nil)
	if err != nil {
		return nil, quantum.NoAgentsFromRequest(rr)
	}
	service, ok := node.Services[rr.Type]
	if !ok {
		return nil, quantum.NoAgentsFromRequest(rr)
	}

	return []resolveResult{
		{
			address: fmt.Sprintf("%s:%d", service.Address, service.Port),
		},
	}, nil
}

func (cr *ClientResolver) resolveClient(results []resolveResult) (conn quantum.ClientConn, err error) {
	// TODO: Do this concurrently, first one to respond wins
	for _, result := range results {
		client := client.New(cr.config)
		conn, err = client.Dial(result.address)
		if err == nil {
			break
		}
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"net/http"
	"strings"

	"gopkg.in/mgo.v2/bson"

	"github.com/gorilla/mux"
)

// GetEventController will answer a JSON of the event
// from the given "id" in the URL. (cf Routes in routes.go)
func GetEventController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	assocationID := vars["id"]
	var res = GetEvent(bson.ObjectIdHex(assocationID))
	json.NewEncoder(w).Encode(res)
}

// GetFutureEventsController will answer a JSON
// containing all future events from "NOW"
func GetFutureEventsController(w http.ResponseWriter, r *http.Request) {
	var res = GetFutureEvents()
	json.NewEncoder(w).Encode(res)
}

// AddEventController will answer the JSON
// of the brand new created event from the JSON body
func AddEventController(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)
	var event Event
	decoder.Decode(&event)
	res := AddEvent(event)
	asso := GetAssociation(event.Association)
	json.NewEncoder(w).Encode(res)
	TriggerNotification("@" + strings.ToLower(asso.Name) + " t'invite à " + event.Name + " 📅", event.ID.Hex())
}

// UpdateEventController will answer the JSON
// of the brand new modified event from the JSON body
func UpdateEventController(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)
	var event Event
	decoder.Decode(&event)
	vars := mux.Vars(r)
	eventID := vars["id"]
	res := UpdateEvent(bson.ObjectIdHex(eventID), event)
	json.NewEncoder(w).Encode(res)
}

// DeleteEventController will answer an empty JSON
// if the deletation has succeed
func DeleteEventController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	event := GetEvent(bson.ObjectIdHex(vars["id"]))
	res := DeleteEvent(event)
	json.NewEncoder(w).Encode(res)
}

// AddParticipantController will answer the JSON
// of the event with the given partipant added
func AddParticipantController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	eventID := bson.ObjectIdHex(vars["id"])
	userID := bson.ObjectIdHex(vars["userID"])
	event, user := AddParticipant(eventID, userID)
	json.NewEncoder(w).Encode(bson.M{"event": event, "user": user})
}

// RemoveParticipantController will answer the JSON
// of the event without the given partipant added
func RemoveParticipantController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	eventID := bson.ObjectIdHex(vars["id"])
	userID := bson.ObjectIdHex(vars["userID"])
	event, user := RemoveParticipant(eventID, userID)
	json.NewEncoder(w).Encode(bson.M{"event": event, "user": user})
}

// // AddImageEventController will set the image of the event and return the event
// func AddImageEventController(w http.ResponseWriter, r *http.Request) {
// 	fileName := UploadImage(r)
// 	if fileName == "error" {
// 		w.Header().Set("status", "400")
// 		fmt.Fprintln(w, "{}")
// 	} else {
// 		vars := mux.Vars(r)
// 		res := SetImageEvent(bson.ObjectIdHex(vars["id"]), fileName)
// 		json.NewEncoder(w).Encode(res)
// 	}
// }
<commit_msg>Add more information to ios notification<commit_after>package main

import (
	"encoding/json"
	"net/http"
	"strings"

	"gopkg.in/mgo.v2/bson"

	"github.com/gorilla/mux"
)

// GetEventController will answer a JSON of the event
// from the given "id" in the URL. (cf Routes in routes.go)
func GetEventController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	assocationID := vars["id"]
	var res = GetEvent(bson.ObjectIdHex(assocationID))
	json.NewEncoder(w).Encode(res)
}

// GetFutureEventsController will answer a JSON
// containing all future events from "NOW"
func GetFutureEventsController(w http.ResponseWriter, r *http.Request) {
	var res = GetFutureEvents()
	json.NewEncoder(w).Encode(res)
}

// AddEventController will answer the JSON
// of the brand new created event from the JSON body
func AddEventController(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)
	var event Event
	decoder.Decode(&event)
	res := AddEvent(event)
	asso := GetAssociation(event.Association)
	json.NewEncoder(w).Encode(res)
	TriggerNotification("@" + strings.ToLower(asso.Name) + " t'invite à " + event.Name + " 📅", res.ID.Hex())
}

// UpdateEventController will answer the JSON
// of the brand new modified event from the JSON body
func UpdateEventController(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)
	var event Event
	decoder.Decode(&event)
	vars := mux.Vars(r)
	eventID := vars["id"]
	res := UpdateEvent(bson.ObjectIdHex(eventID), event)
	json.NewEncoder(w).Encode(res)
}

// DeleteEventController will answer an empty JSON
// if the deletation has succeed
func DeleteEventController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	event := GetEvent(bson.ObjectIdHex(vars["id"]))
	res := DeleteEvent(event)
	json.NewEncoder(w).Encode(res)
}

// AddParticipantController will answer the JSON
// of the event with the given partipant added
func AddParticipantController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	eventID := bson.ObjectIdHex(vars["id"])
	userID := bson.ObjectIdHex(vars["userID"])
	event, user := AddParticipant(eventID, userID)
	json.NewEncoder(w).Encode(bson.M{"event": event, "user": user})
}

// RemoveParticipantController will answer the JSON
// of the event without the given partipant added
func RemoveParticipantController(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	eventID := bson.ObjectIdHex(vars["id"])
	userID := bson.ObjectIdHex(vars["userID"])
	event, user := RemoveParticipant(eventID, userID)
	json.NewEncoder(w).Encode(bson.M{"event": event, "user": user})
}

// // AddImageEventController will set the image of the event and return the event
// func AddImageEventController(w http.ResponseWriter, r *http.Request) {
// 	fileName := UploadImage(r)
// 	if fileName == "error" {
// 		w.Header().Set("status", "400")
// 		fmt.Fprintln(w, "{}")
// 	} else {
// 		vars := mux.Vars(r)
// 		res := SetImageEvent(bson.ObjectIdHex(vars["id"]), fileName)
// 		json.NewEncoder(w).Encode(res)
// 	}
// }
<|endoftext|>


<commit_before>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package control

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"
	"time"

	"github.com/coreos/dbtester/remotestorage"
	"github.com/spf13/cobra"
)

// Command implements 'control' command.
var Command = &cobra.Command{
	Use:   "control",
	Short: "Controls tests.",
	RunE:  commandFunc,
}

var configPath string

func init() {
	Command.PersistentFlags().StringVarP(&configPath, "config", "c", "", "YAML configuration file path.")
}

func commandFunc(cmd *cobra.Command, args []string) error {
	cfg, err := ReadConfig(configPath)
	if err != nil {
		return err
	}
	switch cfg.Database {
	case "etcdv2":
	case "etcdv3":
	case "zookeeper":
	case "zetcd":
	case "consul":
	case "cetcd":
	default:
		return fmt.Errorf("%q is not supported", cfg.Database)
	}
	if !cfg.Step2.SkipStressDatabase {
		switch cfg.Step2.BenchType {
		case "write":
		case "read":
		case "read-oneshot":
		default:
			return fmt.Errorf("%q is not supported", cfg.Step2.BenchType)
		}
	}

	bts, err := ioutil.ReadFile(cfg.GoogleCloudStorageKeyPath)
	if err != nil {
		return err
	}
	cfg.GoogleCloudStorageKey = string(bts)

	cfg.PeerIPString = strings.Join(cfg.PeerIPs, "___") // protoc sorts the 'repeated' type data
	cfg.AgentEndpoints = make([]string, len(cfg.PeerIPs))
	cfg.DatabaseEndpoints = make([]string, len(cfg.PeerIPs))
	for i := range cfg.PeerIPs {
		cfg.AgentEndpoints[i] = fmt.Sprintf("%s:%d", cfg.PeerIPs[i], cfg.AgentPort)
	}
	for i := range cfg.PeerIPs {
		cfg.DatabaseEndpoints[i] = fmt.Sprintf("%s:%d", cfg.PeerIPs[i], cfg.DatabasePort)
	}

	println()
	if !cfg.Step1.SkipStartDatabase {
		plog.Info("step 1: starting databases...")
		if err = step1(cfg); err != nil {
			return err
		}
	}

	if !cfg.Step2.SkipStressDatabase {
		println()
		time.Sleep(5 * time.Second)
		plog.Info("step 2: starting tests...")
		if err = step2(cfg); err != nil {
			return err
		}
	}

	println()
	time.Sleep(5 * time.Second)
	if err := step3(cfg); err != nil {
		return err
	}

	{
		u, err := remotestorage.NewGoogleCloudStorage([]byte(cfg.GoogleCloudStorageKey), cfg.GoogleCloudProjectName)
		if err != nil {
			plog.Fatal(err)
		}
		srcCSVResultPath := cfg.ResultPathTimeSeries
		dstCSVResultPath := filepath.Base(cfg.ResultPathTimeSeries)
		if !strings.HasPrefix(dstCSVResultPath, cfg.TestName) {
			dstCSVResultPath = fmt.Sprintf("%s-%s", cfg.TestName, dstCSVResultPath)
		}
		dstCSVResultPath = filepath.Join(cfg.GoogleCloudStorageSubDirectory, dstCSVResultPath)

		var uerr error
		for k := 0; k < 15; k++ {
			if uerr = u.UploadFile(cfg.GoogleCloudStorageBucketName, srcCSVResultPath, dstCSVResultPath); uerr != nil {
				plog.Printf("#%d: UploadFile error %v", k, uerr)
				time.Sleep(2 * time.Second)
				continue
			}
			break
		}
	}
	{
		u, err := remotestorage.NewGoogleCloudStorage([]byte(cfg.GoogleCloudStorageKey), cfg.GoogleCloudProjectName)
		if err != nil {
			plog.Fatal(err)
		}

		srcCSVResultPath := cfg.ResultPathLog
		dstCSVResultPath := filepath.Base(cfg.ResultPathLog)
		if !strings.HasPrefix(dstCSVResultPath, cfg.TestName) {
			dstCSVResultPath = fmt.Sprintf("%s-%s", cfg.TestName, dstCSVResultPath)
		}
		dstCSVResultPath = filepath.Join(cfg.GoogleCloudStorageSubDirectory, dstCSVResultPath)

		var uerr error
		for k := 0; k < 15; k++ {
			if uerr = u.UploadFile(cfg.GoogleCloudStorageBucketName, srcCSVResultPath, dstCSVResultPath); uerr != nil {
				plog.Printf("#%d: UploadFile error %v", k, uerr)
				time.Sleep(2 * time.Second)
				continue
			}
			break
		}
	}

	return nil
}
<commit_msg>control: add comments<commit_after>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package control

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"
	"time"

	"github.com/coreos/dbtester/remotestorage"
	"github.com/spf13/cobra"
)

// Command implements 'control' command.
var Command = &cobra.Command{
	Use:   "control",
	Short: "Controls tests.",
	RunE:  commandFunc,
}

var configPath string

func init() {
	Command.PersistentFlags().StringVarP(&configPath, "config", "c", "", "YAML configuration file path.")
}

func commandFunc(cmd *cobra.Command, args []string) error {
	cfg, err := ReadConfig(configPath)
	if err != nil {
		return err
	}
	switch cfg.Database {
	case "etcdv2":
	case "etcdv3":
	case "zookeeper":
	case "zetcd":
	case "consul":
	case "cetcd":
	default:
		return fmt.Errorf("%q is not supported", cfg.Database)
	}
	if !cfg.Step2.SkipStressDatabase {
		switch cfg.Step2.BenchType {
		case "write":
		case "read":
		case "read-oneshot":
		default:
			return fmt.Errorf("%q is not supported", cfg.Step2.BenchType)
		}
	}

	bts, err := ioutil.ReadFile(cfg.GoogleCloudStorageKeyPath)
	if err != nil {
		return err
	}
	cfg.GoogleCloudStorageKey = string(bts)

	// protoc sorts the 'repeated' type data
	// encode in string to enforce ordering of IPs
	cfg.PeerIPString = strings.Join(cfg.PeerIPs, "___")
	cfg.AgentEndpoints = make([]string, len(cfg.PeerIPs))
	cfg.DatabaseEndpoints = make([]string, len(cfg.PeerIPs))
	for i := range cfg.PeerIPs {
		cfg.AgentEndpoints[i] = fmt.Sprintf("%s:%d", cfg.PeerIPs[i], cfg.AgentPort)
	}
	for i := range cfg.PeerIPs {
		cfg.DatabaseEndpoints[i] = fmt.Sprintf("%s:%d", cfg.PeerIPs[i], cfg.DatabasePort)
	}

	println()
	if !cfg.Step1.SkipStartDatabase {
		plog.Info("step 1: starting databases...")
		if err = step1(cfg); err != nil {
			return err
		}
	}

	if !cfg.Step2.SkipStressDatabase {
		println()
		time.Sleep(5 * time.Second)
		plog.Info("step 2: starting tests...")
		if err = step2(cfg); err != nil {
			return err
		}
	}

	println()
	time.Sleep(5 * time.Second)
	if err := step3(cfg); err != nil {
		return err
	}

	{
		u, err := remotestorage.NewGoogleCloudStorage([]byte(cfg.GoogleCloudStorageKey), cfg.GoogleCloudProjectName)
		if err != nil {
			plog.Fatal(err)
		}
		srcCSVResultPath := cfg.ResultPathTimeSeries
		dstCSVResultPath := filepath.Base(cfg.ResultPathTimeSeries)
		if !strings.HasPrefix(dstCSVResultPath, cfg.TestName) {
			dstCSVResultPath = fmt.Sprintf("%s-%s", cfg.TestName, dstCSVResultPath)
		}
		dstCSVResultPath = filepath.Join(cfg.GoogleCloudStorageSubDirectory, dstCSVResultPath)

		var uerr error
		for k := 0; k < 15; k++ {
			if uerr = u.UploadFile(cfg.GoogleCloudStorageBucketName, srcCSVResultPath, dstCSVResultPath); uerr != nil {
				plog.Printf("#%d: UploadFile error %v", k, uerr)
				time.Sleep(2 * time.Second)
				continue
			}
			break
		}
	}
	{
		u, err := remotestorage.NewGoogleCloudStorage([]byte(cfg.GoogleCloudStorageKey), cfg.GoogleCloudProjectName)
		if err != nil {
			plog.Fatal(err)
		}

		srcCSVResultPath := cfg.ResultPathLog
		dstCSVResultPath := filepath.Base(cfg.ResultPathLog)
		if !strings.HasPrefix(dstCSVResultPath, cfg.TestName) {
			dstCSVResultPath = fmt.Sprintf("%s-%s", cfg.TestName, dstCSVResultPath)
		}
		dstCSVResultPath = filepath.Join(cfg.GoogleCloudStorageSubDirectory, dstCSVResultPath)

		var uerr error
		for k := 0; k < 15; k++ {
			if uerr = u.UploadFile(cfg.GoogleCloudStorageBucketName, srcCSVResultPath, dstCSVResultPath); uerr != nil {
				plog.Printf("#%d: UploadFile error %v", k, uerr)
				time.Sleep(2 * time.Second)
				continue
			}
			break
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"io"
	"log"
	"os"

	"io/ioutil"

	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"golang.org/x/oauth2/jwt"
	"google.golang.org/api/datastore/v1beta2"
)

var signal = make(chan int)
var secret []byte
var config *jwt.Config
var service *datastore.Service

func init() {
	err := error(nil)
	secret, err = ioutil.ReadFile("../config/Coduno-6b4d6d5a0f06.json")
	if err != nil {
		panic(err)
	}
	config, err = google.JWTConfigFromJSON(secret, datastore.DatastoreScope)
	if err != nil {
		panic(err)
	}
	client := config.Client(context.Background())
	service, err = datastore.New(client)
	if err != nil {
		panic(err)
	}
}

// LogBuildStart sends info to the datastore, informing that a new build
// started
func LogBuildStart(repo string, commit string, user string) {
	insert := new(datastore.CommitRequest)
	insert.Mode = "NON_TRANSACTIONAL"
	path := make([]*datastore.KeyPathElement, 1)
	path[0] = new(datastore.KeyPathElement)
	path[0].Kind = "testrun"
	mutation := new(datastore.Mutation)
	mutation.InsertAutoId = make([]*datastore.Entity, 1)
	mutation.InsertAutoId[0] = new(datastore.Entity)
	mutation.InsertAutoId[0].Properties = make(map[string]datastore.Property)
	mutation.InsertAutoId[0].Properties["repo"] = datastore.Property{StringValue: repo}
	mutation.InsertAutoId[0].Properties["commit"] = datastore.Property{StringValue: commit}
	mutation.InsertAutoId[0].Properties["user"] = datastore.Property{StringValue: user}
	mutation.InsertAutoId[0].Key = new(datastore.Key)
	mutation.InsertAutoId[0].Key.Path = path

	req := service.Datasets.Commit("coduno", insert)
	ret, err := req.Do()
	if err != nil {
		log.Panic(err)
	}
	log.Print(ret.Header)
}

func sendSig() {
	signal <- 1
}

func pipeOutput(out io.ReadCloser, dest io.WriteCloser, logBuf *bytes.Buffer) {
	tempBuf := make([]byte, 1024)
	writeErr := error(nil)
	r, readErr := int(0), error(nil)

	defer out.Close()
	defer dest.Close()
	defer sendSig()

	for readErr == nil {
		r, readErr = out.Read(tempBuf)
		logBuf.Write(tempBuf[0:r])

		if r != 0 && writeErr == nil {
			_, writeErr := dest.Write(tempBuf[0:r])
			if writeErr != nil {
				log.Print(writeErr)
			}
		}
	}
}

func main() {
	if len(os.Args) != 3 {
		log.Fatal("Invalid number of arguments. This should never have happend")
	}

	commit := os.Args[1]
	tmpdir := os.Args[2]
	LogBuildStart("test", commit, tmpdir)
	/*
		cmd := exec.Command("sudo", "docker", "run", "--rm", "-v", tmpdir+":/app", "coduno/base")
		stdout, err := cmd.StdoutPipe()
		if err != nil {
			log.Fatal(err)
		}
		stderr, err := cmd.StderrPipe()
		if err != nil {
			log.Fatal(err)
		}
		stdin, err := cmd.StdinPipe()
		if err != nil {
			log.Fatal(err)
		}

		var b1 bytes.Buffer
		var b2 bytes.Buffer
		cmd.Start()
		go pipeOutput(stdout, stdin, &b1)
		go pipeOutput(stderr, stdin, &b2)

		<-signal
		<-signal
		log.Print(b1.String())
		log.Print(b2.String())
		log.Print(commit)*/
}
<commit_msg>Switched to google cloud api for go<commit_after>package main

import (
	"bytes"
	"io"
	"log"
	"os"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"golang.org/x/oauth2/jwt"

	"io/ioutil"

	"google.golang.org/cloud"
	"google.golang.org/cloud/datastore"
)

// LogData holds all data that is stored for a single run of coduno
type LogData struct {
	Challenge string
	User      string
	Commit    string
	Status    string
	StartTime time.Time
	EndTime   time.Time
	InLog     string
	OutLog    string
	ExtraLog  string
}

var signal = make(chan int)
var secret []byte
var config *jwt.Config
var ctx context.Context

func init() {
	err := error(nil)
	secret, err = ioutil.ReadFile("../config/secret.json")
	if err != nil {
		panic(err)
	}
	config, err = google.JWTConfigFromJSON(secret, datastore.ScopeDatastore, datastore.ScopeUserEmail)
	if err != nil {
		panic(err)
	}

	ctx = cloud.NewContext("coduno", config.Client(oauth2.NoContext))
}

// LogBuildStart sends info to the datastore, informing that a new build
// started
func LogBuildStart(repo string, commit string, user string) {
	key := datastore.NewIncompleteKey(ctx, "testrun", nil)
	_, err := datastore.Put(ctx, key, &LogData{
		Commit:    commit,
		Challenge: repo,
		User:      user,
	})
	if err != nil {
		log.Panic(err)
	}
}

func sendSig() {
	signal <- 1
}

func pipeOutput(out io.ReadCloser, dest io.WriteCloser, logBuf *bytes.Buffer) {
	tempBuf := make([]byte, 1024)
	writeErr := error(nil)
	r, readErr := int(0), error(nil)

	defer out.Close()
	defer dest.Close()
	defer sendSig()

	for readErr == nil {
		r, readErr = out.Read(tempBuf)
		logBuf.Write(tempBuf[0:r])

		if r != 0 && writeErr == nil {
			_, writeErr := dest.Write(tempBuf[0:r])
			if writeErr != nil {
				log.Print(writeErr)
			}
		}
	}
}

func main() {
	if len(os.Args) != 3 {
		log.Fatal("Invalid number of arguments. This should never have happend")
	}

	commit := os.Args[1]
	tmpdir := os.Args[2]
	LogBuildStart("test", commit, tmpdir)
	/*
		cmd := exec.Command("sudo", "docker", "run", "--rm", "-v", tmpdir+":/app", "coduno/base")
		stdout, err := cmd.StdoutPipe()
		if err != nil {
			log.Fatal(err)
		}
		stderr, err := cmd.StderrPipe()
		if err != nil {
			log.Fatal(err)
		}
		stdin, err := cmd.StdinPipe()
		if err != nil {
			log.Fatal(err)
		}

		var b1 bytes.Buffer
		var b2 bytes.Buffer
		cmd.Start()
		go pipeOutput(stdout, stdin, &b1)
		go pipeOutput(stderr, stdin, &b2)

		<-signal
		<-signal
		log.Print(b1.String())
		log.Print(b2.String())
		log.Print(commit)*/
}
<|endoftext|>


<commit_before>package couchdb

import (
	"errors"
	"fmt"

	"github.com/google/go-querystring/query"
)

// ChangesFeedMode is a value for the feed parameter of a ChangesRequest
type ChangesFeedMode string

// ChangesFeedStyle is a value for the style parameter of a ChangesRequest
type ChangesFeedStyle string

const (
	// ChangesModeNormal is the only mode supported by cozy-stack
	ChangesModeNormal ChangesFeedMode = "normal"
	// ChangesStyleAllDocs pass all revisions including conflicts
	ChangesStyleAllDocs ChangesFeedStyle = "all_docs"
	// ChangesStyleMainOnly only pass the winning revision
	ChangesStyleMainOnly ChangesFeedStyle = "main_only"
)

// ValidChangesMode convert any string into a ChangesFeedMode or gives an error
// if the string is invalid.
func ValidChangesMode(feed string) (ChangesFeedMode, error) {
	if feed == "" || feed == string(ChangesModeNormal) {
		return ChangesModeNormal, nil
	}

	err := fmt.Errorf("Unsuported feed value '%s'", feed)
	return ChangesModeNormal, err
}

// ValidChangesStyle convert any string into a ChangesFeedStyle or gives an
// error if the string is invalid.
func ValidChangesStyle(style string) (ChangesFeedStyle, error) {
	if style == "" || style == string(ChangesStyleMainOnly) {
		return ChangesStyleMainOnly, nil
	}
	if style == string(ChangesStyleAllDocs) {
		return ChangesStyleAllDocs, nil
	}
	err := fmt.Errorf("Unsuported style value '%s'", style)
	return ChangesStyleMainOnly, err
}

// A ChangesRequest are all parameters than can be passed to a changes feed
type ChangesRequest struct {
	DocType string `url:"-"`
	// Includes conflicts information in response. Ignored if include_docs isn’t
	// true. Default is false.
	Conflicts bool `url:"conflicts,omitempty"`
	// Return the change results in descending sequence order (most recent change
	// first). Default is false.
	Descending bool `url:"descending,omitempty"`
	// see Changes Feeds. Default is normal.
	Feed ChangesFeedMode `url:"feed,omitempty"`
	// Reference to a filter function from a design document that will filter
	// whole stream emitting only filtered events. See the section Change
	// Notifications in the book CouchDB The Definitive Guide for more
	// information.
	Filter string `url:"filter,omitempty"`
	// Period in milliseconds after which an empty line is sent in the results.
	// Only applicable for longpoll, continuous, and eventsource feeds. Overrides
	// any timeout to keep the feed alive indefinitely. Default is 60000. May be
	// true to use default value.
	Heartbeat int `url:"heartbeat,omitempty"`
	// Include the associated document with each result. If there are conflicts,
	// only the winning revision is returned. Default is false.
	IncludeDocs bool `url:"include_docs,omitempty"`
	// Include the Base64-encoded content of attachments in the documents that
	// are included if include_docs is true. Ignored if include_docs isn’t true.
	// Default is false.
	Attachments bool `url:"attachments,omitempty"`
	// Include encoding information in attachment stubs if include_docs is true
	// and the particular attachment is compressed. Ignored if include_docs isn’t
	// true. Default is false.
	AttEncodingInfo bool `url:"att_encoding_info,omitempty"`
	// Alias of Last-Event-ID header.
	LastEventID int `url:"last,omitempty"`
	// Limit number of result rows to the specified value (note that using 0 here
	//  has the same effect as 1).
	Limit int `url:"limit,omitempty"`
	// Start the results from the change immediately after the given update
	// sequence. Can be valid update sequence or now value. Default is 0.
	Since string `url:"since,omitempty"`
	// Specifies how many revisions are returned in the changes array. The
	// default, main_only, will only return the current “winning” revision;
	// all_docs will return all leaf revisions (including conflicts and deleted
	// former conflicts).
	Style ChangesFeedStyle `url:"style,omitempty"`
	// Maximum period in milliseconds to wait for a change before the response
	// is sent, even if there are no results. Only applicable for longpoll or
	// continuous feeds. Default value is specified by httpd/changes_timeout
	// configuration option. Note that 60000 value is also the default maximum
	// timeout to prevent undetected dead connections.
	Timeout int `url:"timeout,omitempty"`
	// Allows to use view functions as filters. Documents counted as “passed” for
	// view filter in case if map function emits at least one record for them.
	// See _view for more info.
	View string `url:"view,omitempty"`
}

// A ChangesResponse is the response provided by a GetChanges call
type ChangesResponse struct {
	LastSeq string   `json:"last_seq"` // Last change update sequence
	Pending int      `json:"pending"`  // Count of remaining items in the feed
	Results []Change `json:"results"`  // Changes made to a database
}

// A Change is an atomic change in couchdb
type Change struct {
	DocID   string  `json:"id"`
	Seq     string  `json:"seq"`
	Doc     JSONDoc `json:"doc"`
	Changes []struct {
		Rev string `json:"rev"`
	} `json:"changes"`
}

// GetChanges returns a list of change in couchdb
func GetChanges(db Database, req *ChangesRequest) (*ChangesResponse, error) {
	if req.DocType == "" {
		return nil, errors.New("Empty doctype in GetChanges")
	}

	v, err := query.Values(req)
	if err != nil {
		return nil, err
	}

	var response ChangesResponse
	url := makeDBName(db, req.DocType) + "/_changes?" + v.Encode()
	err = makeRequest("GET", url, nil, &response)

	if err != nil {
		return nil, err
	}
	return &response, nil

}
<commit_msg>align bools in ChangesFeed<commit_after>package couchdb

import (
	"errors"
	"fmt"

	"github.com/google/go-querystring/query"
)

// ChangesFeedMode is a value for the feed parameter of a ChangesRequest
type ChangesFeedMode string

// ChangesFeedStyle is a value for the style parameter of a ChangesRequest
type ChangesFeedStyle string

const (
	// ChangesModeNormal is the only mode supported by cozy-stack
	ChangesModeNormal ChangesFeedMode = "normal"
	// ChangesStyleAllDocs pass all revisions including conflicts
	ChangesStyleAllDocs ChangesFeedStyle = "all_docs"
	// ChangesStyleMainOnly only pass the winning revision
	ChangesStyleMainOnly ChangesFeedStyle = "main_only"
)

// ValidChangesMode convert any string into a ChangesFeedMode or gives an error
// if the string is invalid.
func ValidChangesMode(feed string) (ChangesFeedMode, error) {
	if feed == "" || feed == string(ChangesModeNormal) {
		return ChangesModeNormal, nil
	}

	err := fmt.Errorf("Unsuported feed value '%s'", feed)
	return ChangesModeNormal, err
}

// ValidChangesStyle convert any string into a ChangesFeedStyle or gives an
// error if the string is invalid.
func ValidChangesStyle(style string) (ChangesFeedStyle, error) {
	if style == "" || style == string(ChangesStyleMainOnly) {
		return ChangesStyleMainOnly, nil
	}
	if style == string(ChangesStyleAllDocs) {
		return ChangesStyleAllDocs, nil
	}
	err := fmt.Errorf("Unsuported style value '%s'", style)
	return ChangesStyleMainOnly, err
}

// A ChangesRequest are all parameters than can be passed to a changes feed
type ChangesRequest struct {
	DocType string `url:"-"`
	// see Changes Feeds. Default is normal.
	Feed ChangesFeedMode `url:"feed,omitempty"`
	// Maximum period in milliseconds to wait for a change before the response
	// is sent, even if there are no results. Only applicable for longpoll or
	// continuous feeds. Default value is specified by httpd/changes_timeout
	// configuration option. Note that 60000 value is also the default maximum
	// timeout to prevent undetected dead connections.
	Timeout int `url:"timeout,omitempty"`
	// Period in milliseconds after which an empty line is sent in the results.
	// Only applicable for longpoll, continuous, and eventsource feeds. Overrides
	// any timeout to keep the feed alive indefinitely. Default is 60000. May be
	// true to use default value.
	Heartbeat int `url:"heartbeat,omitempty"`
	// Includes conflicts information in response. Ignored if include_docs isn’t
	// true. Default is false.
	Conflicts bool `url:"conflicts,omitempty"`
	// Return the change results in descending sequence order (most recent change
	// first). Default is false.
	Descending bool `url:"descending,omitempty"`
	// Include the associated document with each result. If there are conflicts,
	// only the winning revision is returned. Default is false.
	IncludeDocs bool `url:"include_docs,omitempty"`
	// Include the Base64-encoded content of attachments in the documents that
	// are included if include_docs is true. Ignored if include_docs isn’t true.
	// Default is false.
	Attachments bool `url:"attachments,omitempty"`
	// Include encoding information in attachment stubs if include_docs is true
	// and the particular attachment is compressed. Ignored if include_docs isn’t
	// true. Default is false.
	AttEncodingInfo bool `url:"att_encoding_info,omitempty"`
	// Alias of Last-Event-ID header.
	LastEventID int `url:"last,omitempty"`
	// Limit number of result rows to the specified value (note that using 0 here
	//  has the same effect as 1).
	Limit int `url:"limit,omitempty"`
	// Start the results from the change immediately after the given update
	// sequence. Can be valid update sequence or now value. Default is 0.
	Since string `url:"since,omitempty"`
	// Specifies how many revisions are returned in the changes array. The
	// default, main_only, will only return the current “winning” revision;
	// all_docs will return all leaf revisions (including conflicts and deleted
	// former conflicts).
	Style ChangesFeedStyle `url:"style,omitempty"`
	// Reference to a filter function from a design document that will filter
	// whole stream emitting only filtered events. See the section Change
	// Notifications in the book CouchDB The Definitive Guide for more
	// information.
	Filter string `url:"filter,omitempty"`
	// Allows to use view functions as filters. Documents counted as “passed” for
	// view filter in case if map function emits at least one record for them.
	// See _view for more info.
	View string `url:"view,omitempty"`
}

// A ChangesResponse is the response provided by a GetChanges call
type ChangesResponse struct {
	LastSeq string   `json:"last_seq"` // Last change update sequence
	Pending int      `json:"pending"`  // Count of remaining items in the feed
	Results []Change `json:"results"`  // Changes made to a database
}

// A Change is an atomic change in couchdb
type Change struct {
	DocID   string  `json:"id"`
	Seq     string  `json:"seq"`
	Doc     JSONDoc `json:"doc"`
	Changes []struct {
		Rev string `json:"rev"`
	} `json:"changes"`
}

// GetChanges returns a list of change in couchdb
func GetChanges(db Database, req *ChangesRequest) (*ChangesResponse, error) {
	if req.DocType == "" {
		return nil, errors.New("Empty doctype in GetChanges")
	}

	v, err := query.Values(req)
	if err != nil {
		return nil, err
	}

	var response ChangesResponse
	url := makeDBName(db, req.DocType) + "/_changes?" + v.Encode()
	err = makeRequest("GET", url, nil, &response)

	if err != nil {
		return nil, err
	}
	return &response, nil

}
<|endoftext|>


<commit_before>// +build freebsd

package cpu

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	common "github.com/shirou/gopsutil/common"
)

// sys/resource.h
const (
	CPUser    = 0
	CPNice    = 1
	CPSys     = 2
	CPIntr    = 3
	CPIdle    = 4
	CPUStates = 5
)

// time.h
const (
	ClocksPerSec = 128
)

func CPUTimes(percpu bool) ([]CPUTimesStat, error) {
	var ret []CPUTimesStat

	var sysctlCall string
	var ncpu int
	if percpu {
		sysctlCall = "kern.cp_times"
		ncpu, _ = CPUCounts(true)
	} else {
		sysctlCall = "kern.cp_time"
		ncpu = 1
	}

	cpuTimes, err := common.DoSysctrl(sysctlCall)
	if err != nil {
		return ret, err
	}

	for i := 0; i < ncpu; i++ {
		offset := CPUStates * i
		user, err := strconv.ParseFloat(cpuTimes[CPUser+offset], 32)
		if err != nil {
			return ret, err
		}
		nice, err := strconv.ParseFloat(cpuTimes[CPNice+offset], 32)
		if err != nil {
			return ret, err
		}
		sys, err := strconv.ParseFloat(cpuTimes[CPSys+offset], 32)
		if err != nil {
			return ret, err
		}
		idle, err := strconv.ParseFloat(cpuTimes[CPIdle+offset], 32)
		if err != nil {
			return ret, err
		}
		intr, err := strconv.ParseFloat(cpuTimes[CPIntr+offset], 32)
		if err != nil {
			return ret, err
		}

		c := CPUTimesStat{
			User:   float32(user / ClocksPerSec),
			Nice:   float32(nice / ClocksPerSec),
			System: float32(sys / ClocksPerSec),
			Idle:   float32(idle / ClocksPerSec),
			Irq:    float32(intr / ClocksPerSec),
		}
		if !percpu {
			c.CPU = "cpu-total"
		} else {
			c.CPU = fmt.Sprintf("cpu%d", i)
		}

		ret = append(ret, c)
	}

	return ret, nil
}

// Returns only one CPUInfoStat on FreeBSD
func CPUInfo() ([]CPUInfoStat, error) {
	filename := "/var/run/dmesg.boot"
	lines, _ := common.ReadLines(filename)

	var ret []CPUInfoStat

	c := CPUInfoStat{}
	for _, line := range lines {
		if matches := regexp.MustCompile(`CPU:\s+(.+) \(([\d.]+).+\)`).FindStringSubmatch(line); matches != nil {
			c.ModelName = matches[1]
			t, err := strconv.ParseFloat(matches[2], 64)
			if err != nil {
				return ret, nil
			}
			c.Mhz = t
		} else if matches := regexp.MustCompile(`Origin = "(.+)"  Id = (.+)  Family = (.+)  Model = (.+)  Stepping = (.+)`).FindStringSubmatch(line); matches != nil {
			c.VendorID = matches[1]
			c.Family = matches[3]
			c.Model = matches[4]
			t, err := strconv.ParseInt(matches[5], 10, 32)
			if err != nil {
				return ret, nil
			}
			c.Stepping = int32(t)
		} else if matches := regexp.MustCompile(`Features=.+<(.+)>`).FindStringSubmatch(line); matches != nil {
			for _, v := range strings.Split(matches[1], ",") {
				c.Flags = append(c.Flags, strings.ToLower(v))
			}
		} else if matches := regexp.MustCompile(`Features2=[a-f\dx]+<(.+)>`).FindStringSubmatch(line); matches != nil {
			for _, v := range strings.Split(matches[1], ",") {
				c.Flags = append(c.Flags, strings.ToLower(v))
			}
		} else if matches := regexp.MustCompile(`Logical CPUs per core: (\d+)`).FindStringSubmatch(line); matches != nil {
			// FIXME: no this line?
			t, err := strconv.ParseInt(matches[1], 10, 32)
			if err != nil {
				return ret, nil
			}
			c.Cores = int32(t)
		}

	}

	return append(ret, c), nil
}
<commit_msg>Update cpu_freebsd.go<commit_after>// +build freebsd

package cpu

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	common "github.com/shirou/chendrak/common"
)

// sys/resource.h
const (
	CPUser    = 0
	CPNice    = 1
	CPSys     = 2
	CPIntr    = 3
	CPIdle    = 4
	CPUStates = 5
)

// time.h
const (
	ClocksPerSec = 128
)

func CPUTimes(percpu bool) ([]CPUTimesStat, error) {
	var ret []CPUTimesStat

	var sysctlCall string
	var ncpu int
	if percpu {
		sysctlCall = "kern.cp_times"
		ncpu, _ = CPUCounts(true)
	} else {
		sysctlCall = "kern.cp_time"
		ncpu = 1
	}

	cpuTimes, err := common.DoSysctrl(sysctlCall)
	if err != nil {
		return ret, err
	}

	for i := 0; i < ncpu; i++ {
		offset := CPUStates * i
		user, err := strconv.ParseFloat(cpuTimes[CPUser+offset], 32)
		if err != nil {
			return ret, err
		}
		nice, err := strconv.ParseFloat(cpuTimes[CPNice+offset], 32)
		if err != nil {
			return ret, err
		}
		sys, err := strconv.ParseFloat(cpuTimes[CPSys+offset], 32)
		if err != nil {
			return ret, err
		}
		idle, err := strconv.ParseFloat(cpuTimes[CPIdle+offset], 32)
		if err != nil {
			return ret, err
		}
		intr, err := strconv.ParseFloat(cpuTimes[CPIntr+offset], 32)
		if err != nil {
			return ret, err
		}

		c := CPUTimesStat{
			User:   float32(user / ClocksPerSec),
			Nice:   float32(nice / ClocksPerSec),
			System: float32(sys / ClocksPerSec),
			Idle:   float32(idle / ClocksPerSec),
			Irq:    float32(intr / ClocksPerSec),
		}
		if !percpu {
			c.CPU = "cpu-total"
		} else {
			c.CPU = fmt.Sprintf("cpu%d", i)
		}

		ret = append(ret, c)
	}

	return ret, nil
}

// Returns only one CPUInfoStat on FreeBSD
func CPUInfo() ([]CPUInfoStat, error) {
	filename := "/var/run/dmesg.boot"
	lines, _ := common.ReadLines(filename)

	var ret []CPUInfoStat

	c := CPUInfoStat{}
	for _, line := range lines {
		if matches := regexp.MustCompile(`CPU:\s+(.+) \(([\d.]+).+\)`).FindStringSubmatch(line); matches != nil {
			c.ModelName = matches[1]
			t, err := strconv.ParseFloat(matches[2], 64)
			if err != nil {
				return ret, nil
			}
			c.Mhz = t
		} else if matches := regexp.MustCompile(`Origin = "(.+)"  Id = (.+)  Family = (.+)  Model = (.+)  Stepping = (.+)`).FindStringSubmatch(line); matches != nil {
			c.VendorID = matches[1]
			c.Family = matches[3]
			c.Model = matches[4]
			t, err := strconv.ParseInt(matches[5], 10, 32)
			if err != nil {
				return ret, nil
			}
			c.Stepping = int32(t)
		} else if matches := regexp.MustCompile(`Features=.+<(.+)>`).FindStringSubmatch(line); matches != nil {
			for _, v := range strings.Split(matches[1], ",") {
				c.Flags = append(c.Flags, strings.ToLower(v))
			}
		} else if matches := regexp.MustCompile(`Features2=[a-f\dx]+<(.+)>`).FindStringSubmatch(line); matches != nil {
			for _, v := range strings.Split(matches[1], ",") {
				c.Flags = append(c.Flags, strings.ToLower(v))
			}
		} else if matches := regexp.MustCompile(`Logical CPUs per core: (\d+)`).FindStringSubmatch(line); matches != nil {
			// FIXME: no this line?
			t, err := strconv.ParseInt(matches[1], 10, 32)
			if err != nil {
				return ret, nil
			}
			c.Cores = int32(t)
		}

	}

	return append(ret, c), nil
}
<|endoftext|>


<commit_before>package crawler

import (
	"encoding/json"
	"fmt"
	"github.com/dokterbob/ipfs-search/indexer"
	"github.com/dokterbob/ipfs-search/queue"
	"gopkg.in/ipfs/go-ipfs-api.v1"
	"log"
	"net"
	"net/http"
	"net/url"
	// "path"
	"strings"
	"syscall"
	"time"
)

const (
	// Reconnect time in seconds
	RECONNECT_WAIT = 2
	TIKA_TIMEOUT   = 120
)

type CrawlerArgs struct {
	Hash       string
	Name       string
	Size       uint64
	ParentHash string
	ParentName string // This is legacy, should be removed
}

type Crawler struct {
	sh *shell.Shell
	id *indexer.Indexer
	fq *queue.TaskQueue
	hq *queue.TaskQueue
}

func NewCrawler(sh *shell.Shell, id *indexer.Indexer, fq *queue.TaskQueue, hq *queue.TaskQueue) *Crawler {
	return &Crawler{
		sh: sh,
		id: id,
		fq: fq,
		hq: hq,
	}
}

func hashUrl(hash string) string {
	return fmt.Sprintf("/ipfs/%s", hash)
}

// Update references with name, parent_hash and parent_name. Returns true when updated
func update_references(references []indexer.Reference, name string, parent_hash string) ([]indexer.Reference, bool) {
	if parent_hash == "" {
		// No parent hash, don't bother adding reference
		return references, false
	}

	for i := range references {
		if references[i].ParentHash == parent_hash {
			log.Printf("Reference '%s' for %s exists, not updating", name, parent_hash)
			return references, false
		}
	}

	log.Printf("Adding reference '%s' for %s", name, parent_hash)

	references = append(references, indexer.Reference{
		Name:       name,
		ParentHash: parent_hash,
	})

	return references, true
}

// Handle IPFS errors graceously, returns try again bool and original error
func (c Crawler) handleError(err error, hash string) (bool, error) {
	if _, ok := err.(*shell.Error); ok && strings.Contains(err.Error(), "proto") {
		// We're not recovering from protocol errors, so panic

		// Attempt to index panic to prevent re-indexing
		metadata := map[string]interface{}{
			"error": err.Error(),
		}

		c.id.IndexItem("invalid", hash, metadata)

		panic(err)
	}

	if uerr, ok := err.(*url.Error); ok {
		// URL errors

		log.Printf("URL error %v", uerr)

		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}
		}
	}

	return false, err
}

// Given a particular hash (file or directory), start crawling
func (c Crawler) CrawlHash(hash string, name string, parent_hash string, parent_name string) error {
	var references []indexer.Reference

	references, item_type, err := c.id.GetReferences(hash)
	if err != nil {
		return err
	}

	if references != nil {
		log.Printf("Already indexed '%s'.", hash)

		references, references_updated := update_references(references, name, parent_hash)

		if references_updated {
			log.Printf("Updating references for '%s'.", hash)

			properties := map[string]interface{}{
				"references": references,
			}

			err := c.id.IndexItem(item_type, hash, properties)
			if err != nil {
				return err
			}
		} else {
			log.Printf("Not updating references for '%s'", hash)
		}

		return nil
	} else {
		// Initialize references
		references = []indexer.Reference{
			{
				Name:       name,
				ParentHash: parent_hash,
			}}
	}

	log.Printf("Crawling hash '%s' (%s)", hash, name)

	url := hashUrl(hash)

	var list *shell.UnixLsObject

	try_again := true
	for try_again {
		list, err = c.sh.FileList(url)

		try_again, err = c.handleError(err, hash)

		if try_again {
			log.Printf("Retrying in %d seconds", RECONNECT_WAIT)
			time.Sleep(RECONNECT_WAIT * time.Duration(time.Second))
		}
	}

	if err != nil {
		return err
	}

	switch list.Type {
	case "File":
		// Add to file crawl queue
		// Note: we're expecting no references here, see comment below
		args := CrawlerArgs{
			Hash: hash,
			Name: name,
			Size: list.Size,
		}

		err = c.fq.AddTask(args)
		if err != nil {
			// failed to send the task
			return err
		}
	case "Directory":
		// Index name and size for directory and directory items
		properties := map[string]interface{}{
			"links":      list.Links,
			"size":       list.Size,
			"references": references,
		}

		err := c.id.IndexItem("directory", hash, properties)
		if err != nil {
			return err
		}

		for _, link := range list.Links {
			args := CrawlerArgs{
				Hash:       link.Hash,
				Name:       link.Name,
				Size:       link.Size,
				ParentHash: hash,
			}

			switch link.Type {
			case "File":
				// Add file to crawl queue
				err = c.fq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}

			case "Directory":
				// Add directory to crawl queue
				c.hq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}
			default:
				log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
			}
		}
	default:
		log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
	}

	log.Printf("Finished hash %s", hash)

	return nil
}

func getMetadata(path string, metadata *map[string]interface{}) error {
	const ipfs_tika_url = "http://localhost:8081"

	client := http.Client{
		Timeout: TIKA_TIMEOUT * time.Duration(time.Second),
	}

	resp, err := client.Get(ipfs_tika_url + path)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return fmt.Errorf("Undesired status '%s' from ipfs-tika.", resp.Status)
	}

	// Parse resulting JSON
	if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil {
		return err
	}

	return err
}

// Crawl a single object, known to be a file
func (c Crawler) CrawlFile(hash string, name string, parent_hash string, parent_name string, size uint64) error {
	/* Note: huge duplicaiton with hash crawl code. */
	var references []indexer.Reference

	references, item_type, err := c.id.GetReferences(hash)
	if err != nil {
		return err
	}

	if references != nil {
		log.Printf("Already indexed '%s'.", hash)

		references, references_updated := update_references(references, name, parent_hash)

		if references_updated {
			log.Printf("Updating references for '%s'.", hash)

			properties := map[string]interface{}{
				"references": references,
			}

			err := c.id.IndexItem(item_type, hash, properties)
			if err != nil {
				return err
			}
		} else {
			log.Printf("Not updating references for '%s'", hash)
		}

		return nil
	} else {
		// Initialize references
		references = []indexer.Reference{
			{
				Name:       name,
				ParentHash: parent_hash,
			}}
	}

	log.Printf("Crawling file %s (%s)\n", hash, name)

	metadata := make(map[string]interface{})

	if size > 0 {
		if size > 10*1024*1024 {
			// Fail hard for really large files, for now
			return fmt.Errorf("%s (%s) too large, not indexing (for now).", hash, name)
		}

		var path string
		if name != "" && parent_hash != "" {
			path = fmt.Sprintf("/ipfs/%s/%s", parent_hash, name)
		} else {
			path = fmt.Sprintf("/ipfs/%s", hash)
		}

		try_again := true
		for try_again {
			err = getMetadata(path, &metadata)

			try_again, err = c.handleError(err, hash)

			if try_again {
				log.Printf("Retrying in %d seconds", RECONNECT_WAIT)
				time.Sleep(RECONNECT_WAIT * time.Duration(time.Second))
			}
		}

		if err != nil {
			return err
		}

		// Check for IPFS links in content
		/*
			for raw_url := range metadata.urls {
				url, err := URL.Parse(raw_url)

				if err != nil {
					return err
				}

				if strings.HasPrefix(url.Path, "/ipfs/") {
					// Found IPFS link!
					args := CrawlerArgs{
						Hash:       link.Hash,
						Name:       link.Name,
						Size:       link.Size,
						ParentHash: hash,
					}

				}
			}
		*/
	}

	metadata["size"] = size
	metadata["references"] = references

	err = c.id.IndexItem("file", hash, metadata)
	if err != nil {
		return err
	}

	log.Printf("Finished file %s", hash)

	return nil
}
<commit_msg>Slight cleanup / deduplication of code.<commit_after>package crawler

import (
	"encoding/json"
	"fmt"
	"github.com/dokterbob/ipfs-search/indexer"
	"github.com/dokterbob/ipfs-search/queue"
	"gopkg.in/ipfs/go-ipfs-api.v1"
	"log"
	"net"
	"net/http"
	"net/url"
	// "path"
	"strings"
	"syscall"
	"time"
)

const (
	// Reconnect time in seconds
	RECONNECT_WAIT = 2
	TIKA_TIMEOUT   = 120
)

type CrawlerArgs struct {
	Hash       string
	Name       string
	Size       uint64
	ParentHash string
	ParentName string // This is legacy, should be removed
}

type Crawler struct {
	sh *shell.Shell
	id *indexer.Indexer
	fq *queue.TaskQueue
	hq *queue.TaskQueue
}

func NewCrawler(sh *shell.Shell, id *indexer.Indexer, fq *queue.TaskQueue, hq *queue.TaskQueue) *Crawler {
	return &Crawler{
		sh: sh,
		id: id,
		fq: fq,
		hq: hq,
	}
}

func hashUrl(hash string) string {
	return fmt.Sprintf("/ipfs/%s", hash)
}

// Update references with name, parent_hash and parent_name. Returns true when updated
func update_references(references []indexer.Reference, name string, parent_hash string) ([]indexer.Reference, bool) {
	if parent_hash == "" {
		// No parent hash, don't bother adding reference
		return references, false
	}

	for i := range references {
		if references[i].ParentHash == parent_hash {
			log.Printf("Reference '%s' for %s exists, not updating", name, parent_hash)
			return references, false
		}
	}

	log.Printf("Adding reference '%s' for %s", name, parent_hash)

	references = append(references, indexer.Reference{
		Name:       name,
		ParentHash: parent_hash,
	})

	return references, true
}

// Handle IPFS errors graceously, returns try again bool and original error
func (c Crawler) handleError(err error, hash string) (bool, error) {
	if _, ok := err.(*shell.Error); ok && strings.Contains(err.Error(), "proto") {
		// We're not recovering from protocol errors, so panic

		// Attempt to index panic to prevent re-indexing
		metadata := map[string]interface{}{
			"error": err.Error(),
		}

		c.id.IndexItem("invalid", hash, metadata)

		panic(err)
	}

	if uerr, ok := err.(*url.Error); ok {
		// URL errors

		log.Printf("URL error %v", uerr)

		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}
		}
	}

	return false, err
}

func (c Crawler) index_references(hash string, name string, parent_hash string) ([]indexer.Reference, bool, error) {
	var references []indexer.Reference

	references, item_type, err := c.id.GetReferences(hash)
	if err != nil {
		return nil, false, err
	}

	if references != nil {
		log.Printf("Already indexed '%s'.", hash)

		references, references_updated := update_references(references, name, parent_hash)

		if references_updated {
			log.Printf("Updating references for '%s'.", hash)

			properties := map[string]interface{}{
				"references": references,
			}

			err := c.id.IndexItem(item_type, hash, properties)
			if err != nil {
				return nil, false, err
			}
		} else {
			log.Printf("Not updating references for '%s'", hash)
		}

		return references, true, nil
	} else {
		// Initialize references
		references = []indexer.Reference{
			{
				Name:       name,
				ParentHash: parent_hash,
			}}
	}

	return references, false, nil
}

// Given a particular hash (file or directory), start crawling
func (c Crawler) CrawlHash(hash string, name string, parent_hash string, parent_name string) error {
	references, already_indexed, err := c.index_references(hash, name, parent_hash)

	if err != nil {
		return err
	}

	if already_indexed {
		return nil
	}

	log.Printf("Crawling hash '%s' (%s)", hash, name)

	url := hashUrl(hash)

	var list *shell.UnixLsObject

	try_again := true
	for try_again {
		list, err = c.sh.FileList(url)

		try_again, err = c.handleError(err, hash)

		if try_again {
			log.Printf("Retrying in %d seconds", RECONNECT_WAIT)
			time.Sleep(RECONNECT_WAIT * time.Duration(time.Second))
		}
	}

	if err != nil {
		return err
	}

	switch list.Type {
	case "File":
		// Add to file crawl queue
		// Note: we're expecting no references here, see comment below
		args := CrawlerArgs{
			Hash: hash,
			Name: name,
			Size: list.Size,
		}

		err = c.fq.AddTask(args)
		if err != nil {
			// failed to send the task
			return err
		}
	case "Directory":
		// Index name and size for directory and directory items
		properties := map[string]interface{}{
			"links":      list.Links,
			"size":       list.Size,
			"references": references,
		}

		err := c.id.IndexItem("directory", hash, properties)
		if err != nil {
			return err
		}

		for _, link := range list.Links {
			args := CrawlerArgs{
				Hash:       link.Hash,
				Name:       link.Name,
				Size:       link.Size,
				ParentHash: hash,
			}

			switch link.Type {
			case "File":
				// Add file to crawl queue
				err = c.fq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}

			case "Directory":
				// Add directory to crawl queue
				c.hq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}
			default:
				log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
			}
		}
	default:
		log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
	}

	log.Printf("Finished hash %s", hash)

	return nil
}

func getMetadata(path string, metadata *map[string]interface{}) error {
	const ipfs_tika_url = "http://localhost:8081"

	client := http.Client{
		Timeout: TIKA_TIMEOUT * time.Duration(time.Second),
	}

	resp, err := client.Get(ipfs_tika_url + path)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return fmt.Errorf("Undesired status '%s' from ipfs-tika.", resp.Status)
	}

	// Parse resulting JSON
	if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil {
		return err
	}

	return err
}

// Crawl a single object, known to be a file
func (c Crawler) CrawlFile(hash string, name string, parent_hash string, parent_name string, size uint64) error {
	references, already_indexed, err := c.index_references(hash, name, parent_hash)

	if err != nil {
		return err
	}

	if already_indexed {
		return nil
	}

	log.Printf("Crawling file %s (%s)\n", hash, name)

	metadata := make(map[string]interface{})

	if size > 0 {
		if size > 10*1024*1024 {
			// Fail hard for really large files, for now
			return fmt.Errorf("%s (%s) too large, not indexing (for now).", hash, name)
		}

		var path string
		if name != "" && parent_hash != "" {
			path = fmt.Sprintf("/ipfs/%s/%s", parent_hash, name)
		} else {
			path = fmt.Sprintf("/ipfs/%s", hash)
		}

		try_again := true
		for try_again {
			err = getMetadata(path, &metadata)

			try_again, err = c.handleError(err, hash)

			if try_again {
				log.Printf("Retrying in %d seconds", RECONNECT_WAIT)
				time.Sleep(RECONNECT_WAIT * time.Duration(time.Second))
			}
		}

		if err != nil {
			return err
		}

		// Check for IPFS links in content
		/*
			for raw_url := range metadata.urls {
				url, err := URL.Parse(raw_url)

				if err != nil {
					return err
				}

				if strings.HasPrefix(url.Path, "/ipfs/") {
					// Found IPFS link!
					args := CrawlerArgs{
						Hash:       link.Hash,
						Name:       link.Name,
						Size:       link.Size,
						ParentHash: hash,
					}

				}
			}
		*/
	}

	metadata["size"] = size
	metadata["references"] = references

	err = c.id.IndexItem("file", hash, metadata)
	if err != nil {
		return err
	}

	log.Printf("Finished file %s", hash)

	return nil
}
<|endoftext|>