<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"fmt"

	mf "github.com/manifestival/manifestival"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"knative.dev/operator/pkg/apis/operator/v1alpha1"
)

// JobTransform updates the job with the expected value for the key app in the label
func JobTransform(obj v1alpha1.KComponent) mf.Transformer {
	return func(u *unstructured.Unstructured) error {
		if u.GetKind() == "Job" {
			component := "serving"
			if _, ok := obj.(*v1alpha1.KnativeEventing); ok {
				component = "eventing"
			}
			if u.GetName() == "" {
				u.SetName(fmt.Sprintf("%s%s-%s", u.GetGenerateName(), component, TargetVersion(obj)))
			} else {
				u.SetName(fmt.Sprintf("%s-%s-%s", u.GetName(), component, TargetVersion(obj)))
			}
		}
		return nil
	}
}
<commit_msg>Adding istio ignore annotation transformer for jobs (#236)<commit_after>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"fmt"

	mf "github.com/manifestival/manifestival"
	batchv1 "k8s.io/api/batch/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/client-go/kubernetes/scheme"
	"knative.dev/operator/pkg/apis/operator/v1alpha1"
)

const istioAnnotationName = "sidecar.istio.io/inject"

// JobTransform updates the job with the expected value for the key app in the label
func JobTransform(obj v1alpha1.KComponent) mf.Transformer {
	return func(u *unstructured.Unstructured) error {
		if u.GetKind() == "Job" {
			job := &batchv1.Job{}
			if err := scheme.Scheme.Convert(u, job, nil); err != nil {
				return err
			}

			component := "serving"
			if _, ok := obj.(*v1alpha1.KnativeEventing); ok {
				component = "eventing"
			}
			if job.GetName() == "" {
				job.SetName(fmt.Sprintf("%s%s-%s", job.GetGenerateName(), component, TargetVersion(obj)))
			} else {
				job.SetName(fmt.Sprintf("%s-%s-%s", job.GetName(), component, TargetVersion(obj)))
			}

			addIstioIgnoreAnnotation(job)
			return scheme.Scheme.Convert(job, u, nil)
		}

		return nil
	}
}

func addIstioIgnoreAnnotation(job *batchv1.Job) {
	annotations := job.Spec.Template.GetAnnotations()
	if annotations == nil {
		annotations = make(map[string]string)
	}

	istioAnnotation := annotations[istioAnnotationName]
	if istioAnnotation == "" {
		annotations[istioAnnotationName] = "false"
		job.Spec.Template.SetAnnotations(annotations)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package routes

import (
	"github.com/emicklei/go-restful"
	"github.com/golang/glog"

	"k8s.io/apiserver/pkg/server/mux"
	"k8s.io/kube-openapi/pkg/common"
	"k8s.io/kube-openapi/pkg/handler"
)

// OpenAPI installs spec endpoints for each web service.
type OpenAPI struct {
	Config *common.Config
}

// Install adds the SwaggerUI webservice to the given mux.
func (oa OpenAPI) Install(c *restful.Container, mux *mux.PathRecorderMux) {
	_, err := handler.BuildAndRegisterOpenAPIService("/swagger.json", c.RegisteredWebServices(), oa.Config, mux)
	if err != nil {
		glog.Fatalf("Failed to register open api spec for root: %v", err)
	}
}
<commit_msg>Add new openapi endpoint in aggregator server<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package routes

import (
	restful "github.com/emicklei/go-restful"
	"github.com/golang/glog"

	"k8s.io/apiserver/pkg/server/mux"
	"k8s.io/kube-openapi/pkg/common"
	"k8s.io/kube-openapi/pkg/handler"
)

// OpenAPI installs spec endpoints for each web service.
type OpenAPI struct {
	Config *common.Config
}

// Install adds the SwaggerUI webservice to the given mux.
func (oa OpenAPI) Install(c *restful.Container, mux *mux.PathRecorderMux) {
	// NOTE: [DEPRECATION] We will announce deprecation for format-separated endpoints for OpenAPI spec,
	// and switch to a single /openapi/v2 endpoint in Kubernetes 1.10. The design doc and deprecation process
	// are tracked at: https://docs.google.com/document/d/19lEqE9lc4yHJ3WJAJxS_G7TcORIJXGHyq3wpwcH28nU.
	_, err := handler.BuildAndRegisterOpenAPIService("/swagger.json", c.RegisteredWebServices(), oa.Config, mux)
	if err != nil {
		glog.Fatalf("Failed to register open api spec for root: %v", err)
	}
	_, err = handler.BuildAndRegisterOpenAPIVersionedService("/openapi/v2", c.RegisteredWebServices(), oa.Config, mux)
	if err != nil {
		glog.Fatalf("Failed to register versioned open api spec for root: %v", err)
	}
}
<|endoftext|>


<commit_before>package server

import (
	"net"
	"net/http"
	"net/url"
	"testing"
	"time"

	"github.com/gabstv/sandpiper/pkg/util"
	"github.com/gabstv/sandpiper/route"
	"github.com/gorilla/websocket"
)

type wsTestServer struct {
	upgrader websocket.Upgrader
	t        *testing.T
	send     chan []byte
	ws       *websocket.Conn
	Numm     int
}

func (s *wsTestServer) wswrite() {
	ticker := time.NewTicker(time.Second * 50)
	defer func() {
		ticker.Stop()
		s.ws.Close()
	}()
	for {
		select {
		case msg, ok := <-s.send:
			if !ok {
				s.write(websocket.CloseMessage, []byte{})
				return
			}
			if err := s.write(websocket.TextMessage, msg); err != nil {
				return
			}
		case <-ticker.C:
			if err := s.write(websocket.PingMessage, []byte{}); err != nil {
				return
			}
		}
	}
}

func (s *wsTestServer) wsread() {
	defer func() {
		s.ws.Close()
	}()
	s.ws.SetReadLimit(512)
	s.ws.SetReadDeadline(time.Now().Add(time.Second * 60))
	s.ws.SetPongHandler(func(string) error { s.ws.SetReadDeadline(time.Now().Add(time.Second * 60)); return nil })
	for {
		_, msg, err := s.ws.ReadMessage()
		if err != nil {
			return // probably EOF
		}
		s.t.Logf("Received '%v'", string(msg))
		s.Numm++
	}
}

// write writes a message with the given message type and payload.
func (s *wsTestServer) write(mt int, payload []byte) error {
	s.ws.SetWriteDeadline(time.Now().Add(time.Second * 10))
	return s.ws.WriteMessage(mt, payload)
}

func (t *wsTestServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	t.t.Logf("Headers %v", r.Header)
	if r.Method != "GET" {
		http.Error(w, "Method not allowed", 405)
		return
	}
	ws, err := t.upgrader.Upgrade(w, r, nil)
	if err != nil {
		t.t.Fatalf("Websocket upgrader error! %s", err.Error())
	}
	t.ws = ws
	go t.wswrite()
	t.wsread()
}

func TestWebsocket(t *testing.T) {
	s0 := &wsTestServer{
		upgrader: websocket.Upgrader{
			ReadBufferSize:  1024,
			WriteBufferSize: 1024,
		},
		t:    t,
		send: make(chan []byte, 256),
	}
	go func() {
		err := http.ListenAndServe(":9099", s0)
		if err != nil {
			t.Fatal(err)
		}
	}()
	//
	sv := Default(&Config{
		Debug:         true,
		ListenAddr:    ":9100",
		ListenAddrTLS: ":9101",
	})
	r0 := route.Route{
		Domain: "example.com",
		Server: route.RouteServer{
			OutConnType: route.HTTP,
			OutAddress:  "localhost:9099",
		},
		WsCFG: util.WsConfig{Enabled: true},
	}
	err := sv.Add(r0)
	if err != nil {
		t.Fatal(err)
	}

	uri, _ := url.Parse("ws://localhost:9100")
	h := http.Header{}
	h.Set("X-Sandpiper-Host", "example.com")
	//h.Set("Upgrade", "websocket")

	go func() {
		err := sv.Run()
		if err != nil {
			t.Fatalf("sv.Run ERR: %s", err)
		}
	}()

	time.Sleep(time.Millisecond * 100)

	c, err := net.Dial("tcp", uri.Host)
	if err != nil {
		t.Fatalf("Dial: %v", err)
	}

	ws, _, err := websocket.NewClient(c, uri, h, 1024, 1024)
	if err != nil {
		t.Fatalf("Could not connect %s", err)
	}
	if e0 := ws.WriteMessage(websocket.TextMessage, []byte("hello 1!")); e0 != nil {
		t.Fatalf("ws.WriteMessage %s", e0)
	}
	time.Sleep(time.Second * 1)
	if e0 := ws.WriteMessage(websocket.TextMessage, []byte("hello 2!")); e0 != nil {
		t.Fatalf("ws.WriteMessage %s", e0)
	}
	time.Sleep(time.Second * 7)
	if e0 := ws.WriteMessage(websocket.TextMessage, []byte("hello 3!")); e0 != nil {
		t.Fatalf("ws.WriteMessage %s", e0)
	}
	time.Sleep(time.Second * 1)
	ws.WriteMessage(websocket.CloseMessage, []byte{})
	time.Sleep(time.Second * 1)
	if s0.Numm != 3 {
		t.Fatalf("Should have received 3 messages!")
	}
}
<commit_msg>fix test<commit_after>package server

import (
	"net"
	"net/http"
	"net/url"
	"testing"
	"time"

	"github.com/gabstv/sandpiper/internal/pkg/route"
	"github.com/gabstv/sandpiper/pkg/util"
	"github.com/gorilla/websocket"
)

type wsTestServer struct {
	upgrader websocket.Upgrader
	t        *testing.T
	send     chan []byte
	ws       *websocket.Conn
	Numm     int
}

func (s *wsTestServer) wswrite() {
	ticker := time.NewTicker(time.Second * 50)
	defer func() {
		ticker.Stop()
		s.ws.Close()
	}()
	for {
		select {
		case msg, ok := <-s.send:
			if !ok {
				s.write(websocket.CloseMessage, []byte{})
				return
			}
			if err := s.write(websocket.TextMessage, msg); err != nil {
				return
			}
		case <-ticker.C:
			if err := s.write(websocket.PingMessage, []byte{}); err != nil {
				return
			}
		}
	}
}

func (s *wsTestServer) wsread() {
	defer func() {
		s.ws.Close()
	}()
	s.ws.SetReadLimit(512)
	s.ws.SetReadDeadline(time.Now().Add(time.Second * 60))
	s.ws.SetPongHandler(func(string) error { s.ws.SetReadDeadline(time.Now().Add(time.Second * 60)); return nil })
	for {
		_, msg, err := s.ws.ReadMessage()
		if err != nil {
			return // probably EOF
		}
		s.t.Logf("Received '%v'", string(msg))
		s.Numm++
	}
}

// write writes a message with the given message type and payload.
func (s *wsTestServer) write(mt int, payload []byte) error {
	s.ws.SetWriteDeadline(time.Now().Add(time.Second * 10))
	return s.ws.WriteMessage(mt, payload)
}

func (t *wsTestServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	t.t.Logf("Headers %v", r.Header)
	if r.Method != "GET" {
		http.Error(w, "Method not allowed", 405)
		return
	}
	ws, err := t.upgrader.Upgrade(w, r, nil)
	if err != nil {
		t.t.Fatalf("Websocket upgrader error! %s", err.Error())
	}
	t.ws = ws
	go t.wswrite()
	t.wsread()
}

func TestWebsocket(t *testing.T) {
	s0 := &wsTestServer{
		upgrader: websocket.Upgrader{
			ReadBufferSize:  1024,
			WriteBufferSize: 1024,
		},
		t:    t,
		send: make(chan []byte, 256),
	}
	go func() {
		err := http.ListenAndServe(":9099", s0)
		if err != nil {
			t.Fatal(err)
		}
	}()
	//
	sv := Default(&Config{
		Debug:         true,
		ListenAddr:    ":9100",
		ListenAddrTLS: ":9101",
	})
	r0 := route.Route{
		Domain: "example.com",
		Server: route.RouteServer{
			OutConnType: route.HTTP,
			OutAddress:  "localhost:9099",
		},
		WsCFG: util.WsConfig{Enabled: true},
	}
	err := sv.Add(r0)
	if err != nil {
		t.Fatal(err)
	}

	uri, _ := url.Parse("ws://localhost:9100")
	h := http.Header{}
	h.Set("X-Sandpiper-Host", "example.com")
	//h.Set("Upgrade", "websocket")

	go func() {
		err := sv.Run()
		if err != nil {
			t.Fatalf("sv.Run ERR: %s", err)
		}
	}()

	time.Sleep(time.Millisecond * 100)

	c, err := net.Dial("tcp", uri.Host)
	if err != nil {
		t.Fatalf("Dial: %v", err)
	}

	ws, _, err := websocket.NewClient(c, uri, h, 1024, 1024)
	if err != nil {
		t.Fatalf("Could not connect %s", err)
	}
	if e0 := ws.WriteMessage(websocket.TextMessage, []byte("hello 1!")); e0 != nil {
		t.Fatalf("ws.WriteMessage %s", e0)
	}
	time.Sleep(time.Second * 1)
	if e0 := ws.WriteMessage(websocket.TextMessage, []byte("hello 2!")); e0 != nil {
		t.Fatalf("ws.WriteMessage %s", e0)
	}
	time.Sleep(time.Second * 7)
	if e0 := ws.WriteMessage(websocket.TextMessage, []byte("hello 3!")); e0 != nil {
		t.Fatalf("ws.WriteMessage %s", e0)
	}
	time.Sleep(time.Second * 1)
	ws.WriteMessage(websocket.CloseMessage, []byte{})
	time.Sleep(time.Second * 1)
	if s0.Numm != 3 {
		t.Fatalf("Should have received 3 messages!")
	}
}
<|endoftext|>


<commit_before>package trafficmon

import (
	"bytes"
	"fmt"
	"math"
	"strconv"
	"sync"
	"sync/atomic"
	"time"
	"unsafe"
)

type trafficData struct {
	logarithm bool
	startTime int64
	lastTime  int64
	lastData  int64
	ticks     int64
	interval  int // seconds
	tmp       float64
	min, max  float64
	data      []float64
}

func (d *trafficData) Init(ln, intval int) {
	d.data = make([]float64, ln)
	d.interval = intval
	d.startTime = time.Now().UnixNano()
	d.lastTime = d.startTime
	d.ticks = 1
}

func (d *trafficData) Log(n float64) float64 {
	return math.Log2(n + 1)
}

func round(x float64) int64 {
	if int64(x+0.5) == int64(x) {
		return int64(x)
	}
	return int64(x) + 1
}

func (d *trafficData) Append(data int64) {
	now := time.Now().UnixNano()
	intv := int64(d.interval) * 1e9

AGAIN:
	if now <= d.startTime+d.ticks*intv {
		d.tmp += float64(data - d.lastData)
		d.lastData = data
		d.lastTime = now
	} else {
		ds := float64(data-d.lastData) / (float64(now-d.lastTime) / 1e9) // average data per second
		rem := ds * float64(d.startTime+d.ticks*intv-d.lastTime) / 1e9   // remaining data to next tick
		// fmt.Println(d.lastData, ds, rem)
		d.tmp += rem
		d.lastData += round(rem)
		d.tmp /= float64(d.interval)

		copy(d.data[1:], d.data)
		d.data[0] = d.tmp
		d.tmp = 0

		d.lastTime = d.startTime + d.ticks*intv
		d.ticks++
		goto AGAIN
	}
}

func (d *trafficData) Range() (min float64, avg float64, max float64) {
	min, max = 1e100, 0.0

	for i := len(d.data) - 1; i >= 0; i-- {
		f := d.data[i]
		avg += f

		if f > max {
			max = f
		} else if f < min {
			min = f
		}
	}

	d.min, d.max = min, max
	avg /= float64(len(d.data))
	return
}

func (d *trafficData) Get(index int) float64 {
	f := d.data[index]
	if d.logarithm {
		return d.Log(f)
	}
	return f
}

type Survey struct {
	totalSent   int64
	totalRecved int64
	latency     float64
	latencyMin  int64
	latencyMax  int64
	sent        trafficData
	recved      trafficData
	sync.Mutex
}

func (s *Survey) Init(length, intval int) {
	s.sent.Init(length/intval, intval)
	s.recved.Init(length/intval, intval)
	s.latencyMin = -1
}

func (s *Survey) Send(size int64) *Survey {
	atomic.AddInt64(&s.totalSent, size)
	return s
}

func (s *Survey) Recv(size int64) *Survey {
	atomic.AddInt64(&s.totalRecved, size)
	return s
}

func (s *Survey) Latency(nsec int64) {
	const N = 2
	for {
		o := s.latency
		n := o - o/N + float64(nsec)/N

		oi, ni := *(*uint64)(unsafe.Pointer(&o)), *(*uint64)(unsafe.Pointer(&n))
		if atomic.CompareAndSwapUint64((*uint64)(unsafe.Pointer(&s.latency)), oi, ni) {
			break
		}
	}

	if nsec > s.latencyMax {
		s.latencyMax = nsec
	}

	if nsec < s.latencyMin || s.latencyMin == -1 {
		s.latencyMin = nsec
	}
}

func (s *Survey) Update() {
	s.Lock()
	s.sent.Append(s.totalSent)
	s.recved.Append(s.totalRecved)
	s.Unlock()
}

func (s *Survey) Data() (int64, int64) {
	return s.totalRecved, s.totalSent
}

func (s *Survey) SVG(w, h int, logarithm bool) *bytes.Buffer {
	s.Lock()
	defer s.Unlock()

	ret := &bytes.Buffer{}
	ret.WriteString(fmt.Sprintf(`<svg xmlns="http://www.w3.org/2000/svg" version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 %d %d">`, w, h))
	ret.WriteString(`<style>*{ font-family: "Lucida Console", Monaco, monospace; box-sizing: border-box; }</style>`)
	ret.WriteString(`<defs>`)
	id := strconv.FormatInt(time.Now().Unix(), 16)
	ret.WriteString(`<linearGradient id="traffic-` + id + `-i" x1="0" x2="1" y1="0" y2="0"><stop offset="0%" stop-color="white" stop-opacity="0.7"/><stop offset="100%" stop-color="white" stop-opacity="0"/></linearGradient>`)
	ret.WriteString(`<clipPath id="traffic-` + id + `-c"><rect width="100%" height="100%" fill="none" stroke="none"/></clipPath>`)
	ret.WriteString(`</defs><g clip-path="url(#traffic-` + id + `-c)">`)

	wTick := float64(w) / float64(len(s.sent.data)-1)
	s.sent.logarithm, s.recved.logarithm = logarithm, logarithm
	_, savg, smax := s.sent.Range()
	_, ravg, rmax := s.recved.Range()
	margin := h / 10
	tick := 60 / s.sent.interval
	minutes := len(s.sent.data) / tick

	for i := tick; i < len(s.sent.data); i += tick * 2 {
		x := float64(i) * wTick
		ret.WriteString(fmt.Sprintf(`<rect x="%f" y="0" width="%f" height="%d" fill="#f7f8f9"/>`, x-wTick, wTick*float64(tick), h))
	}

	for i, m := 0, 0; i < len(s.sent.data); i, m = i+tick*5, m+5 {
		x := float64(i) * wTick
		ret.WriteString(fmt.Sprintf(`<text x="%f" y="%d" font-size=".3em">-%d</text>`, x+1, h-2, minutes-m))
	}

	polybegin := func(c string) {
		ret.WriteString(`<polyline stroke="` + c + `" fill="` + c + `" fill-opacity="0.5" stroke-width="0.5px" points="`)
	}

	if delta := smax; delta > 0 {
		if logarithm {
			delta = s.sent.Log(smax)
			margin = h/2 + 1
		}

		hScale := float64(h-margin) / delta
		polybegin(`#F44336`)

		x := 0.0
		for i := len(s.sent.data) - 1; i >= 0; i-- {
			f := int(s.sent.Get(i) * hScale)

			if x1, x2 := x-wTick/2, x+wTick/2; logarithm {
				ret.WriteString(fmt.Sprintf(`%f,%d %f,%d `, x1, h/2-f, x2, h/2-f))
			} else {
				ret.WriteString(fmt.Sprintf(`%f,%d %f,%d `, x1, f, x2, f))
			}
			x += wTick
		}

		if logarithm {
			ret.WriteString(fmt.Sprintf(` %d,%d 0,%d"/>`, w, h/2, h/2))
		} else {
			ret.WriteString(fmt.Sprintf(` %d,%d %d,%d %d,%d -1,-1 -1,0"/>`, w, 0, w+1, 0, w+1, -1))
		}
	}

	if delta := rmax; delta > 0 {
		if logarithm {
			delta = s.sent.Log(rmax)
			margin = h/2 + 1
		}

		hScale := float64(h-margin) / delta
		polybegin(`#00796B`)

		x := 0.0
		for i := len(s.recved.data) - 1; i >= 0; i-- {
			f := int(s.recved.Get(i) * hScale)
			ret.WriteString(fmt.Sprintf(`%f,%d %f,%d `, x-wTick/2, h-f, x+wTick/2, h-f))
			x += wTick
		}

		ret.WriteString(fmt.Sprintf(` %d,%d %d,%d %d,%d -1,%d -1,%d"/>`, w, h, w+1, h, w+1, h+1, h+1, h))
	}

	ret.WriteString(`<rect width="100%" height="100%" fill="url(#traffic-` + id + `-i)"/>`)

	ret.WriteString(`<text font-size="0.33em" style='text-shadow: 0 0 1px #ccc'>`)

	format := func(f float64) string {
		if f < 10 {
			return strconv.FormatFloat(f, 'f', 3, 64)
		} else if f < 100 {
			return strconv.FormatFloat(f, 'f', 2, 64)
		}
		return strconv.FormatFloat(f, 'f', 1, 64)
	}

	sText := `<tspan fill="#303F9F" x=".4em" dy="1.2em">Lt %d ms %d ms %d ms</tspan><tspan fill="#F44336" x=".4em" dy="1.2em">Tx %s KB/s %s KB/s %s KB/s %.2f MB</tspan>`
	rText := `<tspan fill="#00796B" x=".4em" dy="1.2em">Rx %s KB/s %s KB/s %s KB/s %.2f MB</tspan>`
	if logarithm {
		rText = `<tspan y="50%%" style="visibility:hidden">a</tspan>` + rText
	}

	ret.WriteString(fmt.Sprintf(sText, s.latencyMin/1e6, int(s.latency/1e6), s.latencyMax/1e6,
		format(s.sent.data[0]/1024), format(savg/1024), format(smax/1024), float64(s.totalSent)/1024/1024))

	ret.WriteString(fmt.Sprintf(rText, format(s.recved.data[0]/1024), format(ravg/1024), format(rmax/1024), float64(s.totalRecved)/1024/1024))

	ret.WriteString(`</text>`)

	ret.WriteString(fmt.Sprintf(`<polyline stroke-width="1px" stroke="#d1d2d3" fill="none" points="0,0 %d,%d %d,%d %d,%d 0,0"/>`,
		w, 0, w, h, 0, h))

	if logarithm {
		ret.WriteString(`<line x1="0" y1="50%" x2="100%" y2="50%" stroke-width="0.5px" stroke="#d7d8d9"/>`)
	}

	ret.WriteString(`</g></svg>`)
	return ret
}
<commit_msg>Traffic monitor fix<commit_after>package trafficmon

import (
	"math"
	"sync"
	"sync/atomic"
	"time"
	"unsafe"
)

type trafficData struct {
	logarithm bool
	startTime int64
	lastTime  int64
	lastData  int64
	ticks     int64
	interval  int // seconds
	tmp       float64
	min, max  float64
	data      []float64
}

func (d *trafficData) Init(ln, intval int) {
	d.data = make([]float64, ln)
	d.interval = intval
	d.startTime = time.Now().UnixNano()
	d.lastTime = d.startTime
	d.ticks = 1
}

func (d *trafficData) Log(n float64) float64 {
	return math.Log2(n + 1)
}

func round(x float64) int64 {
	if int64(x+0.5) == int64(x) {
		return int64(x)
	}
	return int64(x) + 1
}

func (d *trafficData) Append(data int64) {
	now := time.Now().UnixNano()
	intv := int64(d.interval) * 1e9

AGAIN:
	if now <= d.startTime+d.ticks*intv {
		d.tmp += float64(data - d.lastData)
		d.lastData = data
		d.lastTime = now
	} else {
		ds := float64(data-d.lastData) / (float64(now-d.lastTime) / 1e9) // average data per second
		rem := ds * float64(d.startTime+d.ticks*intv-d.lastTime) / 1e9   // remaining data to next tick
		// fmt.Println(d.lastData, ds, rem)
		d.tmp += rem
		d.lastData += round(rem)
		d.tmp /= float64(d.interval)

		copy(d.data[1:], d.data)
		d.data[0] = d.tmp
		d.tmp = 0

		d.lastTime = d.startTime + d.ticks*intv
		d.ticks++
		goto AGAIN
	}
}

func (d *trafficData) Range() (min float64, avg float64, max float64) {
	min, max = 1e100, 0.0

	for i := len(d.data) - 1; i >= 0; i-- {
		f := d.data[i]
		avg += f

		if f > max {
			max = f
		} else if f < min {
			min = f
		}
	}

	d.min, d.max = min, max
	avg /= float64(len(d.data))
	return
}

func (d *trafficData) Get(index int) float64 {
	f := d.data[index]
	if d.logarithm {
		return d.Log(f)
	}
	return f
}

type Survey struct {
	totalSent   int64
	totalRecved int64
	latency     float64
	latencyMin  int64
	latencyMax  int64
	sent        trafficData
	recved      trafficData
	sync.Mutex
}

func (s *Survey) Init(length, intval int) {
	s.sent.Init(length/intval, intval)
	s.recved.Init(length/intval, intval)
	s.latencyMin = -1
}

func (s *Survey) Send(size int64) *Survey {
	atomic.AddInt64(&s.totalSent, size)
	return s
}

func (s *Survey) Recv(size int64) *Survey {
	atomic.AddInt64(&s.totalRecved, size)
	return s
}

func (s *Survey) Latency(nsec int64) {
	const N = 2
	for {
		o := s.latency
		n := o - o/N + float64(nsec)/N

		oi, ni := *(*uint64)(unsafe.Pointer(&o)), *(*uint64)(unsafe.Pointer(&n))
		if atomic.CompareAndSwapUint64((*uint64)(unsafe.Pointer(&s.latency)), oi, ni) {
			break
		}
	}

	if nsec > s.latencyMax {
		s.latencyMax = nsec
	}

	if nsec < s.latencyMin || s.latencyMin == -1 {
		s.latencyMin = nsec
	}
}

func (s *Survey) Update() {
	s.Lock()
	s.sent.Append(s.totalSent)
	s.recved.Append(s.totalRecved)
	s.Unlock()
}

func (s *Survey) Data() (int64, int64) {
	return s.totalRecved, s.totalSent
}
<|endoftext|>


<commit_before>package space

import (
	"testing"

	"github.com/documize/community/core/uniqueid"
	"github.com/documize/community/domain/test"
	"github.com/documize/community/model/space"
)

// TestSpace tests all space database operations.
func TestSpace(t *testing.T) {
	rt, s, ctx := test.SetupTest()
	spaceID := uniqueid.Generate()
	var err error

	t.Run("Add Space", func(t *testing.T) {
		ctx.Transaction, err = rt.Db.Beginx()
		if err != nil {
			return
		}
		sp := space.Space{}
		sp.RefID = spaceID
		sp.OrgID = ctx.OrgID
		sp.Type = space.ScopePrivate
		sp.UserID = ctx.UserID
		sp.Name = "test"

		err = s.Space.Add(ctx, sp)
		if err != nil {
			ctx.Transaction.Rollback()
			t.Error("failed to delete space")
		}
		ctx.Transaction.Commit()

		sp2, err := s.Space.Get(ctx, sp.RefID)
		if err != nil || sp.Name != sp2.Name {
			t.Error("failed to create space")
		}
	})

	t.Run("Update Space", func(t *testing.T) {
		ctx.Transaction, err = rt.Db.Beginx()

		sp, err := s.Space.Get(ctx, spaceID)
		if err != nil {
			ctx.Transaction.Rollback()
			t.Error("failed to get space")
			return
		}

		sp.Name = "test update"
		err = s.Space.Update(ctx, sp)
		if err != nil {
			ctx.Transaction.Rollback()
			t.Error("failed to update space")
			return
		}

		ctx.Transaction.Commit()

		sp, err = s.Space.Get(ctx, spaceID)
		if err != nil || sp.Name != "test update" {
			t.Error("failed to update space")
		}
	})

	t.Run("Delete Space", func(t *testing.T) {
		ctx.Transaction, err = rt.Db.Beginx()

		_, err = s.Space.Delete(ctx, spaceID)
		if err != nil {
			ctx.Transaction.Rollback()
			t.Error("failed to delete space")
			return
		}

		ctx.Transaction.Commit()
	})
}
<commit_msg>test teardown<commit_after>package space

import (
	"testing"

	"github.com/documize/community/core/uniqueid"
	"github.com/documize/community/domain/test"
	"github.com/documize/community/model/space"
)

// TestSpace tests all space database operations.
func TestSpace(t *testing.T) {
	rt, s, ctx := test.SetupTest()
	spaceID := uniqueid.Generate()
	var err error

	t.Run("Add Space", func(t *testing.T) {
		ctx.Transaction, err = rt.Db.Beginx()
		if err != nil {
			return
		}
		sp := space.Space{}
		sp.RefID = spaceID
		sp.OrgID = ctx.OrgID
		sp.Type = space.ScopePrivate
		sp.UserID = ctx.UserID
		sp.Name = "test"

		err = s.Space.Add(ctx, sp)
		if err != nil {
			ctx.Transaction.Rollback()
			t.Error("failed to delete space")
		}
		ctx.Transaction.Commit()

		sp2, err := s.Space.Get(ctx, sp.RefID)
		if err != nil || sp.Name != sp2.Name {
			t.Error("failed to create space")
		}
	})

	t.Run("Update Space", func(t *testing.T) {
		ctx.Transaction, err = rt.Db.Beginx()

		sp, err := s.Space.Get(ctx, spaceID)
		if err != nil {
			ctx.Transaction.Rollback()
			t.Error("failed to get space")
			return
		}

		sp.Name = "test update"
		err = s.Space.Update(ctx, sp)
		if err != nil {
			ctx.Transaction.Rollback()
			t.Error("failed to update space")
			return
		}

		ctx.Transaction.Commit()

		sp, err = s.Space.Get(ctx, spaceID)
		if err != nil || sp.Name != "test update" {
			t.Error("failed to update space")
		}
	})

	t.Run("Delete Space", func(t *testing.T) {
		ctx.Transaction, err = rt.Db.Beginx()

		_, err = s.Space.Delete(ctx, spaceID)
		if err != nil {
			ctx.Transaction.Rollback()
			t.Error("failed to delete space")
			return
		}

		ctx.Transaction.Commit()
	})

	// teardown code goes here
}
<|endoftext|>


<commit_before>package gce

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/emccode/rexray/core"
	"github.com/emccode/rexray/core/config"
	"github.com/emccode/rexray/core/errors"
	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/compute/v1"
	"io/ioutil"
	"regexp"
	"strconv"
	"strings"
)

const providerName = "gce"

// The GCE storage driver.
type driver struct {
	client  *compute.Service
	r       *core.RexRay
	zone    string
	project string
}

func ef() errors.Fields {
	return errors.Fields{
		"provider": providerName,
	}
}

func eff(fields errors.Fields) map[string]interface{} {
	errFields := map[string]interface{}{
		"provider": providerName,
	}
	if fields != nil {
		for k, v := range fields {
			errFields[k] = v
		}
	}
	return errFields
}

func init() {
	core.RegisterDriver(providerName, newDriver)
	config.Register(configRegistration())
}

func newDriver() core.Driver {
	return &driver{}
}

func (d *driver) Init(r *core.RexRay) error {
	d.r = r

	var err error

	d.zone = d.r.Config.GetString("gce.zone")
	d.project = d.r.Config.GetString("gce.project")
	serviceAccountJSON, err := ioutil.ReadFile(d.r.Config.GetString("gce.keyfile"))
	if err != nil {
		log.WithField("provider", providerName).Fatalf("Could not read service account credentials file, %s => {%s}", d.r.Config.GetString("gce.keyfile"), err)
	}

	config, err := google.JWTConfigFromJSON(serviceAccountJSON,
		compute.ComputeScope,
	)
	client, err := compute.New(config.Client(context.Background()))

	if err != nil {
		log.WithField("provider", providerName).Fatalf("Could not create compute client => {%s}", err)
	}
	d.client = client
	log.WithField("provider", providerName).Info("storage driver initialized")

	return nil
}

func (d *driver) Name() string {
	return providerName
}

func (d *driver) GetVolumeMapping() ([]*core.BlockDevice, error) {
	log.WithField("provider", providerName).Debug("GetVolumeMapping")

	diskMap := make(map[string]*compute.Disk)
	disks, err := d.client.Disks.List(d.project, d.zone).Do()
	if err != nil {
		return []*core.BlockDevice{}, err
	}
	for _, disk := range disks.Items {
		log.WithField("provider", providerName).Debugf("%s", disk.SelfLink)
		diskMap[disk.SelfLink] = disk
	}

	instances, err := d.client.Instances.List(d.project, d.zone).Do()
	if err != nil {
		return []*core.BlockDevice{}, err
	}
	var ret []*core.BlockDevice
	for _, instance := range instances.Items {
		for _, disk := range instance.Disks {
			log.WithField("provider", providerName).Debugf("%s", disk.Source)
			ret = append(ret, &core.BlockDevice{
				ProviderName: "gce",
				InstanceID:   strconv.FormatUint(instance.Id, 10),
				VolumeID:     strconv.FormatUint(diskMap[disk.Source].Id, 10),
				DeviceName:   disk.DeviceName,
				Region:       diskMap[disk.Source].Zone,
				Status:       diskMap[disk.Source].Status,
				NetworkName:  disk.Source,
			})

		}
	}
	return ret, nil
}

func (d *driver) GetInstance() (*core.Instance, error) {
	log.WithField("provider", providerName).Debug("GetInstance")
	return nil, nil
}

func (d *driver) CreateSnapshot(
	runAsync bool,
	snapshotName, volumeID, description string) ([]*core.Snapshot, error) {

	log.WithField("provider", providerName).Debug("CreateSnapshot")
	return nil, nil

}

func (d *driver) GetSnapshot(
	volumeID, snapshotID, snapshotName string) ([]*core.Snapshot, error) {

	log.WithField("provider", providerName).Debug("GetSnapshot")
	return nil, nil
}

func (d *driver) RemoveSnapshot(snapshotID string) error {
	log.WithField("provider", providerName).Debug("RemoveSnapshot")
	return nil
}

func (d *driver) GetDeviceNextAvailable() (string, error) {
	letters := []string{
		"a", "b", "c", "d", "e", "f", "g", "h",
		"i", "j", "k", "l", "m", "n", "o", "p"}

	blockDeviceNames := make(map[string]bool)

	blockDeviceMapping, err := d.GetVolumeMapping()
	if err != nil {
		return "", err
	}

	for _, blockDevice := range blockDeviceMapping {
		re, _ := regexp.Compile(`^/dev/xvd([a-z])`)
		res := re.FindStringSubmatch(blockDevice.DeviceName)
		if len(res) > 0 {
			blockDeviceNames[res[1]] = true
		}
	}

	localDevices, err := getLocalDevices()
	if err != nil {
		return "", err
	}

	for _, localDevice := range localDevices {
		re, _ := regexp.Compile(`^xvd([a-z])`)
		res := re.FindStringSubmatch(localDevice)
		if len(res) > 0 {
			blockDeviceNames[res[1]] = true
		}
	}

	for _, letter := range letters {
		if !blockDeviceNames[letter] {
			nextDeviceName := "/dev/xvd" + letter
			log.Println("Got next device name: " + nextDeviceName)
			return nextDeviceName, nil
		}
	}
	return "", errors.New("No available device")
}

func getLocalDevices() (deviceNames []string, err error) {
	file := "/proc/partitions"
	contentBytes, err := ioutil.ReadFile(file)
	if err != nil {
		return []string{}, err
	}

	content := string(contentBytes)

	lines := strings.Split(content, "\n")
	for _, line := range lines[2:] {
		fields := strings.Fields(line)
		if len(fields) == 4 {
			deviceNames = append(deviceNames, fields[3])
		}
	}

	return deviceNames, nil
}

func (d *driver) CreateVolume(
	runAsync bool, volumeName, volumeID, snapshotID, volumeType string,
	IOPS, size int64, availabilityZone string) (*core.Volume, error) {
	log.WithField("provider", providerName).Debug("CreateVolume")
	return nil, nil

}

func (d *driver) createVolumeCreateSnapshot(
	volumeID string, snapshotID string) (string, error) {
	log.WithField("provider", providerName).Debug("CreateVolumeCreateSnapshot")
	return "", nil

}

func (d *driver) GetVolume(
	volumeID, volumeName string) ([]*core.Volume, error) {
	log.WithField("provider", providerName).Debugf("GetVolume :%s %s", volumeID, volumeName)

	query := d.client.Disks.List(d.project, d.zone)
	if volumeID != "" {
		query.Filter(fmt.Sprintf("id eq %s", volumeID))
	}
	if volumeName != "" {
		query.Filter(fmt.Sprintf("name eq %s", volumeName))
	}
	var attachments []*core.VolumeAttachment
	instances, err := d.client.Instances.List(d.project, d.zone).Do()
	if err != nil {
		return []*core.Volume{}, err
	}
	for _, instance := range instances.Items {
		for _, disk := range instance.Disks {
			attachment := &core.VolumeAttachment{
				InstanceID: strconv.FormatUint(instance.Id, 10),
				DeviceName: disk.DeviceName,
				Status:     disk.Mode,
				VolumeID:   disk.Source,
			}
			attachments = append(attachments, attachment)

		}
	}

	disks, err := query.Do()
	if err != nil {
		return []*core.Volume{}, err
	}
	var volumesSD []*core.Volume
	for _, disk := range disks.Items {
		var diskAttachments []*core.VolumeAttachment
		for _, attachment := range attachments {
			if attachment.VolumeID == disk.SelfLink {
				diskAttachments = append(diskAttachments, &core.VolumeAttachment{
					InstanceID: attachment.InstanceID,
					DeviceName: attachment.DeviceName,
					Status:     attachment.Status,
					VolumeID:   strconv.FormatUint(disk.Id, 10),
				})
			}
		}
		volumeSD := &core.Volume{
			Name:             disk.Name,
			VolumeID:         strconv.FormatUint(disk.Id, 10),
			AvailabilityZone: disk.Zone,
			Status:           disk.Status,
			VolumeType:       disk.Kind,
			NetworkName:      disk.SelfLink,
			IOPS:             0,
			Size:             strconv.FormatInt(disk.SizeGb, 10),
			Attachments:      diskAttachments,
		}
		volumesSD = append(volumesSD, volumeSD)

	}
	return volumesSD, nil
}

func (d *driver) GetVolumeAttach(
	volumeID, instanceID string) ([]*core.VolumeAttachment, error) {
	log.WithField("provider", providerName).Debugf("GetVolumeAttach :%s %s", volumeID, instanceID)
	var attachments []*core.VolumeAttachment
	query := d.client.Instances.List(d.project, d.zone)
	if instanceID != "" {
		query.Filter(fmt.Sprintf("id eq %s", instanceID))
	}
	instances, err := query.Do()
	if err != nil {
		return []*core.VolumeAttachment{}, err
	}
	for _, instance := range instances.Items {
		for _, disk := range instance.Disks {
			attachment := &core.VolumeAttachment{
				InstanceID: strconv.FormatUint(instance.Id, 10),
				DeviceName: disk.DeviceName,
				Status:     disk.Mode,
				VolumeID:   disk.Source,
			}
			attachments = append(attachments, attachment)

		}
	}
	return attachments, nil
}

func (d *driver) waitSnapshotComplete(snapshotID string) error {
	return nil
}

func (d *driver) waitVolumeComplete(volumeID string) error {
	return nil
}

func (d *driver) waitVolumeAttach(volumeID, instanceID string) error {
	return nil
}

func (d *driver) waitVolumeDetach(volumeID string) error {
	return nil
}

func (d *driver) RemoveVolume(volumeID string) error {
	return nil
}

func (d *driver) AttachVolume(
	runAsync bool,
	volumeID, instanceID string) ([]*core.VolumeAttachment, error) {
	log.WithField("provider", providerName).Debug("AttachVolume")
	return nil, nil

}

func (d *driver) DetachVolume(
	runAsync bool,
	volumeID, blank string) error {
	log.WithField("provider", providerName).Debug("DetachVolume")
	return nil
}

func (d *driver) CopySnapshot(runAsync bool,
	volumeID, snapshotID, snapshotName, destinationSnapshotName,
	destinationRegion string) (*core.Snapshot, error) {
	log.WithField("provider", providerName).Debug("CopySnapshot")
	return nil, nil
}

func configRegistration() *config.Registration {
	r := config.NewRegistration("Google GCE")
	r.Key(config.String, "", "", "", "gce.zone")
	r.Key(config.String, "", "", "", "gce.project")
	r.Key(config.String, "", "", "", "gce.keyfile")
	return r
}
<commit_msg> Implement new method<commit_after>package gce

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/emccode/rexray/core"
	"github.com/emccode/rexray/core/config"
	"github.com/emccode/rexray/core/errors"
	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/compute/v1"
	"io/ioutil"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"strings"
)

const providerName = "gce"

// The GCE storage driver.
type driver struct {
	currentInstanceId string
	client            *compute.Service
	r                 *core.RexRay
	zone              string
	project           string
}

func ef() errors.Fields {
	return errors.Fields{
		"provider": providerName,
	}
}

func eff(fields errors.Fields) map[string]interface{} {
	errFields := map[string]interface{}{
		"provider": providerName,
	}
	if fields != nil {
		for k, v := range fields {
			errFields[k] = v
		}
	}
	return errFields
}

func init() {
	core.RegisterDriver(providerName, newDriver)
	config.Register(configRegistration())
}

func newDriver() core.Driver {
	return &driver{}
}

func (d *driver) Init(r *core.RexRay) error {
	d.r = r

	var err error

	d.zone = d.r.Config.GetString("gce.zone")
	d.project = d.r.Config.GetString("gce.project")
	serviceAccountJSON, err := ioutil.ReadFile(d.r.Config.GetString("gce.keyfile"))
	if err != nil {
		log.WithField("provider", providerName).Fatalf("Could not read service account credentials file, %s => {%s}", d.r.Config.GetString("gce.keyfile"), err)
		return err
	}

	config, err := google.JWTConfigFromJSON(serviceAccountJSON,
		compute.ComputeScope,
	)
	client, err := compute.New(config.Client(context.Background()))

	if err != nil {
		log.WithField("provider", providerName).Fatalf("Could not create compute client => {%s}", err)
	}
	d.client = client
	d.currentInstanceId = getCurrentInstanceId()
	log.WithField("provider", providerName).Info("storage driver initialized")
	return nil
}

func getCurrentInstanceId() (string, error) {
	conn, err := net.DialTimeout("tcp", "metadata.google.internal:80", 50*time.Millisecond)
	if err != nil {
		return "", fmt.Errorf("Error: %v\n", err)
	}
	defer conn.Close()

	url := "http://metadata.google.internal/computeMetadata/v1/instance/id"
	client := &http.Client{}
	req, _ := http.NewRequest("GET", url, nil)
	req.Header.Set("Metadata-Flavor", "Google")
	resp, err := client.Do(req)
	if err != nil {
		return "", fmt.Errorf("Error: %v\n", err)
	}

	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", fmt.Errorf("Error: %v\n", err)
	}
	return data, nil
}

func (d *driver) Name() string {
	return providerName
}

func (d *driver) GetVolumeMapping() ([]*core.BlockDevice, error) {
	log.WithField("provider", providerName).Debug("GetVolumeMapping")

	diskMap := make(map[string]*compute.Disk)
	disks, err := d.client.Disks.List(d.project, d.zone).Do()
	if err != nil {
		return []*core.BlockDevice{}, err
	}
	for _, disk := range disks.Items {
		diskMap[disk.SelfLink] = disk
	}

	instances, err := d.client.Instances.List(d.project, d.zone).Do()
	if err != nil {
		return []*core.BlockDevice{}, err
	}
	var ret []*core.BlockDevice
	for _, instance := range instances.Items {
		for _, disk := range instance.Disks {
			ret = append(ret, &core.BlockDevice{
				ProviderName: "gce",
				InstanceID:   strconv.FormatUint(instance.Id, 10),
				VolumeID:     strconv.FormatUint(diskMap[disk.Source].Id, 10),
				DeviceName:   disk.DeviceName,
				Region:       diskMap[disk.Source].Zone,
				Status:       diskMap[disk.Source].Status,
				NetworkName:  disk.Source,
			})

		}
	}
	return ret, nil
}

func (d *driver) GetInstance() (*core.Instance, error) {
	log.WithField("provider", providerName).Debug("GetInstance")
	var attachments []*core.VolumeAttachment
	query := d.client.Instances.List(d.project, d.zone)
	query.Filter(fmt.Sprintf("id eq %s", d.currentInstanceId))
	instances, err := query.Do()
	if err != nil {
		return []*core.Instance{}, err
	}
	var ret []*core.Instance
	for _, instance := range instances.Items {
		return &core.Instance{
			ProviderName: "gce",
			InstanceID:   strconv.FormatUint(instance.Id, 10),
			Region:       instance.Zone,
			Region:       instance.Status,
			NetworkName:  instance.Name,
		}),nil

	}
	return nil, nil
}

func (d *driver) CreateSnapshot(
	runAsync bool,
	snapshotName, volumeID, description string) ([]*core.Snapshot, error) {

	log.WithField("provider", providerName).Debug("CreateSnapshot")
	return nil, nil

}

func (d *driver) GetSnapshot(
	volumeID, snapshotID, snapshotName string) ([]*core.Snapshot, error) {

	log.WithField("provider", providerName).Debug("GetSnapshot")
	return nil, nil
}

func (d *driver) RemoveSnapshot(snapshotID string) error {
	log.WithField("provider", providerName).Debug("RemoveSnapshot")
	return nil
}

func (d *driver) GetDeviceNextAvailable() (string, error) {
	letters := []string{
		"a", "b", "c", "d", "e", "f", "g", "h",
		"i", "j", "k", "l", "m", "n", "o", "p"}

	blockDeviceNames := make(map[string]bool)

	blockDeviceMapping, err := d.GetVolumeMapping()
	if err != nil {
		return "", err
	}

	for _, blockDevice := range blockDeviceMapping {
		re, _ := regexp.Compile(`^/dev/xvd([a-z])`)
		res := re.FindStringSubmatch(blockDevice.DeviceName)
		if len(res) > 0 {
			blockDeviceNames[res[1]] = true
		}
	}

	localDevices, err := getLocalDevices()
	if err != nil {
		return "", err
	}

	for _, localDevice := range localDevices {
		re, _ := regexp.Compile(`^xvd([a-z])`)
		res := re.FindStringSubmatch(localDevice)
		if len(res) > 0 {
			blockDeviceNames[res[1]] = true
		}
	}

	for _, letter := range letters {
		if !blockDeviceNames[letter] {
			nextDeviceName := "/dev/xvd" + letter
			log.Println("Got next device name: " + nextDeviceName)
			return nextDeviceName, nil
		}
	}
	return "", errors.New("No available device")
}

func getLocalDevices() (deviceNames []string, err error) {
	file := "/proc/partitions"
	contentBytes, err := ioutil.ReadFile(file)
	if err != nil {
		return []string{}, err
	}

	content := string(contentBytes)

	lines := strings.Split(content, "\n")
	for _, line := range lines[2:] {
		fields := strings.Fields(line)
		if len(fields) == 4 {
			deviceNames = append(deviceNames, fields[3])
		}
	}

	return deviceNames, nil
}

func (d *driver) CreateVolume(
	runAsync bool, volumeName, volumeID, snapshotID, volumeType string,
	IOPS, size int64, availabilityZone string) (*core.Volume, error) {
	log.WithField("provider", providerName).Debug("CreateVolume")
	return nil, nil

}

func (d *driver) createVolumeCreateSnapshot(
	volumeID string, snapshotID string) (string, error) {
	log.WithField("provider", providerName).Debug("CreateVolumeCreateSnapshot")
	return "", nil

}

func (d *driver) GetVolume(
	volumeID, volumeName string) ([]*core.Volume, error) {
	log.WithField("provider", providerName).Debugf("GetVolume :%s %s", volumeID, volumeName)

	query := d.client.Disks.List(d.project, d.zone)
	if volumeID != "" {
		query.Filter(fmt.Sprintf("id eq %s", volumeID))
	}
	if volumeName != "" {
		query.Filter(fmt.Sprintf("name eq %s", volumeName))
	}
	var attachments []*core.VolumeAttachment
	instances, err := d.client.Instances.List(d.project, d.zone).Do()
	if err != nil {
		return []*core.Volume{}, err
	}
	for _, instance := range instances.Items {
		for _, disk := range instance.Disks {
			attachment := &core.VolumeAttachment{
				InstanceID: strconv.FormatUint(instance.Id, 10),
				DeviceName: disk.DeviceName,
				Status:     disk.Mode,
				VolumeID:   disk.Source,
			}
			attachments = append(attachments, attachment)

		}
	}

	disks, err := query.Do()
	if err != nil {
		return []*core.Volume{}, err
	}
	var volumesSD []*core.Volume
	for _, disk := range disks.Items {
		var diskAttachments []*core.VolumeAttachment
		for _, attachment := range attachments {
			if attachment.VolumeID == disk.SelfLink {
				diskAttachments = append(diskAttachments, &core.VolumeAttachment{
					InstanceID: attachment.InstanceID,
					DeviceName: attachment.DeviceName,
					Status:     attachment.Status,
					VolumeID:   strconv.FormatUint(disk.Id, 10),
				})
			}
		}
		volumeSD := &core.Volume{
			Name:             disk.Name,
			VolumeID:         strconv.FormatUint(disk.Id, 10),
			AvailabilityZone: disk.Zone,
			Status:           disk.Status,
			VolumeType:       disk.Kind,
			NetworkName:      disk.SelfLink,
			IOPS:             0,
			Size:             strconv.FormatInt(disk.SizeGb, 10),
			Attachments:      diskAttachments,
		}
		volumesSD = append(volumesSD, volumeSD)

	}
	return volumesSD, nil
}

func (d *driver) GetVolumeAttach(
	volumeID, instanceID string) ([]*core.VolumeAttachment, error) {
	log.WithField("provider", providerName).Debugf("GetVolumeAttach :%s %s", volumeID, instanceID)
	var attachments []*core.VolumeAttachment
	query := d.client.Instances.List(d.project, d.zone)
	if instanceID != "" {
		query.Filter(fmt.Sprintf("id eq %s", instanceID))
	}
	instances, err := query.Do()
	if err != nil {
		return []*core.VolumeAttachment{}, err
	}
	for _, instance := range instances.Items {
		for _, disk := range instance.Disks {
			attachment := &core.VolumeAttachment{
				InstanceID: strconv.FormatUint(instance.Id, 10),
				DeviceName: disk.DeviceName,
				Status:     disk.Mode,
				VolumeID:   disk.Source,
			}
			attachments = append(attachments, attachment)

		}
	}
	return attachments, nil
}

func (d *driver) waitSnapshotComplete(snapshotID string) error {
	return nil
}

func (d *driver) waitVolumeComplete(volumeID string) error {
	return nil
}

func (d *driver) waitVolumeAttach(volumeID, instanceID string) error {
	return nil
}

func (d *driver) waitVolumeDetach(volumeID string) error {
	return nil
}

func (d *driver) RemoveVolume(volumeID string) error {
	return nil
}

func (d *driver) AttachVolume(
	runAsync bool,
	volumeID, instanceID string) ([]*core.VolumeAttachment, error) {
	log.WithField("provider", providerName).Debug("AttachVolume")
	return nil, nil

}

func (d *driver) DetachVolume(
	runAsync bool,
	volumeID, blank string) error {
	log.WithField("provider", providerName).Debug("DetachVolume")
	return nil
}

func (d *driver) CopySnapshot(runAsync bool,
	volumeID, snapshotID, snapshotName, destinationSnapshotName,
	destinationRegion string) (*core.Snapshot, error) {
	log.WithField("provider", providerName).Debug("CopySnapshot")
	return nil, nil
}

func configRegistration() *config.Registration {
	r := config.NewRegistration("Google GCE")
	r.Key(config.String, "", "", "", "gce.zone")
	r.Key(config.String, "", "", "", "gce.project")
	r.Key(config.String, "", "", "", "gce.keyfile")
	return r
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "net/http"
  "os"
  "io"
  "io/ioutil"
  "log"
)

var (
    Trace   *log.Logger
    Info    *log.Logger
    Warning *log.Logger
    Error   *log.Logger
)

func Init(
    traceHandle io.Writer,
    infoHandle io.Writer,
    warningHandle io.Writer,
    errorHandle io.Writer) {

    Trace = log.New(traceHandle,
        "TRACE: ",
        log.Ldate|log.Ltime|log.Lshortfile)

    Info = log.New(infoHandle,
        "INFO: ",
        log.Ldate|log.Ltime|log.Lshortfile)

    Warning = log.New(warningHandle,
        "WARNING: ",
        log.Ldate|log.Ltime|log.Lshortfile)

    Error = log.New(errorHandle,
        "ERROR: ",
        log.Ldate|log.Ltime|log.Lshortfile)
}

func handler(w http.ResponseWriter, r *http.Request) {
  h, _ := os.Hostname()
  fmt.Fprintf(w, "Hi there, I'm served from %s!", h)
  Info.Println("Request served by:", h)
}

func main() {
  // Logging is good. Sending to stdout / stderr allows both Docker and
  // supervisord to deal with the logs.
  Init(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr)

  // Make port configurable. This may change depending on where / how it is
  // deployed
  port := os.Getenv("PORT")
  if port == "" {
      port = "8484"
  }
  Warning.Println("Listening on port:", port)

  http.HandleFunc("/", handler)
  http.ListenAndServe(":" + port , nil)
}
<commit_msg>Add basic 404 handling, logging and new line at end of output<commit_after>package main

import (
  "fmt"
  "net/http"
  "os"
  "io"
  "io/ioutil"
  "log"
)

var (
    Trace   *log.Logger
    Info    *log.Logger
    Warning *log.Logger
    Error   *log.Logger
)

func Init(
    traceHandle io.Writer,
    infoHandle io.Writer,
    warningHandle io.Writer,
    errorHandle io.Writer) {

    Trace = log.New(traceHandle,
        "TRACE: ",
        log.Ldate|log.Ltime|log.Lshortfile)

    Info = log.New(infoHandle,
        "INFO: ",
        log.Ldate|log.Ltime|log.Lshortfile)

    Warning = log.New(warningHandle,
        "WARNING: ",
        log.Ldate|log.Ltime|log.Lshortfile)

    Error = log.New(errorHandle,
        "ERROR: ",
        log.Ldate|log.Ltime|log.Lshortfile)
}

func handler(w http.ResponseWriter, r *http.Request) {
  // Create log line from request as will be used a number of times
  rlog := "Proxy: " + r.RemoteAddr + " RealIP: " + r.Header.Get("X-Real-Ip") +
          " Host: " + r.Host + " Method: " + r.Method + " Request: " + r.RequestURI +
          " User-Agent: " + r.Header.Get("User-Agent")

  //Really simple 404 / logging at ERROR level
  if r.URL.Path != "/" {
    http.NotFound(w, r)
    Error.Println(rlog)
    return
  }

  h, _ := os.Hostname()
  fmt.Fprintf(w, "Hi there, I'm served from %s!\n", h)
  Info.Println(rlog)
}

func main() {
  // Logging is good. Sending to stdout / stderr allows both Docker and
  // supervisord to deal with the logs.
  Init(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr)

  // Make port configurable. This may change depending on where / how it is
  // deployed
  port := os.Getenv("PORT")
  if port == "" {
      port = "8484"
  }

  Warning.Println("Listening on port:", port)
  http.HandleFunc("/", handler)
  http.ListenAndServe(":" + port , nil)
}
<|endoftext|>


<commit_before>package api

import (
	"log"
	"net/http"

	"github.com/mdlayher/wavepipe/data"

	"github.com/go-martini/martini"
	"github.com/martini-contrib/render"
)

// LogoutResponse represents the JSON response for /api/logouts
type LogoutResponse struct {
	Error *Error `json:"error"`
}

// GetLogout destroys a new session from the wavepipe API, and returns a HTTP status and JSON
func GetLogout(r render.Render, req *http.Request, session *data.Session, params martini.Params) {
	// Output struct for logouts request
	res := LogoutResponse{}

	// Check API version
	if version, ok := params["version"]; ok {
		// Check if this API call is supported in the advertised version
		if !apiVersionSet.Has(version) {
			res.Error = new(Error)
			res.Error.Code = 400
			res.Error.Message = "unsupported API version: " + version
			r.JSON(400, res)
			return
		}
	}

	// Destroy the current API session
	if err := session.Delete(); err != nil {
		log.Println(err)

		res.Error = new(Error)
		res.Error.Code = 500
		res.Error.Message = "server error"
		r.JSON(500, res)
		return
	}

	// Build response
	res.Error = nil

	// HTTP 200 OK with JSON
	r.JSON(200, res)
	return
}
<commit_msg>api/logout, add shortcut methods for rendering errors and server errors<commit_after>package api

import (
	"log"
	"net/http"

	"github.com/mdlayher/wavepipe/data"

	"github.com/go-martini/martini"
	"github.com/martini-contrib/render"
)

// LogoutResponse represents the JSON response for /api/logouts
type LogoutResponse struct {
	Error *Error `json:"error"`
	render  render.Render `json:"-"`
}

// RenderError renders a JSON error message with the specified HTTP status code and message
func (l *LogoutResponse) RenderError(code int, message string) {
	// Generate error
	l.Error = new(Error)
	l.Error.Code = code
	l.Error.Message = message

	// Render with specified HTTP status code
	l.render.JSON(code, l)
}

// ServerError is a shortcut to render a HTTP 500 with generic "server error" message
func (l *LogoutResponse) ServerError() {
	l.RenderError(500, "server error")
	return
}

// GetLogout destroys a new session from the wavepipe API, and returns a HTTP status and JSON
func GetLogout(r render.Render, req *http.Request, session *data.Session, params martini.Params) {
	// Output struct for logouts request
	res := LogoutResponse{render: r}

	// Check API version
	if version, ok := params["version"]; ok {
		// Check if this API call is supported in the advertised version
		if !apiVersionSet.Has(version) {
			res.RenderError(400, "unsupported API version: "+version)
			return
		}
	}

	// Destroy the current API session
	if err := session.Delete(); err != nil {
		log.Println(err)
		res.ServerError()
		return
	}

	// Build response
	res.Error = nil

	// HTTP 200 OK with JSON
	r.JSON(200, res)
	return
}
<|endoftext|>


<commit_before>// package idletiming provides mechanisms for adding idle timeouts to net.Conn
// and net.Listener.
package idletiming

import (
	"errors"
	"io"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/getlantern/golog"
)

var (
	log = golog.LoggerFor("idletiming")

	// ErrIdled is return when attempting to use a network connection that was
	// closed because of idling.
	ErrIdled = errors.New("Use of idled network connection")
)

// Conn creates a new net.Conn wrapping the given net.Conn that times out after
// the specified period. Once a connection has timed out, any pending reads or
// writes will return io.EOF and the underlying connection will be closed.
//
// idleTimeout specifies how long to wait for inactivity before considering
// connection idle.
//
// If onIdle is specified, it will be called to indicate when the connection has
// idled and been closed.
func Conn(conn net.Conn, idleTimeout time.Duration, onIdle func()) *IdleTimingConn {
	c := &IdleTimingConn{
		conn:             conn,
		idleTimeout:      idleTimeout,
		halfIdleTimeout:  time.Duration(idleTimeout.Nanoseconds() / 2),
		activeCh:         make(chan bool, 1),
		closedCh:         make(chan bool, 1),
		lastActivityTime: int64(time.Now().UnixNano()),
	}

	go func() {
		timer := time.NewTimer(idleTimeout)
		defer timer.Stop()
		for {
			select {
			case <-c.activeCh:
				// We're active, continue
				timer.Reset(idleTimeout)
				atomic.StoreInt64(&c.lastActivityTime, time.Now().UnixNano())
				continue
			case <-timer.C:
				c.Close()
				if onIdle != nil {
					onIdle()
				}
				return
			case <-c.closedCh:
				return
			}
		}
	}()

	return c
}

// IdleTimingConn is a net.Conn that wraps another net.Conn and that times out
// if idle for more than idleTimeout.
type IdleTimingConn struct {
	// Keep it at the top to make sure 64-bit alignment, see
	// https://golang.org/pkg/sync/atomic/#pkg-note-BUG
	lastActivityTime int64
	readDeadline     guardedTime
	writeDeadline    guardedTime

	conn             net.Conn
	idleTimeout      time.Duration
	halfIdleTimeout  time.Duration
	activeCh         chan bool
	closedCh         chan bool
	closeMutex       sync.RWMutex // prevents Close() from interfering with io operations
	closed           bool
	hasReadAfterIdle int32
}

// TimesOutIn returns how much time is left before this connection will time
// out, assuming there is no further activity.
func (c *IdleTimingConn) TimesOutIn() time.Duration {
	return c.TimesOutAt().Sub(time.Now())
}

// TimesOutAt returns the time at which this connection will time out, assuming
// there is no further activity
func (c *IdleTimingConn) TimesOutAt() time.Time {
	return time.Unix(0, atomic.LoadInt64(&c.lastActivityTime)).Add(c.idleTimeout)
}

// Read implements the method from io.Reader
func (c *IdleTimingConn) Read(b []byte) (int, error) {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	if err := c.checkClosedFirstTime(&c.hasReadAfterIdle, io.EOF); err != nil {
		return 0, err
	}

	totalN := 0
	readDeadline := c.readDeadline.Get()

	// Continually read while we can, always setting a deadline that's less than
	// our idleTimeout so that we can update our active status before we hit the
	// idleTimeout.
	for {
		maxDeadline := time.Now().Add(c.halfIdleTimeout)
		if !readDeadline.IsZero() && !maxDeadline.Before(readDeadline) {
			// Caller's deadline is before ours, use it
			if err := c.conn.SetReadDeadline(readDeadline); err != nil {
				log.Tracef("Unable to set read deadline: %v", err)
			}
			n, err := c.conn.Read(b)
			c.markActive(n)
			totalN = totalN + n
			return totalN, err
		} else {
			// Use our own deadline
			if err := c.conn.SetReadDeadline(maxDeadline); err != nil {
				log.Tracef("Unable to set read deadline: %v", err)
			}
			n, err := c.conn.Read(b)
			c.markActive(n)
			totalN = totalN + n
			hitMaxDeadline := isTimeout(err) && !time.Now().Before(maxDeadline)
			if hitMaxDeadline {
				// Ignore timeouts when encountering deadline based on
				// IdleTimeout
				err = nil
			}
			if n == 0 || !hitMaxDeadline {
				return totalN, err
			}
			b = b[n:]
		}
	}
}

// Write implements the method from io.Reader
func (c *IdleTimingConn) Write(b []byte) (int, error) {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	if err := c.checkClosed(); err != nil {
		return 0, err
	}

	totalN := 0
	writeDeadline := c.writeDeadline.Get()

	// Continually write while we can, always setting a deadline that's less
	// than our idleTimeout so that we can update our active status before we
	// hit the idleTimeout.
	for {
		maxDeadline := time.Now().Add(c.halfIdleTimeout)
		if !writeDeadline.IsZero() && !maxDeadline.Before(writeDeadline) {
			// Caller's deadline is before ours, use it
			if err := c.conn.SetWriteDeadline(writeDeadline); err != nil {
				log.Tracef("Unable to set write deadline: %v", err)
			}
			n, err := c.conn.Write(b)
			c.markActive(n)
			totalN = totalN + n
			return totalN, err
		} else {
			// Use our own deadline
			if err := c.conn.SetWriteDeadline(maxDeadline); err != nil {
				log.Tracef("Unable to set write deadline: %v", err)
			}
			n, err := c.conn.Write(b)
			c.markActive(n)
			totalN = totalN + n
			hitMaxDeadline := isTimeout(err) && !time.Now().Before(maxDeadline)
			if hitMaxDeadline {
				// Ignore timeouts when encountering deadline based on
				// IdleTimeout
				err = nil
			}
			if n == 0 || !hitMaxDeadline {
				return totalN, err
			}
			b = b[n:]
		}
	}
}

// Close this IdleTimingConn. This will close the underlying net.Conn as well,
// returning the error from calling its Close method.
func (c *IdleTimingConn) Close() error {
	c.closeMutex.Lock()
	defer c.closeMutex.Unlock()

	if err := c.checkClosed(); err != nil {
		return err
	}

	c.closed = true

	select {
	case c.closedCh <- true:
		// close accepted
	default:
		// already closing, ignore
	}
	return c.conn.Close()
}

func (c *IdleTimingConn) LocalAddr() net.Addr {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	return c.conn.LocalAddr()
}

func (c *IdleTimingConn) RemoteAddr() net.Addr {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	return c.conn.RemoteAddr()
}

func (c *IdleTimingConn) SetDeadline(t time.Time) error {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	if err := c.checkClosed(); err != nil {
		return err
	}

	if err := c.SetReadDeadline(t); err != nil {
		log.Tracef("Unable to set read deadline: %v", err)
	}
	if err := c.SetWriteDeadline(t); err != nil {
		log.Tracef("Unable to set write deadline: %v", err)
	}
	return nil
}

func (c *IdleTimingConn) SetReadDeadline(t time.Time) error {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	if err := c.checkClosed(); err != nil {
		return err
	}

	c.readDeadline.Set(t)
	return nil
}

func (c *IdleTimingConn) SetWriteDeadline(t time.Time) error {
	c.closeMutex.RLock()
	defer c.closeMutex.RUnlock()

	if err := c.checkClosed(); err != nil {
		return err
	}

	c.writeDeadline.Set(t)
	return nil
}

func (c *IdleTimingConn) markActive(n int) bool {
	if n > 0 {
		select {
		case c.activeCh <- true:
			// ok
		default:
			// still waiting to process previous markActive
		}
		return true
	}
	return false
}

func (c *IdleTimingConn) checkClosed() error {
	return c.checkClosedFirstTime(nil, nil)
}

func (c *IdleTimingConn) checkClosedFirstTime(hasDone *int32, firstTimeError error) error {
	if c.closed {
		if hasDone != nil && atomic.CompareAndSwapInt32(hasDone, 0, 1) {
			return firstTimeError
		}
		return ErrIdled
	}
	return nil
}

func isTimeout(err error) bool {
	if netErr, ok := err.(net.Error); ok {
		return netErr.Timeout()
	}
	return false
}

type guardedTime struct {
	sync.RWMutex
	t time.Time
}

func (g *guardedTime) Get() time.Time {
	g.RLock()
	retval := g.t
	g.RUnlock()
	return retval
}

func (g *guardedTime) Set(t time.Time) {
	g.Lock()
	g.t = t
	g.Unlock()
}
<commit_msg>Removed closeMutex<commit_after>// package idletiming provides mechanisms for adding idle timeouts to net.Conn
// and net.Listener.
package idletiming

import (
	"errors"
	"io"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/getlantern/golog"
)

var (
	log = golog.LoggerFor("idletiming")

	// ErrIdled is return when attempting to use a network connection that was
	// closed because of idling.
	ErrIdled = errors.New("Use of idled network connection")
)

// Conn creates a new net.Conn wrapping the given net.Conn that times out after
// the specified period. Once a connection has timed out, any pending reads or
// writes will return io.EOF and the underlying connection will be closed.
//
// idleTimeout specifies how long to wait for inactivity before considering
// connection idle.
//
// If onIdle is specified, it will be called to indicate when the connection has
// idled and been closed.
func Conn(conn net.Conn, idleTimeout time.Duration, onIdle func()) *IdleTimingConn {
	c := &IdleTimingConn{
		conn:             conn,
		idleTimeout:      idleTimeout,
		halfIdleTimeout:  time.Duration(idleTimeout.Nanoseconds() / 2),
		activeCh:         make(chan bool, 1),
		closedCh:         make(chan bool, 1),
		lastActivityTime: int64(time.Now().UnixNano()),
	}

	go func() {
		timer := time.NewTimer(idleTimeout)
		defer timer.Stop()
		for {
			select {
			case <-c.activeCh:
				// We're active, continue
				timer.Reset(idleTimeout)
				atomic.StoreInt64(&c.lastActivityTime, time.Now().UnixNano())
				continue
			case <-timer.C:
				c.Close()
				if onIdle != nil {
					onIdle()
				}
				return
			case <-c.closedCh:
				return
			}
		}
	}()

	return c
}

// IdleTimingConn is a net.Conn that wraps another net.Conn and that times out
// if idle for more than idleTimeout.
type IdleTimingConn struct {
	// Keep it at the top to make sure 64-bit alignment, see
	// https://golang.org/pkg/sync/atomic/#pkg-note-BUG
	lastActivityTime int64
	readDeadline     guardedTime
	writeDeadline    guardedTime

	conn             net.Conn
	idleTimeout      time.Duration
	halfIdleTimeout  time.Duration
	activeCh         chan bool
	closedCh         chan bool
	closed           int32
	hasReadAfterIdle int32
}

// TimesOutIn returns how much time is left before this connection will time
// out, assuming there is no further activity.
func (c *IdleTimingConn) TimesOutIn() time.Duration {
	return c.TimesOutAt().Sub(time.Now())
}

// TimesOutAt returns the time at which this connection will time out, assuming
// there is no further activity
func (c *IdleTimingConn) TimesOutAt() time.Time {
	return time.Unix(0, atomic.LoadInt64(&c.lastActivityTime)).Add(c.idleTimeout)
}

// Read implements the method from io.Reader
func (c *IdleTimingConn) Read(b []byte) (int, error) {
	if err := c.checkClosedFirstTime(&c.hasReadAfterIdle, io.EOF); err != nil {
		return 0, err
	}

	totalN := 0
	readDeadline := c.readDeadline.Get()

	// Continually read while we can, always setting a deadline that's less than
	// our idleTimeout so that we can update our active status before we hit the
	// idleTimeout.
	for {
		maxDeadline := time.Now().Add(c.halfIdleTimeout)
		if !readDeadline.IsZero() && !maxDeadline.Before(readDeadline) {
			// Caller's deadline is before ours, use it
			if err := c.conn.SetReadDeadline(readDeadline); err != nil {
				log.Tracef("Unable to set read deadline: %v", err)
			}
			n, err := c.conn.Read(b)
			c.markActive(n)
			totalN = totalN + n
			return totalN, err
		} else {
			// Use our own deadline
			if err := c.conn.SetReadDeadline(maxDeadline); err != nil {
				log.Tracef("Unable to set read deadline: %v", err)
			}
			n, err := c.conn.Read(b)
			c.markActive(n)
			totalN = totalN + n
			hitMaxDeadline := isTimeout(err) && !time.Now().Before(maxDeadline)
			if hitMaxDeadline {
				// Ignore timeouts when encountering deadline based on
				// IdleTimeout
				err = nil
			}
			if n == 0 || !hitMaxDeadline {
				return totalN, err
			}
			b = b[n:]
		}
	}
}

// Write implements the method from io.Reader
func (c *IdleTimingConn) Write(b []byte) (int, error) {
	if err := c.checkClosed(); err != nil {
		return 0, err
	}

	totalN := 0
	writeDeadline := c.writeDeadline.Get()

	// Continually write while we can, always setting a deadline that's less
	// than our idleTimeout so that we can update our active status before we
	// hit the idleTimeout.
	for {
		maxDeadline := time.Now().Add(c.halfIdleTimeout)
		if !writeDeadline.IsZero() && !maxDeadline.Before(writeDeadline) {
			// Caller's deadline is before ours, use it
			if err := c.conn.SetWriteDeadline(writeDeadline); err != nil {
				log.Tracef("Unable to set write deadline: %v", err)
			}
			n, err := c.conn.Write(b)
			c.markActive(n)
			totalN = totalN + n
			return totalN, err
		} else {
			// Use our own deadline
			if err := c.conn.SetWriteDeadline(maxDeadline); err != nil {
				log.Tracef("Unable to set write deadline: %v", err)
			}
			n, err := c.conn.Write(b)
			c.markActive(n)
			totalN = totalN + n
			hitMaxDeadline := isTimeout(err) && !time.Now().Before(maxDeadline)
			if hitMaxDeadline {
				// Ignore timeouts when encountering deadline based on
				// IdleTimeout
				err = nil
			}
			if n == 0 || !hitMaxDeadline {
				return totalN, err
			}
			b = b[n:]
		}
	}
}

// Close this IdleTimingConn. This will close the underlying net.Conn as well,
// returning the error from calling its Close method.
func (c *IdleTimingConn) Close() error {
	if err := c.checkClosed(); err != nil {
		return err
	}

	atomic.StoreInt32(&c.closed, 1)

	select {
	case c.closedCh <- true:
		// close accepted
	default:
		// already closing, ignore
	}
	return c.conn.Close()
}

func (c *IdleTimingConn) LocalAddr() net.Addr {
	return c.conn.LocalAddr()
}

func (c *IdleTimingConn) RemoteAddr() net.Addr {
	return c.conn.RemoteAddr()
}

func (c *IdleTimingConn) SetDeadline(t time.Time) error {
	if err := c.checkClosed(); err != nil {
		return err
	}

	if err := c.SetReadDeadline(t); err != nil {
		log.Tracef("Unable to set read deadline: %v", err)
	}
	if err := c.SetWriteDeadline(t); err != nil {
		log.Tracef("Unable to set write deadline: %v", err)
	}
	return nil
}

func (c *IdleTimingConn) SetReadDeadline(t time.Time) error {
	if err := c.checkClosed(); err != nil {
		return err
	}

	c.readDeadline.Set(t)
	return nil
}

func (c *IdleTimingConn) SetWriteDeadline(t time.Time) error {
	if err := c.checkClosed(); err != nil {
		return err
	}

	c.writeDeadline.Set(t)
	return nil
}

func (c *IdleTimingConn) markActive(n int) bool {
	if n > 0 {
		select {
		case c.activeCh <- true:
			// ok
		default:
			// still waiting to process previous markActive
		}
		return true
	}
	return false
}

func (c *IdleTimingConn) checkClosed() error {
	return c.checkClosedFirstTime(nil, nil)
}

func (c *IdleTimingConn) checkClosedFirstTime(hasDone *int32, firstTimeError error) error {
	if atomic.LoadInt32(&c.closed) == 1 {
		if hasDone != nil && atomic.CompareAndSwapInt32(hasDone, 0, 1) {
			return firstTimeError
		}
		return ErrIdled
	}
	return nil
}

func isTimeout(err error) bool {
	if netErr, ok := err.(net.Error); ok {
		return netErr.Timeout()
	}
	return false
}

type guardedTime struct {
	sync.RWMutex
	t time.Time
}

func (g *guardedTime) Get() time.Time {
	g.RLock()
	retval := g.t
	g.RUnlock()
	return retval
}

func (g *guardedTime) Set(t time.Time) {
	g.Lock()
	g.t = t
	g.Unlock()
}
<|endoftext|>


<commit_before>// Package old implements a plugin to remember URLs and announce duplicates.
package old

import (
	"fmt"
	"log"
	"regexp"
	"strings"
	"time"

	"github.com/StalkR/goircbot/bot"
	"github.com/StalkR/goircbot/lib/nohl"
	"github.com/fluffle/goirc/client"
)

var (
	linkRE    = regexp.MustCompile(`(?:^|\s)(https?://[^#\s]+)`)
	backlogRE = regexp.MustCompile("<[+%@&~]?[a-zA-Z0-9_`^\\[\\]-]+>")
)

func readURLs(b *bot.Bot, line *client.Line, o *Old, ignore map[string]bool) {
	target := line.Args[0]
	if !strings.HasPrefix(target, "#") {
		return
	}
	if _, ignore := ignore[line.Nick]; ignore {
		return
	}
	text := line.Args[1]
	if backlogRE.MatchString(text) {
		return
	}

	matches := linkRE.FindAllStringSubmatch(text, -1)
	if matches == nil {
		return
	}
	for _, submatches := range matches {
		url := submatches[1]
		i, err := o.Old(url)
		if err != nil {
			if err = o.Add(url, target, line.Nick); err != nil {
				log.Print(err)
			}
			return
		}
		duration := time.Since(i.Time) / time.Second * time.Second
		nick := nohl.Nick(b, target, i.Nick)
		b.Conn.Privmsg(target, fmt.Sprintf("old! first shared by %v %v ago", nick, duration))
	}
}

// Register registers the plugin with a bot.
func Register(b *bot.Bot, oldfile string, ignore []string) {
	ignoremap := make(map[string]bool)
	for _, nick := range ignore {
		ignoremap[nick] = true
	}

	o := load(oldfile)

	b.Conn.HandleFunc("privmsg",
		func(conn *client.Conn, line *client.Line) { readURLs(b, line, o, ignoremap) })

	if len(oldfile) > 0 {
		b.AddCron("old-save", bot.Cron{
			Handler:  func(b *bot.Bot) { save(oldfile, o) },
			Duration: time.Minute})
	}

	// Every day, clean URLs older than a year so it does not grow infinitely.
	b.AddCron("old-clean", bot.Cron{
		Handler:  func(b *bot.Bot) { o.Clean(time.Hour * 24 * 365) },
		Duration: time.Hour * 24})
}
<commit_msg>plugins/old: use duration (for real)<commit_after>// Package old implements a plugin to remember URLs and announce duplicates.
package old

import (
	"fmt"
	"log"
	"regexp"
	"strings"
	"time"

	"github.com/StalkR/goircbot/bot"
	"github.com/StalkR/goircbot/lib/duration"
	"github.com/StalkR/goircbot/lib/nohl"
	"github.com/fluffle/goirc/client"
)

var (
	linkRE    = regexp.MustCompile(`(?:^|\s)(https?://[^#\s]+)`)
	backlogRE = regexp.MustCompile("<[+%@&~]?[a-zA-Z0-9_`^\\[\\]-]+>")
)

func readURLs(b *bot.Bot, line *client.Line, o *Old, ignore map[string]bool) {
	target := line.Args[0]
	if !strings.HasPrefix(target, "#") {
		return
	}
	if _, ignore := ignore[line.Nick]; ignore {
		return
	}
	text := line.Args[1]
	if backlogRE.MatchString(text) {
		return
	}

	matches := linkRE.FindAllStringSubmatch(text, -1)
	if matches == nil {
		return
	}
	for _, submatches := range matches {
		url := submatches[1]
		i, err := o.Old(url)
		if err != nil {
			if err = o.Add(url, target, line.Nick); err != nil {
				log.Print(err)
			}
			return
		}
		ago := duration.Format(time.Since(i.Time))
		nick := nohl.Nick(b, target, i.Nick)
		b.Conn.Privmsg(target, fmt.Sprintf("old! first shared by %v %v ago", nick, ago))
	}
}

// Register registers the plugin with a bot.
func Register(b *bot.Bot, oldfile string, ignore []string) {
	ignoremap := make(map[string]bool)
	for _, nick := range ignore {
		ignoremap[nick] = true
	}

	o := load(oldfile)

	b.Conn.HandleFunc("privmsg",
		func(conn *client.Conn, line *client.Line) { readURLs(b, line, o, ignoremap) })

	if len(oldfile) > 0 {
		b.AddCron("old-save", bot.Cron{
			Handler:  func(b *bot.Bot) { save(oldfile, o) },
			Duration: time.Minute})
	}

	// Every day, clean URLs older than a year so it does not grow infinitely.
	b.AddCron("old-clean", bot.Cron{
		Handler:  func(b *bot.Bot) { o.Clean(time.Hour * 24 * 365) },
		Duration: time.Hour * 24})
}
<|endoftext|>


<commit_before>package standard

import (
	. "github.com/balzaczyy/golucene/core/analysis"
	. "github.com/balzaczyy/golucene/core/analysis/tokenattributes"
	"github.com/balzaczyy/golucene/core/util"
	"io"
)

// standard/StandardTokenizer.java

const (
	ALPHANUM        = 0
	NUM             = 6
	ACRONYM_DEP     = 8 // deprecated 3.1
	SOUTHEAST_ASIAN = 9
	IDEOGRAPHIC     = 10
	HIRAGANA        = 11
	KATAKANA        = 12
	HANGUL          = 13
)

/* String token types that correspond to token type int constants */
var TOKEN_TYPES = []string{
	"<ALPHANUM>",
	"<APOSTROPHE>",
	"<ACRONYM>",
	"<COMPANY>",
	"<EMAIL>",
	"<HOST>",
	"<NUM>",
	"<CJ>",
	"<ACRONYM_DEP>",
	"<SOUTHEAST_ASIAN>",
	"<IDEOGRAPHIC>",
	"<HIRAGANA>",
	"<KATAKANA>",
	"<HANGUL>",
}

/*
A grammar-based tokenizer constructed with JFlex.

As of Lucene version 3.1, this class implements the Word Break rules
from the Unicode Text Segmentation algorithm, as specified in Unicode
standard Annex #29.

Many applications have specific tokenizer needs. If this tokenizer
does not suit your application, please consider copying this source
code directory to your project and maintaining your own grammar-based
tokenizer.

Version

You must specify the required Version compatibility when creating
StandardTokenizer:

	- As of 3.4, Hiragana and Han characters are no longer wrongly
	split from their combining characters. If you use a previous
	version number, you get the exact broken behavior for backwards
	compatibility.
	- As of 3.1, StandardTokenizer implements Unicode text segmentation.
	If you use a previous version number, you get the exact behavior of
	ClassicTokenizer for backwards compatibility.
*/
type StandardTokenizer struct {
	*Tokenizer
	input io.ReadCloser

	// A private instance of the JFlex-constructed scanner
	scanner StandardTokenizerInterface

	skippedPositions int
	maxTokenLength   int

	// this tokenizer generates three attributes:
	// term offset, positionIncrement and type

	termAtt    CharTermAttribute
	offsetAtt  OffsetAttribute
	posIncrAtt PositionIncrementAttribute
	typeAtt    TypeAttribute
}

/*
Creates a new instance of the StandardTokenizer. Attaches the input
to the newly created JFlex scanner.
*/
func newStandardTokenizer(matchVersion util.Version, input io.ReadCloser) *StandardTokenizer {
	ans := &StandardTokenizer{
		Tokenizer: NewTokenizer(input),
		input:     input,
	}
	ans.init(matchVersion)
	return ans
}

func (t *StandardTokenizer) init(matchVersion util.Version) {
	// GoLucene support >=4.5 only
	t.scanner = newStandardTokenizerImpl(nil)
}

func (t *StandardTokenizer) IncrementToken() (bool, error) {
	t.Attributes().Clear()
	t.skippedPositions = 0

	for {
		tokenType, err := t.scanner.nextToken()
		if tokenType == YYEOF || err != nil {
			return false, err
		}

		if t.scanner.yylength() <= t.maxTokenLength {
			t.posIncrAtt.SetPositionIncrement(t.skippedPositions + 1)
			t.scanner.text(t.termAtt)
			start := t.scanner.yychar()
			t.offsetAtt.SetOffset(t.CorrectOffset(start), t.CorrectOffset(start+t.termAtt.Length()))
			// This 'if' should be removed in the next release. For now,
			// it converts invalid acronyms to HOST. When removed, only the
			// 'else' part should remain.
			if tokenType == ACRONYM_DEP {
				panic("not implemented yet")
			} else {
				t.typeAtt.SetType(TOKEN_TYPES[tokenType])
			}
			return true, nil
		} else {
			// When we skip a too-long term, we still increment the positionincrement
			t.skippedPositions++
		}
	}
}

func (t *StandardTokenizer) End() error {
	panic("not implemented yet")
}

func (t *StandardTokenizer) Reset() error {
	t.scanner.yyreset(t.input)
	t.skippedPositions = 0
	return nil
}

// standard/StandardTokenizerInterface.java

/* This character denotes the end of file */
const YYEOF = -1

/* Internal interface for supporting versioned grammars. */
type StandardTokenizerInterface interface {
	// Copies the matched text into the CharTermAttribute
	text(CharTermAttribute)
	// Returns the current position.
	yychar() int
	// Resets the scanner to read from a new input stream.
	// Does not close the old reader.
	//
	// All internal variables are reset, the old input stream cannot be
	// reused (internal buffer) is discarded and lost). Lexical state
	// is set to ZZ_INITIAL.
	yyreset(io.ReadCloser)
	// Returns the length of the matched text region.
	yylength() int
	// Resumes scanning until the next regular expression is matched,
	// the end of input is encountered or an I/O-Error occurs.
	nextToken() (int, error)
}
<commit_msg>fix NPE<commit_after>package standard

import (
	. "github.com/balzaczyy/golucene/core/analysis"
	. "github.com/balzaczyy/golucene/core/analysis/tokenattributes"
	"github.com/balzaczyy/golucene/core/util"
	"io"
)

// standard/StandardTokenizer.java

const (
	ALPHANUM        = 0
	NUM             = 6
	ACRONYM_DEP     = 8 // deprecated 3.1
	SOUTHEAST_ASIAN = 9
	IDEOGRAPHIC     = 10
	HIRAGANA        = 11
	KATAKANA        = 12
	HANGUL          = 13
)

/* String token types that correspond to token type int constants */
var TOKEN_TYPES = []string{
	"<ALPHANUM>",
	"<APOSTROPHE>",
	"<ACRONYM>",
	"<COMPANY>",
	"<EMAIL>",
	"<HOST>",
	"<NUM>",
	"<CJ>",
	"<ACRONYM_DEP>",
	"<SOUTHEAST_ASIAN>",
	"<IDEOGRAPHIC>",
	"<HIRAGANA>",
	"<KATAKANA>",
	"<HANGUL>",
}

/*
A grammar-based tokenizer constructed with JFlex.

As of Lucene version 3.1, this class implements the Word Break rules
from the Unicode Text Segmentation algorithm, as specified in Unicode
standard Annex #29.

Many applications have specific tokenizer needs. If this tokenizer
does not suit your application, please consider copying this source
code directory to your project and maintaining your own grammar-based
tokenizer.

Version

You must specify the required Version compatibility when creating
StandardTokenizer:

	- As of 3.4, Hiragana and Han characters are no longer wrongly
	split from their combining characters. If you use a previous
	version number, you get the exact broken behavior for backwards
	compatibility.
	- As of 3.1, StandardTokenizer implements Unicode text segmentation.
	If you use a previous version number, you get the exact behavior of
	ClassicTokenizer for backwards compatibility.
*/
type StandardTokenizer struct {
	*Tokenizer
	input io.ReadCloser

	// A private instance of the JFlex-constructed scanner
	scanner StandardTokenizerInterface

	skippedPositions int
	maxTokenLength   int

	// this tokenizer generates three attributes:
	// term offset, positionIncrement and type

	termAtt    CharTermAttribute
	offsetAtt  OffsetAttribute
	posIncrAtt PositionIncrementAttribute
	typeAtt    TypeAttribute
}

/*
Creates a new instance of the StandardTokenizer. Attaches the input
to the newly created JFlex scanner.
*/
func newStandardTokenizer(matchVersion util.Version, input io.ReadCloser) *StandardTokenizer {
	ans := &StandardTokenizer{
		Tokenizer:      NewTokenizer(input),
		input:          input,
		maxTokenLength: DEFAULT_MAX_TOKEN_LENGTH,
	}
	ans.termAtt = ans.Attributes().Add("CharTermAttribute").(CharTermAttribute)
	ans.offsetAtt = ans.Attributes().Add("OffsetAttribute").(OffsetAttribute)
	ans.posIncrAtt = ans.Attributes().Add("PositionIncrementAttribute").(PositionIncrementAttribute)
	ans.typeAtt = ans.Attributes().Add("TypeAttribute").(TypeAttribute)
	ans.init(matchVersion)
	return ans
}

func (t *StandardTokenizer) init(matchVersion util.Version) {
	// GoLucene support >=4.5 only
	t.scanner = newStandardTokenizerImpl(nil)
}

func (t *StandardTokenizer) IncrementToken() (bool, error) {
	t.Attributes().Clear()
	t.skippedPositions = 0

	for {
		tokenType, err := t.scanner.nextToken()
		if tokenType == YYEOF || err != nil {
			return false, err
		}

		if t.scanner.yylength() <= t.maxTokenLength {
			t.posIncrAtt.SetPositionIncrement(t.skippedPositions + 1)
			t.scanner.text(t.termAtt)
			start := t.scanner.yychar()
			t.offsetAtt.SetOffset(t.CorrectOffset(start), t.CorrectOffset(start+t.termAtt.Length()))
			// This 'if' should be removed in the next release. For now,
			// it converts invalid acronyms to HOST. When removed, only the
			// 'else' part should remain.
			if tokenType == ACRONYM_DEP {
				panic("not implemented yet")
			} else {
				t.typeAtt.SetType(TOKEN_TYPES[tokenType])
			}
			return true, nil
		} else {
			// When we skip a too-long term, we still increment the positionincrement
			t.skippedPositions++
		}
	}
}

func (t *StandardTokenizer) End() error {
	panic("not implemented yet")
}

func (t *StandardTokenizer) Reset() error {
	t.scanner.yyreset(t.input)
	t.skippedPositions = 0
	return nil
}

// standard/StandardTokenizerInterface.java

/* This character denotes the end of file */
const YYEOF = -1

/* Internal interface for supporting versioned grammars. */
type StandardTokenizerInterface interface {
	// Copies the matched text into the CharTermAttribute
	text(CharTermAttribute)
	// Returns the current position.
	yychar() int
	// Resets the scanner to read from a new input stream.
	// Does not close the old reader.
	//
	// All internal variables are reset, the old input stream cannot be
	// reused (internal buffer) is discarded and lost). Lexical state
	// is set to ZZ_INITIAL.
	yyreset(io.ReadCloser)
	// Returns the length of the matched text region.
	yylength() int
	// Resumes scanning until the next regular expression is matched,
	// the end of input is encountered or an I/O-Error occurs.
	nextToken() (int, error)
}
<|endoftext|>