<commit_before>package herd

import (
	"fmt"
	"io"
	"sort"
	"time"
)

type rDuration time.Duration

func (d rDuration) selector(sub *Sub) bool {
	if time.Since(sub.lastReachableTime) <= time.Duration(d) {
		return true
	}
	return false
}

func (herd *Herd) writeHtml(writer io.Writer) {
	numSubs := herd.countSelectedSubs(nil)
	fmt.Fprintf(writer, "Time since current cycle start: %s
\n",
		time.Since(herd.currentScanStartTime))
	if numSubs < 1 {
		fmt.Fprintf(writer, "Duration of previous cycle: %s
\n",
			herd.previousScanDuration)
	} else {
		fmt.Fprintf(writer, "Duration of previous cycle: %s (%s/sub)
\n",
			herd.previousScanDuration,
			herd.previousScanDuration/time.Duration(numSubs))
	}
	fmt.Fprintf(writer, "Image server: %s
\n",
		herd.imageServerAddress, herd.imageServerAddress)
	fmt.Fprintf(writer,
		"Number of subs: %d
\n",
		numSubs)
	numSubs = herd.countSelectedSubs(selectAliveSub)
	fmt.Fprintf(writer,
		"Number of alive subs: %d
\n",
		numSubs)
	fmt.Fprint(writer, "Number of reachable subs in last: ")
	herd.writeReachableSubsLink(writer, time.Minute, "1 min", "1m")
	herd.writeReachableSubsLink(writer, time.Minute*10, "10 min", "10m")
	herd.writeReachableSubsLink(writer, time.Hour, "1 hour", "1h")
	herd.writeReachableSubsLink(writer, time.Hour*24, "1 day", "1d")
	fmt.Fprintln(writer, "
")
	numSubs = herd.countSelectedSubs(selectDeviantSub)
	fmt.Fprintf(writer,
		"Number of deviant subs: %d
\n",
		numSubs)
	numSubs = herd.countSelectedSubs(selectCompliantSub)
	fmt.Fprintf(writer,
		"Number of compliant subs: %d
\n",
		numSubs)
	subs := herd.getSelectedSubs(nil)
	connectDurations := getConnectDurations(subs)
	shortPollDurations := getPollDurations(subs, false)
	fullPollDurations := getPollDurations(subs, true)
	showDurationStats(writer, connectDurations, "Connect")
	showDurationStats(writer, shortPollDurations, "Short poll")
	showDurationStats(writer, fullPollDurations, "Full poll")
	fmt.Fprintf(writer, "Connection slots: %d out of %d
\n",
		len(herd.connectionSemaphore), cap(herd.connectionSemaphore))
	fmt.Fprintf(writer, "Poll slots: %d out of %d
\n",
		len(herd.pollSemaphore), cap(herd.pollSemaphore))
}

func (herd *Herd) writeReachableSubsLink(writer io.Writer,
	duration time.Duration, durationString string, query string) {
	numSubs := herd.countSelectedSubs(rDuration(duration).selector)
	fmt.Fprintf(writer, "%s",
		query, durationString)
	fmt.Fprintf(writer, "(%d), ",
		query, numSubs)
}

func selectAliveSub(sub *Sub) bool {
	switch sub.status {
	case statusUnknown:
		return false
	case statusConnecting:
		return false
	case statusDNSError:
		return false
	case statusConnectionRefused:
		return false
	case statusFailedToConnect:
		return false
	case statusFailedToPoll:
		return false
	}
	return true
}

func selectDeviantSub(sub *Sub) bool {
	if sub.status == statusUpdating {
		return true
	}
	return false
}

func selectCompliantSub(sub *Sub) bool {
	if sub.status == statusSynced {
		return true
	}
	return false
}

func getConnectDurations(subs []*Sub) []time.Duration {
	durations := make([]time.Duration, 0, len(subs))
	for _, sub := range subs {
		if sub.lastConnectDuration > 0 {
			durations = append(durations, sub.lastConnectDuration)
		}
	}
	sort.Sort(durationList(durations))
	return durations
}

func getPollDurations(subs []*Sub, full bool) []time.Duration {
	durations := make([]time.Duration, 0, len(subs))
	for _, sub := range subs {
		var duration time.Duration
		if full {
			duration = sub.lastFullPollDuration
		} else {
			duration = sub.lastShortPollDuration
		}
		if duration > 0 {
			durations = append(durations, duration)
		}
	}
	sort.Sort(durationList(durations))
	return durations
}

type durationList []time.Duration

func (dl durationList) Len() int {
	return len(dl)
}

func (dl durationList) Less(i, j int) bool {
	return dl[i] < dl[j]
}

func (dl durationList) Swap(i, j int) {
	dl[i], dl[j] = dl[j], dl[i]
}

func showDurationStats(writer io.Writer, durations []time.Duration,
	durationType string) {
	if len(durations) < 1 {
		return
	}
	var avgDuration time.Duration
	for _, duration := range durations {
		avgDuration += duration
	}
	avgDuration /= time.Duration(len(durations))
	medDuration := durations[len(durations)/2]
	unit := "ns"
	scale := 1.0
	switch {
	case medDuration > 1e9:
		unit = "s"
		scale = 1e-9
	case medDuration > 1e6:
		unit = "ms"
		scale = 1e-6
	case medDuration > 1e3:
		unit = "µs"
		scale = 1e-3
	}
	fmt.Fprintf(writer,
		"%s durations: %.3f/%.3f/%.3f/%.3f %s (avg/med/min/max)
\n",
		durationType,
		float64(avgDuration)*scale, float64(medDuration)*scale,
		float64(durations[0])*scale, float64(durations[len(durations)-1])*scale,
		unit)
}
<commit_msg>Remove trailing comma in dominator status page.<commit_after>package herd

import (
	"fmt"
	"io"
	"sort"
	"time"
)

type rDuration time.Duration

func (d rDuration) selector(sub *Sub) bool {
	if time.Since(sub.lastReachableTime) <= time.Duration(d) {
		return true
	}
	return false
}

func (herd *Herd) writeHtml(writer io.Writer) {
	numSubs := herd.countSelectedSubs(nil)
	fmt.Fprintf(writer, "Time since current cycle start: %s
\n",
		time.Since(herd.currentScanStartTime))
	if numSubs < 1 {
		fmt.Fprintf(writer, "Duration of previous cycle: %s
\n",
			herd.previousScanDuration)
	} else {
		fmt.Fprintf(writer, "Duration of previous cycle: %s (%s/sub)
\n",
			herd.previousScanDuration,
			herd.previousScanDuration/time.Duration(numSubs))
	}
	fmt.Fprintf(writer, "Image server: %s
\n",
		herd.imageServerAddress, herd.imageServerAddress)
	fmt.Fprintf(writer,
		"Number of subs: %d
\n",
		numSubs)
	numSubs = herd.countSelectedSubs(selectAliveSub)
	fmt.Fprintf(writer,
		"Number of alive subs: %d
\n",
		numSubs)
	fmt.Fprint(writer, "Number of reachable subs in last: ")
	herd.writeReachableSubsLink(writer, time.Minute, "1 min", "1m", true)
	herd.writeReachableSubsLink(writer, time.Minute*10, "10 min", "10m", true)
	herd.writeReachableSubsLink(writer, time.Hour, "1 hour", "1h", true)
	herd.writeReachableSubsLink(writer, time.Hour*24, "1 day", "1d", false)
	numSubs = herd.countSelectedSubs(selectDeviantSub)
	fmt.Fprintf(writer,
		"Number of deviant subs: %d
\n",
		numSubs)
	numSubs = herd.countSelectedSubs(selectCompliantSub)
	fmt.Fprintf(writer,
		"Number of compliant subs: %d
\n",
		numSubs)
	subs := herd.getSelectedSubs(nil)
	connectDurations := getConnectDurations(subs)
	shortPollDurations := getPollDurations(subs, false)
	fullPollDurations := getPollDurations(subs, true)
	showDurationStats(writer, connectDurations, "Connect")
	showDurationStats(writer, shortPollDurations, "Short poll")
	showDurationStats(writer, fullPollDurations, "Full poll")
	fmt.Fprintf(writer, "Connection slots: %d out of %d
\n",
		len(herd.connectionSemaphore), cap(herd.connectionSemaphore))
	fmt.Fprintf(writer, "Poll slots: %d out of %d
\n",
		len(herd.pollSemaphore), cap(herd.pollSemaphore))
}

func (herd *Herd) writeReachableSubsLink(writer io.Writer,
	duration time.Duration, durationString string, query string,
	moreToCome bool) {
	numSubs := herd.countSelectedSubs(rDuration(duration).selector)
	fmt.Fprintf(writer, "%s",
		query, durationString)
	fmt.Fprintf(writer, "(%d)",
		query, numSubs)
	if moreToCome {
		fmt.Fprint(writer, ", ")
	} else {
		fmt.Fprintln(writer, "
")
	}
}

func selectAliveSub(sub *Sub) bool {
	switch sub.status {
	case statusUnknown:
		return false
	case statusConnecting:
		return false
	case statusDNSError:
		return false
	case statusConnectionRefused:
		return false
	case statusFailedToConnect:
		return false
	case statusFailedToPoll:
		return false
	}
	return true
}

func selectDeviantSub(sub *Sub) bool {
	if sub.status == statusUpdating {
		return true
	}
	return false
}

func selectCompliantSub(sub *Sub) bool {
	if sub.status == statusSynced {
		return true
	}
	return false
}

func getConnectDurations(subs []*Sub) []time.Duration {
	durations := make([]time.Duration, 0, len(subs))
	for _, sub := range subs {
		if sub.lastConnectDuration > 0 {
			durations = append(durations, sub.lastConnectDuration)
		}
	}
	sort.Sort(durationList(durations))
	return durations
}

func getPollDurations(subs []*Sub, full bool) []time.Duration {
	durations := make([]time.Duration, 0, len(subs))
	for _, sub := range subs {
		var duration time.Duration
		if full {
			duration = sub.lastFullPollDuration
		} else {
			duration = sub.lastShortPollDuration
		}
		if duration > 0 {
			durations = append(durations, duration)
		}
	}
	sort.Sort(durationList(durations))
	return durations
}

type durationList []time.Duration

func (dl durationList) Len() int {
	return len(dl)
}

func (dl durationList) Less(i, j int) bool {
	return dl[i] < dl[j]
}

func (dl durationList) Swap(i, j int) {
	dl[i], dl[j] = dl[j], dl[i]
}

func showDurationStats(writer io.Writer, durations []time.Duration,
	durationType string) {
	if len(durations) < 1 {
		return
	}
	var avgDuration time.Duration
	for _, duration := range durations {
		avgDuration += duration
	}
	avgDuration /= time.Duration(len(durations))
	medDuration := durations[len(durations)/2]
	unit := "ns"
	scale := 1.0
	switch {
	case medDuration > 1e9:
		unit = "s"
		scale = 1e-9
	case medDuration > 1e6:
		unit = "ms"
		scale = 1e-6
	case medDuration > 1e3:
		unit = "µs"
		scale = 1e-3
	}
	fmt.Fprintf(writer,
		"%s durations: %.3f/%.3f/%.3f/%.3f %s (avg/med/min/max)
\n",
		durationType,
		float64(avgDuration)*scale, float64(medDuration)*scale,
		float64(durations[0])*scale, float64(durations[len(durations)-1])*scale,
		unit)
}
<|endoftext|>


<commit_before>package gocbcore

import (
	"errors"
	"fmt"
	"sync"
)

var (
	errPipelineClosed = errors.New("pipeline has been closed")
	errPipelineFull   = errors.New("pipeline is too full")
)

type memdGetClientFn func() (*memdClient, error)

type memdPipeline struct {
	address     string
	getClientFn memdGetClientFn
	maxItems    int
	queue       *memdOpQueue
	maxClients  int
	clients     []*memdPipelineClient
}

func newPipeline(address string, maxClients, maxItems int, getClientFn memdGetClientFn) *memdPipeline {
	return &memdPipeline{
		address:     address,
		getClientFn: getClientFn,
		maxClients:  maxClients,
		maxItems:    maxItems,
		queue:       newMemdOpQueue(),
	}
}

func newDeadPipeline(maxItems int) *memdPipeline {
	return newPipeline("", 0, maxItems, nil)
}

func (pipeline memdPipeline) debugString() string {
	var outStr string

	if pipeline.address != "" {
		outStr += fmt.Sprintf("Address: %s\n", pipeline.address)
		outStr += fmt.Sprintf("Max Clients: %d\n", pipeline.maxClients)
		outStr += fmt.Sprintf("Num Clients: %d\n", len(pipeline.clients))
		outStr += fmt.Sprintf("Max Items: %d\n", pipeline.maxItems)
	} else {
		outStr += "Dead-Server Queue\n"
	}

	outStr += "Op Queue:\n"
	outStr += reindentLog("  ", pipeline.queue.debugString())

	return outStr
}

func (pipeline memdPipeline) Address() string {
	return pipeline.address
}

func (pipeline *memdPipeline) StartClients() {
	for len(pipeline.clients) < pipeline.maxClients {
		client := newMemdPipelineClient(pipeline)
		pipeline.clients = append(pipeline.clients, client)

		go client.Run()
	}
}

func (pipeline *memdPipeline) sendRequest(req *memdQRequest, maxItems int) error {
	err := pipeline.queue.Push(req, maxItems)
	if err == errOpQueueClosed {
		return errPipelineClosed
	} else if err == errOpQueueFull {
		return errPipelineFull
	} else if err != nil {
		return err
	}

	return nil
}

func (pipeline *memdPipeline) RequeueRequest(req *memdQRequest) error {
	return pipeline.sendRequest(req, 0)
}

func (pipeline *memdPipeline) SendRequest(req *memdQRequest) error {
	return pipeline.sendRequest(req, pipeline.maxItems)
}

// Performs a takeover of another pipeline.  Note that this does not
//  take over the requests queued in the old pipeline, and those must
//  be drained and processed separately.
func (pipeline *memdPipeline) Takeover(oldPipeline *memdPipeline) {
	if oldPipeline.address != pipeline.address {
		logErrorf("Attempted pipeline takeover for differing address")

		// We try to 'gracefully' error here by resolving all the requests as
		//  errors, but allowing the application to continue.
		err := oldPipeline.Close()
		if err != nil {
			// Log and continue with this non-fatal error.
			logDebugf("Failed to shutdown old pipeline (%s)", err)
		}

		// Drain all the requests as an internal error so they are not lost
		oldPipeline.Drain(func(req *memdQRequest) {
			req.tryCallback(nil, ErrInternalError)
		})

		return
	}

	// Migrate all the clients to the new pipeline
	pipeline.clients = oldPipeline.clients
	oldPipeline.clients = nil
	for _, client := range pipeline.clients {
		client.ReassignTo(pipeline)
	}

	// Shut down the old pipelines queue, this will force all the
	//  clients to 'refresh' their consumer, and pick up the new
	//  pipeline queue from the new pipeline.
	oldPipeline.queue.Close()
}

func (pipeline *memdPipeline) Close() error {
	// Shut down all the clients
	var errs MultiError
	for _, pipecli := range pipeline.clients {
		err := pipecli.Close()
		if err != nil {
			errs.add(err)
		}
	}

	// Kill the queue, forcing everyone to stop
	pipeline.queue.Close()

	return errs.get()
}

func (pipeline *memdPipeline) Drain(cb func(*memdQRequest)) {
	pipeline.queue.Drain(cb)
}

type memdPipelineClient struct {
	parent     *memdPipeline
	clientDead bool
	consumer   *memdOpConsumer
	lock       sync.Mutex
}

func newMemdPipelineClient(parent *memdPipeline) *memdPipelineClient {
	return &memdPipelineClient{
		parent: parent,
	}
}

func (pipecli *memdPipelineClient) ReassignTo(parent *memdPipeline) {
	pipecli.lock.Lock()
	pipecli.parent = parent
	consumer := pipecli.consumer
	pipecli.lock.Unlock()

	consumer.Close()
}

func (pipecli *memdPipelineClient) ioLoop(client *memdClient) {
	pipecli.lock.Lock()
	if pipecli.parent == nil {
		logErrorf("Pipeline ioLoop started with no parent")
		pipecli.lock.Unlock()

		err := client.Close()
		if err != nil {
			logErrorf("Failed to shut down broken ioLoop client (%s)", err)
		}

		return
	}
	address := pipecli.parent.address
	pipecli.lock.Unlock()

	killSig := make(chan struct{})

	go func() {
		logDebugf("Pipeline `%s/%p` client watcher starting...", address, pipecli)

		<-client.CloseNotify()

		pipecli.lock.Lock()
		pipecli.clientDead = true
		consumer := pipecli.consumer
		pipecli.lock.Unlock()

		consumer.Close()

		killSig <- struct{}{}
	}()

	logDebugf("Pipeline `%s/%p` IO loop starting...", address, pipecli)

	var localConsumer *memdOpConsumer
	for {
		if localConsumer == nil {
			pipecli.lock.Lock()

			if pipecli.parent == nil {
				// This pipelineClient has been shut down
				pipecli.lock.Unlock()
				break
			}

			if pipecli.clientDead {
				// The client has disconnected from the server
				pipecli.lock.Unlock()
				break
			}

			localConsumer = pipecli.parent.queue.Consumer()
			pipecli.consumer = localConsumer
			pipecli.lock.Unlock()
		}

		req := localConsumer.Pop()
		if req == nil {
			localConsumer = nil
			continue
		}

		err := client.SendRequest(req)
		if err != nil {
			logDebugf("Server write error: %v", err)

			// We need to alert the caller that there was a network error
			req.Callback(nil, req, ErrNetwork)

			// Stop looping
			break
		}
	}

	// Ensure the connection is fully closed
	err := client.Close()
	if err != nil {
		// We just log here, since this is a non-fatal error.
		logErrorf("Failed to shut down client socket (%s)", err)
	}

	// We must wait for the close wait goroutine to die as well before we can continue.
	<-killSig
}

func (pipecli *memdPipelineClient) Run() {
	for {
		pipecli.lock.Lock()
		pipeline := pipecli.parent
		pipecli.clientDead = false
		pipecli.lock.Unlock()

		if pipeline == nil {
			break
		}

		client, err := pipeline.getClientFn()
		if err != nil {
			continue
		}

		// Runs until the connection has died (for whatever reason)
		pipecli.ioLoop(client)
	}
}

func (pipecli *memdPipelineClient) Close() error {
	pipecli.lock.Lock()
	pipecli.parent = nil
	consumer := pipecli.consumer
	pipecli.lock.Unlock()

	if consumer != nil {
		consumer.Close()
	}

	return nil
}
<commit_msg>Improve pipeline management logging.<commit_after>package gocbcore

import (
	"errors"
	"fmt"
	"sync"
)

var (
	errPipelineClosed = errors.New("pipeline has been closed")
	errPipelineFull   = errors.New("pipeline is too full")
)

type memdGetClientFn func() (*memdClient, error)

type memdPipeline struct {
	address     string
	getClientFn memdGetClientFn
	maxItems    int
	queue       *memdOpQueue
	maxClients  int
	clients     []*memdPipelineClient
}

func newPipeline(address string, maxClients, maxItems int, getClientFn memdGetClientFn) *memdPipeline {
	return &memdPipeline{
		address:     address,
		getClientFn: getClientFn,
		maxClients:  maxClients,
		maxItems:    maxItems,
		queue:       newMemdOpQueue(),
	}
}

func newDeadPipeline(maxItems int) *memdPipeline {
	return newPipeline("", 0, maxItems, nil)
}

func (pipeline memdPipeline) debugString() string {
	var outStr string

	if pipeline.address != "" {
		outStr += fmt.Sprintf("Address: %s\n", pipeline.address)
		outStr += fmt.Sprintf("Max Clients: %d\n", pipeline.maxClients)
		outStr += fmt.Sprintf("Num Clients: %d\n", len(pipeline.clients))
		outStr += fmt.Sprintf("Max Items: %d\n", pipeline.maxItems)
	} else {
		outStr += "Dead-Server Queue\n"
	}

	outStr += "Op Queue:\n"
	outStr += reindentLog("  ", pipeline.queue.debugString())

	return outStr
}

func (pipeline memdPipeline) Address() string {
	return pipeline.address
}

func (pipeline *memdPipeline) StartClients() {
	for len(pipeline.clients) < pipeline.maxClients {
		client := newMemdPipelineClient(pipeline)
		pipeline.clients = append(pipeline.clients, client)

		go client.Run()
	}
}

func (pipeline *memdPipeline) sendRequest(req *memdQRequest, maxItems int) error {
	err := pipeline.queue.Push(req, maxItems)
	if err == errOpQueueClosed {
		return errPipelineClosed
	} else if err == errOpQueueFull {
		return errPipelineFull
	} else if err != nil {
		return err
	}

	return nil
}

func (pipeline *memdPipeline) RequeueRequest(req *memdQRequest) error {
	return pipeline.sendRequest(req, 0)
}

func (pipeline *memdPipeline) SendRequest(req *memdQRequest) error {
	return pipeline.sendRequest(req, pipeline.maxItems)
}

// Performs a takeover of another pipeline.  Note that this does not
//  take over the requests queued in the old pipeline, and those must
//  be drained and processed separately.
func (pipeline *memdPipeline) Takeover(oldPipeline *memdPipeline) {
	if oldPipeline.address != pipeline.address {
		logErrorf("Attempted pipeline takeover for differing address")

		// We try to 'gracefully' error here by resolving all the requests as
		//  errors, but allowing the application to continue.
		err := oldPipeline.Close()
		if err != nil {
			// Log and continue with this non-fatal error.
			logDebugf("Failed to shutdown old pipeline (%s)", err)
		}

		// Drain all the requests as an internal error so they are not lost
		oldPipeline.Drain(func(req *memdQRequest) {
			req.tryCallback(nil, ErrInternalError)
		})

		return
	}

	// Migrate all the clients to the new pipeline
	pipeline.clients = oldPipeline.clients
	oldPipeline.clients = nil
	for _, client := range pipeline.clients {
		client.ReassignTo(pipeline)
	}

	// Shut down the old pipelines queue, this will force all the
	//  clients to 'refresh' their consumer, and pick up the new
	//  pipeline queue from the new pipeline.
	oldPipeline.queue.Close()
}

func (pipeline *memdPipeline) Close() error {
	// Shut down all the clients
	var errs MultiError
	for _, pipecli := range pipeline.clients {
		err := pipecli.Close()
		if err != nil {
			errs.add(err)
		}
	}

	// Kill the queue, forcing everyone to stop
	pipeline.queue.Close()

	return errs.get()
}

func (pipeline *memdPipeline) Drain(cb func(*memdQRequest)) {
	pipeline.queue.Drain(cb)
}

type memdPipelineClient struct {
	parent     *memdPipeline
	clientDead bool
	consumer   *memdOpConsumer
	lock       sync.Mutex
}

func newMemdPipelineClient(parent *memdPipeline) *memdPipelineClient {
	return &memdPipelineClient{
		parent: parent,
	}
}

func (pipecli *memdPipelineClient) ReassignTo(parent *memdPipeline) {
	pipecli.lock.Lock()
	pipecli.parent = parent
	consumer := pipecli.consumer
	pipecli.lock.Unlock()

	consumer.Close()
}

func (pipecli *memdPipelineClient) ioLoop(client *memdClient) {
	pipecli.lock.Lock()
	if pipecli.parent == nil {
		logErrorf("Pipeline ioLoop started with no parent")
		pipecli.lock.Unlock()

		err := client.Close()
		if err != nil {
			logErrorf("Failed to shut down broken ioLoop client (%s)", err)
		}

		return
	}
	address := pipecli.parent.address
	pipecli.lock.Unlock()

	killSig := make(chan struct{})

	go func() {
		logDebugf("Pipeline `%s/%p` client watcher starting...", address, pipecli)

		<-client.CloseNotify()

		logDebugf("Pipeline `%s/%p` client died", address, pipecli)

		pipecli.lock.Lock()
		pipecli.clientDead = true
		consumer := pipecli.consumer
		pipecli.lock.Unlock()

		logDebugf("Pipeline `%s/%p` closing consumer %p", address, pipecli, consumer)
		consumer.Close()

		killSig <- struct{}{}
	}()

	logDebugf("Pipeline `%s/%p` IO loop starting...", address, pipecli)

	var localConsumer *memdOpConsumer
	for {
		if localConsumer == nil {
			logDebugf("Pipeline `%s/%p` fetching new consumer", address, pipecli)

			pipecli.lock.Lock()

			if pipecli.parent == nil {
				logDebugf("Pipeline `%s/%p` found no parent", address, pipecli)

				// This pipelineClient has been shut down
				pipecli.lock.Unlock()
				break
			}

			if pipecli.clientDead {
				// The client has disconnected from the server
				pipecli.lock.Unlock()
				break
			}

			localConsumer = pipecli.parent.queue.Consumer()
			pipecli.consumer = localConsumer
			pipecli.lock.Unlock()
		}

		req := localConsumer.Pop()
		if req == nil {
			localConsumer = nil
			continue
		}

		err := client.SendRequest(req)
		if err != nil {
			logDebugf("Server write error: %v", err)

			// We need to alert the caller that there was a network error
			req.Callback(nil, req, ErrNetwork)

			// Stop looping
			break
		}
	}

	// Ensure the connection is fully closed
	err := client.Close()
	if err != nil {
		// We just log here, since this is a non-fatal error.
		logErrorf("Failed to shut down client socket (%s)", err)
	}

	// We must wait for the close wait goroutine to die as well before we can continue.
	<-killSig
}

func (pipecli *memdPipelineClient) Run() {
	for {
		logDebugf("Pipeline Client `%p` preparing for new client loop", pipecli)

		pipecli.lock.Lock()
		pipeline := pipecli.parent
		pipecli.clientDead = false
		pipecli.lock.Unlock()

		logDebugf("Pipeline Client `%p` is on parent: %p", pipecli, pipeline)

		if pipeline == nil {
			break
		}

		logDebugf("Pipeline Client `%p` retrieving new client connection", pipecli)
		client, err := pipeline.getClientFn()
		if err != nil {
			continue
		}

		// Runs until the connection has died (for whatever reason)
		logDebugf("Pipeline Client `%p` starting new client loop", pipecli)
		pipecli.ioLoop(client)
	}
}

func (pipecli *memdPipelineClient) Close() error {
	logDebugf("Pipeline Client `%p` received close request", pipecli)

	pipecli.lock.Lock()
	pipecli.parent = nil
	consumer := pipecli.consumer
	pipecli.lock.Unlock()

	if consumer != nil {
		consumer.Close()
	}

	return nil
}
<|endoftext|>


<commit_before>package lib

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/influxdata/influxdb/client/v2"

	"github.com/subutai-io/base/agent/config"
)

var (
	lxcnic      map[string]string
	traff       = []string{"in", "out"}
	cgtype      = []string{"cpuacct", "memory"}
	metrics     = []string{"total", "used", "available"}
	btrfsmounts = []string{"rootfs", "home", "var", "opt"}
	cpu         = []string{"user", "nice", "system", "idle", "iowait"}
	lxcmemory   = map[string]bool{"cache": true, "rss": true, "Cached": true, "MemFree": true}
	memory      = map[string]bool{"Active": true, "Buffers": true, "Cached": true, "MemFree": true}
)

func Collect() {
	for {
		CollectStats()
		time.Sleep(time.Second * 30)
	}
}

func CollectStats() {
	clnt, bp, err := initInfluxdb()
	if err == nil {
		netStat(clnt, bp)
		cgroupStat(clnt, bp)
		btrfsStat(clnt, bp)
		diskFree(clnt, bp)
		cpuStat(clnt, bp)
		memStat(clnt, bp)
	} else {
		fmt.Println(err)
	}
}

func initInfluxdb() (clnt client.Client, bp client.BatchPoints, err error) {
	clnt, err = client.NewHTTPClient(client.HTTPConfig{
		Addr:               "https://" + config.Influxdb.Server + ":8086",
		Username:           config.Influxdb.User,
		Password:           config.Influxdb.Pass,
		InsecureSkipVerify: true,
	})
	if err != nil {
		return
	}
	bp, _ = client.NewBatchPoints(client.BatchPointsConfig{
		Database:        config.Influxdb.Db,
		RetentionPolicy: "hour",
	})
	return
}

func parsefile(hostname, lxc, cgtype, filename string, clnt client.Client, bp client.BatchPoints) {
	file, err := os.Open(filename)
	if err != nil {
		return
	}
	defer file.Close()
	scanner := bufio.NewScanner(bufio.NewReader(file))
	for scanner.Scan() {
		line := strings.Split(scanner.Text(), " ")
		value, _ := strconv.ParseInt(line[1], 10, 62)
		if cgtype == "memory" && lxcmemory[line[0]] {
			point, _ := client.NewPoint("lxc_"+cgtype,
				map[string]string{"hostname": lxc, "type": line[0]},
				map[string]interface{}{"value": value / int64(runtime.NumCPU())},
				time.Now())
			bp.AddPoint(point)
		} else if cgtype == "cpuacct" {
			point, _ := client.NewPoint("lxc_cpu",
				map[string]string{"hostname": lxc, "type": line[0]},
				map[string]interface{}{"value": value / int64(runtime.NumCPU())},
				time.Now())
			bp.AddPoint(point)
		}
	}
	clnt.Write(bp)
}

func cgroupStat(clnt client.Client, bp client.BatchPoints) {
	hostname, _ := os.Hostname()
	for _, item := range cgtype {
		path := "/sys/fs/cgroup/" + item + "/lxc/"
		files, _ := ioutil.ReadDir(path)
		for _, f := range files {
			if f.IsDir() {
				parsefile(hostname, f.Name(), item, path+f.Name()+"/"+item+".stat", clnt, bp)
			}
		}
	}
}

func grepnic(filename string) string {
	regex, err := regexp.Compile("lxc.network.veth.pair")
	if err != nil {
		return ""
	}
	fh, err := os.Open(filename)
	f := bufio.NewReader(fh)
	if err != nil {
		return ""
	}
	defer fh.Close()
	buf := make([]byte, 64)
	for {
		buf, _, err = f.ReadLine()
		if err != nil {
			return ""
		}
		if regex.MatchString(string(buf)) {
			return string(buf)
		}
	}
}

func lxclist() map[string]string {
	files, _ := ioutil.ReadDir(config.Agent.LxcPrefix)
	list := make(map[string]string)
	for _, f := range files {
		line := grepnic(config.Agent.LxcPrefix + f.Name() + "/config")
		if line != "" {
			nic := strings.Split(line, "=")
			if len(nic) >= 2 {
				list[strings.Fields(nic[1])[0]] = f.Name()
			}
		}
	}
	return list
}

func netStat(clnt client.Client, bp client.BatchPoints) {
	hostname, _ := os.Hostname()
	lxcnic = lxclist()
	file, err := os.Open("/proc/net/dev")
	if err != nil {
		return
	}
	defer file.Close()
	scanner := bufio.NewScanner(bufio.NewReader(file))
	lc := 0
	traffic := make([]int, 2)
	for scanner.Scan() {
		lc++
		line := strings.Fields(scanner.Text())
		if lc > 2 {
			traffic[0], _ = strconv.Atoi(line[1])
			traffic[1], _ = strconv.Atoi(line[10])
			nicname := strings.Split(line[0], ":")[0]
			metric := "host_net"
			if lxcnic[nicname] != "" {
				metric = "lxc_net"
				hostname = lxcnic[nicname]
			}
			for i := range traffic {
				point, _ := client.NewPoint(metric,
					map[string]string{"hostname": hostname, "iface": nicname, "type": traff[i]},
					map[string]interface{}{"value": traffic[i]},
					time.Now())
				bp.AddPoint(point)
			}
		}
	}
	clnt.Write(bp)
}

func btrfsStat(clnt client.Client, bp client.BatchPoints) {
	list := make(map[string]string)
	out, _ := exec.Command("btrfs", "subvolume", "list", config.Agent.LxcPrefix).Output()
	scanner := bufio.NewScanner(bytes.NewReader(out))
	for scanner.Scan() {
		line := strings.Fields(scanner.Text())
		list["0/"+line[1]] = line[8]
	}
	out, _ = exec.Command("btrfs", "qgroup", "show", "-r", "--raw", config.Agent.LxcPrefix).Output()
	scanner = bufio.NewScanner(bytes.NewReader(out))
	for scanner.Scan() {
		line := strings.Fields(scanner.Text())
		path := strings.Split(list[line[0]], "/")
		if len(path) > 3 {
			value, _ := strconv.Atoi(line[2])
			point, _ := client.NewPoint("lxc_disk",
				map[string]string{"hostname": path[2], "mount": path[3], "type": "used"},
				map[string]interface{}{"value": value},
				time.Now())
			bp.AddPoint(point)
		}
	}
	clnt.Write(bp)
}

func diskFree(clnt client.Client, bp client.BatchPoints) {
	hostname, _ := os.Hostname()
	out, _ := exec.Command("df", "-B1").Output()
	scanner := bufio.NewScanner(bytes.NewReader(out))
	for scanner.Scan() {
		line := strings.Fields(scanner.Text())
		if strings.HasPrefix(line[0], "/dev") {
			for i := range metrics {
				value, _ := strconv.Atoi(line[i+1])
				point, _ := client.NewPoint("host_disk",
					map[string]string{"hostname": hostname, "mount": line[5], "type": metrics[i]},
					map[string]interface{}{"value": value},
					time.Now())
				bp.AddPoint(point)
			}
		}
	}
	clnt.Write(bp)
}

func memStat(clnt client.Client, bp client.BatchPoints) {
	hostname, _ := os.Hostname()
	file, err := os.Open("/proc/meminfo")
	if err != nil {
		return
	}
	defer file.Close()
	scanner := bufio.NewScanner(bufio.NewReader(file))
	for scanner.Scan() {
		line := strings.Fields(strings.Replace(scanner.Text(), ":", "", -1))
		value, _ := strconv.ParseInt(line[1], 10, 62)
		if memory[line[0]] {
			point, _ := client.NewPoint("host_memory",
				map[string]string{"hostname": hostname, "type": line[0]},
				map[string]interface{}{"value": value * 1024},
				time.Now())
			bp.AddPoint(point)
		}
	}
	clnt.Write(bp)
}

func cpuStat(clnt client.Client, bp client.BatchPoints) {
	hostname, _ := os.Hostname()
	file, err := os.Open("/proc/stat")
	if err != nil {
		return
	}
	defer file.Close()
	scanner := bufio.NewScanner(bufio.NewReader(file))
	for scanner.Scan() {
		line := strings.Fields(scanner.Text())
		if line[0] == "cpu" {
			for i := range cpu {
				value, _ := strconv.Atoi(line[i+1])
				point, _ := client.NewPoint("host_cpu",
					map[string]string{"hostname": hostname, "type": cpu[i]},
					map[string]interface{}{"value": value / runtime.NumCPU()},
					time.Now())
				bp.AddPoint(point)
			}
		}
	}
	clnt.Write(bp)
}
<commit_msg>Fixed collection network metrics. #340<commit_after>package lib

import (
	"bufio"
	"bytes"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/influxdata/influxdb/client/v2"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/log"
)

var (
	lxcnic      map[string]string
	traff       = []string{"in", "out"}
	cgtype      = []string{"cpuacct", "memory"}
	metrics     = []string{"total", "used", "available"}
	btrfsmounts = []string{"rootfs", "home", "var", "opt"}
	cpu         = []string{"user", "nice", "system", "idle", "iowait"}
	lxcmemory   = map[string]bool{"cache": true, "rss": true, "Cached": true, "MemFree": true}
	memory      = map[string]bool{"Active": true, "Buffers": true, "Cached": true, "MemFree": true}
)

func Collect() {
	for {
		collectStats()
		time.Sleep(time.Second * 30)
	}
}

func collectStats() {
	clnt, bp, err := initInfluxdb()
	if !log.Check(log.WarnLevel, "Initialization InfluxDB", err) {
		netStat(clnt, bp)
		cgroupStat(clnt, bp)
		btrfsStat(clnt, bp)
		diskFree(clnt, bp)
		cpuStat(clnt, bp)
		memStat(clnt, bp)
	}
}

func initInfluxdb() (clnt client.Client, bp client.BatchPoints, err error) {
	clnt, err = client.NewHTTPClient(client.HTTPConfig{
		Addr:               "https://" + config.Influxdb.Server + ":8086",
		Username:           config.Influxdb.User,
		Password:           config.Influxdb.Pass,
		InsecureSkipVerify: true,
	})
	if err != nil {
		return
	}
	bp, _ = client.NewBatchPoints(client.BatchPointsConfig{
		Database:        config.Influxdb.Db,
		RetentionPolicy: "hour",
	})
	return
}

func parsefile(hostname, lxc, cgtype, filename string, clnt client.Client, bp client.BatchPoints) {
	file, err := os.Open(filename)
	if err != nil {
		return
	}
	defer file.Close()
	scanner := bufio.NewScanner(bufio.NewReader(file))
	for scanner.Scan() {
		line := strings.Split(scanner.Text(), " ")
		value, _ := strconv.ParseInt(line[1], 10, 62)
		if cgtype == "memory" && lxcmemory[line[0]] {
			point, _ := client.NewPoint("lxc_"+cgtype,
				map[string]string{"hostname": lxc, "type": line[0]},
				map[string]interface{}{"value": value / int64(runtime.NumCPU())},
				time.Now())
			bp.AddPoint(point)
		} else if cgtype == "cpuacct" {
			point, _ := client.NewPoint("lxc_cpu",
				map[string]string{"hostname": lxc, "type": line[0]},
				map[string]interface{}{"value": value / int64(runtime.NumCPU())},
				time.Now())
			bp.AddPoint(point)
		}
	}
	clnt.Write(bp)
}

func cgroupStat(clnt client.Client, bp client.BatchPoints) {
	hostname, _ := os.Hostname()
	for _, item := range cgtype {
		path := "/sys/fs/cgroup/" + item + "/lxc/"
		files, _ := ioutil.ReadDir(path)
		for _, f := range files {
			if f.IsDir() {
				parsefile(hostname, f.Name(), item, path+f.Name()+"/"+item+".stat", clnt, bp)
			}
		}
	}
}

func grepnic(filename string) string {
	regex, err := regexp.Compile("lxc.network.veth.pair")
	if err != nil {
		return ""
	}
	fh, err := os.Open(filename)
	f := bufio.NewReader(fh)
	if err != nil {
		return ""
	}
	defer fh.Close()
	buf := make([]byte, 64)
	for {
		buf, _, err = f.ReadLine()
		if err != nil {
			return ""
		}
		if regex.MatchString(string(buf)) {
			return string(buf)
		}
	}
}

func lxclist() map[string]string {
	files, _ := ioutil.ReadDir(config.Agent.LxcPrefix)
	list := make(map[string]string)
	for _, f := range files {
		line := grepnic(config.Agent.LxcPrefix + f.Name() + "/config")
		if line != "" {
			nic := strings.Split(line, "=")
			if len(nic) >= 2 {
				list[strings.Fields(nic[1])[0]] = f.Name()
			}
		}
	}
	return list
}

func netStat(clnt client.Client, bp client.BatchPoints) {
	lxcnic = lxclist()
	file, err := os.Open("/proc/net/dev")
	if err != nil {
		return
	}
	defer file.Close()
	scanner := bufio.NewScanner(bufio.NewReader(file))
	lc := 0
	traffic := make([]int, 2)
	for scanner.Scan() {
		hostname, _ := os.Hostname()
		lc++
		line := strings.Fields(scanner.Text())
		if lc > 2 {
			traffic[0], _ = strconv.Atoi(line[1])
			traffic[1], _ = strconv.Atoi(line[9])
			nicname := strings.Split(line[0], ":")[0]
			metric := "host_net"
			if lxcnic[nicname] != "" {
				metric = "lxc_net"
				hostname = lxcnic[nicname]
			}
			for i := range traffic {
				point, _ := client.NewPoint(metric,
					map[string]string{"hostname": hostname, "iface": nicname, "type": traff[i]},
					map[string]interface{}{"value": traffic[i]},
					time.Now())
				bp.AddPoint(point)
			}
		}
	}
	clnt.Write(bp)
}

func btrfsStat(clnt client.Client, bp client.BatchPoints) {
	list := make(map[string]string)
	out, _ := exec.Command("btrfs", "subvolume", "list", config.Agent.LxcPrefix).Output()
	scanner := bufio.NewScanner(bytes.NewReader(out))
	for scanner.Scan() {
		line := strings.Fields(scanner.Text())
		list["0/"+line[1]] = line[8]
	}
	out, _ = exec.Command("btrfs", "qgroup", "show", "-r", "--raw", config.Agent.LxcPrefix).Output()
	scanner = bufio.NewScanner(bytes.NewReader(out))
	for scanner.Scan() {
		line := strings.Fields(scanner.Text())
		path := strings.Split(list[line[0]], "/")
		if len(path) > 3 {
			value, _ := strconv.Atoi(line[2])
			point, _ := client.NewPoint("lxc_disk",
				map[string]string{"hostname": path[2], "mount": path[3], "type": "used"},
				map[string]interface{}{"value": value},
				time.Now())
			bp.AddPoint(point)
		}
	}
	clnt.Write(bp)
}

func diskFree(clnt client.Client, bp client.BatchPoints) {
	hostname, _ := os.Hostname()
	out, _ := exec.Command("df", "-B1").Output()
	scanner := bufio.NewScanner(bytes.NewReader(out))
	for scanner.Scan() {
		line := strings.Fields(scanner.Text())
		if strings.HasPrefix(line[0], "/dev") {
			for i := range metrics {
				value, _ := strconv.Atoi(line[i+1])
				point, _ := client.NewPoint("host_disk",
					map[string]string{"hostname": hostname, "mount": line[5], "type": metrics[i]},
					map[string]interface{}{"value": value},
					time.Now())
				bp.AddPoint(point)
			}
		}
	}
	clnt.Write(bp)
}

func memStat(clnt client.Client, bp client.BatchPoints) {
	hostname, _ := os.Hostname()
	file, err := os.Open("/proc/meminfo")
	if err != nil {
		return
	}
	defer file.Close()
	scanner := bufio.NewScanner(bufio.NewReader(file))
	for scanner.Scan() {
		line := strings.Fields(strings.Replace(scanner.Text(), ":", "", -1))
		value, _ := strconv.ParseInt(line[1], 10, 62)
		if memory[line[0]] {
			point, _ := client.NewPoint("host_memory",
				map[string]string{"hostname": hostname, "type": line[0]},
				map[string]interface{}{"value": value * 1024},
				time.Now())
			bp.AddPoint(point)
		}
	}
	clnt.Write(bp)
}

func cpuStat(clnt client.Client, bp client.BatchPoints) {
	hostname, _ := os.Hostname()
	file, err := os.Open("/proc/stat")
	if err != nil {
		return
	}
	defer file.Close()
	scanner := bufio.NewScanner(bufio.NewReader(file))
	for scanner.Scan() {
		line := strings.Fields(scanner.Text())
		if line[0] == "cpu" {
			for i := range cpu {
				value, _ := strconv.Atoi(line[i+1])
				point, _ := client.NewPoint("host_cpu",
					map[string]string{"hostname": hostname, "type": cpu[i]},
					map[string]interface{}{"value": value / runtime.NumCPU()},
					time.Now())
				bp.AddPoint(point)
			}
		}
	}
	clnt.Write(bp)
}
<|endoftext|>


<commit_before>package agent

import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/mime"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/googleapi"
	storage "google.golang.org/api/storage/v1"
)

type GSUploader struct {
	// The destination which includes the GS bucket name and the path.
	// gs://my-bucket-name/foo/bar
	Destination string

	// Whether or not HTTP calls shoud be debugged
	DebugHTTP bool

	// The GS service
	Service *storage.Service
}

func (u *GSUploader) Setup(destination string, debugHTTP bool) error {
	u.Destination = destination
	u.DebugHTTP = debugHTTP

	client, err := u.getClient(storage.DevstorageFullControlScope)
	if err != nil {
		return errors.New(fmt.Sprintf("Error creating Google Cloud Storage client: %v", err))
	}
	service, err := storage.New(client)
	if err != nil {
		return err
	}
	u.Service = service

	return nil
}

func (u *GSUploader) URL(artifact *api.Artifact) string {
	// If not publicly accessible, GET on this URL results in 403.
	var artifactURL = &url.URL{
		Scheme: "https",
		Host:   "storage.googleapis.com",
		Path:   u.BucketName() + "/" + u.artifactPath(artifact),
	}
	return artifactURL.String()
}

func (u *GSUploader) Upload(artifact *api.Artifact) error {
	permission := os.Getenv("BUILDKITE_GS_ACL")

	// The dirtiest validation method ever...
	if permission != "" &&
		permission != "authenticatedRead" &&
		permission != "private" &&
		permission != "projectPrivate" &&
		permission != "publicRead" &&
		permission != "publicReadWrite" {
		logger.Fatal("Invalid GS ACL `%s`", permission)
	}

	if permission == "" {
		logger.Debug("Uploading \"%s\" to bucket \"%s\" with default permission",
			u.artifactPath(artifact), u.BucketName())
	} else {
		logger.Debug("Uploading \"%s\" to bucket \"%s\" with permission \"%s\"",
			u.artifactPath(artifact), u.BucketName(), permission)
	}
	object := &storage.Object{
		Name:        u.artifactPath(artifact),
		ContentType: u.mimeType(artifact),
	}
	file, err := os.Open(artifact.AbsolutePath)
	if err != nil {
		return errors.New(fmt.Sprintf("Failed to open file \"%q\" (%v)", artifact.AbsolutePath, err))
	}
	call := u.Service.Objects.Insert(u.BucketName(), object)
	if permission != "" {
		call = call.PredefinedAcl(permission)
	}
	if res, err := call.Media(file, googleapi.ContentType("")).Do(); err == nil {
		logger.Debug("Created object %v at location %v\n\n", res.Name, res.SelfLink)
	} else {
		return errors.New(fmt.Sprintf("Failed to PUT file \"%s\" (%v)", u.artifactPath(artifact), err))
	}

	return nil
}

func (u *GSUploader) artifactPath(artifact *api.Artifact) string {
	parts := []string{u.BucketPath(), artifact.Path}

	return strings.Join(parts, "/")
}

func (u *GSUploader) BucketPath() string {
	return strings.Join(u.destinationParts()[1:len(u.destinationParts())], "/")
}

func (u *GSUploader) BucketName() string {
	return u.destinationParts()[0]
}

func (u *GSUploader) destinationParts() []string {
	trimmed := strings.TrimPrefix(u.Destination, "gs://")

	return strings.Split(trimmed, "/")
}

func (u *GSUploader) getClient(scope string) (*http.Client, error) {
	if os.Getenv("BUILDKITE_GS_APPLICATION_CREDENTIALS") != "" {
		data, err := ioutil.ReadFile(os.Getenv("BUILDKITE_GS_APPLICATION_CREDENTIALS"))
		if err != nil {
			return nil, err
		}
		conf, err := google.JWTConfigFromJSON(data, scope)
		if err != nil {
			return nil, err
		}
		return conf.Client(oauth2.NoContext), nil
	}
	return google.DefaultClient(context.Background(), scope)
}

func (u *GSUploader) mimeType(a *api.Artifact) string {
	extension := filepath.Ext(a.Path)
	mimeType := mime.TypeByExtension(extension)

	if mimeType != "" {
		return mimeType
	} else {
		return "binary/octet-stream"
	}
}
<commit_msg>Add BUILDKITE_GCS_ACCESS_HOST for GCS Host choice<commit_after>package agent

import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/mime"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/googleapi"
	storage "google.golang.org/api/storage/v1"
)

type GSUploader struct {
	// The destination which includes the GS bucket name and the path.
	// gs://my-bucket-name/foo/bar
	Destination string

	// Whether or not HTTP calls shoud be debugged
	DebugHTTP bool

	// The GS service
	Service *storage.Service
}

func (u *GSUploader) Setup(destination string, debugHTTP bool) error {
	u.Destination = destination
	u.DebugHTTP = debugHTTP

	client, err := u.getClient(storage.DevstorageFullControlScope)
	if err != nil {
		return errors.New(fmt.Sprintf("Error creating Google Cloud Storage client: %v", err))
	}
	service, err := storage.New(client)
	if err != nil {
		return err
	}
	u.Service = service

	return nil
}

func (u *GSUploader) URL(artifact *api.Artifact) string {
	host := "storage.googleapis.com"
	if os.Getenv("BUILDKITE_GCS_ACCESS_HOST") != "" {
		host = os.Getenv("BUILDKITE_GCS_ACCESS_HOST")
	}

	var artifactURL = &url.URL{
		Scheme: "https",
		Host:   host,
		Path:   u.BucketName() + "/" + u.artifactPath(artifact),
	}
	return artifactURL.String()
}

func (u *GSUploader) Upload(artifact *api.Artifact) error {
	permission := os.Getenv("BUILDKITE_GS_ACL")

	// The dirtiest validation method ever...
	if permission != "" &&
		permission != "authenticatedRead" &&
		permission != "private" &&
		permission != "projectPrivate" &&
		permission != "publicRead" &&
		permission != "publicReadWrite" {
		logger.Fatal("Invalid GS ACL `%s`", permission)
	}

	if permission == "" {
		logger.Debug("Uploading \"%s\" to bucket \"%s\" with default permission",
			u.artifactPath(artifact), u.BucketName())
	} else {
		logger.Debug("Uploading \"%s\" to bucket \"%s\" with permission \"%s\"",
			u.artifactPath(artifact), u.BucketName(), permission)
	}
	object := &storage.Object{
		Name:        u.artifactPath(artifact),
		ContentType: u.mimeType(artifact),
	}
	file, err := os.Open(artifact.AbsolutePath)
	if err != nil {
		return errors.New(fmt.Sprintf("Failed to open file \"%q\" (%v)", artifact.AbsolutePath, err))
	}
	call := u.Service.Objects.Insert(u.BucketName(), object)
	if permission != "" {
		call = call.PredefinedAcl(permission)
	}
	if res, err := call.Media(file, googleapi.ContentType("")).Do(); err == nil {
		logger.Debug("Created object %v at location %v\n\n", res.Name, res.SelfLink)
	} else {
		return errors.New(fmt.Sprintf("Failed to PUT file \"%s\" (%v)", u.artifactPath(artifact), err))
	}

	return nil
}

func (u *GSUploader) artifactPath(artifact *api.Artifact) string {
	parts := []string{u.BucketPath(), artifact.Path}

	return strings.Join(parts, "/")
}

func (u *GSUploader) BucketPath() string {
	return strings.Join(u.destinationParts()[1:len(u.destinationParts())], "/")
}

func (u *GSUploader) BucketName() string {
	return u.destinationParts()[0]
}

func (u *GSUploader) destinationParts() []string {
	trimmed := strings.TrimPrefix(u.Destination, "gs://")

	return strings.Split(trimmed, "/")
}

func (u *GSUploader) getClient(scope string) (*http.Client, error) {
	if os.Getenv("BUILDKITE_GS_APPLICATION_CREDENTIALS") != "" {
		data, err := ioutil.ReadFile(os.Getenv("BUILDKITE_GS_APPLICATION_CREDENTIALS"))
		if err != nil {
			return nil, err
		}
		conf, err := google.JWTConfigFromJSON(data, scope)
		if err != nil {
			return nil, err
		}
		return conf.Client(oauth2.NoContext), nil
	}
	return google.DefaultClient(context.Background(), scope)
}

func (u *GSUploader) mimeType(a *api.Artifact) string {
	extension := filepath.Ext(a.Path)
	mimeType := mime.TypeByExtension(extension)

	if mimeType != "" {
		return mimeType
	} else {
		return "binary/octet-stream"
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os/exec"
	"regexp"
	"strings"
)

type messageBlock struct {
	Message   string `json:"message,omitempty"`
	From      string `json:"from,omitempty"`
	Room      string `json:"room,omitempty"`
	Mentioned bool   `json:"mentioned,omitempty"`
}

func (m *messageBlock) handleMessage(source string,
	dispatch chan<- *dispatcherRequest) {

	logger.Debug.Println("Message: ", m.Message)
	logger.Debug.Println("From: ", m.From)
	logger.Debug.Println("Room: ", m.Room)

	prefixMatch := false

	if len(m.Message) > conf.prefixLen &&
		m.Message[0:conf.prefixLen] == conf.Prefix {

		prefixMatch = true
	}

	if prefixMatch {
		trimmed := strings.TrimLeft(m.Message[conf.prefixLen:], " ")
		for _, pr := range prefixPResponders {
			if match, err := regexp.MatchString(pr.Match,
				trimmed); err == nil && match {

				logger.Debug.Println("Matched:", pr.Match)

				output, err := exec.Command(pr.Cmd, pr.Args...).Output()

				if err != nil {
					logger.Error.Println("Passive responder error:", err)
				} else {
					logger.Debug.Println("Passive responder executed:",
						string(output))

					dispatch <- &dispatcherRequest{
						Query: &query{
							Type:   "message",
							Source: "PR:" + pr.Match,
							To:     source,
							Message: &messageBlock{
								Message: strings.TrimRight(string(output), "\n"),
								Room:    m.Room,
							},
						},
					}
				}

				if !pr.FallThrough {
					break
				}
			}
		}
	} else {
		// for _, pr := range noPrefixPResponders {
		// }
	}
}
<commit_msg>move responder code into separate file<commit_after>package main

import (
	"strings"
)

type messageBlock struct {
	Message   string `json:"message,omitempty"`
	From      string `json:"from,omitempty"`
	Room      string `json:"room,omitempty"`
	Mentioned bool   `json:"mentioned,omitempty"`
	Mention   string `json:"mention,omitempty"`
}

func (m *messageBlock) handleMessage(source string,
	dispatch chan<- *dispatcherRequest) {

	logger.Debug.Println("Message: ", m.Message)
	logger.Debug.Println("From: ", m.From)
	logger.Debug.Println("Room: ", m.Room)

	prefixMatch := false

	if len(m.Message) > conf.prefixLen &&
		m.Message[0:conf.prefixLen] == conf.Prefix {

		prefixMatch = true
	}

	if prefixMatch {
		trimmed := strings.TrimLeft(m.Message[conf.prefixLen:], " ")
		triggerPassiveResponders(prefixPResponders, trimmed, source, m.Room,
			m.Mention, m.Mentioned, dispatch)
	} else {
		triggerPassiveResponders(noPrefixPResponders, m.Message, source, m.Room,
			m.Mention, m.Mentioned, dispatch)
	}
}
<|endoftext|>


<commit_before>package middleware

import (
	"bufio"
	"fmt"
	"net"
	"net/http"
)

// ResponseWriter is a wrapper around http.ResponseWriter that provides extra information about
// the response.
type ResponseWriter interface {
	http.ResponseWriter
	http.Flusher
	// Status returns the status code of the response or 0 if the response has not been written.
	Status() int
}

// NewResponseWriter creates a ResponseWriter that wraps an http.ResponseWriter
func NewResponseWriter(rw http.ResponseWriter) ResponseWriter {
	return &responseWriter{rw, 0}
}

type responseWriter struct {
	http.ResponseWriter
	status int
}

func (rw *responseWriter) WriteHeader(s int) {
	rw.status = s
	rw.ResponseWriter.WriteHeader(s)
}

func (rw *responseWriter) Status() int {
	return rw.status
}

func (rw *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	hijacker, ok := rw.ResponseWriter.(http.Hijacker)
	if !ok {
		return nil, nil, fmt.Errorf("the ResponseWriter doesn't support the Hijacker interface")
	}
	return hijacker.Hijack()
}

func (rw *responseWriter) CloseNotify() <-chan bool {
	return rw.ResponseWriter.(http.CloseNotifier).CloseNotify()
}

func (rw *responseWriter) Flush() {
	flusher, ok := rw.ResponseWriter.(http.Flusher)
	if ok {
		flusher.Flush()
	}
}
<commit_msg>Default to StatusOK for response writer<commit_after>package middleware

import (
	"bufio"
	"fmt"
	"net"
	"net/http"
)

// ResponseWriter is a wrapper around http.ResponseWriter that provides extra information about
// the response.
type ResponseWriter interface {
	http.ResponseWriter
	http.Flusher
	// Status returns the status code of the response or 200 if the response has not been written.
	Status() int
}

// NewResponseWriter creates a ResponseWriter that wraps an http.ResponseWriter
func NewResponseWriter(rw http.ResponseWriter) ResponseWriter {
	return &responseWriter{rw, http.StatusOK}
}

type responseWriter struct {
	http.ResponseWriter
	status int
}

func (rw *responseWriter) WriteHeader(s int) {
	rw.status = s
	rw.ResponseWriter.WriteHeader(s)
}

func (rw *responseWriter) Status() int {
	return rw.status
}

func (rw *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	hijacker, ok := rw.ResponseWriter.(http.Hijacker)
	if !ok {
		return nil, nil, fmt.Errorf("the ResponseWriter doesn't support the Hijacker interface")
	}
	return hijacker.Hijack()
}

func (rw *responseWriter) CloseNotify() <-chan bool {
	return rw.ResponseWriter.(http.CloseNotifier).CloseNotify()
}

func (rw *responseWriter) Flush() {
	flusher, ok := rw.ResponseWriter.(http.Flusher)
	if ok {
		flusher.Flush()
	}
}
<|endoftext|>


<commit_before>package cairo

// #include <stdlib.h>
// #include <cairo.h>
// #include <cairo-gobject.h>
// #include <cairo-pdf.h>
import "C"

import (
	"runtime"
	"unsafe"
)

// TODO(jrick) SetUserData (depends on UserDataKey and DestroyFunc)

// TODO(jrick) GetUserData (depends on UserDataKey)

/*
 * cairo_surface_t
 */

// Surface is a representation of Cairo's cairo_surface_t.
type Surface struct {
	surface *C.cairo_surface_t
}

func NewSurfaceFromPNG(fileName string) (*Surface, error) {

	cstr := C.CString(fileName)
	defer C.free(unsafe.Pointer(cstr))

	surfaceNative := C.cairo_image_surface_create_from_png(cstr)

	status := Status(C.cairo_surface_status(surfaceNative))
	if status != STATUS_SUCCESS {
		return nil, ErrorStatus(status)
	}

	return &Surface{surfaceNative}, nil
}

// CreateImageSurface is a wrapper around cairo_image_surface_create().
func CreateImageSurface(format Format, width, height int) *Surface {
	c := C.cairo_image_surface_create(C.cairo_format_t(format),
		C.int(width), C.int(height))
	s := wrapSurface(c)
	runtime.SetFinalizer(s, (*Surface).destroy)
	return s
}

/// Create a new PDF surface.
func CreatePDFSurface(fileName string, width float64, height float64) (*Surface, error) {
	cstr := C.CString(fileName)
	defer C.free(unsafe.Pointer(cstr))

	surfaceNative := C.cairo_pdf_surface_create(cstr, C.double(width), C.double(height))

	status := Status(C.cairo_surface_status(surfaceNative))
	if status != STATUS_SUCCESS {
		return nil, ErrorStatus(status)
	}

	s := wrapSurface(surfaceNative)

	runtime.SetFinalizer(s, (*Surface).destroy)

	return s, nil
}

// native returns a pointer to the underlying cairo_surface_t.
func (v *Surface) native() *C.cairo_surface_t {
	if v == nil {
		return nil
	}
	return v.surface
}

// Native returns a pointer to the underlying cairo_surface_t.
func (v *Surface) Native() uintptr {
	return uintptr(unsafe.Pointer(v.native()))
}

func marshalSurface(p uintptr) (interface{}, error) {
	c := C.g_value_get_boxed((*C.GValue)(unsafe.Pointer(p)))
	return WrapSurface(uintptr(c)), nil
}

func wrapSurface(surface *C.cairo_surface_t) *Surface {
	return &Surface{surface}
}

// NewSurface creates a gotk3 cairo Surface from a pointer to a
// C cairo_surface_t.  This is primarily designed for use with other
// gotk3 packages and should be avoided by applications.
func NewSurface(s uintptr, needsRef bool) *Surface {
	surface := WrapSurface(s)
	if needsRef {
		surface.reference()
	}
	runtime.SetFinalizer(surface, (*Surface).destroy)
	return surface
}

func WrapSurface(s uintptr) *Surface {
	ptr := (*C.cairo_surface_t)(unsafe.Pointer(s))
	return wrapSurface(ptr)
}

// Closes the surface. The surface must not be used afterwards.
func (v *Surface) Close() {
	v.destroy()
}

// CreateSimilar is a wrapper around cairo_surface_create_similar().
func (v *Surface) CreateSimilar(content Content, width, height int) *Surface {
	c := C.cairo_surface_create_similar(v.native(),
		C.cairo_content_t(content), C.int(width), C.int(height))
	s := wrapSurface(c)
	runtime.SetFinalizer(s, (*Surface).destroy)
	return s
}

// TODO cairo_surface_create_similar_image (since 1.12)

// CreateForRectangle is a wrapper around cairo_surface_create_for_rectangle().
func (v *Surface) CreateForRectangle(x, y, width, height float64) *Surface {
	c := C.cairo_surface_create_for_rectangle(v.native(), C.double(x),
		C.double(y), C.double(width), C.double(height))
	s := wrapSurface(c)
	runtime.SetFinalizer(s, (*Surface).destroy)
	return s
}

// reference is a wrapper around cairo_surface_reference().
func (v *Surface) reference() {
	v.surface = C.cairo_surface_reference(v.native())
}

// destroy is a wrapper around cairo_surface_destroy().
func (v *Surface) destroy() {
	if v.surface != nil {
		C.cairo_surface_destroy(v.native())
		v.surface = nil
	}
}

// Status is a wrapper around cairo_surface_status().
func (v *Surface) Status() Status {
	c := C.cairo_surface_status(v.native())
	return Status(c)
}

// Flush is a wrapper around cairo_surface_flush().
func (v *Surface) Flush() {
	C.cairo_surface_flush(v.native())
}

// TODO(jrick) GetDevice (requires Device bindings)

// TODO(jrick) GetFontOptions (require FontOptions bindings)

// TODO(jrick) GetContent (requires Content bindings)

// MarkDirty is a wrapper around cairo_surface_mark_dirty().
func (v *Surface) MarkDirty() {
	C.cairo_surface_mark_dirty(v.native())
}

// MarkDirtyRectangle is a wrapper around cairo_surface_mark_dirty_rectangle().
func (v *Surface) MarkDirtyRectangle(x, y, width, height int) {
	C.cairo_surface_mark_dirty_rectangle(v.native(), C.int(x), C.int(y),
		C.int(width), C.int(height))
}

// SetDeviceOffset is a wrapper around cairo_surface_set_device_offset().
func (v *Surface) SetDeviceOffset(x, y float64) {
	C.cairo_surface_set_device_offset(v.native(), C.double(x), C.double(y))
}

// GetDeviceOffset is a wrapper around cairo_surface_get_device_offset().
func (v *Surface) GetDeviceOffset() (x, y float64) {
	var xOffset, yOffset C.double
	C.cairo_surface_get_device_offset(v.native(), &xOffset, &yOffset)
	return float64(xOffset), float64(yOffset)
}

// SetFallbackResolution is a wrapper around
// cairo_surface_set_fallback_resolution().
func (v *Surface) SetFallbackResolution(xPPI, yPPI float64) {
	C.cairo_surface_set_fallback_resolution(v.native(), C.double(xPPI),
		C.double(yPPI))
}

// GetFallbackResolution is a wrapper around
// cairo_surface_get_fallback_resolution().
func (v *Surface) GetFallbackResolution() (xPPI, yPPI float64) {
	var x, y C.double
	C.cairo_surface_get_fallback_resolution(v.native(), &x, &y)
	return float64(x), float64(y)
}

// GetType is a wrapper around cairo_surface_get_type().
func (v *Surface) GetType() SurfaceType {
	c := C.cairo_surface_get_type(v.native())
	return SurfaceType(c)
}

// TODO(jrick) SetUserData (depends on UserDataKey and DestroyFunc)

// TODO(jrick) GetUserData (depends on UserDataKey)

// CopyPage is a wrapper around cairo_surface_copy_page().
func (v *Surface) CopyPage() {
	C.cairo_surface_copy_page(v.native())
}

// ShowPage is a wrapper around cairo_surface_show_page().
func (v *Surface) ShowPage() {
	C.cairo_surface_show_page(v.native())
}

// HasShowTextGlyphs is a wrapper around cairo_surface_has_show_text_glyphs().
func (v *Surface) HasShowTextGlyphs() bool {
	c := C.cairo_surface_has_show_text_glyphs(v.native())
	return gobool(c)
}

// TODO(jrick) SetMimeData (depends on DestroyFunc)

// GetMimeData is a wrapper around cairo_surface_get_mime_data().  The
// returned mimetype data is returned as a Go byte slice.
func (v *Surface) GetMimeData(mimeType MimeType) []byte {
	cstr := C.CString(string(mimeType))
	defer C.free(unsafe.Pointer(cstr))
	var data *C.uchar
	var length C.ulong
	C.cairo_surface_get_mime_data(v.native(), cstr, &data, &length)
	return C.GoBytes(unsafe.Pointer(data), C.int(length))
}

// WriteToPNG is a wrapper around cairo_surface_write_png(). It writes the Cairo
// surface to the given file in PNG format.
func (v *Surface) WriteToPNG(fileName string) error {
	cstr := C.CString(fileName)
	defer C.free(unsafe.Pointer(cstr))

	status := Status(C.cairo_surface_write_to_png(v.surface, cstr))

	if status != STATUS_SUCCESS {
		return ErrorStatus(status)
	}

	return nil
}

// TODO(jrick) SupportsMimeType (since 1.12)

// TODO(jrick) MapToImage (since 1.12)

// TODO(jrick) UnmapImage (since 1.12)

// GetHeight is a wrapper around cairo_image_surface_get_height().
func (v *Surface) GetHeight() int {
	return int(C.cairo_image_surface_get_height(v.surface))
}

// GetWidth is a wrapper around cairo_image_surface_get_width().
func (v *Surface) GetWidth() int {
	return int(C.cairo_image_surface_get_width(v.surface))
}

// GetData is a wrapper around cairo_image_surface_get_data().
func (v *Surface) GetData() unsafe.Pointer {
	return unsafe.Pointer(C.cairo_image_surface_get_data(v.surface))
}
<commit_msg>add a wrapper for cairo_image_surface_create_for_data<commit_after>package cairo

// #include <stdlib.h>
// #include <cairo.h>
// #include <cairo-gobject.h>
// #include <cairo-pdf.h>
import "C"

import (
	"runtime"
	"unsafe"
)

// TODO(jrick) SetUserData (depends on UserDataKey and DestroyFunc)

// TODO(jrick) GetUserData (depends on UserDataKey)

/*
 * cairo_surface_t
 */

// Surface is a representation of Cairo's cairo_surface_t.
type Surface struct {
	surface *C.cairo_surface_t
}

func NewSurfaceFromPNG(fileName string) (*Surface, error) {

	cstr := C.CString(fileName)
	defer C.free(unsafe.Pointer(cstr))

	surfaceNative := C.cairo_image_surface_create_from_png(cstr)

	status := Status(C.cairo_surface_status(surfaceNative))
	if status != STATUS_SUCCESS {
		return nil, ErrorStatus(status)
	}

	return &Surface{surfaceNative}, nil
}

// CreateImageSurfaceForData is a wrapper around cairo_image_surface_create_for_data().
func CreateImageSurfaceForData(data []byte, format Format, width, height, stride int) (*Surface, error) {
	surfaceNative := C.cairo_image_surface_create_for_data((*C.guchar)(unsafe.Pointer(&data[0])),
		C.cairo_format_t(format), C.int(width), C.int(height), C.int(stride))

	status := Status(C.cairo_surface_status(surfaceNative))
	if status != STATUS_SUCCESS {
		return nil, ErrorStatus(status)
	}

	s := wrapSurface(surfaceNative)

	runtime.SetFinalizer(s, (*Surface).destroy)

	return s, nil
}

// CreateImageSurface is a wrapper around cairo_image_surface_create().
func CreateImageSurface(format Format, width, height int) *Surface {
	c := C.cairo_image_surface_create(C.cairo_format_t(format),
		C.int(width), C.int(height))
	s := wrapSurface(c)
	runtime.SetFinalizer(s, (*Surface).destroy)
	return s
}

/// Create a new PDF surface.
func CreatePDFSurface(fileName string, width float64, height float64) (*Surface, error) {
	cstr := C.CString(fileName)
	defer C.free(unsafe.Pointer(cstr))

	surfaceNative := C.cairo_pdf_surface_create(cstr, C.double(width), C.double(height))

	status := Status(C.cairo_surface_status(surfaceNative))
	if status != STATUS_SUCCESS {
		return nil, ErrorStatus(status)
	}

	s := wrapSurface(surfaceNative)

	runtime.SetFinalizer(s, (*Surface).destroy)

	return s, nil
}

// native returns a pointer to the underlying cairo_surface_t.
func (v *Surface) native() *C.cairo_surface_t {
	if v == nil {
		return nil
	}
	return v.surface
}

// Native returns a pointer to the underlying cairo_surface_t.
func (v *Surface) Native() uintptr {
	return uintptr(unsafe.Pointer(v.native()))
}

func marshalSurface(p uintptr) (interface{}, error) {
	c := C.g_value_get_boxed((*C.GValue)(unsafe.Pointer(p)))
	return WrapSurface(uintptr(c)), nil
}

func wrapSurface(surface *C.cairo_surface_t) *Surface {
	return &Surface{surface}
}

// NewSurface creates a gotk3 cairo Surface from a pointer to a
// C cairo_surface_t.  This is primarily designed for use with other
// gotk3 packages and should be avoided by applications.
func NewSurface(s uintptr, needsRef bool) *Surface {
	surface := WrapSurface(s)
	if needsRef {
		surface.reference()
	}
	runtime.SetFinalizer(surface, (*Surface).destroy)
	return surface
}

func WrapSurface(s uintptr) *Surface {
	ptr := (*C.cairo_surface_t)(unsafe.Pointer(s))
	return wrapSurface(ptr)
}

// Closes the surface. The surface must not be used afterwards.
func (v *Surface) Close() {
	v.destroy()
}

// CreateSimilar is a wrapper around cairo_surface_create_similar().
func (v *Surface) CreateSimilar(content Content, width, height int) *Surface {
	c := C.cairo_surface_create_similar(v.native(),
		C.cairo_content_t(content), C.int(width), C.int(height))
	s := wrapSurface(c)
	runtime.SetFinalizer(s, (*Surface).destroy)
	return s
}

// TODO cairo_surface_create_similar_image (since 1.12)

// CreateForRectangle is a wrapper around cairo_surface_create_for_rectangle().
func (v *Surface) CreateForRectangle(x, y, width, height float64) *Surface {
	c := C.cairo_surface_create_for_rectangle(v.native(), C.double(x),
		C.double(y), C.double(width), C.double(height))
	s := wrapSurface(c)
	runtime.SetFinalizer(s, (*Surface).destroy)
	return s
}

// reference is a wrapper around cairo_surface_reference().
func (v *Surface) reference() {
	v.surface = C.cairo_surface_reference(v.native())
}

// destroy is a wrapper around cairo_surface_destroy().
func (v *Surface) destroy() {
	if v.surface != nil {
		C.cairo_surface_destroy(v.native())
		v.surface = nil
	}
}

// Status is a wrapper around cairo_surface_status().
func (v *Surface) Status() Status {
	c := C.cairo_surface_status(v.native())
	return Status(c)
}

// Flush is a wrapper around cairo_surface_flush().
func (v *Surface) Flush() {
	C.cairo_surface_flush(v.native())
}

// TODO(jrick) GetDevice (requires Device bindings)

// TODO(jrick) GetFontOptions (require FontOptions bindings)

// TODO(jrick) GetContent (requires Content bindings)

// MarkDirty is a wrapper around cairo_surface_mark_dirty().
func (v *Surface) MarkDirty() {
	C.cairo_surface_mark_dirty(v.native())
}

// MarkDirtyRectangle is a wrapper around cairo_surface_mark_dirty_rectangle().
func (v *Surface) MarkDirtyRectangle(x, y, width, height int) {
	C.cairo_surface_mark_dirty_rectangle(v.native(), C.int(x), C.int(y),
		C.int(width), C.int(height))
}

// SetDeviceOffset is a wrapper around cairo_surface_set_device_offset().
func (v *Surface) SetDeviceOffset(x, y float64) {
	C.cairo_surface_set_device_offset(v.native(), C.double(x), C.double(y))
}

// GetDeviceOffset is a wrapper around cairo_surface_get_device_offset().
func (v *Surface) GetDeviceOffset() (x, y float64) {
	var xOffset, yOffset C.double
	C.cairo_surface_get_device_offset(v.native(), &xOffset, &yOffset)
	return float64(xOffset), float64(yOffset)
}

// SetFallbackResolution is a wrapper around
// cairo_surface_set_fallback_resolution().
func (v *Surface) SetFallbackResolution(xPPI, yPPI float64) {
	C.cairo_surface_set_fallback_resolution(v.native(), C.double(xPPI),
		C.double(yPPI))
}

// GetFallbackResolution is a wrapper around
// cairo_surface_get_fallback_resolution().
func (v *Surface) GetFallbackResolution() (xPPI, yPPI float64) {
	var x, y C.double
	C.cairo_surface_get_fallback_resolution(v.native(), &x, &y)
	return float64(x), float64(y)
}

// GetType is a wrapper around cairo_surface_get_type().
func (v *Surface) GetType() SurfaceType {
	c := C.cairo_surface_get_type(v.native())
	return SurfaceType(c)
}

// TODO(jrick) SetUserData (depends on UserDataKey and DestroyFunc)

// TODO(jrick) GetUserData (depends on UserDataKey)

// CopyPage is a wrapper around cairo_surface_copy_page().
func (v *Surface) CopyPage() {
	C.cairo_surface_copy_page(v.native())
}

// ShowPage is a wrapper around cairo_surface_show_page().
func (v *Surface) ShowPage() {
	C.cairo_surface_show_page(v.native())
}

// HasShowTextGlyphs is a wrapper around cairo_surface_has_show_text_glyphs().
func (v *Surface) HasShowTextGlyphs() bool {
	c := C.cairo_surface_has_show_text_glyphs(v.native())
	return gobool(c)
}

// TODO(jrick) SetMimeData (depends on DestroyFunc)

// GetMimeData is a wrapper around cairo_surface_get_mime_data().  The
// returned mimetype data is returned as a Go byte slice.
func (v *Surface) GetMimeData(mimeType MimeType) []byte {
	cstr := C.CString(string(mimeType))
	defer C.free(unsafe.Pointer(cstr))
	var data *C.uchar
	var length C.ulong
	C.cairo_surface_get_mime_data(v.native(), cstr, &data, &length)
	return C.GoBytes(unsafe.Pointer(data), C.int(length))
}

// WriteToPNG is a wrapper around cairo_surface_write_png(). It writes the Cairo
// surface to the given file in PNG format.
func (v *Surface) WriteToPNG(fileName string) error {
	cstr := C.CString(fileName)
	defer C.free(unsafe.Pointer(cstr))

	status := Status(C.cairo_surface_write_to_png(v.surface, cstr))

	if status != STATUS_SUCCESS {
		return ErrorStatus(status)
	}

	return nil
}

// TODO(jrick) SupportsMimeType (since 1.12)

// TODO(jrick) MapToImage (since 1.12)

// TODO(jrick) UnmapImage (since 1.12)

// GetHeight is a wrapper around cairo_image_surface_get_height().
func (v *Surface) GetHeight() int {
	return int(C.cairo_image_surface_get_height(v.surface))
}

// GetWidth is a wrapper around cairo_image_surface_get_width().
func (v *Surface) GetWidth() int {
	return int(C.cairo_image_surface_get_width(v.surface))
}

// GetData is a wrapper around cairo_image_surface_get_data().
func (v *Surface) GetData() unsafe.Pointer {
	return unsafe.Pointer(C.cairo_image_surface_get_data(v.surface))
}
<|endoftext|>


<commit_before>package driver

import (
	//gonet "net"
	//"strconv"
	//"errors"
	//"strings"
	"net"
	"fmt"
	"crypto/rand"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/go-plugins-helpers/ipam"
	"github.com/vishvananda/netlink"
	"github.com/llimllib/ipaddress"

	//"golang.org/x/net/context"
	//dockerclient "github.com/docker/engine-api/client"
	//dockertypes "github.com/docker/engine-api/types"
)

type Driver struct {
	ipam.Ipam
}

func NewDriver() (*Driver, error) {
	log.Debugf("NewDriver")
	d := &Driver{}
	return d, nil
}

func (d *Driver) GetCapabilities() (*ipam.CapabilitiesResponse, error) {
	log.Debugf("GetCapabilities")
	return &ipam.CapabilitiesResponse{
		RequiresMACAddress: false,
	}, nil
}

func (d *Driver) GetDefaultAddressSpaces() (*ipam.AddressSpacesResponse, error) {
	log.Debugf("GetDefaultAddressSpaces")
	return &ipam.AddressSpacesResponse{
		LocalDefaultAddressSpace: "arp-ipam-default",
		GlobalDefaultAddressSpace: "arp-ipam-default",
	}, nil
}

func (d *Driver) RequestPool(r *ipam.RequestPoolRequest) (*ipam.RequestPoolResponse, error) {
	log.Debugf("RequestPool: %v", r)
	if r.Pool == "" {
		log.Errorf("Automatic pool assignment not supported")
		return nil, fmt.Errorf("Automatic pool assignment not supported")
	}
	if r.V6 {
		log.Errorf("Automatic V6 pool assignment not supported.")
		return nil, fmt.Errorf("Automatic V6 pool assignment not supported.")
	}
	if r.SubPool != "" {
		log.Errorf("SubPool not supported.")
		return nil, fmt.Errorf("SubPool not supported.")
	}
	return &ipam.RequestPoolResponse{
		PoolID: r.Pool,
		Pool: r.Pool,
	}, nil
}

func (d *Driver) ReleasePool(r *ipam.ReleasePoolRequest) error {
	log.Debugf("ReleasePool: %v", r)
	return nil
}

func (d *Driver) RequestAddress(r *ipam.RequestAddressRequest) (*ipam.RequestAddressResponse, error) {
	log.Debugf("RequestAddress: %v", r)

	n, err := netlink.ParseIPNet(r.PoolID)
	if err != nil {
		log.Errorf("Unable to parse PoolID: %v", r.PoolID)
		log.Errorf("err: %v", err)
		return nil, err
	}

	res := &ipam.RequestAddressResponse{}

	//FIXME checkneigh

	if r.Address != "" {
		addr := net.ParseIP(r.Address)
		if addr == nil {
			log.Errorf("Unable to parse address: %v", r.Address)
			return nil, fmt.Errorf("Unable to parse address: %v", r.Address)
		}

		check, err := tryAddress(&addr)
		if err != nil {
			log.Errorf("err: ", err)
			return nil, err
		}

		if check {
			log.Errorf("Address already in use: %v", addr)
			return nil, fmt.Errorf("Address already in use: %v", addr)
		}
		ret_addr := net.IPNet{ IP: addr, Mask: n.Mask }
		res.Address = ret_addr.String()
		return res, nil
	}

	triedAddresses := make(map[string]struct{})
	var e struct{}
	triedAddresses[string(ipaddress.LastAddress(n))] = e
	triedAddresses[string(n.IP)] = e
	ones, maskSize := n.Mask.Size()
	var totalAddresses int
	totalAddresses = 1<<uint8(maskSize - ones)
	for len(triedAddresses) < totalAddresses {
		try, err := randAddr(n)
		if err != nil {
			log.Errorf("Error generating random address: %v", err)
			return nil, err
		}
		if _, ok := triedAddresses[string(try)]; ok { continue }

		check, err := tryAddress(&try)
		if err != nil {
			log.Errorf("err: ", err)
			return nil, err
		}

		if !check {
			ret_addr := net.IPNet{ IP: try, Mask: n.Mask }
			res.Address = ret_addr.String()
			return res, nil
		}
		triedAddresses[string(try)] = e
	}

	log.Errorf("All avaliable addresses are in use")
	return nil, fmt.Errorf("All avaliable addresses are in use")
}

func randAddr(n *net.IPNet) (net.IP, error) {
	// ip & (mask | random) should generate a random ip
	ip := n.IP.To4()
	if ip != nil {
		rand_bytes := make([]byte, 4)
		_, err := rand.Read(rand_bytes)
		if err != nil {
			return nil, err
		}
		return net.IP{
			ip[0] & (n.Mask[0] | rand_bytes[0]),
			ip[1] & (n.Mask[1] | rand_bytes[1]),
			ip[2] & (n.Mask[2] | rand_bytes[2]),
			ip[3] & (n.Mask[3] | rand_bytes[3]),
		}, nil
	}

	rand_bytes := make([]byte, 16)
	_, err := rand.Read(rand_bytes)
	if err != nil {
		return nil, err
	}
	return net.IP{
		ip[0] & (n.Mask[0] | rand_bytes[0]),
		ip[1] & (n.Mask[1] | rand_bytes[1]),
		ip[2] & (n.Mask[2] | rand_bytes[2]),
		ip[3] & (n.Mask[3] | rand_bytes[3]),
		ip[4] & (n.Mask[4] | rand_bytes[4]),
		ip[5] & (n.Mask[5] | rand_bytes[5]),
		ip[6] & (n.Mask[6] | rand_bytes[6]),
		ip[7] & (n.Mask[7] | rand_bytes[7]),
		ip[8] & (n.Mask[8] | rand_bytes[8]),
		ip[9] & (n.Mask[9] | rand_bytes[9]),
		ip[10] & (n.Mask[10] | rand_bytes[10]),
		ip[11] & (n.Mask[11] | rand_bytes[11]),
		ip[12] & (n.Mask[12] | rand_bytes[12]),
		ip[13] & (n.Mask[13] | rand_bytes[13]),
		ip[14] & (n.Mask[14] | rand_bytes[14]),
		ip[15] & (n.Mask[15] | rand_bytes[15]),
	}, nil
}

func tryAddress(ip *net.IP) (bool, error) {
	firstcheck, err := checkNeigh(ip)
	if err != nil {
		return true, err
	}
	if firstcheck {
		return firstcheck, err
	}
	err = probe(ip)
	if err != nil {
		return true, err
	}
	return checkNeigh(ip)
}

func probe(ip *net.IP) (error) {
	conn, err := net.Dial("udp", ip.String() + ":8765")
	defer conn.Close()
	if err != nil {
		return err
	}
	conn.Write([]byte("probe"))
	return nil
}

// Check neighbor table for IP. Return true if the address is in the neighbor cache
func checkNeigh(ip *net.IP) (bool, error) {
	NeighLoop:
		for {
			var neighs []netlink.Neigh
			var err error
			if ip.To4() != nil {
				neighs, err = netlink.NeighList(0, netlink.FAMILY_V4)
			} else {
				neighs, err = netlink.NeighList(0, netlink.FAMILY_V6)
			}
			if err != nil {
				log.Errorf("Error getting ip neighbors")
				return true, err
			}

			for _, neigh := range neighs {
				if &neigh.IP == ip {
					if neigh.HardwareAddr != nil {
						return true, nil
					}
					if neigh.State == netlink.NUD_INCOMPLETE {
						// Break and try again, the kernel is still trying to resolve
						continue NeighLoop
					}
					return false, nil
				}
			}
			return false, nil
		}
}

func (d *Driver) ReleaseAddress(r *ipam.ReleaseAddressRequest) error {
	log.Debugf("ReleaseAddress: %v", r)
	return nil
}
<commit_msg>impliment own logic for lastaddress and fix random address logic<commit_after>package driver

import (
	//gonet "net"
	//"strconv"
	//"errors"
	//"strings"
	"net"
	"fmt"
	"crypto/rand"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/go-plugins-helpers/ipam"
	"github.com/vishvananda/netlink"

	//"golang.org/x/net/context"
	//dockerclient "github.com/docker/engine-api/client"
	//dockertypes "github.com/docker/engine-api/types"
)

type Driver struct {
	ipam.Ipam
}

func NewDriver() (*Driver, error) {
	log.Debugf("NewDriver")
	d := &Driver{}
	return d, nil
}

func (d *Driver) GetCapabilities() (*ipam.CapabilitiesResponse, error) {
	log.Debugf("GetCapabilities")
	return &ipam.CapabilitiesResponse{
		RequiresMACAddress: false,
	}, nil
}

func (d *Driver) GetDefaultAddressSpaces() (*ipam.AddressSpacesResponse, error) {
	log.Debugf("GetDefaultAddressSpaces")
	return &ipam.AddressSpacesResponse{
		LocalDefaultAddressSpace: "arp-ipam-default",
		GlobalDefaultAddressSpace: "arp-ipam-default",
	}, nil
}

func (d *Driver) RequestPool(r *ipam.RequestPoolRequest) (*ipam.RequestPoolResponse, error) {
	log.Debugf("RequestPool: %v", r)
	if r.Pool == "" {
		log.Errorf("Automatic pool assignment not supported")
		return nil, fmt.Errorf("Automatic pool assignment not supported")
	}
	if r.V6 {
		log.Errorf("Automatic V6 pool assignment not supported.")
		return nil, fmt.Errorf("Automatic V6 pool assignment not supported.")
	}
	if r.SubPool != "" {
		log.Errorf("SubPool not supported.")
		return nil, fmt.Errorf("SubPool not supported.")
	}
	return &ipam.RequestPoolResponse{
		PoolID: r.Pool,
		Pool: r.Pool,
	}, nil
}

func (d *Driver) ReleasePool(r *ipam.ReleasePoolRequest) error {
	log.Debugf("ReleasePool: %v", r)
	return nil
}

func (d *Driver) RequestAddress(r *ipam.RequestAddressRequest) (*ipam.RequestAddressResponse, error) {
	log.Debugf("RequestAddress: %v", r)

	n, err := netlink.ParseIPNet(r.PoolID)
	if err != nil {
		log.Errorf("Unable to parse PoolID: %v", r.PoolID)
		log.Errorf("err: %v", err)
		return nil, err
	}

	res := &ipam.RequestAddressResponse{}

	//FIXME checkneigh

	if r.Address != "" {
		log.Debugf("Specific Address Requested: %v", r.Address)
		addr := net.ParseIP(r.Address)
		if addr == nil {
			log.Errorf("Unable to parse address: %v", r.Address)
			return nil, fmt.Errorf("Unable to parse address: %v", r.Address)
		}

		check, err := tryAddress(&addr)
		if err != nil {
			log.Errorf("err: ", err)
			return nil, err
		}

		if check {
			log.Errorf("Address already in use: %v", addr)
			return nil, fmt.Errorf("Address already in use: %v", addr)
		}
		ret_addr := net.IPNet{ IP: addr, Mask: n.Mask }
		res.Address = ret_addr.String()
		return res, nil
	}

	log.Debugf("Random Address Requested in network %v", n)
	triedAddresses := make(map[string]struct{})
	var e struct{}
	lastAddress := lastAddr(n)
	log.Debugf("Excluding last address: %v", lastAddress)
	triedAddresses[string(lastAddress)] = e
	firstAddress := firstAddr(n)
	log.Debugf("Excluding network address: %v", firstAddress)
	triedAddresses[string(firstAddress)] = e
	ones, maskSize := n.Mask.Size()
	var totalAddresses int
	totalAddresses = 1<<uint8(maskSize - ones)
	log.Debugf("Address avaliable to try: %v", totalAddresses)
	for len(triedAddresses) < totalAddresses {
		try, err := randAddr(n)
		log.Debugf("Trying random address: %v", try)
		if err != nil {
			log.Errorf("Error generating random address: %v", err)
			return nil, err
		}
		log.Debugf("Checking triedAddresses: %v", triedAddresses)
		if _, ok := triedAddresses[string(try)]; ok {
			log.Debugf("Address already tried: %v", try)
			continue
		}

		check, err := tryAddress(&try)
		if err != nil {
			log.Errorf("err: ", err)
			return nil, err
		}

		if !check {
			log.Debugf("Returning address: %v", try)
			ret_addr := net.IPNet{ IP: try, Mask: n.Mask }
			res.Address = ret_addr.String()
			return res, nil
		}
		triedAddresses[string(try)] = e
		log.Debugf("Address in use: %v", try)
	}

	log.Errorf("All avaliable addresses are in use")
	return nil, fmt.Errorf("All avaliable addresses are in use")
}

func lastAddr(n *net.IPNet) net.IP {
	ip := n.IP.To4()
	if ip != nil {
		log.Debugf("Type IPv4")
		return net.IP{
			ip[0] | ^n.Mask[0],
			ip[1] | ^n.Mask[1],
			ip[2] | ^n.Mask[2],
			ip[3] | ^n.Mask[3],
		}
	}

	return net.IP{
		ip[0] | ^n.Mask[0],
		ip[1] | ^n.Mask[1],
		ip[2] | ^n.Mask[2],
		ip[3] | ^n.Mask[3],
		ip[4] | ^n.Mask[4],
		ip[5] | ^n.Mask[5],
		ip[6] | ^n.Mask[6],
		ip[7] | ^n.Mask[7],
		ip[8] | ^n.Mask[8],
		ip[9] | ^n.Mask[9],
		ip[10] | ^n.Mask[10],
		ip[11] | ^n.Mask[11],
		ip[12] | ^n.Mask[12],
		ip[13] | ^n.Mask[13],
		ip[14] | ^n.Mask[14],
		ip[15] | ^n.Mask[15],
	}
}

func firstAddr(n *net.IPNet) net.IP {
	ip := n.IP.To4()
	if ip != nil {
		log.Debugf("Type IPv4")
		return net.IP{
			ip[0] & n.Mask[0],
			ip[1] & n.Mask[1],
			ip[2] & n.Mask[2],
			ip[3] & n.Mask[3],
		}
	}

	return net.IP{
		ip[0] & n.Mask[0],
		ip[1] & n.Mask[1],
		ip[2] & n.Mask[2],
		ip[3] & n.Mask[3],
		ip[4] & n.Mask[4],
		ip[5] & n.Mask[5],
		ip[6] & n.Mask[6],
		ip[7] & n.Mask[7],
		ip[8] & n.Mask[8],
		ip[9] & n.Mask[9],
		ip[10] & n.Mask[10],
		ip[11] & n.Mask[11],
		ip[12] & n.Mask[12],
		ip[13] & n.Mask[13],
		ip[14] & n.Mask[14],
		ip[15] & n.Mask[15],
	}
}

func randAddr(n *net.IPNet) (net.IP, error) {
	// ip & (mask | random) should generate a random ip
	log.Debugf("Generating random address for network: %v", n)
	ip := n.IP.To4()
	if ip != nil {
		log.Debugf("Type IPv4")
		rand_bytes := make([]byte, 4)
		_, err := rand.Read(rand_bytes)
		log.Debugf("Random bytes: %v", rand_bytes)
		if err != nil {
			return nil, err
		}
		return net.IP{
			ip[0] | (^n.Mask[0] & rand_bytes[0]),
			ip[1] | (^n.Mask[1] & rand_bytes[1]),
			ip[2] | (^n.Mask[2] & rand_bytes[2]),
			ip[3] | (^n.Mask[3] & rand_bytes[3]),
		}, nil
	}

	rand_bytes := make([]byte, 16)
	_, err := rand.Read(rand_bytes)
	if err != nil {
		return nil, err
	}
	return net.IP{
		ip[0] | (^n.Mask[0] & rand_bytes[0]),
		ip[1] | (^n.Mask[1] & rand_bytes[1]),
		ip[2] | (^n.Mask[2] & rand_bytes[2]),
		ip[3] | (^n.Mask[3] & rand_bytes[3]),
		ip[4] | (^n.Mask[4] & rand_bytes[4]),
		ip[5] | (^n.Mask[5] & rand_bytes[5]),
		ip[6] | (^n.Mask[6] & rand_bytes[6]),
		ip[7] | (^n.Mask[7] & rand_bytes[7]),
		ip[8] | (^n.Mask[8] & rand_bytes[8]),
		ip[9] | (^n.Mask[9] & rand_bytes[9]),
		ip[10] | (^n.Mask[10] & rand_bytes[10]),
		ip[11] | (^n.Mask[11] & rand_bytes[11]),
		ip[12] | (^n.Mask[12] & rand_bytes[12]),
		ip[13] | (^n.Mask[13] & rand_bytes[13]),
		ip[14] | (^n.Mask[14] & rand_bytes[14]),
		ip[15] | (^n.Mask[15] & rand_bytes[15]),
	}, nil
}

func tryAddress(ip *net.IP) (bool, error) {
	firstcheck, err := checkNeigh(ip)
	if err != nil {
		return true, err
	}
	if firstcheck {
		return firstcheck, err
	}
	err = probe(ip)
	if err != nil {
		return true, err
	}
	return checkNeigh(ip)
}

func probe(ip *net.IP) (error) {
	conn, err := net.Dial("udp", ip.String() + ":8765")
	defer conn.Close()
	if err != nil {
		return err
	}
	conn.Write([]byte("probe"))
	return nil
}

// Check neighbor table for IP. Return true if the address is in the neighbor cache
func checkNeigh(ip *net.IP) (bool, error) {
	NeighLoop:
		for {
			var neighs []netlink.Neigh
			var err error
			if ip.To4() != nil {
				neighs, err = netlink.NeighList(0, netlink.FAMILY_V4)
			} else {
				neighs, err = netlink.NeighList(0, netlink.FAMILY_V6)
			}
			if err != nil {
				log.Errorf("Error getting ip neighbors")
				return true, err
			}

			for _, neigh := range neighs {
				if &neigh.IP == ip {
					if neigh.HardwareAddr != nil {
						return true, nil
					}
					if neigh.State == netlink.NUD_INCOMPLETE {
						// Break and try again, the kernel is still trying to resolve
						continue NeighLoop
					}
					return false, nil
				}
			}
			return false, nil
		}
}

func (d *Driver) ReleaseAddress(r *ipam.ReleaseAddressRequest) error {
	log.Debugf("ReleaseAddress: %v", r)
	return nil
}
<|endoftext|>


<commit_before>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2013 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
    "testing"
    "database/sql"
)

func TestConnect(t *testing.T) {
    conn, err := sql.Open("firebirdsql", "sysdba:masterkey@localhost:3050/tmp/go_test.fdb")
    if err != nil {
        t.Fatalf("Error connecting: %v", err)
    }
    var sql string
    var n int

    sql = "SELECT Count(*) FROM rdb$relations where rdb$relation_name='FOO'"
    err = conn.QueryRow(sql).Scan(&n)
    if err != nil {
        t.Fatalf("Error QueryRow: %v", err)
    }
    if n > 0 {
        conn.Exec("DROP TABLE foo")
    }

    sql = `
        CREATE TABLE foo (
            a INTEGER NOT NULL,
            b VARCHAR(30) NOT NULL UNIQUE,
            c VARCHAR(1024),
            d DECIMAL(16,3) DEFAULT -0.123,
            e DATE DEFAULT '1967-08-11',
            f TIMESTAMP DEFAULT '1967-08-11 23:45:01',
            g TIME DEFAULT '23:45:01',
            h BLOB SUB_TYPE 1,
            i DOUBLE PRECISION DEFAULT 0.0,
            j FLOAT DEFAULT 0.0,
            PRIMARY KEY (a),
            CONSTRAINT CHECK_A CHECK (a <> 0)
        )
    `
    conn.Exec(sql)
    // 3 records insert
    conn.Exec("insert into foo(a, b, c,h) values (1, 'a', 'b','This is a memo')")
    conn.Exec("insert into foo(a, b, c, e, g, i, j) values (2, 'A', 'B', '1999-01-25', '00:00:01', 0.1, 0.1)")
    conn.Exec("insert into foo(a, b, c, e, g, i, j) values (3, 'X', 'Y', '2001-07-05', '00:01:02', 0.2, 0.2)")

    err = conn.QueryRow("select count(*) cnt from foo").Scan(&n)
    if err != nil {
        t.Fatalf("Error QueryRow: %v", err)
    }
    if n != 3 {
        t.Fatalf("Error bad record count: %v", n)
    }

    var name string
    err = conn.QueryRow("select b from foo where a=1").Scan(&name)

    defer conn.Close()

}
<commit_msg>fetch all record test (hung up)<commit_after>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2013 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
    "fmt"
    "testing"
    "database/sql"
)

func TestConnect(t *testing.T) {
    conn, err := sql.Open("firebirdsql", "sysdba:masterkey@localhost:3050/tmp/go_test.fdb")
    if err != nil {
        t.Fatalf("Error connecting: %v", err)
    }
    var sql string
    var n int

    sql = "SELECT Count(*) FROM rdb$relations where rdb$relation_name='FOO'"
    err = conn.QueryRow(sql).Scan(&n)
    if err != nil {
        t.Fatalf("Error QueryRow: %v", err)
    }
    if n > 0 {
        conn.Exec("DROP TABLE foo")
    }

    sql = `
        CREATE TABLE foo (
            a INTEGER NOT NULL,
            b VARCHAR(30) NOT NULL UNIQUE,
            c VARCHAR(1024),
            d DECIMAL(16,3) DEFAULT -0.123,
            e DATE DEFAULT '1967-08-11',
            f TIMESTAMP DEFAULT '1967-08-11 23:45:01',
            g TIME DEFAULT '23:45:01',
            h BLOB SUB_TYPE 1,
            i DOUBLE PRECISION DEFAULT 0.0,
            j FLOAT DEFAULT 0.0,
            PRIMARY KEY (a),
            CONSTRAINT CHECK_A CHECK (a <> 0)
        )
    `
    conn.Exec(sql)
    // 3 records insert
    conn.Exec("insert into foo(a, b, c,h) values (1, 'a', 'b','This is a memo')")
    conn.Exec("insert into foo(a, b, c, e, g, i, j) values (2, 'A', 'B', '1999-01-25', '00:00:01', 0.1, 0.1)")
    conn.Exec("insert into foo(a, b, c, e, g, i, j) values (3, 'X', 'Y', '2001-07-05', '00:01:02', 0.2, 0.2)")

    err = conn.QueryRow("select count(*) cnt from foo").Scan(&n)
    if err != nil {
        t.Fatalf("Error QueryRow: %v", err)
    }
    if n != 3 {
        t.Fatalf("Error bad record count: %v", n)
    }

    rows, err := conn.Query("select a, b, c, i, j from foo")
    fmt.Println("a")
    for rows.Next() {
        fmt.Println("b")
    }

    defer conn.Close()

}
<|endoftext|>


<commit_before>package store

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/sethgrid/pester"
	log "github.com/sirupsen/logrus"
)

const DefaultHttpTries = 7 // ~2min total of trying with exponential backoff (0 and 1 both mean 1 try total)

func MakePesterClient() *pester.Client {
	client := pester.New()
	client.Backoff = pester.ExponentialBackoff
	client.MaxRetries = DefaultHttpTries
	client.LogHook = func(e pester.ErrEntry) {
		log.Infof("Retrying after failed attempt: %+v", e)
	}
	return client
}

func MakeHTTPStore(rootURI string) Store {
	return MakeCustomHTTPStore(rootURI, MakePesterClient(), &TTLConfig{TTL: DefaultTTL, TTLKey: DefaultTTLKey, TTLFormat: DefaultTTLFormat})
}

func MakeCustomHTTPStore(rootURI string, client Client, ttlc *TTLConfig) Store {
	if !strings.HasSuffix(rootURI, "/") {
		rootURI = rootURI + "/"
	}
	if ttlc == nil {
		ttlc = &TTLConfig{TTLKey: DefaultTTLKey, TTLFormat: DefaultTTLFormat}
	}
	log.Infof("Making new HTTP Store with root URI: %s", rootURI)
	return &httpStore{rootURI, client, *ttlc}
}

type Client interface {
	Do(req *http.Request) (resp *http.Response, err error)
}

type httpStore struct {
	rootURI string
	client  Client
	ttlc    TTLConfig
}

func (s *httpStore) getTTLValue(resp *http.Response) *TTLValue {
	expire := resp.Header.Get(s.ttlc.TTLKey)
	ttl, err := time.Parse(s.ttlc.TTLFormat, expire)
	if err != nil {
		return nil
	}
	return &TTLValue{TTL: ttl, TTLKey: s.ttlc.TTLKey}
}

func (s *httpStore) OpenForRead(name string) (*Resource, error) {
	return s.openForRead(name, false)
}

func (s *httpStore) openForRead(name string, existCheck bool) (*Resource, error) {
	label := "Read"
	if existCheck {
		label = "Exist"
	}
	uri := s.rootURI + name
	log.Infof("%sing %s", label, uri)

	var req *http.Request
	if existCheck {
		req, _ = http.NewRequest("HEAD", uri, nil)
	} else {
		req, _ = http.NewRequest("GET", uri, nil)
	}

	resp, err := s.client.Do(req)
	if err != nil {
		if !existCheck {
			log.Infof("%s error: %s %v", label, uri, err)
		}
		return nil, err
	}

	if resp.StatusCode == http.StatusOK {
		log.Infof("%s result %s %v", label, uri, resp.StatusCode)
		ttlv := s.getTTLValue(resp)
		var rc io.ReadCloser
		if existCheck {
			rc = ioutil.NopCloser(resp.Body)
		} else {
			rc = resp.Body
		}
		return NewResource(rc, resp.ContentLength, ttlv), nil
	}
	log.Infof("%s response status error: %s %v", label, uri, resp.Status)
	if !existCheck {
		resp.Body.Close()
	}
	if resp.StatusCode == http.StatusNotFound {
		return nil, os.ErrNotExist
	} else if resp.StatusCode == http.StatusBadRequest {
		return nil, os.ErrInvalid
	}
	return nil, fmt.Errorf("could not open: %+v", resp)
}

func (s *httpStore) Exists(name string) (bool, error) {
	r, err := s.openForRead(name, true)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		log.Infof("Exists error: %s %v", name, err)
		return false, err
	}
	log.Infof("Exists ok: %s", name)
	r.Close()
	if r.TTLValue != nil && r.TTLValue.TTL.Before(time.Now()) {
		return false, nil
	}
	return true, nil
}

func (s *httpStore) Write(name string, resource *Resource) error {
	if resource == nil {
		log.Info("Writing nil resource is a no op.")
		return nil
	}
	if strings.Contains(name, "/") {
		log.Infof("Write error: %s '/' not allowed", name)
		return errors.New("'/' not allowed in name when writing bundles.")
	}
	uri := s.rootURI + name

	post := func() (*http.Response, error) {
		req, err := http.NewRequest("POST", uri, resource)
		if err != nil {
			return nil, err
		}
		req.ContentLength = resource.Length
		req.Header.Set("Content-Type", "text/plain")
		ttl := resource.TTLValue
		if ttl == nil {
			ttl = &TTLValue{TTL: time.Now().Add(s.ttlc.TTL), TTLKey: s.ttlc.TTLKey}
		}
		if ttl.TTLKey != "" {
			req.Header[ttl.TTLKey] = []string{ttl.TTL.Format(s.ttlc.TTLFormat)}
		}
		log.Infof("Writing %s: length: %d header: %v", uri, req.ContentLength, req.Header)
		return s.client.Do(req)
	}

	resp, err := post()
	if err != nil {
		log.Infof("Write error: %s %v", uri, err)
	} else {
		resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			log.Infof("Write response status error: %s -- %s", uri, resp.Status)
			return errors.New(resp.Status)
		}
	}
	return err
}

func (s *httpStore) Root() string {
	return s.rootURI
}
<commit_msg>Make retry logs errors (#561)<commit_after>package store

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/sethgrid/pester"
	log "github.com/sirupsen/logrus"
)

const DefaultHttpTries = 7 // ~2min total of trying with exponential backoff (0 and 1 both mean 1 try total)

func MakePesterClient() *pester.Client {
	client := pester.New()
	client.Backoff = pester.ExponentialBackoff
	client.MaxRetries = DefaultHttpTries
	client.LogHook = func(e pester.ErrEntry) {
		log.Errorf("Retrying after failed attempt: %+v", e)
	}
	return client
}

func MakeHTTPStore(rootURI string) Store {
	return MakeCustomHTTPStore(rootURI, MakePesterClient(), &TTLConfig{TTL: DefaultTTL, TTLKey: DefaultTTLKey, TTLFormat: DefaultTTLFormat})
}

func MakeCustomHTTPStore(rootURI string, client Client, ttlc *TTLConfig) Store {
	if !strings.HasSuffix(rootURI, "/") {
		rootURI = rootURI + "/"
	}
	if ttlc == nil {
		ttlc = &TTLConfig{TTLKey: DefaultTTLKey, TTLFormat: DefaultTTLFormat}
	}
	log.Infof("Making new HTTP Store with root URI: %s", rootURI)
	return &httpStore{rootURI, client, *ttlc}
}

type Client interface {
	Do(req *http.Request) (resp *http.Response, err error)
}

type httpStore struct {
	rootURI string
	client  Client
	ttlc    TTLConfig
}

func (s *httpStore) getTTLValue(resp *http.Response) *TTLValue {
	expire := resp.Header.Get(s.ttlc.TTLKey)
	ttl, err := time.Parse(s.ttlc.TTLFormat, expire)
	if err != nil {
		return nil
	}
	return &TTLValue{TTL: ttl, TTLKey: s.ttlc.TTLKey}
}

func (s *httpStore) OpenForRead(name string) (*Resource, error) {
	return s.openForRead(name, false)
}

func (s *httpStore) openForRead(name string, existCheck bool) (*Resource, error) {
	label := "Read"
	if existCheck {
		label = "Exist"
	}
	uri := s.rootURI + name
	log.Infof("%sing %s", label, uri)

	var req *http.Request
	if existCheck {
		req, _ = http.NewRequest("HEAD", uri, nil)
	} else {
		req, _ = http.NewRequest("GET", uri, nil)
	}

	resp, err := s.client.Do(req)
	if err != nil {
		if !existCheck {
			log.Infof("%s error: %s %v", label, uri, err)
		}
		return nil, err
	}

	if resp.StatusCode == http.StatusOK {
		log.Infof("%s result %s %v", label, uri, resp.StatusCode)
		ttlv := s.getTTLValue(resp)
		var rc io.ReadCloser
		if existCheck {
			rc = ioutil.NopCloser(resp.Body)
		} else {
			rc = resp.Body
		}
		return NewResource(rc, resp.ContentLength, ttlv), nil
	}
	log.Errorf("%s response status error: %s %v", label, uri, resp.Status)
	if !existCheck {
		resp.Body.Close()
	}
	if resp.StatusCode == http.StatusNotFound {
		return nil, os.ErrNotExist
	} else if resp.StatusCode == http.StatusBadRequest {
		return nil, os.ErrInvalid
	}
	return nil, fmt.Errorf("could not open: %+v", resp)
}

func (s *httpStore) Exists(name string) (bool, error) {
	r, err := s.openForRead(name, true)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		log.Errorf("Exists error: %s %v", name, err)
		return false, err
	}
	log.Infof("Exists ok: %s", name)
	r.Close()
	if r.TTLValue != nil && r.TTLValue.TTL.Before(time.Now()) {
		return false, nil
	}
	return true, nil
}

func (s *httpStore) Write(name string, resource *Resource) error {
	if resource == nil {
		log.Info("Writing nil resource is a no op.")
		return nil
	}
	if strings.Contains(name, "/") {
		log.Errorf("Write error: %s '/' not allowed", name)
		return errors.New("'/' not allowed in name when writing bundles.")
	}
	uri := s.rootURI + name

	post := func() (*http.Response, error) {
		req, err := http.NewRequest("POST", uri, resource)
		if err != nil {
			return nil, err
		}
		req.ContentLength = resource.Length
		req.Header.Set("Content-Type", "text/plain")
		ttl := resource.TTLValue
		if ttl == nil {
			ttl = &TTLValue{TTL: time.Now().Add(s.ttlc.TTL), TTLKey: s.ttlc.TTLKey}
		}
		if ttl.TTLKey != "" {
			req.Header[ttl.TTLKey] = []string{ttl.TTL.Format(s.ttlc.TTLFormat)}
		}
		log.Infof("Writing %s: length: %d header: %v", uri, req.ContentLength, req.Header)
		return s.client.Do(req)
	}

	resp, err := post()
	if err != nil {
		log.Errorf("Write error: %s %v", uri, err)
	} else {
		resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			log.Errorf("Write response status error: %s -- %s", uri, resp.Status)
			return errors.New(resp.Status)
		}
	}
	return err
}

func (s *httpStore) Root() string {
	return s.rootURI
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Andreas Auernhammer. All rights reserved.
// Use of this source code is governed by a license that can be
// found in the LICENSE file.

package aead

import (
	"crypto/aes"
	"crypto/cipher"
	"errors"

	"github.com/aead/chacha20poly1305"
)

const (
	AES_GCM Algorithm = 1 + iota
	ChaCha20Poly1305
	ChaCha20Poly1305_IETF
	XChaCha20Poly1305
)

var (
	errUnknownAlgorithm = errors.New("libsodium/aead: unknown algorithm")
	errInvalidKeySize   = errors.New("libsodium/aead: bad key length")
)

type Algorithm uint

func NewCipher(alg Algorithm, key []byte) (cipher.AEAD, error) {
	if len(key) != 32 {
		return nil, errInvalidKeySize
	}

	switch alg {
	default:
		return nil, errUnknownAlgorithm
	case AES_GCM:
		c, err := aes.NewCipher(key)
		if err != nil {
			return nil, err
		}
		return cipher.NewGCM(c)
	case ChaCha20Poly1305:
		return chacha20poly1305.NewCipher(key)
	case ChaCha20Poly1305_IETF:
		return chacha20poly1305.NewIETFCipher(key)
	case XChaCha20Poly1305:
		return chacha20poly1305.NewXCipher(key)
	}
}

func Encrypt(alg Algorithm, dst, src, additionalData, nonce, key []byte) ([]byte, error) {
	c, err := NewCipher(alg, key)
	if err != nil {
		return nil, err
	}
	return c.Seal(dst[:0], nonce, src, additionalData), nil
}

func Decrypt(alg Algorithm, dst, src, additionalData, nonce, key []byte) ([]byte, error) {
	c, err := NewCipher(alg, key)
	if err != nil {
		return nil, err
	}
	return c.Open(dst[:0], nonce, src, additionalData)
}
<commit_msg>aead: simplify API and add documentation<commit_after>// Copyright (c) 2017 Andreas Auernhammer. All rights reserved.
// Use of this source code is governed by a license that can be
// found in the LICENSE file.

package aead

import (
	"crypto/cipher"
	"errors"

	"github.com/aead/chacha20poly1305"
)

const TagSize = 16

const (
	ChaCha20Poly1305 Algorithm = 1 + iota
	ChaCha20Poly1305_IETF
	XChaCha20Poly1305
	unknown
)

var (
	errUnknownAlgorithm = errors.New("libsodium/aead: unknown algorithm")
	errInvalidKeySize   = errors.New("libsodium/aead: bad key length")
)

type Algorithm uint

func (alg Algorithm) isKnown() bool {
	return alg > 0 && alg < unknown
}

func (alg Algorithm) newCipher(key []byte) (cipher.AEAD, error) {
	if len(key) != 32 {
		return nil, errInvalidKeySize
	}
	if !alg.isKnown() {
		return nil, errUnknownAlgorithm
	}

	switch alg {
	default:
		return chacha20poly1305.NewCipher(key)
	case ChaCha20Poly1305_IETF:
		return chacha20poly1305.NewIETFCipher(key)
	case XChaCha20Poly1305:
		return chacha20poly1305.NewXCipher(key)
	}
}

// Encrypt encrypts and authenticates the plaintext and writes the result to ciphertext.
// The authentication tag is appended to the ciphertext - therefore the ciphertext must be
// 16 bytes longer than the plaintext. The additionalData is not encrypted but authenticated
// and can be nil.
// The nonce must be unique for one specific key and should either be randomly genereated every
// time or genereated randomly once and incremented continuously (like a counter) - depending
// on the size of the nonce. See the nonce generation guidelines for details.
// A non-nil error indicates, that the encryption operation failed.
func (alg Algorithm) Encrypt(ciphertext, plaintext, additionalData, nonce, key []byte) (err error) {
	c, err := alg.newCipher(key)
	if err != nil {
		return
	}
	c.Seal(ciphertext[:0], nonce, plaintext, additionalData)
	return
}

// Decrypt decrypts and checks integrity of the ciphertext and writes the result to plaintext.
// This function is the inverse operation of Encrypt.
// A non-nil error indicates, that the encryption operation failed - especially modifed ciphertext
// or additionalData leads to different authentication tag and causes this function to fail with additionalData
// non-nil error.
func (alg Algorithm) Decrypt(plaintext, ciphertext, additionalData, nonce, key []byte) (err error) {
	c, err := alg.newCipher(key)
	if err != nil {
		return
	}
	_, err = c.Open(plaintext[:0], nonce, ciphertext, additionalData)
	return
}
<|endoftext|>