<commit_before>package transport

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
)

// ReadSeekCloser combines io.ReadSeeker with io.Closer.
type ReadSeekCloser interface {
	io.ReadSeeker
	io.Closer
}

// NewHTTPReadSeeker handles reading from an HTTP endpoint using a GET
// request. When seeking and starting a read from a non-zero offset
// the a "Range" header will be added which sets the offset.
// TODO(dmcgowan): Move this into a separate utility package
func NewHTTPReadSeeker(client *http.Client, url string, errorHandler func(*http.Response) error) ReadSeekCloser {
	return &httpReadSeeker{
		client:       client,
		url:          url,
		errorHandler: errorHandler,
	}
}

type httpReadSeeker struct {
	client *http.Client
	url    string

	// errorHandler creates an error from an unsuccessful HTTP response.
	// This allows the error to be created with the HTTP response body
	// without leaking the body through a returned error.
	errorHandler func(*http.Response) error

	size int64

	// rc is the remote read closer.
	rc io.ReadCloser
	// brd is a buffer for internal buffered io.
	brd *bufio.Reader
	// readerOffset tracks the offset as of the last read.
	readerOffset int64
	// seekOffset allows Seek to override the offset. Seek changes
	// seekOffset instead of changing readOffset directly so that
	// connection resets can be delayed and possibly avoided if the
	// seek is undone (i.e. seeking to the end and then back to the
	// beginning).
	seekOffset int64
	err        error
}

func (hrs *httpReadSeeker) Read(p []byte) (n int, err error) {
	if hrs.err != nil {
		return 0, hrs.err
	}

	// If we seeked to a different position, we need to reset the
	// connection. This logic is here instead of Seek so that if
	// a seek is undone before the next read, the connection doesn't
	// need to be closed and reopened. A common example of this is
	// seeking to the end to determine the length, and then seeking
	// back to the original position.
	if hrs.readerOffset != hrs.seekOffset {
		hrs.reset()
	}

	hrs.readerOffset = hrs.seekOffset

	rd, err := hrs.reader()
	if err != nil {
		return 0, err
	}

	n, err = rd.Read(p)
	hrs.seekOffset += int64(n)
	hrs.readerOffset += int64(n)

	// Simulate io.EOF error if we reach filesize.
	if err == nil && hrs.size >= 0 && hrs.readerOffset >= hrs.size {
		err = io.EOF
	}

	return n, err
}

func (hrs *httpReadSeeker) Seek(offset int64, whence int) (int64, error) {
	if hrs.err != nil {
		return 0, hrs.err
	}

	_, err := hrs.reader()
	if err != nil {
		return 0, err
	}

	newOffset := hrs.seekOffset

	switch whence {
	case os.SEEK_CUR:
		newOffset += int64(offset)
	case os.SEEK_END:
		if hrs.size < 0 {
			return 0, errors.New("content length not known")
		}
		newOffset = hrs.size + int64(offset)
	case os.SEEK_SET:
		newOffset = int64(offset)
	}

	if newOffset < 0 {
		err = errors.New("cannot seek to negative position")
	} else {
		hrs.seekOffset = newOffset
	}

	return hrs.seekOffset, err
}

func (hrs *httpReadSeeker) Close() error {
	if hrs.err != nil {
		return hrs.err
	}

	// close and release reader chain
	if hrs.rc != nil {
		hrs.rc.Close()
	}

	hrs.rc = nil
	hrs.brd = nil

	hrs.err = errors.New("httpLayer: closed")

	return nil
}

func (hrs *httpReadSeeker) reset() {
	if hrs.err != nil {
		return
	}
	if hrs.rc != nil {
		hrs.rc.Close()
		hrs.rc = nil
	}
}

func (hrs *httpReadSeeker) reader() (io.Reader, error) {
	if hrs.err != nil {
		return nil, hrs.err
	}

	if hrs.rc != nil {
		return hrs.brd, nil
	}

	req, err := http.NewRequest("GET", hrs.url, nil)
	if err != nil {
		return nil, err
	}

	if hrs.readerOffset > 0 {
		// TODO(stevvooe): Get this working correctly.

		// If we are at different offset, issue a range request from there.
		req.Header.Add("Range", "1-")
		// TODO: get context in here
		// context.GetLogger(hrs.context).Infof("Range: %s", req.Header.Get("Range"))
	}

	resp, err := hrs.client.Do(req)
	if err != nil {
		return nil, err
	}

	// Normally would use client.SuccessStatus, but that would be a cyclic
	// import
	if resp.StatusCode >= 200 && resp.StatusCode <= 399 {
		hrs.rc = resp.Body
		if resp.StatusCode == http.StatusOK {
			hrs.size = resp.ContentLength
		} else {
			hrs.size = -1
		}
	} else {
		defer resp.Body.Close()
		if hrs.errorHandler != nil {
			return nil, hrs.errorHandler(resp)
		}
		return nil, fmt.Errorf("unexpected status resolving reader: %v", resp.Status)
	}

	if hrs.brd == nil {
		hrs.brd = bufio.NewReader(hrs.rc)
	} else {
		hrs.brd.Reset(hrs.rc)
	}

	return hrs.brd, nil
}
<commit_msg>Support range requests in the client's httpReadSeeker<commit_after>package transport

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"regexp"
	"strconv"
)

var (
	contentRangeRegexp = regexp.MustCompile(`bytes ([0-9]+)-([0-9]+)/([0-9]+|\\*)`)

	// ErrWrongCodeForByteRange is returned if the client sends a request
	// with a Range header but the server returns a 2xx or 3xx code other
	// than 206 Partial Content.
	ErrWrongCodeForByteRange = errors.New("expected HTTP 206 from byte range request")
)

// ReadSeekCloser combines io.ReadSeeker with io.Closer.
type ReadSeekCloser interface {
	io.ReadSeeker
	io.Closer
}

// NewHTTPReadSeeker handles reading from an HTTP endpoint using a GET
// request. When seeking and starting a read from a non-zero offset
// the a "Range" header will be added which sets the offset.
// TODO(dmcgowan): Move this into a separate utility package
func NewHTTPReadSeeker(client *http.Client, url string, errorHandler func(*http.Response) error) ReadSeekCloser {
	return &httpReadSeeker{
		client:       client,
		url:          url,
		errorHandler: errorHandler,
	}
}

type httpReadSeeker struct {
	client *http.Client
	url    string

	// errorHandler creates an error from an unsuccessful HTTP response.
	// This allows the error to be created with the HTTP response body
	// without leaking the body through a returned error.
	errorHandler func(*http.Response) error

	size int64

	// rc is the remote read closer.
	rc io.ReadCloser
	// readerOffset tracks the offset as of the last read.
	readerOffset int64
	// seekOffset allows Seek to override the offset. Seek changes
	// seekOffset instead of changing readOffset directly so that
	// connection resets can be delayed and possibly avoided if the
	// seek is undone (i.e. seeking to the end and then back to the
	// beginning).
	seekOffset int64
	err        error
}

func (hrs *httpReadSeeker) Read(p []byte) (n int, err error) {
	if hrs.err != nil {
		return 0, hrs.err
	}

	// If we seeked to a different position, we need to reset the
	// connection. This logic is here instead of Seek so that if
	// a seek is undone before the next read, the connection doesn't
	// need to be closed and reopened. A common example of this is
	// seeking to the end to determine the length, and then seeking
	// back to the original position.
	if hrs.readerOffset != hrs.seekOffset {
		hrs.reset()
	}

	hrs.readerOffset = hrs.seekOffset

	rd, err := hrs.reader()
	if err != nil {
		return 0, err
	}

	n, err = rd.Read(p)
	hrs.seekOffset += int64(n)
	hrs.readerOffset += int64(n)

	return n, err
}

func (hrs *httpReadSeeker) Seek(offset int64, whence int) (int64, error) {
	if hrs.err != nil {
		return 0, hrs.err
	}

	lastReaderOffset := hrs.readerOffset

	if whence == os.SEEK_SET && hrs.rc == nil {
		// If no request has been made yet, and we are seeking to an
		// absolute position, set the read offset as well to avoid an
		// unnecessary request.
		hrs.readerOffset = offset
	}

	_, err := hrs.reader()
	if err != nil {
		hrs.readerOffset = lastReaderOffset
		return 0, err
	}

	newOffset := hrs.seekOffset

	switch whence {
	case os.SEEK_CUR:
		newOffset += offset
	case os.SEEK_END:
		if hrs.size < 0 {
			return 0, errors.New("content length not known")
		}
		newOffset = hrs.size + offset
	case os.SEEK_SET:
		newOffset = offset
	}

	if newOffset < 0 {
		err = errors.New("cannot seek to negative position")
	} else {
		hrs.seekOffset = newOffset
	}

	return hrs.seekOffset, err
}

func (hrs *httpReadSeeker) Close() error {
	if hrs.err != nil {
		return hrs.err
	}

	// close and release reader chain
	if hrs.rc != nil {
		hrs.rc.Close()
	}

	hrs.rc = nil

	hrs.err = errors.New("httpLayer: closed")

	return nil
}

func (hrs *httpReadSeeker) reset() {
	if hrs.err != nil {
		return
	}
	if hrs.rc != nil {
		hrs.rc.Close()
		hrs.rc = nil
	}
}

func (hrs *httpReadSeeker) reader() (io.Reader, error) {
	if hrs.err != nil {
		return nil, hrs.err
	}

	if hrs.rc != nil {
		return hrs.rc, nil
	}

	req, err := http.NewRequest("GET", hrs.url, nil)
	if err != nil {
		return nil, err
	}

	if hrs.readerOffset > 0 {
		// If we are at different offset, issue a range request from there.
		req.Header.Add("Range", fmt.Sprintf("bytes=%d-", hrs.readerOffset))
		// TODO: get context in here
		// context.GetLogger(hrs.context).Infof("Range: %s", req.Header.Get("Range"))
	}

	resp, err := hrs.client.Do(req)
	if err != nil {
		return nil, err
	}

	// Normally would use client.SuccessStatus, but that would be a cyclic
	// import
	if resp.StatusCode >= 200 && resp.StatusCode <= 399 {
		if hrs.readerOffset > 0 {
			if resp.StatusCode != http.StatusPartialContent {
				return nil, ErrWrongCodeForByteRange
			}

			contentRange := resp.Header.Get("Content-Range")
			if contentRange == "" {
				return nil, errors.New("no Content-Range header found in HTTP 206 response")
			}

			submatches := contentRangeRegexp.FindStringSubmatch(contentRange)
			if len(submatches) < 4 {
				return nil, fmt.Errorf("could not parse Content-Range header: %s", contentRange)
			}

			startByte, err := strconv.ParseUint(submatches[1], 10, 64)
			if err != nil {
				return nil, fmt.Errorf("could not parse start of range in Content-Range header: %s", contentRange)
			}

			if startByte != uint64(hrs.readerOffset) {
				return nil, fmt.Errorf("received Content-Range starting at offset %d instead of requested %d", startByte, hrs.readerOffset)
			}

			endByte, err := strconv.ParseUint(submatches[2], 10, 64)
			if err != nil {
				return nil, fmt.Errorf("could not parse end of range in Content-Range header: %s", contentRange)
			}

			if submatches[3] == "*" {
				hrs.size = -1
			} else {
				size, err := strconv.ParseUint(submatches[3], 10, 64)
				if err != nil {
					return nil, fmt.Errorf("could not parse total size in Content-Range header: %s", contentRange)
				}

				if endByte+1 != size {
					return nil, fmt.Errorf("range in Content-Range stops before the end of the content: %s", contentRange)
				}

				hrs.size = int64(size)
			}
		} else if resp.StatusCode == http.StatusOK {
			hrs.size = resp.ContentLength
		} else {
			hrs.size = -1
		}
		hrs.rc = resp.Body
	} else {
		defer resp.Body.Close()
		if hrs.errorHandler != nil {
			return nil, hrs.errorHandler(resp)
		}
		return nil, fmt.Errorf("unexpected status resolving reader: %v", resp.Status)
	}

	return hrs.rc, nil
}
<|endoftext|>


<commit_before>package rpcserver

import (
	"context"
	"fmt"
	"github.com/getlantern/bytemap"
	"github.com/getlantern/errors"
	"github.com/getlantern/golog"
	"github.com/getlantern/wal"
	"github.com/getlantern/zenodb"
	"github.com/getlantern/zenodb/common"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/encoding"
	"github.com/getlantern/zenodb/planner"
	"github.com/getlantern/zenodb/rpc"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
	"net"
	"time"
)

var (
	log = golog.LoggerFor("zenodb.rpc")
)

type Opts struct {
	// Password, if specified, is the password that clients must present in order
	// to access the server.
	Password string
}

// DB is an interface for database-like things (implemented by common.DB).
type DB interface {
	InsertRaw(stream string, ts time.Time, dims bytemap.ByteMap, vals bytemap.ByteMap) error

	Query(sqlString string, isSubQuery bool, subQueryResults [][]interface{}, includeMemStore bool) (core.FlatRowSource, error)

	Follow(f *common.Follow, cb func([]byte, wal.Offset) error)

	RegisterQueryHandler(partition int, query planner.QueryClusterFN)
}

func Serve(db DB, l net.Listener, opts *Opts) error {
	l = &rpc.SnappyListener{l}
	gs := grpc.NewServer(grpc.CustomCodec(rpc.Codec))
	gs.RegisterService(&rpc.ServiceDesc, &server{db, opts.Password})
	return gs.Serve(l)
}

type server struct {
	db       DB
	password string
}

func (s *server) Insert(stream grpc.ServerStream) error {
	// No need to authorize, anyone can insert

	now := time.Now()
	streamName := ""

	report := &rpc.InsertReport{
		Errors: make(map[int]string),
	}

	i := -1
	for {
		i++
		insert := &rpc.Insert{}
		err := stream.RecvMsg(insert)
		if err != nil {
			return fmt.Errorf("Error reading insert: %v", err)
		}
		if insert.EndOfInserts {
			// We're done inserting
			return stream.SendMsg(report)
		}
		report.Received++

		if streamName == "" {
			streamName = insert.Stream
			if streamName == "" {
				return fmt.Errorf("Please specify a stream")
			}
		}

		if len(insert.Dims) == 0 {
			report.Errors[i] = fmt.Sprintf("Need at least one dim")
			continue
		}
		if len(insert.Vals) == 0 {
			report.Errors[i] = fmt.Sprintf("Need at least one val")
			continue
		}
		var ts time.Time
		if insert.TS == 0 {
			ts = now
		} else {
			ts = encoding.TimeFromInt(insert.TS)
		}

		// TODO: make sure we don't barf on invalid bytemaps here
		insertErr := s.db.InsertRaw(streamName, ts, bytemap.ByteMap(insert.Dims), bytemap.ByteMap(insert.Vals))
		if insertErr != nil {
			report.Errors[i] = fmt.Sprintf("Unable to insert: %v", insertErr)
			continue
		}
		report.Succeeded++
	}
}

func (s *server) Query(q *rpc.Query, stream grpc.ServerStream) error {
	authorizeErr := s.authorize(stream)
	if authorizeErr != nil {
		return authorizeErr
	}

	source, err := s.db.Query(q.SQLString, q.IsSubQuery, q.SubQueryResults, q.IncludeMemStore)
	if err != nil {
		return err
	}

	rr := &rpc.RemoteQueryResult{}
	stats, err := source.Iterate(stream.Context(), func(fields core.Fields) error {
		// Send query metadata
		md := zenodb.MetaDataFor(source, fields)
		return stream.SendMsg(md)
	}, func(row *core.FlatRow) (bool, error) {
		rr.Row = row
		return true, stream.SendMsg(rr)
	})
	if err != nil {
		return err
	}

	// Send end of results
	rr.Row = nil
	if stats != nil {
		rr.Stats = stats.(*common.QueryStats)
	}
	rr.EndOfResults = true
	return stream.SendMsg(rr)
}

func (s *server) Follow(f *common.Follow, stream grpc.ServerStream) error {
	authorizeErr := s.authorize(stream)
	if authorizeErr != nil {
		return authorizeErr
	}

	log.Debugf("Follower %d joined", f.PartitionNumber)
	defer log.Debugf("Follower %d left", f.PartitionNumber)
	s.db.Follow(f, func(data []byte, newOffset wal.Offset) error {
		return stream.SendMsg(&rpc.Point{data, newOffset})
	})
	return nil
}

func (s *server) HandleRemoteQueries(r *rpc.RegisterQueryHandler, stream grpc.ServerStream) error {
	initialResultCh := make(chan *rpc.RemoteQueryResult)
	initialErrCh := make(chan error)
	finalErrCh := make(chan error)

	finish := func(err error) {
		select {
		case finalErrCh <- err:
			// ok
			log.Debugf("Posted final err: %v", err)
		default:
			// ignore
			log.Debug("Already had final err!")
		}
	}

	s.db.RegisterQueryHandler(r.Partition, func(ctx context.Context, sqlString string, isSubQuery bool, subQueryResults [][]interface{}, unflat bool, onFields core.OnFields, onRow core.OnRow, onFlatRow core.OnFlatRow) (interface{}, error) {
		q := &rpc.Query{
			SQLString:       sqlString,
			IsSubQuery:      isSubQuery,
			SubQueryResults: subQueryResults,
			Unflat:          unflat,
			IncludeMemStore: common.ShouldIncludeMemStore(ctx),
		}
		q.Deadline, q.HasDeadline = ctx.Deadline()
		sendErr := stream.SendMsg(q)

		m, recvErr := <-initialResultCh, <-initialErrCh

		// Check send error after reading initial result to avoid blocking
		// unnecessarily
		if sendErr != nil {
			err := errors.New("Unable to send query: %v", sendErr)
			finish(err)
			return nil, err
		}

		var finalErr error

		first := true
	receiveLoop:
		for {
			// Process current result
			if recvErr != nil {
				m.Error = recvErr.Error()
				finalErr = errors.New("Unable to receive result: %v", recvErr)
				break
			}

			if first {
				// First message contains only fields information
				onFields(m.Fields)
				first = false
			} else {
				// Subsequent messages contain data
				if m.EndOfResults {
					break
				}
				var more bool
				var err error
				if unflat {
					more, err = onRow(m.Key, m.Vals)
				} else {
					more, err = onFlatRow(m.Row)
				}
				if !more || err != nil {
					finalErr = err
					break receiveLoop
				}
			}

			// Read next result
			m = &rpc.RemoteQueryResult{}
			recvErr = stream.RecvMsg(m)
		}

		finish(finalErr)
		return m.Stats, finalErr
	})

	// Block on reading initial result to keep connection open
	m := &rpc.RemoteQueryResult{}
	err := stream.RecvMsg(m)
	initialResultCh <- m
	initialErrCh <- err

	if err == nil {
		// Wait for final error so we don't close the connection prematurely
		err = <-finalErrCh
	}
	log.Debugf("Returning err?: %v", err)
	return err
}

func (s *server) authorize(stream grpc.ServerStream) error {
	if s.password == "" {
		log.Debug("No password specified, allowing access to world")
		return nil
	}
	md, ok := metadata.FromIncomingContext(stream.Context())
	if !ok {
		return log.Error("No metadata provided, unable to authenticate")
	}
	passwords := md[rpc.PasswordKey]
	for _, password := range passwords {
		if password == s.password {
			// authorized
			return nil
		}
	}
	return log.Error("None of the provided passwords matched, not authorized!")
}
<commit_msg>Added more logging for query error handling<commit_after>package rpcserver

import (
	"context"
	"fmt"
	"github.com/getlantern/bytemap"
	"github.com/getlantern/errors"
	"github.com/getlantern/golog"
	"github.com/getlantern/wal"
	"github.com/getlantern/zenodb"
	"github.com/getlantern/zenodb/common"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/encoding"
	"github.com/getlantern/zenodb/planner"
	"github.com/getlantern/zenodb/rpc"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
	"net"
	"time"
)

var (
	log = golog.LoggerFor("zenodb.rpc")
)

type Opts struct {
	// Password, if specified, is the password that clients must present in order
	// to access the server.
	Password string
}

// DB is an interface for database-like things (implemented by common.DB).
type DB interface {
	InsertRaw(stream string, ts time.Time, dims bytemap.ByteMap, vals bytemap.ByteMap) error

	Query(sqlString string, isSubQuery bool, subQueryResults [][]interface{}, includeMemStore bool) (core.FlatRowSource, error)

	Follow(f *common.Follow, cb func([]byte, wal.Offset) error)

	RegisterQueryHandler(partition int, query planner.QueryClusterFN)
}

func Serve(db DB, l net.Listener, opts *Opts) error {
	l = &rpc.SnappyListener{l}
	gs := grpc.NewServer(grpc.CustomCodec(rpc.Codec))
	gs.RegisterService(&rpc.ServiceDesc, &server{db, opts.Password})
	return gs.Serve(l)
}

type server struct {
	db       DB
	password string
}

func (s *server) Insert(stream grpc.ServerStream) error {
	// No need to authorize, anyone can insert

	now := time.Now()
	streamName := ""

	report := &rpc.InsertReport{
		Errors: make(map[int]string),
	}

	i := -1
	for {
		i++
		insert := &rpc.Insert{}
		err := stream.RecvMsg(insert)
		if err != nil {
			return fmt.Errorf("Error reading insert: %v", err)
		}
		if insert.EndOfInserts {
			// We're done inserting
			return stream.SendMsg(report)
		}
		report.Received++

		if streamName == "" {
			streamName = insert.Stream
			if streamName == "" {
				return fmt.Errorf("Please specify a stream")
			}
		}

		if len(insert.Dims) == 0 {
			report.Errors[i] = fmt.Sprintf("Need at least one dim")
			continue
		}
		if len(insert.Vals) == 0 {
			report.Errors[i] = fmt.Sprintf("Need at least one val")
			continue
		}
		var ts time.Time
		if insert.TS == 0 {
			ts = now
		} else {
			ts = encoding.TimeFromInt(insert.TS)
		}

		// TODO: make sure we don't barf on invalid bytemaps here
		insertErr := s.db.InsertRaw(streamName, ts, bytemap.ByteMap(insert.Dims), bytemap.ByteMap(insert.Vals))
		if insertErr != nil {
			report.Errors[i] = fmt.Sprintf("Unable to insert: %v", insertErr)
			continue
		}
		report.Succeeded++
	}
}

func (s *server) Query(q *rpc.Query, stream grpc.ServerStream) error {
	authorizeErr := s.authorize(stream)
	if authorizeErr != nil {
		return authorizeErr
	}

	source, err := s.db.Query(q.SQLString, q.IsSubQuery, q.SubQueryResults, q.IncludeMemStore)
	if err != nil {
		return err
	}

	rr := &rpc.RemoteQueryResult{}
	stats, err := source.Iterate(stream.Context(), func(fields core.Fields) error {
		// Send query metadata
		md := zenodb.MetaDataFor(source, fields)
		return stream.SendMsg(md)
	}, func(row *core.FlatRow) (bool, error) {
		rr.Row = row
		return true, stream.SendMsg(rr)
	})
	if err != nil {
		return err
	}

	// Send end of results
	rr.Row = nil
	if stats != nil {
		rr.Stats = stats.(*common.QueryStats)
	}
	rr.EndOfResults = true
	return stream.SendMsg(rr)
}

func (s *server) Follow(f *common.Follow, stream grpc.ServerStream) error {
	authorizeErr := s.authorize(stream)
	if authorizeErr != nil {
		return authorizeErr
	}

	log.Debugf("Follower %d joined", f.PartitionNumber)
	defer log.Debugf("Follower %d left", f.PartitionNumber)
	s.db.Follow(f, func(data []byte, newOffset wal.Offset) error {
		return stream.SendMsg(&rpc.Point{data, newOffset})
	})
	return nil
}

func (s *server) HandleRemoteQueries(r *rpc.RegisterQueryHandler, stream grpc.ServerStream) error {
	initialResultCh := make(chan *rpc.RemoteQueryResult)
	initialErrCh := make(chan error, 1)
	finalErrCh := make(chan error, 1)

	finish := func(err error) {
		select {
		case finalErrCh <- err:
			// ok
		default:
			// ignore
		}
	}

	s.db.RegisterQueryHandler(r.Partition, func(ctx context.Context, sqlString string, isSubQuery bool, subQueryResults [][]interface{}, unflat bool, onFields core.OnFields, onRow core.OnRow, onFlatRow core.OnFlatRow) (interface{}, error) {
		q := &rpc.Query{
			SQLString:       sqlString,
			IsSubQuery:      isSubQuery,
			SubQueryResults: subQueryResults,
			Unflat:          unflat,
			IncludeMemStore: common.ShouldIncludeMemStore(ctx),
		}
		q.Deadline, q.HasDeadline = ctx.Deadline()
		sendErr := stream.SendMsg(q)

		m, recvErr := <-initialResultCh, <-initialErrCh

		// Check send error after reading initial result to avoid blocking
		// unnecessarily
		if sendErr != nil {
			err := errors.New("Unable to send query: %v", sendErr)
			finish(err)
			return nil, err
		}

		var finalErr error

		first := true
	receiveLoop:
		for {
			// Process current result
			if recvErr != nil {
				m.Error = recvErr.Error()
				finalErr = errors.New("Unable to receive result: %v", recvErr)
				break
			}

			if first {
				// First message contains only fields information
				onFields(m.Fields)
				first = false
			} else {
				// Subsequent messages contain data
				if m.EndOfResults {
					break
				}
				var more bool
				var err error
				if unflat {
					more, err = onRow(m.Key, m.Vals)
				} else {
					more, err = onFlatRow(m.Row)
				}
				if !more || err != nil {
					finalErr = err
					break receiveLoop
				}
			}

			// Read next result
			m = &rpc.RemoteQueryResult{}
			recvErr = stream.RecvMsg(m)
		}

		finish(finalErr)
		return m.Stats, finalErr
	})

	// Block on reading initial result to keep connection open
	m := &rpc.RemoteQueryResult{}
	err := stream.RecvMsg(m)
	initialResultCh <- m
	initialErrCh <- err

	if err == nil {
		// Wait for final error so we don't close the connection prematurely
		err = <-finalErrCh
	}
	log.Debugf("Returning err?: %v", err)
	return err
}

func (s *server) authorize(stream grpc.ServerStream) error {
	if s.password == "" {
		log.Debug("No password specified, allowing access to world")
		return nil
	}
	md, ok := metadata.FromIncomingContext(stream.Context())
	if !ok {
		return log.Error("No metadata provided, unable to authenticate")
	}
	passwords := md[rpc.PasswordKey]
	for _, password := range passwords {
		if password == s.password {
			// authorized
			return nil
		}
	}
	return log.Error("None of the provided passwords matched, not authorized!")
}
<|endoftext|>


<commit_before>package http

import (
	"context"
	"encoding/json"
	"io"
	"net/http"
	"strings"
	"time"

	"github.com/julienschmidt/httprouter"

	"github.com/zitryss/aye-and-nay/domain/model"
	"github.com/zitryss/aye-and-nay/pkg/errors"
)

func newController(
	serv model.Servicer,
) controller {
	conf := newContrConfig()
	return controller{conf, serv}
}

type controller struct {
	conf contrConfig
	serv model.Servicer
}

func (c *controller) handleAlbum() httprouter.Handle {
	input := func(r *http.Request, ps httprouter.Params) (context.Context, albumRequest, error) {
		ctx := r.Context()
		ct := r.Header.Get("Content-Type")
		if !strings.HasPrefix(ct, "multipart/form-data") {
			return nil, albumRequest{}, errors.Wrap(model.ErrWrongContentType)
		}
		maxBodySize := int64(c.conf.maxNumberOfFiles) * c.conf.maxFileSize
		if r.ContentLength > maxBodySize {
			return nil, albumRequest{}, errors.Wrap(model.ErrBodyTooLarge)
		}
		err := r.ParseMultipartForm(r.ContentLength)
		if err != nil {
			return nil, albumRequest{}, errors.Wrap(err)
		}
		fhs := r.MultipartForm.File["images"]
		if len(fhs) < 2 {
			return nil, albumRequest{}, errors.Wrap(model.ErrNotEnoughImages)
		}
		if len(fhs) > c.conf.maxNumberOfFiles {
			return nil, albumRequest{}, errors.Wrap(model.ErrTooManyImages)
		}
		req := albumRequest{ff: make([]model.File, 0, len(fhs)), multi: r.MultipartForm}
		defer func() {
			for _, f := range req.ff {
				_ = f.Reader.(io.Closer).Close()
			}
			_ = req.multi.RemoveAll()
		}()
		for _, fh := range fhs {
			if fh.Size > c.conf.maxFileSize {
				return nil, albumRequest{}, errors.Wrap(model.ErrImageTooLarge)
			}
			f, err := fh.Open()
			if err != nil {
				return nil, albumRequest{}, errors.Wrap(err)
			}
			b := make([]byte, 512)
			_, err = f.Read(b)
			if err != nil {
				_ = f.Close()
				return nil, albumRequest{}, errors.Wrap(err)
			}
			_, err = f.Seek(0, io.SeekStart)
			if err != nil {
				_ = f.Close()
				return nil, albumRequest{}, errors.Wrap(err)
			}
			typ := http.DetectContentType(b)
			if !strings.HasPrefix(typ, "image/") {
				_ = f.Close()
				return nil, albumRequest{}, errors.Wrap(model.ErrNotImage)
			}
			req.ff = append(req.ff, model.File{Reader: f, Size: fh.Size})
		}
		vals := r.MultipartForm.Value["duration"]
		if len(vals) == 0 {
			return nil, albumRequest{}, errors.Wrap(model.ErrDurationNotSet)
		}
		dur, err := time.ParseDuration(vals[0])
		if err != nil {
			return nil, albumRequest{}, errors.Wrap(model.ErrDurationInvalid)
		}
		req.dur = dur
		return ctx, req, nil
	}
	process := func(ctx context.Context, req albumRequest) (albumResponse, error) {
		defer func() {
			for _, f := range req.ff {
				_ = f.Reader.(io.Closer).Close()
			}
			_ = req.multi.RemoveAll()
		}()
		album, err := c.serv.Album(ctx, req.ff, req.dur)
		if err != nil {
			return albumResponse{}, errors.Wrap(err)
		}
		resp := albumResponse{}
		resp.Album.Id = album
		return resp, nil
	}
	output := func(ctx context.Context, w http.ResponseWriter, resp albumResponse) error {
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		w.WriteHeader(201)
		err := json.NewEncoder(w).Encode(resp)
		if err != nil {
			return errors.Wrap(err)
		}
		return nil
	}
	return handleHttpRouterError(
		func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error {
			ctx, req, err := input(r, ps)
			if err != nil {
				return errors.Wrap(err)
			}
			resp, err := process(ctx, req)
			if err != nil {
				return errors.Wrap(err)
			}
			err = output(ctx, w, resp)
			if err != nil {
				return errors.Wrap(err)
			}
			return nil
		},
	)
}

func (c *controller) handleReady() httprouter.Handle {
	input := func(r *http.Request, ps httprouter.Params) (context.Context, readyRequest, error) {
		ctx := r.Context()
		req := readyRequest{}
		req.album.id = ps.ByName("album")
		return ctx, req, nil
	}
	process := func(ctx context.Context, req readyRequest) (readyResponse, error) {
		p, err := c.serv.Progress(ctx, req.album.id)
		if err != nil {
			return readyResponse{}, errors.Wrap(err)
		}
		resp := readyResponse{}
		resp.Album.Progress = p
		return resp, nil
	}
	output := func(ctx context.Context, w http.ResponseWriter, resp readyResponse) error {
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		err := json.NewEncoder(w).Encode(resp)
		if err != nil {
			return errors.Wrap(err)
		}
		return nil
	}
	return handleHttpRouterError(
		func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error {
			ctx, req, err := input(r, ps)
			if err != nil {
				return errors.Wrap(err)
			}
			resp, err := process(ctx, req)
			if err != nil {
				return errors.Wrap(err)
			}
			err = output(ctx, w, resp)
			if err != nil {
				return errors.Wrap(err)
			}
			return nil
		},
	)
}

func (c *controller) handlePair() httprouter.Handle {
	input := func(r *http.Request, ps httprouter.Params) (context.Context, pairRequest, error) {
		ctx := r.Context()
		req := pairRequest{}
		req.album.id = ps.ByName("album")
		return ctx, req, nil
	}
	process := func(ctx context.Context, req pairRequest) (pairResponse, error) {
		img1, img2, err := c.serv.Pair(ctx, req.album.id)
		if err != nil {
			return pairResponse{}, errors.Wrap(err)
		}
		resp := pairResponse{}
		resp.Album.Img1.Src = img1.Src
		resp.Album.Img1.Token = img1.Token
		resp.Album.Img2.Src = img2.Src
		resp.Album.Img2.Token = img2.Token
		return resp, nil
	}
	output := func(ctx context.Context, w http.ResponseWriter, resp pairResponse) error {
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		err := json.NewEncoder(w).Encode(resp)
		if err != nil {
			return errors.Wrap(err)
		}
		return nil
	}
	return handleHttpRouterError(
		func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error {
			ctx, req, err := input(r, ps)
			if err != nil {
				return errors.Wrap(err)
			}
			resp, err := process(ctx, req)
			if err != nil {
				return errors.Wrap(err)
			}
			err = output(ctx, w, resp)
			if err != nil {
				return errors.Wrap(err)
			}
			return nil
		},
	)
}

func (c *controller) handleVote() httprouter.Handle {
	input := func(r *http.Request, ps httprouter.Params) (context.Context, voteRequest, error) {
		ctx := r.Context()
		req := voteRequest{}
		req.Album.id = ps.ByName("album")
		ct := r.Header.Get("Content-Type")
		if !strings.HasPrefix(ct, "application/json") {
			return nil, voteRequest{}, errors.Wrap(model.ErrWrongContentType)
		}
		err := json.NewDecoder(r.Body).Decode(&req)
		if err != nil {
			return nil, voteRequest{}, errors.Wrap(err)
		}
		return ctx, req, nil
	}
	process := func(ctx context.Context, req voteRequest) (voteResponse, error) {
		err := c.serv.Vote(ctx, req.Album.id, req.Album.ImgFrom.Token, req.Album.ImgTo.Token)
		if err != nil {
			return voteResponse{}, errors.Wrap(err)
		}
		resp := voteResponse{}
		return resp, nil
	}
	output := func(ctx context.Context, w http.ResponseWriter, resp voteResponse) error {
		return nil
	}
	return handleHttpRouterError(
		func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error {
			ctx, req, err := input(r, ps)
			if err != nil {
				return errors.Wrap(err)
			}
			resp, err := process(ctx, req)
			if err != nil {
				return errors.Wrap(err)
			}
			err = output(ctx, w, resp)
			if err != nil {
				return errors.Wrap(err)
			}
			return nil
		},
	)
}

func (c *controller) handleTop() httprouter.Handle {
	input := func(r *http.Request, ps httprouter.Params) (context.Context, topRequest, error) {
		ctx := r.Context()
		req := topRequest{}
		req.album.id = ps.ByName("album")
		return ctx, req, nil
	}
	process := func(ctx context.Context, req topRequest) (topResponse, error) {
		imgs, err := c.serv.Top(ctx, req.album.id)
		if err != nil {
			return topResponse{}, errors.Wrap(err)
		}
		resp := topResponse{}
		resp.Album.Images = make([]image, 0, len(imgs))
		for _, img := range imgs {
			image := image{img.Src, img.Rating}
			resp.Album.Images = append(resp.Album.Images, image)
		}
		return resp, nil
	}
	output := func(ctx context.Context, w http.ResponseWriter, resp topResponse) error {
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		err := json.NewEncoder(w).Encode(resp)
		if err != nil {
			return errors.Wrap(err)
		}
		return nil
	}
	return handleHttpRouterError(
		func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error {
			ctx, req, err := input(r, ps)
			if err != nil {
				return errors.Wrap(err)
			}
			resp, err := process(ctx, req)
			if err != nil {
				return errors.Wrap(err)
			}
			err = output(ctx, w, resp)
			if err != nil {
				return errors.Wrap(err)
			}
			return nil
		},
	)
}
<commit_msg>Adapt controller to uint64<commit_after>package http

import (
	"context"
	"encoding/json"
	"io"
	"net/http"
	"strings"
	"time"

	"github.com/julienschmidt/httprouter"

	"github.com/zitryss/aye-and-nay/domain/model"
	"github.com/zitryss/aye-and-nay/pkg/base64"
	"github.com/zitryss/aye-and-nay/pkg/errors"
)

func newController(
	serv model.Servicer,
) controller {
	conf := newContrConfig()
	return controller{conf, serv}
}

type controller struct {
	conf contrConfig
	serv model.Servicer
}

func (c *controller) handleAlbum() httprouter.Handle {
	input := func(r *http.Request, ps httprouter.Params) (context.Context, albumRequest, error) {
		ctx := r.Context()
		ct := r.Header.Get("Content-Type")
		if !strings.HasPrefix(ct, "multipart/form-data") {
			return nil, albumRequest{}, errors.Wrap(model.ErrWrongContentType)
		}
		maxBodySize := int64(c.conf.maxNumberOfFiles) * c.conf.maxFileSize
		if r.ContentLength > maxBodySize {
			return nil, albumRequest{}, errors.Wrap(model.ErrBodyTooLarge)
		}
		err := r.ParseMultipartForm(r.ContentLength)
		if err != nil {
			return nil, albumRequest{}, errors.Wrap(err)
		}
		fhs := r.MultipartForm.File["images"]
		if len(fhs) < 2 {
			return nil, albumRequest{}, errors.Wrap(model.ErrNotEnoughImages)
		}
		if len(fhs) > c.conf.maxNumberOfFiles {
			return nil, albumRequest{}, errors.Wrap(model.ErrTooManyImages)
		}
		req := albumRequest{ff: make([]model.File, 0, len(fhs)), multi: r.MultipartForm}
		defer func() {
			for _, f := range req.ff {
				_ = f.Reader.(io.Closer).Close()
			}
			_ = req.multi.RemoveAll()
		}()
		for _, fh := range fhs {
			if fh.Size > c.conf.maxFileSize {
				return nil, albumRequest{}, errors.Wrap(model.ErrImageTooLarge)
			}
			f, err := fh.Open()
			if err != nil {
				return nil, albumRequest{}, errors.Wrap(err)
			}
			b := make([]byte, 512)
			_, err = f.Read(b)
			if err != nil {
				_ = f.Close()
				return nil, albumRequest{}, errors.Wrap(err)
			}
			_, err = f.Seek(0, io.SeekStart)
			if err != nil {
				_ = f.Close()
				return nil, albumRequest{}, errors.Wrap(err)
			}
			typ := http.DetectContentType(b)
			if !strings.HasPrefix(typ, "image/") {
				_ = f.Close()
				return nil, albumRequest{}, errors.Wrap(model.ErrNotImage)
			}
			req.ff = append(req.ff, model.File{Reader: f, Size: fh.Size})
		}
		vals := r.MultipartForm.Value["duration"]
		if len(vals) == 0 {
			return nil, albumRequest{}, errors.Wrap(model.ErrDurationNotSet)
		}
		dur, err := time.ParseDuration(vals[0])
		if err != nil {
			return nil, albumRequest{}, errors.Wrap(model.ErrDurationInvalid)
		}
		req.dur = dur
		return ctx, req, nil
	}
	process := func(ctx context.Context, req albumRequest) (albumResponse, error) {
		defer func() {
			for _, f := range req.ff {
				_ = f.Reader.(io.Closer).Close()
			}
			_ = req.multi.RemoveAll()
		}()
		album, err := c.serv.Album(ctx, req.ff, req.dur)
		if err != nil {
			return albumResponse{}, errors.Wrap(err)
		}
		resp := albumResponse{}
		albumB64 := base64.FromUint64(album)
		resp.Album.Id = albumB64
		return resp, nil
	}
	output := func(ctx context.Context, w http.ResponseWriter, resp albumResponse) error {
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		w.WriteHeader(201)
		err := json.NewEncoder(w).Encode(resp)
		if err != nil {
			return errors.Wrap(err)
		}
		return nil
	}
	return handleHttpRouterError(
		func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error {
			ctx, req, err := input(r, ps)
			if err != nil {
				return errors.Wrap(err)
			}
			resp, err := process(ctx, req)
			if err != nil {
				return errors.Wrap(err)
			}
			err = output(ctx, w, resp)
			if err != nil {
				return errors.Wrap(err)
			}
			return nil
		},
	)
}

func (c *controller) handleReady() httprouter.Handle {
	input := func(r *http.Request, ps httprouter.Params) (context.Context, readyRequest, error) {
		ctx := r.Context()
		req := readyRequest{}
		req.album.id = ps.ByName("album")
		return ctx, req, nil
	}
	process := func(ctx context.Context, req readyRequest) (readyResponse, error) {
		album, err := base64.ToUint64(req.album.id)
		if err != nil {
			return readyResponse{}, errors.Wrap(err)
		}
		p, err := c.serv.Progress(ctx, album)
		if err != nil {
			return readyResponse{}, errors.Wrap(err)
		}
		resp := readyResponse{}
		resp.Album.Progress = p
		return resp, nil
	}
	output := func(ctx context.Context, w http.ResponseWriter, resp readyResponse) error {
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		err := json.NewEncoder(w).Encode(resp)
		if err != nil {
			return errors.Wrap(err)
		}
		return nil
	}
	return handleHttpRouterError(
		func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error {
			ctx, req, err := input(r, ps)
			if err != nil {
				return errors.Wrap(err)
			}
			resp, err := process(ctx, req)
			if err != nil {
				return errors.Wrap(err)
			}
			err = output(ctx, w, resp)
			if err != nil {
				return errors.Wrap(err)
			}
			return nil
		},
	)
}

func (c *controller) handlePair() httprouter.Handle {
	input := func(r *http.Request, ps httprouter.Params) (context.Context, pairRequest, error) {
		ctx := r.Context()
		req := pairRequest{}
		req.album.id = ps.ByName("album")
		return ctx, req, nil
	}
	process := func(ctx context.Context, req pairRequest) (pairResponse, error) {
		album, err := base64.ToUint64(req.album.id)
		if err != nil {
			return pairResponse{}, errors.Wrap(err)
		}
		img1, img2, err := c.serv.Pair(ctx, album)
		if err != nil {
			return pairResponse{}, errors.Wrap(err)
		}
		resp := pairResponse{}
		resp.Album.Img1.Src = img1.Src
		img1TokenB64 := base64.FromUint64(img1.Token)
		resp.Album.Img1.Token = img1TokenB64
		resp.Album.Img2.Src = img2.Src
		img2TokenB64 := base64.FromUint64(img2.Token)
		resp.Album.Img2.Token = img2TokenB64
		return resp, nil
	}
	output := func(ctx context.Context, w http.ResponseWriter, resp pairResponse) error {
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		err := json.NewEncoder(w).Encode(resp)
		if err != nil {
			return errors.Wrap(err)
		}
		return nil
	}
	return handleHttpRouterError(
		func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error {
			ctx, req, err := input(r, ps)
			if err != nil {
				return errors.Wrap(err)
			}
			resp, err := process(ctx, req)
			if err != nil {
				return errors.Wrap(err)
			}
			err = output(ctx, w, resp)
			if err != nil {
				return errors.Wrap(err)
			}
			return nil
		},
	)
}

func (c *controller) handleVote() httprouter.Handle {
	input := func(r *http.Request, ps httprouter.Params) (context.Context, voteRequest, error) {
		ctx := r.Context()
		req := voteRequest{}
		req.Album.id = ps.ByName("album")
		ct := r.Header.Get("Content-Type")
		if !strings.HasPrefix(ct, "application/json") {
			return nil, voteRequest{}, errors.Wrap(model.ErrWrongContentType)
		}
		err := json.NewDecoder(r.Body).Decode(&req)
		if err != nil {
			return nil, voteRequest{}, errors.Wrap(err)
		}
		return ctx, req, nil
	}
	process := func(ctx context.Context, req voteRequest) (voteResponse, error) {
		album, err := base64.ToUint64(req.Album.id)
		if err != nil {
			return voteResponse{}, errors.Wrap(err)
		}
		imgFromToken, err := base64.ToUint64(req.Album.ImgFrom.Token)
		if err != nil {
			return voteResponse{}, errors.Wrap(err)
		}
		imgToToken, err := base64.ToUint64(req.Album.ImgTo.Token)
		if err != nil {
			return voteResponse{}, errors.Wrap(err)
		}
		err = c.serv.Vote(ctx, album, imgFromToken, imgToToken)
		if err != nil {
			return voteResponse{}, errors.Wrap(err)
		}
		resp := voteResponse{}
		return resp, nil
	}
	output := func(ctx context.Context, w http.ResponseWriter, resp voteResponse) error {
		return nil
	}
	return handleHttpRouterError(
		func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error {
			ctx, req, err := input(r, ps)
			if err != nil {
				return errors.Wrap(err)
			}
			resp, err := process(ctx, req)
			if err != nil {
				return errors.Wrap(err)
			}
			err = output(ctx, w, resp)
			if err != nil {
				return errors.Wrap(err)
			}
			return nil
		},
	)
}

func (c *controller) handleTop() httprouter.Handle {
	input := func(r *http.Request, ps httprouter.Params) (context.Context, topRequest, error) {
		ctx := r.Context()
		req := topRequest{}
		req.album.id = ps.ByName("album")
		return ctx, req, nil
	}
	process := func(ctx context.Context, req topRequest) (topResponse, error) {
		album, err := base64.ToUint64(req.album.id)
		if err != nil {
			return topResponse{}, errors.Wrap(err)
		}
		imgs, err := c.serv.Top(ctx, album)
		if err != nil {
			return topResponse{}, errors.Wrap(err)
		}
		resp := topResponse{}
		resp.Album.Images = make([]image, 0, len(imgs))
		for _, img := range imgs {
			image := image{img.Src, img.Rating}
			resp.Album.Images = append(resp.Album.Images, image)
		}
		return resp, nil
	}
	output := func(ctx context.Context, w http.ResponseWriter, resp topResponse) error {
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		err := json.NewEncoder(w).Encode(resp)
		if err != nil {
			return errors.Wrap(err)
		}
		return nil
	}
	return handleHttpRouterError(
		func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) error {
			ctx, req, err := input(r, ps)
			if err != nil {
				return errors.Wrap(err)
			}
			resp, err := process(ctx, req)
			if err != nil {
				return errors.Wrap(err)
			}
			err = output(ctx, w, resp)
			if err != nil {
				return errors.Wrap(err)
			}
			return nil
		},
	)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"time"

	"github.com/nathanjsweet/zsocket"
	fi "github.com/nathanjsweet/zsocket/fakeinterface"
	"github.com/nathanjsweet/zsocket/nettypes"
)

func main() {
	zs, err := zsocket.NewZSocket(18, zsocket.ENABLE_RX|zsocket.ENABLE_TX, 256, zsocket.MAX_ORDER, 4, nettypes.All)
	if err != nil {
		panic(err)
	}
	fip := net.IPAddr{net.IP([]byte{192, 0, 0, 3}), ""}
	fak, err := fi.NewFakeInterface("fake1", net.HardwareAddr([]byte{0x1e, 0x44, 0x70, 0x3c, 0x66, 0xe3}), &fip, 30000)
	go func() {
		for {
			fmt.Printf("%s\n", fak)
			time.Sleep(time.Second * time.Duration(10))
		}
	}()
	if err != nil {
		panic(err)
	}
	go func() {
		panic(zs.Listen(func(f *nettypes.Frame, frameLen uint16) {
			fmt.Println("20:")
			fmt.Println(f.String(frameLen, 0))
			processFrame(f, frameLen)
			_, err := fak.WriteToBuffer(*f, frameLen)
			if err != nil {
				panic(err)
			}
			_, err, errs := fak.FlushFrames()
			if err != nil {
				panic(err)
			}
			if len(errs) > 0 {
				panic(errs)
			}
		}))
	}()
	err = fak.Listen(func(f *nettypes.Frame, frameLen uint16) {
		fmt.Println("fake:")
		fmt.Println(f.String(frameLen, 0))
		processFrame(f, frameLen)
		_, err := zs.WriteToBuffer(*f, frameLen)
		if err != nil {
			panic(err)
		}
		_, err, errs := zs.FlushFrames()
		if err != nil {
			panic(err)
		}
		if len(errs) > 0 {
			panic(errs)
		}
	})
	if err != nil {
		panic(err)
	}
}

func processFrame(f *nettypes.Frame, frameLen uint16) {
	if f.MACEthertype(nettypes.NotTagged) == nettypes.IPv4 {
		ln := frameLen
		mPay, mOff := f.MACPayload(nettypes.NotTagged)
		ln -= mOff
		ip := nettypes.IPv4_P(mPay)
		iPay, iOff := ip.Payload()
		ln -= iOff
		switch ip.Protocol() {
		case nettypes.TCP:
			tcp := nettypes.TCP_P(iPay)
			tcp.SetChecksum(tcp.CalculateChecksum(ln, ip.SourceIP(), ip.DestinationIP()))
		case nettypes.UDP:
			udp := nettypes.UDP_P(iPay)
			udp.SetChecksum(udp.CalculateChecksum())
		}
	}
}
<commit_msg>Fixes to update it to the newer spec<commit_after>package main

import (
	"fmt"
	"net"
	"time"

	"github.com/nathanjsweet/zsocket"
	fi "github.com/nathanjsweet/zsocket/fakeinterface"
	"github.com/nathanjsweet/zsocket/nettypes"
)

func main() {
	zs, err := zsocket.NewZSocket(18, zsocket.ENABLE_RX|zsocket.ENABLE_TX, 32768, 128, nettypes.All)
	if err != nil {
		panic(err)
	}
	fip := net.IPAddr{net.IP([]byte{192, 0, 0, 3}), ""}
	fak, err := fi.NewFakeInterface("fake1", net.HardwareAddr([]byte{0x1e, 0x44, 0x70, 0x3c, 0x66, 0xe3}), &fip, 30000)
	go func() {
		for {
			fmt.Printf("%s\n", fak)
			time.Sleep(time.Second * time.Duration(10))
		}
	}()
	if err != nil {
		panic(err)
	}
	go func() {
		panic(zs.Listen(func(f *nettypes.Frame, frameLen, capturedLen uint16) {
			fmt.Println("20:")
			fmt.Println(f.String(frameLen, 0))
			processFrame(f, frameLen)
			_, err := fak.WriteToBuffer(*f, frameLen)
			if err != nil {
				panic(err)
			}
			_, err, errs := fak.FlushFrames()
			if err != nil {
				panic(err)
			}
			if len(errs) > 0 {
				panic(errs)
			}
		}))
	}()
	err = fak.Listen(func(f *nettypes.Frame, frameLen uint16) {
		fmt.Println("fake:")
		fmt.Println(f.String(frameLen, 0))
		processFrame(f, frameLen)
		_, err := zs.WriteToBuffer(*f, frameLen)
		if err != nil {
			panic(err)
		}
		_, err, errs := zs.FlushFrames()
		if err != nil {
			panic(err)
		}
		if len(errs) > 0 {
			panic(errs)
		}
	})
	if err != nil {
		panic(err)
	}
}

func processFrame(f *nettypes.Frame, frameLen uint16) {
	if f.MACEthertype(nettypes.NotTagged) == nettypes.IPv4 {
		ln := frameLen
		mPay, mOff := f.MACPayload(nettypes.NotTagged)
		ln -= mOff
		ip := nettypes.IPv4_P(mPay)
		iPay, iOff := ip.Payload()
		ln -= iOff
		switch ip.Protocol() {
		case nettypes.TCP:
			tcp := nettypes.TCP_P(iPay)
			tcp.SetChecksum(tcp.CalculateChecksum(ln, ip.SourceIP(), ip.DestinationIP()))
		case nettypes.UDP:
			udp := nettypes.UDP_P(iPay)
			udp.SetChecksum(udp.CalculateChecksum())
		}
	}
}
<|endoftext|>


<commit_before>// +build api

/*
 * This file is part of the libvirt-go project
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * Copyright (c) 2013 Alex Zorin
 * Copyright (C) 2016 Red Hat, Inc.
 *
 */

package libvirt

import (
	"bufio"
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"sort"
	"strings"
	"testing"
)

/*
 * We search for 'C.XXXXX' in *.go. Any false positives need
 * to be listed in one of these vars below
 */
var (
	ignoreFuncs = []string{
		/* Obsolete we use virConnectDomainEventRegisterAny instead */
		"virConnectDomainEventRegister",

		/* Wrapped in connect_cfuncs.go instead */
		"virConnectOpenAuth",
		"virConnectRegisterCloseCallback",

		/* Auth callback typedef */
		"virConnectAuthCallbackPtr",

		/* Not thread safe, so not exposed */
		"virConnCopyLastError",
		"virConnGetLastError",
		"virConnResetLastError",
		"virConnSetErrorFunc",

		/* Only needed at C level */
		"virCopyLastError",
		"virFreeError",
		"virGetLastErrorMessage",
		"virResetLastError",
		"virSaveLastError",
		"virDefaultErrorFunc",

		/* Deprecated in favour of virDomainCreateXML */
		"virDomainCreateLinux",

		/* Connect callback typedef */
		"virConnectCloseFunc",

		/* Data free callback typedef */
		"virFreeCallback",

		/* Error callback typedef */
		"virErrorFunc",

		/* Domain event callback typedefs */
		"virConnectDomainEventAgentLifecycleCallback",
		"virConnectDomainEventBalloonChangeCallback",
		"virConnectDomainEventBlockJobCallback",
		"virConnectDomainEventCallback",
		"virConnectDomainEventDeregister",
		"virConnectDomainEventDeviceAddedCallback",
		"virConnectDomainEventDeviceRemovalFailedCallback",
		"virConnectDomainEventDeviceRemovedCallback",
		"virConnectDomainEventDiskChangeCallback",
		"virConnectDomainEventGraphicsCallback",
		"virConnectDomainEventIOErrorCallback",
		"virConnectDomainEventIOErrorReasonCallback",
		"virConnectDomainEventJobCompletedCallback",
		"virConnectDomainEventMigrationIterationCallback",
		"virConnectDomainEventPMSuspendCallback",
		"virConnectDomainEventPMSuspendDiskCallback",
		"virConnectDomainEventPMWakeupCallback",
		"virConnectDomainEventRTCChangeCallback",
		"virConnectDomainEventTrayChangeCallback",
		"virConnectDomainEventTunableCallback",
		"virConnectDomainEventWatchdogCallback",
		"virConnectDomainEventMetadataChangeCallback",
		"virConnectDomainEventBlockThresholdCallback",

		/* Network event callback typedefs */
		"virConnectNetworkEventGenericCallback",
		"virConnectNetworkEventLifecycleCallback",

		/* Node device event callback typedefs */
		"virConnectNodeDeviceEventGenericCallback",
		"virConnectNodeDeviceEventLifecycleCallback",

		/* Secret event callback typedefs */
		"virConnectSecretEventGenericCallback",
		"virConnectSecretEventLifecycleCallback",

		/* Storage pool event callback typedefs */
		"virConnectStoragePoolEventGenericCallback",
		"virConnectStoragePoolEventLifecycleCallback",

		/* Stream event callback typedef */
		"virStreamEventCallback",

		/* Event loop callback typedefs */
		"virEventAddHandleFunc",
		"virEventAddTimeoutFunc",
		"virEventHandleCallback",
		"virEventRemoveHandleFunc",
		"virEventRemoveTimeoutFunc",
		"virEventTimeoutCallback",
		"virEventUpdateHandleFunc",
		"virEventUpdateTimeoutFunc",

		/* Typedefs that don't need exposing as is */
		"virStreamSinkFunc",
		"virStreamSourceFunc",

		/* Only needed at C level */
		"virDomainGetConnect",
		"virDomainSnapshotGetConnect",
		"virDomainSnapshotGetDomain",
		"virInterfaceGetConnect",
		"virNetworkGetConnect",
		"virSecretGetConnect",
		"virStoragePoolGetConnect",
		"virStorageVolGetConnect",

		/* Only needed at C level */
		"virTypedParamsAddBoolean",
		"virTypedParamsAddDouble",
		"virTypedParamsAddFromString",
		"virTypedParamsAddInt",
		"virTypedParamsAddLLong",
		"virTypedParamsAddString",
		"virTypedParamsAddStringList",
		"virTypedParamsAddUInt",
		"virTypedParamsAddULLong",
		"virTypedParamsGet",
		"virTypedParamsGetBoolean",
		"virTypedParamsGetDouble",
		"virTypedParamsGetInt",
		"virTypedParamsGetLLong",
		"virTypedParamsGetString",
		"virTypedParamsGetUInt",
		"virTypedParamsGetULLong",
		"virTypedParamsFree",
	}

	ignoreMacros = []string{
		/* Can't be used as they contain a C format string
		 * that is not supported in go */
		"VIR_DOMAIN_TUNABLE_CPU_IOTHREADSPIN",
		"VIR_DOMAIN_TUNABLE_CPU_VCPUPIN",

		/* Compat defines for obsolete types */
		"_virBlkioParameter",
		"_virMemoryParameter",
		"_virSchedParameter",

		/* Remaped to a funcs at Go level */
		"LIBVIR_CHECK_VERSION",
		"VIR_NODEINFO_MAXCPUS",

		/* Obsoleted by VIR_TYPED_PARAM_FIELD_LENGTH */
		"VIR_DOMAIN_BLKIO_FIELD_LENGTH",
		"VIR_DOMAIN_BLOCK_STATS_FIELD_LENGTH",
		"VIR_DOMAIN_MEMORY_FIELD_LENGTH",
		"VIR_DOMAIN_SCHED_FIELD_LENGTH",
		"VIR_NODE_CPU_STATS_FIELD_LENGTH",
		"VIR_NODE_MEMORY_STATS_FIELD_LENGTH",

		/* Only needed at C level */
		"VIR_COPY_CPUMAP",
		"VIR_CPU_MAPLEN",
		"VIR_CPU_USABLE",
		"VIR_CPU_USED",
		"VIR_DOMAIN_EVENT_CALLBACK",
		"VIR_GET_CPUMAP",
		"VIR_NETWORK_EVENT_CALLBACK",
		"VIR_NODE_DEVICE_EVENT_CALLBACK",
		"VIR_SECRET_EVENT_CALLBACK",
		"VIR_SECURITY_DOI_BUFLEN",
		"VIR_SECURITY_LABEL_BUFLEN",
		"VIR_SECURITY_MODEL_BUFLEN",
		"VIR_STORAGE_POOL_EVENT_CALLBACK",
		"VIR_UNUSE_CPU",
		"VIR_USE_CPU",
	}

	ignoreEnums = []string{

		// Deprecated in favour of VIR_TYPED_PARAM_*
		"VIR_DOMAIN_BLKIO_PARAM_BOOLEAN",
		"VIR_DOMAIN_BLKIO_PARAM_DOUBLE",
		"VIR_DOMAIN_BLKIO_PARAM_INT",
		"VIR_DOMAIN_BLKIO_PARAM_LLONG",
		"VIR_DOMAIN_BLKIO_PARAM_UINT",
		"VIR_DOMAIN_BLKIO_PARAM_ULLONG",
		"VIR_DOMAIN_MEMORY_PARAM_BOOLEAN",
		"VIR_DOMAIN_MEMORY_PARAM_DOUBLE",
		"VIR_DOMAIN_MEMORY_PARAM_INT",
		"VIR_DOMAIN_MEMORY_PARAM_LLONG",
		"VIR_DOMAIN_MEMORY_PARAM_UINT",
		"VIR_DOMAIN_MEMORY_PARAM_ULLONG",
		"VIR_DOMAIN_SCHED_FIELD_BOOLEAN",
		"VIR_DOMAIN_SCHED_FIELD_DOUBLE",
		"VIR_DOMAIN_SCHED_FIELD_INT",
		"VIR_DOMAIN_SCHED_FIELD_LLONG",
		"VIR_DOMAIN_SCHED_FIELD_UINT",
		"VIR_DOMAIN_SCHED_FIELD_ULLONG",

		"VIR_TYPED_PARAM_STRING_OKAY",
	}
)

type CharsetISO88591er struct {
	r   io.ByteReader
	buf *bytes.Buffer
}

func NewCharsetISO88591(r io.Reader) *CharsetISO88591er {
	buf := bytes.Buffer{}
	return &CharsetISO88591er{r.(io.ByteReader), &buf}
}

func (cs *CharsetISO88591er) Read(p []byte) (n int, err error) {
	for _ = range p {
		if r, err := cs.r.ReadByte(); err != nil {
			break
		} else {
			cs.buf.WriteRune(rune(r))
		}
	}
	return cs.buf.Read(p)
}

func isCharset(charset string, names []string) bool {
	charset = strings.ToLower(charset)
	for _, n := range names {
		if charset == strings.ToLower(n) {
			return true
		}
	}
	return false
}

func IsCharsetISO88591(charset string) bool {
	// http://www.iana.org/assignments/character-sets
	// (last updated 2010-11-04)
	names := []string{
		// Name
		"ISO_8859-1:1987",
		// Alias (preferred MIME name)
		"ISO-8859-1",
		// Aliases
		"iso-ir-100",
		"ISO_8859-1",
		"latin1",
		"l1",
		"IBM819",
		"CP819",
		"csISOLatin1",
	}
	return isCharset(charset, names)
}

func CharsetReader(charset string, input io.Reader) (io.Reader, error) {
	if IsCharsetISO88591(charset) {
		return NewCharsetISO88591(input), nil
	}
	return input, nil
}

type APIExport struct {
	Type   string `xml:"type,attr"`
	Symbol string `xml:"symbol,attr"`
}

type APIFile struct {
	Name    string      `xml:"name,attr"`
	Exports []APIExport `xml:"exports"`
}

type API struct {
	XMLName xml.Name  `xml:"api"`
	Files   []APIFile `xml:"files>file"`
}

func GetAPIPath() string {
	cmd := exec.Command("pkg-config", "--variable=libvirt_api", "libvirt")

	cmdOutput := &bytes.Buffer{}
	cmd.Stdout = cmdOutput

	err := cmd.Run()
	if err != nil {
		panic(err)
	}

	return strings.TrimSpace(string(cmdOutput.Bytes()))
}

func GetAPI(path string) *API {
	file, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	defer file.Close()

	decoder := xml.NewDecoder(file)
	decoder.CharsetReader = CharsetReader
	var api API
	err = decoder.Decode(&api)
	if err != nil {
		panic(err)
	}

	return &api
}

func GetSourceFiles() []string {
	files, _ := ioutil.ReadDir(".")

	src := make([]string, 0)
	for _, f := range files {
		if strings.HasSuffix(f.Name(), ".go") &&
			!strings.HasSuffix(f.Name(), "_test.go") {
			src = append(src, f.Name())
		}
	}

	return src
}

func GetAPISymbols(api *API) (funcs map[string]bool, macros map[string]bool, enums map[string]bool) {

	funcs = make(map[string]bool)
	macros = make(map[string]bool)
	enums = make(map[string]bool)
	for _, file := range api.Files {
		for _, export := range file.Exports {
			if export.Type == "function" {
				funcs[export.Symbol] = true
			} else if export.Type == "enum" {
				if !strings.HasSuffix(export.Symbol, "_LAST") {
					enums[export.Symbol] = true
				}
			} else if export.Type == "macro" {
				macros[export.Symbol] = true
			}
		}
	}

	return
}

func ProcessFile(path string) []string {
	file, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	defer file.Close()

	re, err := regexp.Compile("C\\.((vir|VIR|LIBVIR)[a-zA-Z0-9_]+?)(Compat|_cgo)?\\b")
	if err != nil {
		panic(err)
	}

	scanner := bufio.NewScanner(file)
	symbols := make([]string, 0)
	for scanner.Scan() {
		line := scanner.Text()

		match := re.FindStringSubmatch(line)
		if match != nil {
			symbols = append(symbols, match[1])
		}
	}

	if err := scanner.Err(); err != nil {
		panic(err)
	}

	return symbols
}

func RecordUsage(symbols []string, funcs map[string]bool, macros map[string]bool, enums map[string]bool) {

	for _, name := range symbols {
		_, ok := funcs[name]
		if ok {
			funcs[name] = false
			continue
		}

		_, ok = macros[name]
		if ok {
			macros[name] = false
			continue
		}

		_, ok = enums[name]
		if ok {
			enums[name] = false
			continue
		}
	}
}

func ReportMissing(missingNames map[string]bool, symtype string) bool {
	missing := make([]string, 0)
	for key, value := range missingNames {
		if value {
			missing = append(missing, key)
		}
	}

	sort.Strings(missing)

	for _, name := range missing {
		fmt.Println("Missing " + symtype + " '" + name + "'")
	}

	return len(missing) != 0
}

func SetIgnores(ignores []string, symbols map[string]bool) {
	for _, name := range ignores {
		symbols[name] = false
	}
}

func TestAPICoverage(t *testing.T) {
	path := GetAPIPath()
	api := GetAPI(path)

	funcs, macros, enums := GetAPISymbols(api)

	SetIgnores(ignoreFuncs, funcs)
	SetIgnores(ignoreMacros, macros)
	SetIgnores(ignoreEnums, enums)

	src := GetSourceFiles()

	for _, path := range src {
		symbols := ProcessFile(path)

		RecordUsage(symbols, funcs, macros, enums)
	}

	missing := false
	if ReportMissing(funcs, "function") {
		missing = true
	}
	if ReportMissing(macros, "macro") {
		missing = true
	}
	if ReportMissing(enums, "enum") {
		missing = true
	}
	if missing {
		panic("Missing symbols found")
	}
}
<commit_msg>Extend API test suite to cover LXC & QEMU specific APIs<commit_after>// +build api

/*
 * This file is part of the libvirt-go project
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * Copyright (c) 2013 Alex Zorin
 * Copyright (C) 2016 Red Hat, Inc.
 *
 */

package libvirt

import (
	"bufio"
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"sort"
	"strings"
	"testing"
)

/*
 * We search for 'C.XXXXX' in *.go. Any false positives need
 * to be listed in one of these vars below
 */
var (
	ignoreFuncs = []string{
		/* Obsolete we use virConnectDomainEventRegisterAny instead */
		"virConnectDomainEventRegister",

		/* Wrapped in connect_cfuncs.go instead */
		"virConnectOpenAuth",
		"virConnectRegisterCloseCallback",

		/* Auth callback typedef */
		"virConnectAuthCallbackPtr",

		/* Not thread safe, so not exposed */
		"virConnCopyLastError",
		"virConnGetLastError",
		"virConnResetLastError",
		"virConnSetErrorFunc",

		/* Only needed at C level */
		"virCopyLastError",
		"virFreeError",
		"virGetLastErrorMessage",
		"virResetLastError",
		"virSaveLastError",
		"virDefaultErrorFunc",

		/* Deprecated in favour of virDomainCreateXML */
		"virDomainCreateLinux",

		/* Connect callback typedef */
		"virConnectCloseFunc",

		/* Data free callback typedef */
		"virFreeCallback",

		/* Error callback typedef */
		"virErrorFunc",

		/* Domain event callback typedefs */
		"virConnectDomainEventAgentLifecycleCallback",
		"virConnectDomainEventBalloonChangeCallback",
		"virConnectDomainEventBlockJobCallback",
		"virConnectDomainEventCallback",
		"virConnectDomainEventDeregister",
		"virConnectDomainEventDeviceAddedCallback",
		"virConnectDomainEventDeviceRemovalFailedCallback",
		"virConnectDomainEventDeviceRemovedCallback",
		"virConnectDomainEventDiskChangeCallback",
		"virConnectDomainEventGraphicsCallback",
		"virConnectDomainEventIOErrorCallback",
		"virConnectDomainEventIOErrorReasonCallback",
		"virConnectDomainEventJobCompletedCallback",
		"virConnectDomainEventMigrationIterationCallback",
		"virConnectDomainEventPMSuspendCallback",
		"virConnectDomainEventPMSuspendDiskCallback",
		"virConnectDomainEventPMWakeupCallback",
		"virConnectDomainEventRTCChangeCallback",
		"virConnectDomainEventTrayChangeCallback",
		"virConnectDomainEventTunableCallback",
		"virConnectDomainEventWatchdogCallback",
		"virConnectDomainEventMetadataChangeCallback",
		"virConnectDomainEventBlockThresholdCallback",
		"virConnectDomainQemuMonitorEventCallback",

		/* Network event callback typedefs */
		"virConnectNetworkEventGenericCallback",
		"virConnectNetworkEventLifecycleCallback",

		/* Node device event callback typedefs */
		"virConnectNodeDeviceEventGenericCallback",
		"virConnectNodeDeviceEventLifecycleCallback",

		/* Secret event callback typedefs */
		"virConnectSecretEventGenericCallback",
		"virConnectSecretEventLifecycleCallback",

		/* Storage pool event callback typedefs */
		"virConnectStoragePoolEventGenericCallback",
		"virConnectStoragePoolEventLifecycleCallback",

		/* Stream event callback typedef */
		"virStreamEventCallback",

		/* Event loop callback typedefs */
		"virEventAddHandleFunc",
		"virEventAddTimeoutFunc",
		"virEventHandleCallback",
		"virEventRemoveHandleFunc",
		"virEventRemoveTimeoutFunc",
		"virEventTimeoutCallback",
		"virEventUpdateHandleFunc",
		"virEventUpdateTimeoutFunc",

		/* Typedefs that don't need exposing as is */
		"virStreamSinkFunc",
		"virStreamSourceFunc",

		/* Only needed at C level */
		"virDomainGetConnect",
		"virDomainSnapshotGetConnect",
		"virDomainSnapshotGetDomain",
		"virInterfaceGetConnect",
		"virNetworkGetConnect",
		"virSecretGetConnect",
		"virStoragePoolGetConnect",
		"virStorageVolGetConnect",

		/* Only needed at C level */
		"virTypedParamsAddBoolean",
		"virTypedParamsAddDouble",
		"virTypedParamsAddFromString",
		"virTypedParamsAddInt",
		"virTypedParamsAddLLong",
		"virTypedParamsAddString",
		"virTypedParamsAddStringList",
		"virTypedParamsAddUInt",
		"virTypedParamsAddULLong",
		"virTypedParamsGet",
		"virTypedParamsGetBoolean",
		"virTypedParamsGetDouble",
		"virTypedParamsGetInt",
		"virTypedParamsGetLLong",
		"virTypedParamsGetString",
		"virTypedParamsGetUInt",
		"virTypedParamsGetULLong",
		"virTypedParamsFree",
	}

	ignoreMacros = []string{
		/* Can't be used as they contain a C format string
		 * that is not supported in go */
		"VIR_DOMAIN_TUNABLE_CPU_IOTHREADSPIN",
		"VIR_DOMAIN_TUNABLE_CPU_VCPUPIN",

		/* Compat defines for obsolete types */
		"_virBlkioParameter",
		"_virMemoryParameter",
		"_virSchedParameter",

		/* Remaped to a funcs at Go level */
		"LIBVIR_CHECK_VERSION",
		"VIR_NODEINFO_MAXCPUS",

		/* Obsoleted by VIR_TYPED_PARAM_FIELD_LENGTH */
		"VIR_DOMAIN_BLKIO_FIELD_LENGTH",
		"VIR_DOMAIN_BLOCK_STATS_FIELD_LENGTH",
		"VIR_DOMAIN_MEMORY_FIELD_LENGTH",
		"VIR_DOMAIN_SCHED_FIELD_LENGTH",
		"VIR_NODE_CPU_STATS_FIELD_LENGTH",
		"VIR_NODE_MEMORY_STATS_FIELD_LENGTH",

		/* Only needed at C level */
		"VIR_COPY_CPUMAP",
		"VIR_CPU_MAPLEN",
		"VIR_CPU_USABLE",
		"VIR_CPU_USED",
		"VIR_DOMAIN_EVENT_CALLBACK",
		"VIR_GET_CPUMAP",
		"VIR_NETWORK_EVENT_CALLBACK",
		"VIR_NODE_DEVICE_EVENT_CALLBACK",
		"VIR_SECRET_EVENT_CALLBACK",
		"VIR_SECURITY_DOI_BUFLEN",
		"VIR_SECURITY_LABEL_BUFLEN",
		"VIR_SECURITY_MODEL_BUFLEN",
		"VIR_STORAGE_POOL_EVENT_CALLBACK",
		"VIR_UNUSE_CPU",
		"VIR_USE_CPU",
	}

	ignoreEnums = []string{

		// Deprecated in favour of VIR_TYPED_PARAM_*
		"VIR_DOMAIN_BLKIO_PARAM_BOOLEAN",
		"VIR_DOMAIN_BLKIO_PARAM_DOUBLE",
		"VIR_DOMAIN_BLKIO_PARAM_INT",
		"VIR_DOMAIN_BLKIO_PARAM_LLONG",
		"VIR_DOMAIN_BLKIO_PARAM_UINT",
		"VIR_DOMAIN_BLKIO_PARAM_ULLONG",
		"VIR_DOMAIN_MEMORY_PARAM_BOOLEAN",
		"VIR_DOMAIN_MEMORY_PARAM_DOUBLE",
		"VIR_DOMAIN_MEMORY_PARAM_INT",
		"VIR_DOMAIN_MEMORY_PARAM_LLONG",
		"VIR_DOMAIN_MEMORY_PARAM_UINT",
		"VIR_DOMAIN_MEMORY_PARAM_ULLONG",
		"VIR_DOMAIN_SCHED_FIELD_BOOLEAN",
		"VIR_DOMAIN_SCHED_FIELD_DOUBLE",
		"VIR_DOMAIN_SCHED_FIELD_INT",
		"VIR_DOMAIN_SCHED_FIELD_LLONG",
		"VIR_DOMAIN_SCHED_FIELD_UINT",
		"VIR_DOMAIN_SCHED_FIELD_ULLONG",

		"VIR_TYPED_PARAM_STRING_OKAY",
	}
)

type CharsetISO88591er struct {
	r   io.ByteReader
	buf *bytes.Buffer
}

func NewCharsetISO88591(r io.Reader) *CharsetISO88591er {
	buf := bytes.Buffer{}
	return &CharsetISO88591er{r.(io.ByteReader), &buf}
}

func (cs *CharsetISO88591er) Read(p []byte) (n int, err error) {
	for _ = range p {
		if r, err := cs.r.ReadByte(); err != nil {
			break
		} else {
			cs.buf.WriteRune(rune(r))
		}
	}
	return cs.buf.Read(p)
}

func isCharset(charset string, names []string) bool {
	charset = strings.ToLower(charset)
	for _, n := range names {
		if charset == strings.ToLower(n) {
			return true
		}
	}
	return false
}

func IsCharsetISO88591(charset string) bool {
	// http://www.iana.org/assignments/character-sets
	// (last updated 2010-11-04)
	names := []string{
		// Name
		"ISO_8859-1:1987",
		// Alias (preferred MIME name)
		"ISO-8859-1",
		// Aliases
		"iso-ir-100",
		"ISO_8859-1",
		"latin1",
		"l1",
		"IBM819",
		"CP819",
		"csISOLatin1",
	}
	return isCharset(charset, names)
}

func CharsetReader(charset string, input io.Reader) (io.Reader, error) {
	if IsCharsetISO88591(charset) {
		return NewCharsetISO88591(input), nil
	}
	return input, nil
}

type APIExport struct {
	Type   string `xml:"type,attr"`
	Symbol string `xml:"symbol,attr"`
}

type APIFile struct {
	Name    string      `xml:"name,attr"`
	Exports []APIExport `xml:"exports"`
}

type API struct {
	XMLName xml.Name  `xml:"api"`
	Files   []APIFile `xml:"files>file"`
}

func GetAPIPath(varname, modname string) string {
	cmd := exec.Command("pkg-config", "--variable="+varname, modname)

	cmdOutput := &bytes.Buffer{}
	cmd.Stdout = cmdOutput

	err := cmd.Run()
	if err != nil {
		panic(err)
	}

	return strings.TrimSpace(string(cmdOutput.Bytes()))
}

func GetAPI(path string) *API {
	file, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	defer file.Close()

	decoder := xml.NewDecoder(file)
	decoder.CharsetReader = CharsetReader
	var api API
	err = decoder.Decode(&api)
	if err != nil {
		panic(err)
	}

	return &api
}

func GetSourceFiles() []string {
	files, _ := ioutil.ReadDir(".")

	src := make([]string, 0)
	for _, f := range files {
		if strings.HasSuffix(f.Name(), ".go") &&
			!strings.HasSuffix(f.Name(), "_test.go") {
			src = append(src, f.Name())
		}
	}

	return src
}

func GetAPISymbols(api *API, funcs map[string]bool, macros map[string]bool, enums map[string]bool) {

	for _, file := range api.Files {
		for _, export := range file.Exports {
			if export.Type == "function" {
				funcs[export.Symbol] = true
			} else if export.Type == "enum" {
				if !strings.HasSuffix(export.Symbol, "_LAST") {
					enums[export.Symbol] = true
				}
			} else if export.Type == "macro" {
				macros[export.Symbol] = true
			}
		}
	}

	return
}

func ProcessFile(path string) []string {
	file, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	defer file.Close()

	re, err := regexp.Compile("C\\.((vir|VIR|LIBVIR)[a-zA-Z0-9_]+?)(Compat|_cgo)?\\b")
	if err != nil {
		panic(err)
	}

	scanner := bufio.NewScanner(file)
	symbols := make([]string, 0)
	for scanner.Scan() {
		line := scanner.Text()

		match := re.FindStringSubmatch(line)
		if match != nil {
			symbols = append(symbols, match[1])
		}
	}

	if err := scanner.Err(); err != nil {
		panic(err)
	}

	return symbols
}

func RecordUsage(symbols []string, funcs map[string]bool, macros map[string]bool, enums map[string]bool) {

	for _, name := range symbols {
		_, ok := funcs[name]
		if ok {
			funcs[name] = false
			continue
		}

		_, ok = macros[name]
		if ok {
			macros[name] = false
			continue
		}

		_, ok = enums[name]
		if ok {
			enums[name] = false
			continue
		}
	}
}

func ReportMissing(missingNames map[string]bool, symtype string) bool {
	missing := make([]string, 0)
	for key, value := range missingNames {
		if value {
			missing = append(missing, key)
		}
	}

	sort.Strings(missing)

	for _, name := range missing {
		fmt.Println("Missing " + symtype + " '" + name + "'")
	}

	return len(missing) != 0
}

func SetIgnores(ignores []string, symbols map[string]bool) {
	for _, name := range ignores {
		symbols[name] = false
	}
}

func TestAPICoverage(t *testing.T) {
	funcs := make(map[string]bool)
	macros := make(map[string]bool)
	enums := make(map[string]bool)

	path := GetAPIPath("libvirt_api", "libvirt")
	lxcpath := GetAPIPath("libvirt_lxc_api", "libvirt-lxc")
	qemupath := GetAPIPath("libvirt_qemu_api", "libvirt-qemu")

	api := GetAPI(path)
	lxcapi := GetAPI(lxcpath)
	qemuapi := GetAPI(qemupath)

	GetAPISymbols(api, funcs, macros, enums)
	GetAPISymbols(lxcapi, funcs, macros, enums)
	GetAPISymbols(qemuapi, funcs, macros, enums)

	SetIgnores(ignoreFuncs, funcs)
	SetIgnores(ignoreMacros, macros)
	SetIgnores(ignoreEnums, enums)

	src := GetSourceFiles()

	for _, path := range src {
		symbols := ProcessFile(path)

		RecordUsage(symbols, funcs, macros, enums)
	}

	missing := false
	if ReportMissing(funcs, "function") {
		missing = true
	}
	if ReportMissing(macros, "macro") {
		missing = true
	}
	if ReportMissing(enums, "enum") {
		missing = true
	}
	if missing {
		panic("Missing symbols found")
	}
}
<|endoftext|>


<commit_before>package main

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "launchpad.net/gocheck"

	"testing"
)

// hook up gocheck into the "go test" runner.
func Test(t *testing.T) { TestingT(t) }

func TestNewspeak(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Newspeak Suite")
}
<commit_msg>add gocheck benchmark<commit_after>package main

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "launchpad.net/gocheck"

	"testing"
)

// hook up gocheck into the "go test" runner.
func Test(t *testing.T) { TestingT(t) }

type NewspeakSuite struct{}
var _ = Suite(&NewspeakSuite{})

// unit tests
func TestNewspeak(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Newspeak Suite")
}

// benchmarks
func (s *NewspeakSuite) BenchmarkLogic(c *C) {
	for i := 0; i < c.N; i++ {
		// logic to benchmark
	}
}
<|endoftext|>


<commit_before>package emitter

import (
	"fmt"
	"net"
	"net/http"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/metric"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

type PrometheusEmitter struct {
	buildsStarted     prometheus.Counter
	buildsFinished    prometheus.Counter
	buildsSucceeded   prometheus.Counter
	buildsErrored     prometheus.Counter
	buildsFailed      prometheus.Counter
	buildsAborted     prometheus.Counter
	buildsFinishedVec *prometheus.CounterVec
	buildDurationsVec *prometheus.HistogramVec

	workerContainers *prometheus.GaugeVec
	workerVolumes    *prometheus.GaugeVec

	httpRequestsDuration *prometheus.HistogramVec
}

type PrometheusConfig struct {
	BindIP   string `long:"prometheus-bind-ip" description:"IP to listen on to expose Prometheus metrics."`
	BindPort string `long:"prometheus-bind-port" description:"Port to listen on to expose Prometheus metrics."`
}

func init() {
	metric.RegisterEmitter(&PrometheusConfig{})
}

func (config *PrometheusConfig) Description() string { return "Prometheus" }
func (config *PrometheusConfig) IsConfigured() bool {
	return config.BindPort != "" && config.BindIP != ""
}
func (config *PrometheusConfig) bind() string {
	return fmt.Sprintf("%s:%s", config.BindIP, config.BindPort)
}

func (config *PrometheusConfig) NewEmitter() (metric.Emitter, error) {
	buildsStarted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "started_total",
		Help:      "Total number of Concourse builds started.",
	})
	prometheus.MustRegister(buildsStarted)

	buildsFinished := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "finished_total",
		Help:      "Total number of Concourse builds finished.",
	})
	prometheus.MustRegister(buildsFinished)

	buildsSucceeded := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "succeeded_total",
		Help:      "Total number of Concourse builds succeeded.",
	})
	prometheus.MustRegister(buildsSucceeded)

	buildsErrored := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "errored_total",
		Help:      "Total number of Concourse builds errored.",
	})
	prometheus.MustRegister(buildsErrored)

	buildsFailed := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "failed_total",
		Help:      "Total number of Concourse builds failed.",
	})
	prometheus.MustRegister(buildsFailed)

	buildsAborted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "aborted_total",
		Help:      "Total number of Concourse builds aborted.",
	})
	prometheus.MustRegister(buildsAborted)

	buildsFinishedVec := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "finished",
			Help:      "Count of builds finished across various dimensions.",
		},
		[]string{"team", "pipeline", "status"},
	)
	prometheus.MustRegister(buildsFinishedVec)
	buildDurationsVec := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "duration_seconds",
			Help:      "Build time in seconds",
		},
		[]string{"team", "pipeline"},
	)
	prometheus.MustRegister(buildDurationsVec)

	workerContainers := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "containers",
			Help:      "Number of containers per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerContainers)
	workerVolumes := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "volumes",
			Help:      "Number of volumes per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerVolumes)

	httpRequestsDuration := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "http_responses",
			Name:      "duration_seconds",
			Help:      "Response time in seconds",
		},
		[]string{"path", "method"},
	)
	prometheus.MustRegister(httpRequestsDuration)

	listener, err := net.Listen("tcp", config.bind())
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, promhttp.Handler())

	return &PrometheusEmitter{
		buildsStarted:     buildsStarted,
		buildsFinished:    buildsFinished,
		buildsFinishedVec: buildsFinishedVec,
		buildDurationsVec: buildDurationsVec,
		buildsSucceeded:   buildsSucceeded,
		buildsErrored:     buildsErrored,
		buildsFailed:      buildsFailed,
		buildsAborted:     buildsAborted,

		workerContainers: workerContainers,
		workerVolumes:    workerVolumes,

		httpRequestsDuration: httpRequestsDuration,
	}, nil
}

// Emit processes incoming metrics.
// In order to provide idiomatic Prometheus metrics, we'll have to convert the various
// Event types (differentiated by the less-than-ideal string Name field) into different
// Prometheus metrics.
func (emitter *PrometheusEmitter) Emit(logger lager.Logger, event metric.Event) {
	switch event.Name {
	case "build started":
		emitter.buildsStarted.Inc()
	case "build finished":
		emitter.buildFinishedMetrics(logger, event)
	case "worker containers":
		emitter.workerContainersMetrics(logger, event)
	case "worker volumes":
		emitter.workerVolumesMetrics(logger, event)
	case "http response time":
		emitter.httpResponseTimeMetrics(logger, event)
	default:
		// unless we have a specific metric, we do nothing
	}
}

func (emitter *PrometheusEmitter) buildFinishedMetrics(logger lager.Logger, event metric.Event) {
	// concourse_builds_finished_total
	emitter.buildsFinished.Inc()

	// concourse_builds_finished
	team, exists := event.Attributes["team_name"]
	if !exists {
		logger.Error("failed-to-find-team-name-in-event", fmt.Errorf("expected team_name to exist in event.Attributes"))
	}

	pipeline, exists := event.Attributes["pipeline"]
	if !exists {
		logger.Error("failed-to-find-pipeline-in-event", fmt.Errorf("expected pipeline to exist in event.Attributes"))
	}

	buildStatus, exists := event.Attributes["build_status"]
	if !exists {
		logger.Error("failed-to-find-build_status-in-event", fmt.Errorf("expected build_status to exist in event.Attributes"))
	}
	emitter.buildsFinishedVec.WithLabelValues(team, pipeline, buildStatus).Inc()

	// concourse_builds_(aborted|succeeded|failed|errored)_total
	switch buildStatus {
	case string(db.BuildStatusAborted):
		// concourse_builds_aborted_total
		emitter.buildsAborted.Inc()
	case string(db.BuildStatusSucceeded):
		// concourse_builds_succeeded_total
		emitter.buildsSucceeded.Inc()
	case string(db.BuildStatusFailed):
		// concourse_builds_failed_total
		emitter.buildsFailed.Inc()
	case string(db.BuildStatusErrored):
		// concourse_builds_errored_total
		emitter.buildsErrored.Inc()
	}

	// concourse_builds_duration_seconds
	duration, ok := event.Value.(float64)
	if !ok {
		logger.Error("build-finished-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}
	// seconds are the standard prometheus base unit for time
	duration = duration / 1000
	emitter.buildDurationsVec.WithLabelValues(team, pipeline).Observe(duration)
}

func (emitter *PrometheusEmitter) workerContainersMetrics(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	containers, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerContainers.WithLabelValues(worker).Set(float64(containers))
}

func (emitter *PrometheusEmitter) workerVolumesMetrics(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	workers, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerContainers.WithLabelValues(worker).Set(float64(workers))
}

func (emitter *PrometheusEmitter) httpResponseTimeMetrics(logger lager.Logger, event metric.Event) {
	path, exists := event.Attributes["path"]
	if !exists {
		logger.Error("failed-to-find-path-in-event", fmt.Errorf("expected path to exist in event.Attributes"))
	}
	method, exists := event.Attributes["method"]
	if !exists {
		logger.Error("failed-to-find-method-in-event", fmt.Errorf("expected method to exist in event.Attributes"))
	}

	responseTime, ok := event.Value.(float64)
	if !ok {
		logger.Error("http-response-time-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}

	emitter.httpRequestsDuration.WithLabelValues(path, method).Observe(responseTime / 1000)
}
<commit_msg>Fix typo in prometheus worker volumes metric.<commit_after>package emitter

import (
	"fmt"
	"net"
	"net/http"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/metric"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

type PrometheusEmitter struct {
	buildsStarted     prometheus.Counter
	buildsFinished    prometheus.Counter
	buildsSucceeded   prometheus.Counter
	buildsErrored     prometheus.Counter
	buildsFailed      prometheus.Counter
	buildsAborted     prometheus.Counter
	buildsFinishedVec *prometheus.CounterVec
	buildDurationsVec *prometheus.HistogramVec

	workerContainers *prometheus.GaugeVec
	workerVolumes    *prometheus.GaugeVec

	httpRequestsDuration *prometheus.HistogramVec
}

type PrometheusConfig struct {
	BindIP   string `long:"prometheus-bind-ip" description:"IP to listen on to expose Prometheus metrics."`
	BindPort string `long:"prometheus-bind-port" description:"Port to listen on to expose Prometheus metrics."`
}

func init() {
	metric.RegisterEmitter(&PrometheusConfig{})
}

func (config *PrometheusConfig) Description() string { return "Prometheus" }
func (config *PrometheusConfig) IsConfigured() bool {
	return config.BindPort != "" && config.BindIP != ""
}
func (config *PrometheusConfig) bind() string {
	return fmt.Sprintf("%s:%s", config.BindIP, config.BindPort)
}

func (config *PrometheusConfig) NewEmitter() (metric.Emitter, error) {
	buildsStarted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "started_total",
		Help:      "Total number of Concourse builds started.",
	})
	prometheus.MustRegister(buildsStarted)

	buildsFinished := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "finished_total",
		Help:      "Total number of Concourse builds finished.",
	})
	prometheus.MustRegister(buildsFinished)

	buildsSucceeded := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "succeeded_total",
		Help:      "Total number of Concourse builds succeeded.",
	})
	prometheus.MustRegister(buildsSucceeded)

	buildsErrored := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "errored_total",
		Help:      "Total number of Concourse builds errored.",
	})
	prometheus.MustRegister(buildsErrored)

	buildsFailed := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "failed_total",
		Help:      "Total number of Concourse builds failed.",
	})
	prometheus.MustRegister(buildsFailed)

	buildsAborted := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "concourse",
		Subsystem: "builds",
		Name:      "aborted_total",
		Help:      "Total number of Concourse builds aborted.",
	})
	prometheus.MustRegister(buildsAborted)

	buildsFinishedVec := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "finished",
			Help:      "Count of builds finished across various dimensions.",
		},
		[]string{"team", "pipeline", "status"},
	)
	prometheus.MustRegister(buildsFinishedVec)
	buildDurationsVec := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "builds",
			Name:      "duration_seconds",
			Help:      "Build time in seconds",
		},
		[]string{"team", "pipeline"},
	)
	prometheus.MustRegister(buildDurationsVec)

	workerContainers := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "containers",
			Help:      "Number of containers per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerContainers)
	workerVolumes := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "concourse",
			Subsystem: "workers",
			Name:      "volumes",
			Help:      "Number of volumes per worker",
		},
		[]string{"worker"},
	)
	prometheus.MustRegister(workerVolumes)

	httpRequestsDuration := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "concourse",
			Subsystem: "http_responses",
			Name:      "duration_seconds",
			Help:      "Response time in seconds",
		},
		[]string{"path", "method"},
	)
	prometheus.MustRegister(httpRequestsDuration)

	listener, err := net.Listen("tcp", config.bind())
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, promhttp.Handler())

	return &PrometheusEmitter{
		buildsStarted:     buildsStarted,
		buildsFinished:    buildsFinished,
		buildsFinishedVec: buildsFinishedVec,
		buildDurationsVec: buildDurationsVec,
		buildsSucceeded:   buildsSucceeded,
		buildsErrored:     buildsErrored,
		buildsFailed:      buildsFailed,
		buildsAborted:     buildsAborted,

		workerContainers: workerContainers,
		workerVolumes:    workerVolumes,

		httpRequestsDuration: httpRequestsDuration,
	}, nil
}

// Emit processes incoming metrics.
// In order to provide idiomatic Prometheus metrics, we'll have to convert the various
// Event types (differentiated by the less-than-ideal string Name field) into different
// Prometheus metrics.
func (emitter *PrometheusEmitter) Emit(logger lager.Logger, event metric.Event) {
	switch event.Name {
	case "build started":
		emitter.buildsStarted.Inc()
	case "build finished":
		emitter.buildFinishedMetrics(logger, event)
	case "worker containers":
		emitter.workerContainersMetrics(logger, event)
	case "worker volumes":
		emitter.workerVolumesMetrics(logger, event)
	case "http response time":
		emitter.httpResponseTimeMetrics(logger, event)
	default:
		// unless we have a specific metric, we do nothing
	}
}

func (emitter *PrometheusEmitter) buildFinishedMetrics(logger lager.Logger, event metric.Event) {
	// concourse_builds_finished_total
	emitter.buildsFinished.Inc()

	// concourse_builds_finished
	team, exists := event.Attributes["team_name"]
	if !exists {
		logger.Error("failed-to-find-team-name-in-event", fmt.Errorf("expected team_name to exist in event.Attributes"))
	}

	pipeline, exists := event.Attributes["pipeline"]
	if !exists {
		logger.Error("failed-to-find-pipeline-in-event", fmt.Errorf("expected pipeline to exist in event.Attributes"))
	}

	buildStatus, exists := event.Attributes["build_status"]
	if !exists {
		logger.Error("failed-to-find-build_status-in-event", fmt.Errorf("expected build_status to exist in event.Attributes"))
	}
	emitter.buildsFinishedVec.WithLabelValues(team, pipeline, buildStatus).Inc()

	// concourse_builds_(aborted|succeeded|failed|errored)_total
	switch buildStatus {
	case string(db.BuildStatusAborted):
		// concourse_builds_aborted_total
		emitter.buildsAborted.Inc()
	case string(db.BuildStatusSucceeded):
		// concourse_builds_succeeded_total
		emitter.buildsSucceeded.Inc()
	case string(db.BuildStatusFailed):
		// concourse_builds_failed_total
		emitter.buildsFailed.Inc()
	case string(db.BuildStatusErrored):
		// concourse_builds_errored_total
		emitter.buildsErrored.Inc()
	}

	// concourse_builds_duration_seconds
	duration, ok := event.Value.(float64)
	if !ok {
		logger.Error("build-finished-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}
	// seconds are the standard prometheus base unit for time
	duration = duration / 1000
	emitter.buildDurationsVec.WithLabelValues(team, pipeline).Observe(duration)
}

func (emitter *PrometheusEmitter) workerContainersMetrics(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	containers, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerContainers.WithLabelValues(worker).Set(float64(containers))
}

func (emitter *PrometheusEmitter) workerVolumesMetrics(logger lager.Logger, event metric.Event) {
	worker, exists := event.Attributes["worker"]
	if !exists {
		logger.Error("failed-to-find-worker-in-event", fmt.Errorf("expected worker to exist in event.Attributes"))
	}

	volumes, ok := event.Value.(int)
	if !ok {
		logger.Error("worker-volumes-event-value-type-mismatch", fmt.Errorf("expected event.Value to be an int"))
	}

	emitter.workerVolumes.WithLabelValues(worker).Set(float64(volumes))
}

func (emitter *PrometheusEmitter) httpResponseTimeMetrics(logger lager.Logger, event metric.Event) {
	path, exists := event.Attributes["path"]
	if !exists {
		logger.Error("failed-to-find-path-in-event", fmt.Errorf("expected path to exist in event.Attributes"))
	}
	method, exists := event.Attributes["method"]
	if !exists {
		logger.Error("failed-to-find-method-in-event", fmt.Errorf("expected method to exist in event.Attributes"))
	}

	responseTime, ok := event.Value.(float64)
	if !ok {
		logger.Error("http-response-time-event-value-type-mismatch", fmt.Errorf("expected event.Value to be a float64"))
	}

	emitter.httpRequestsDuration.WithLabelValues(path, method).Observe(responseTime / 1000)
}
<|endoftext|>