<commit_before>package fingerprint

import (
	"fmt"
	"log"
	"time"

	client "github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/nomad/structs"
)

const (
	cgroupAvailable   = "available"
	cgroupUnavailable = "unavailable"
)

type CGroupFingerprint struct {
	logger             *log.Logger
	lastState          string
	mountPointDetector MountPointDetector
}

// An interface to isolate calls to the cgroup library
// This facilitates testing where we can implement
// fake mount points to test varios code paths
type MountPointDetector interface {
	MountPoint() (string, error)
}

// Implements the interface detector which calls net directly
type DefaultMountPointDetector struct {
}

// Call out to the default cgroup library
func (b *DefaultMountPointDetector) MountPoint() (string, error) {
	return FindCgroupMountpointDir()
}

func NewCGroupFingerprint(logger *log.Logger) Fingerprint {
	f := &CGroupFingerprint{
		logger:             logger,
		lastState:          cgroupUnavailable,
		mountPointDetector: &DefaultMountPointDetector{},
	}
	return f
}

func (f *CGroupFingerprint) Fingerprint(cfg *client.Config, node *structs.Node) (bool, error) {
	// Try to find a valid cgroup mount point
	mount, err := f.mountPointDetector.MountPoint()
	if err != nil {
		return false, fmt.Errorf("Failed to discover cgroup mount point: %s", err)
	}

	// Check if a cgroup mount point was found
	if mount == "" {
		// Clear any attributes from the previous fingerprint.
		f.clearCGroupAttributes(node)

		if f.lastState == cgroupAvailable {
			f.logger.Printf("[INFO] fingerprint.cgroups: cgroups are unavailable")
		}
		f.lastState = cgroupUnavailable
		return true, nil
	}

	node.Attributes["cgroup.mountpoint"] = mount

	if f.lastState == cgroupUnavailable {
		f.logger.Printf("[INFO] fingerprint.cgroups: cgroups are available")
	}
	f.lastState = cgroupAvailable
	return true, nil
}

func (f *CGroupFingerprint) clearCGroupAttributes(n *structs.Node) {
	delete(n.Attributes, "cgroup.mountpoint")
}

func (f *CGroupFingerprint) Periodic() (bool, time.Duration) {
	return true, 15 * time.Second
}
<commit_msg>Fix comment<commit_after>package fingerprint

import (
	"fmt"
	"log"
	"time"

	client "github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/nomad/structs"
)

const (
	cgroupAvailable   = "available"
	cgroupUnavailable = "unavailable"
)

type CGroupFingerprint struct {
	logger             *log.Logger
	lastState          string
	mountPointDetector MountPointDetector
}

// An interface to isolate calls to the cgroup library
// This facilitates testing where we can implement
// fake mount points to test various code paths
type MountPointDetector interface {
	MountPoint() (string, error)
}

// Implements the interface detector which calls the cgroups library directly
type DefaultMountPointDetector struct {
}

// Call out to the default cgroup library
func (b *DefaultMountPointDetector) MountPoint() (string, error) {
	return FindCgroupMountpointDir()
}

func NewCGroupFingerprint(logger *log.Logger) Fingerprint {
	f := &CGroupFingerprint{
		logger:             logger,
		lastState:          cgroupUnavailable,
		mountPointDetector: &DefaultMountPointDetector{},
	}
	return f
}

func (f *CGroupFingerprint) Fingerprint(cfg *client.Config, node *structs.Node) (bool, error) {
	// Try to find a valid cgroup mount point
	mount, err := f.mountPointDetector.MountPoint()
	if err != nil {
		return false, fmt.Errorf("Failed to discover cgroup mount point: %s", err)
	}

	// Check if a cgroup mount point was found
	if mount == "" {
		// Clear any attributes from the previous fingerprint.
		f.clearCGroupAttributes(node)

		if f.lastState == cgroupAvailable {
			f.logger.Printf("[INFO] fingerprint.cgroups: cgroups are unavailable")
		}
		f.lastState = cgroupUnavailable
		return true, nil
	}

	node.Attributes["cgroup.mountpoint"] = mount

	if f.lastState == cgroupUnavailable {
		f.logger.Printf("[INFO] fingerprint.cgroups: cgroups are available")
	}
	f.lastState = cgroupAvailable
	return true, nil
}

func (f *CGroupFingerprint) clearCGroupAttributes(n *structs.Node) {
	delete(n.Attributes, "cgroup.mountpoint")
}

func (f *CGroupFingerprint) Periodic() (bool, time.Duration) {
	return true, 15 * time.Second
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"testing"
)

func TestRotter(t *testing.T) {
	content := bytes.NewReader([]byte("hello"))
	channel := make(chan byte)
	bytes := make([]byte, 0)

	go Rotter(content, channel)

	for {
		buffer, ok := <-channel
		if !ok {
			break
		}
		bytes = append(bytes, buffer)
	}

	if string(bytes) != "uryyb" {
		t.Error("")
	}
}

func TestRot13(t *testing.T) {
	r13 := []byte("abcdefghijklmnopqrstuvwxyz!5")
	R13 := []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ*4")
	e13 := []byte("nopqrstuvwxyzabcdefghijklm!5")
	E13 := []byte("NOPQRSTUVWXYZABCDEFGHIJKLM*4")

	for i, v := range r13 {
		if Rot13(v) != e13[i] {
			t.Error("rot13ing", string(v), "did not equal", string(e13[i]))
		}
	}

	for i, v := range R13 {
		if Rot13(v) != E13[i] {
			t.Error("Oh yes")
		}
	}
}
<commit_msg>improve the testing errors<commit_after>package main

import (
	"bytes"
	"fmt"
	"testing"
)

func TestRotter(t *testing.T) {
	content := bytes.NewReader([]byte("hello"))
	channel := make(chan byte)
	bytes := make([]byte, 0)

	go Rotter(content, channel)

	for {
		buffer, ok := <-channel
		if !ok {
			break
		}
		bytes = append(bytes, buffer)
	}

	if string(bytes) != "uryyb" {
		t.Error("")
	}
}

func neq_err(v1, v2, v3 byte) string {
	return fmt.Sprintf("rot13ing '%v' did not eql '%v', it equaled '%v'", string(v1), string(v2), string(v3))
}

func TestRot13(t *testing.T) {
	r13 := []byte("abcdefghijklmnopqrstuvwxyz!5")
	R13 := []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ*4")
	e13 := []byte("nopqrstuvwxyzabcdefghijklm!5")
	E13 := []byte("NOPQRSTUVWXYZABCDEFGHIJKLM*4")

	for i, v := range r13 {
		if val := Rot13(v); val != e13[i] {
			t.Error(neq_err(v, val, e13[i]))
		}
	}

	for i, v := range R13 {
		if val := Rot13(v); val != E13[i] {
			t.Error(neq_err(v, val, E13[i]))
		}
	}
}
<|endoftext|>


<commit_before>package routes

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/boatilus/peppercorn/cookie"
	"github.com/boatilus/peppercorn/paths"
	"github.com/boatilus/peppercorn/posts"
	"github.com/boatilus/peppercorn/session"
	"github.com/boatilus/peppercorn/templates"
	"github.com/boatilus/peppercorn/users"
	"github.com/boatilus/peppercorn/utility"
	"github.com/pressly/chi"
)

// IndexHandler is called for the `/` (index) route and
func IndexGetHandler(w http.ResponseWriter, req *http.Request) {
	var data struct {
		CurrentUser *users.User
		PostCount   int
		Posts       []posts.Zip
	}

	data.CurrentUser = users.FromContext(req.Context())
	if data.CurrentUser == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	var err error

	// TODO: We can run these following two queries in parallel.
	data.PostCount, err = posts.Count()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	data.Posts, err = posts.GetRangeJoined(1, uint64(data.CurrentUser.PPP))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	templates.Index.Execute(w, data)
}

func SignInGetHandler(w http.ResponseWriter, req *http.Request) {
	templates.SignIn.Execute(w, nil)
}

func SignOutGetHandler(w http.ResponseWriter, req *http.Request) {
	// Destroy session
	c, err := req.Cookie(session.GetKey())
	if err != nil {

	}

	sid, err := cookie.Decode(c)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	log.Printf("Destroying session for SID \"%s\"", sid)

	// Setting the Max-Age attribute to -1 effectively destroys the cookie, but we'll also null the
	// content if the client decides to ignore Max-Age
	c.MaxAge = -1
	c.Value = ""

	http.SetCookie(w, c)

	if err = session.Destroy(sid); err != nil {
		log.Printf("Error in deleting session with SID \"%s\"", sid)
	}

	http.Redirect(w, req, paths.Get.SignIn, http.StatusTemporaryRedirect)
}

// Of the format: /page/{num}
func PageGetHandler(w http.ResponseWriter, req *http.Request) {
	num := chi.URLParam(req, "num")

	n, err := strconv.ParseUint(num, 10, 64)
	if err != nil {
		msg := fmt.Sprintf("Bad request for route 'page/%v'. Expected '%v' to be a positive integer", num, num)

		http.Error(w, msg, http.StatusBadRequest)
		return
	}

	// TODO: Replace with session data
	user, err := users.GetByID("9b00b4c6-fdcd-44f3-b797-fe009ddd9042")
	if err != nil {
		http.Error(w, err.Error(), http.StatusForbidden)
		return
	}

	if user.PPP == 0 {
		user.PPP = 10
	}

	start := (n * uint64(user.PPP)) - uint64(user.PPP) + 1
	ps, err := posts.GetRange(start, uint64(user.PPP))
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}

	if len(ps) == 0 {
		http.NotFound(w, req)
		return
	} else {
		io.WriteString(w, "Number of posts found: "+strconv.Itoa(len(ps)))
	}
}

// SingleHandler is called for GET requests for the `/post/{num}` route and renders a single post
// by its computed post number.
func SingleGetHandler(w http.ResponseWriter, req *http.Request) {
	num := chi.URLParam(req, "num")

	n, err := strconv.ParseUint(num, 10, 64)
	if err != nil {
		msg := fmt.Sprintf("Bad request for route '/post/%v'. Expected '%v' to be a positive integer", num, num)

		http.Error(w, msg, http.StatusBadRequest)
		return
	}

	p, err := posts.GetOne(n)
	if err != nil {
		http.NotFound(w, req)
		return
	}

	io.WriteString(w, p.Content)
}

func CountGetHandler(w http.ResponseWriter, _ *http.Request) {
	n, err := posts.Count()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	io.WriteString(w, strconv.Itoa(n))
}

// MeGetHandler is the handler
func MeGetHandler(w http.ResponseWriter, req *http.Request) {
	u := users.FromContext(req.Context())
	if u == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	ss, err := session.GetByUser(u.ID)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Reduce the session data retrieved into something more easily-consumable.
	type sessionData struct {
		Device    string
		IP        string
		Timestamp string
	}

	var sessions []sessionData

	now := time.Now()

	for i := range ss {
		data := utility.ParseUserAgent(ss[i].UserAgent)

		s := sessionData{
			Device:    fmt.Sprintf("%s on %s", data.Browser, data.OS),
			IP:        ss[i].IP,
			Timestamp: utility.FormatTime(ss[i].Timestamp, now),
		}

		sessions = append(sessions, s)
	}

	obEmail := utility.ObfuscateEmail(u.Email) // Obfuscate email

	o := struct {
		ObfuscatedEmail string
		Name            string
		Title           string
		Avatar          string
		PPP             string
		Sessions        []sessionData
	}{
		obEmail,
		u.Name,
		u.Title,
		u.Avatar,
		strconv.FormatUint(uint64(u.PPP), 10),
		sessions,
	}

	templates.Me.Execute(w, &o)
}

// MeRevokeGetHandler is the handler called from the /me route to destroy a single session by :num,
// or all sessions with "all"
func MeRevokeGetHandler(w http.ResponseWriter, req *http.Request) {
	u := users.FromContext(req.Context())
	if u == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	i, err := strconv.ParseUint(chi.URLParam(req, "num"), 10, 64)
	if err != nil || i < 0 {
		http.Error(w, "", http.StatusBadRequest)
		return
	}

	if err := session.DestroyByIndex(u.ID, i); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	http.Redirect(w, req, paths.Get.Me, http.StatusSeeOther)
}
<commit_msg>Begin adding pagination to index route<commit_after>package routes

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/boatilus/peppercorn/cookie"
	"github.com/boatilus/peppercorn/paths"
	"github.com/boatilus/peppercorn/posts"
	"github.com/boatilus/peppercorn/session"
	"github.com/boatilus/peppercorn/templates"
	"github.com/boatilus/peppercorn/users"
	"github.com/boatilus/peppercorn/utility"
	"github.com/pressly/chi"
)

// IndexHandler is called for the `/` (index) route and
func IndexGetHandler(w http.ResponseWriter, req *http.Request) {
	var data struct {
		CurrentUser *users.User
		PostCount   int
		Posts       []posts.Zip
	}

	data.CurrentUser = users.FromContext(req.Context())
	if data.CurrentUser == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	var err error

	// TODO: We can run these following two queries in parallel.
	data.PostCount, err = posts.Count()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	numPages := data.PostCount / int(data.CurrentUser.PPP)
	if numPages == 0 {
		numPages = 1
	}

	log.Print("numPages: ", numPages)

	pages := []string

	data.Posts, err = posts.GetRangeJoined(1, uint64(data.CurrentUser.PPP))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	templates.Index.Execute(w, data)
}

func SignInGetHandler(w http.ResponseWriter, req *http.Request) {
	templates.SignIn.Execute(w, nil)
}

func SignOutGetHandler(w http.ResponseWriter, req *http.Request) {
	// Destroy session
	c, err := req.Cookie(session.GetKey())
	if err != nil {

	}

	sid, err := cookie.Decode(c)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	log.Printf("Destroying session for SID \"%s\"", sid)

	// Setting the Max-Age attribute to -1 effectively destroys the cookie, but we'll also null the
	// content if the client decides to ignore Max-Age
	c.MaxAge = -1
	c.Value = ""

	http.SetCookie(w, c)

	if err = session.Destroy(sid); err != nil {
		log.Printf("Error in deleting session with SID \"%s\"", sid)
	}

	http.Redirect(w, req, paths.Get.SignIn, http.StatusTemporaryRedirect)
}

// Of the format: /page/{num}
func PageGetHandler(w http.ResponseWriter, req *http.Request) {
	num := chi.URLParam(req, "num")

	n, err := strconv.ParseUint(num, 10, 64)
	if err != nil {
		msg := fmt.Sprintf("Bad request for route 'page/%v'. Expected '%v' to be a positive integer", num, num)

		http.Error(w, msg, http.StatusBadRequest)
		return
	}

	// TODO: Replace with session data
	user, err := users.GetByID("9b00b4c6-fdcd-44f3-b797-fe009ddd9042")
	if err != nil {
		http.Error(w, err.Error(), http.StatusForbidden)
		return
	}

	if user.PPP == 0 {
		user.PPP = 10
	}

	start := (n * uint64(user.PPP)) - uint64(user.PPP) + 1
	ps, err := posts.GetRange(start, uint64(user.PPP))
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}

	if len(ps) == 0 {
		http.NotFound(w, req)
		return
	} else {
		io.WriteString(w, "Number of posts found: "+strconv.Itoa(len(ps)))
	}
}

// SingleHandler is called for GET requests for the `/post/{num}` route and renders a single post
// by its computed post number.
func SingleGetHandler(w http.ResponseWriter, req *http.Request) {
	num := chi.URLParam(req, "num")

	n, err := strconv.ParseUint(num, 10, 64)
	if err != nil {
		msg := fmt.Sprintf("Bad request for route '/post/%v'. Expected '%v' to be a positive integer", num, num)

		http.Error(w, msg, http.StatusBadRequest)
		return
	}

	p, err := posts.GetOne(n)
	if err != nil {
		http.NotFound(w, req)
		return
	}

	io.WriteString(w, p.Content)
}

func CountGetHandler(w http.ResponseWriter, _ *http.Request) {
	n, err := posts.Count()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	io.WriteString(w, strconv.Itoa(n))
}

// MeGetHandler is the handler
func MeGetHandler(w http.ResponseWriter, req *http.Request) {
	u := users.FromContext(req.Context())
	if u == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	ss, err := session.GetByUser(u.ID)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Reduce the session data retrieved into something more easily-consumable.
	type sessionData struct {
		Device    string
		IP        string
		Timestamp string
	}

	var sessions []sessionData

	now := time.Now()

	for i := range ss {
		data := utility.ParseUserAgent(ss[i].UserAgent)

		s := sessionData{
			Device:    fmt.Sprintf("%s on %s", data.Browser, data.OS),
			IP:        ss[i].IP,
			Timestamp: utility.FormatTime(ss[i].Timestamp, now),
		}

		sessions = append(sessions, s)
	}

	obEmail := utility.ObfuscateEmail(u.Email) // Obfuscate email

	o := struct {
		ObfuscatedEmail string
		Name            string
		Title           string
		Avatar          string
		PPP             string
		Sessions        []sessionData
	}{
		obEmail,
		u.Name,
		u.Title,
		u.Avatar,
		strconv.FormatUint(uint64(u.PPP), 10),
		sessions,
	}

	templates.Me.Execute(w, &o)
}

// MeRevokeGetHandler is the handler called from the /me route to destroy a single session by :num,
// or all sessions with "all"
func MeRevokeGetHandler(w http.ResponseWriter, req *http.Request) {
	u := users.FromContext(req.Context())
	if u == nil {
		http.Error(w, "Could not read user data from request context", http.StatusInternalServerError)
		return
	}

	i, err := strconv.ParseUint(chi.URLParam(req, "num"), 10, 64)
	if err != nil || i < 0 {
		http.Error(w, "", http.StatusBadRequest)
		return
	}

	if err := session.DestroyByIndex(u.ID, i); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	http.Redirect(w, req, paths.Get.Me, http.StatusSeeOther)
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package rpc

import (
	"errors"
	"fmt"
	"io"
	"launchpad.net/juju-core/log"
	"reflect"
	"sync"
)

// A Codec implements reading and writing of messages in an RPC
// session.  The RPC code calls WriteMessage to write a message to the
// connection and calls ReadHeader and ReadBody in pairs to read
// messages.
type Codec interface {
	// ReadHeader reads a message header into hdr.
	ReadHeader(hdr *Header) error

	// ReadBody reads a message body into the given body value.  The
	// isRequest parameter specifies whether the message being read
	// is a request; if not, it's a response.  The body value will
	// be a non-nil struct pointer, or nil to signify that the body
	// should be read and discarded.
	ReadBody(body interface{}, isRequest bool) error

	// WriteMessage writes a message with the given header and body.
	// The body will always be a struct.
	WriteMessage(hdr *Header, body interface{}) error

	// Close closes the codec. It may be called concurrently
	// and should cause the Read methods to unblock.
	Close() error
}

// Header is a header written before every RPC call.  Since RPC requests
// can be initiated from either side, the header may represent a request
// from the other side or a response to an outstanding request.
type Header struct {
	// RequestId holds the sequence number of the request.
	RequestId uint64

	// Type holds the type of object to act on.
	Type string

	// Id holds the id of the object to act on.
	Id string

	// Request holds the action to invoke on the remote object.
	Request string

	// Error holds the error, if any.
	Error string

	// ErrorCode holds the code of the error, if any.
	ErrorCode string
}

// IsRequest returns whether the header represents an RPC request.  If
// it is not a request, it is a response.
func (hdr *Header) IsRequest() bool {
	return hdr.Type != "" || hdr.Request != ""
}

// Note that we use "client request" and "server request" to name
// requests initiated locally and remotely respectively.

// Conn represents an RPC endpoint.  It can both initiate and receive
// RPC requests.  There may be multiple outstanding Calls associated
// with a single Client, and a Client may be used by multiple goroutines
// simultaneously.
type Conn struct {
	// codec holds the underlying RPC connection.
	codec Codec

	// srvPending represents the current server requests.
	srvPending sync.WaitGroup

	// sending guards the write side of the codec - it ensures
	// that codec.WriteMessage is not called concurrently.
	// It also guards shutdown.
	sending sync.Mutex

	// mutex guards the following values.
	mutex sync.Mutex

	// rootValue holds the value to use to serve RPC requests, if any.
	rootValue reflect.Value

	// transformErrors is used to transform returned errors.
	transformErrors func(error) error

	// reqId holds the latest client request id.
	reqId uint64

	// clientPending holds all pending client requests.
	clientPending map[uint64]*Call

	// closing is set when the connection is shutting down via
	// Close.  When this is set, no more client or server requests
	// will be initiated.
	closing bool

	// shutdown is set when the input loop terminates. When this
	// is set, no more client requests will be sent to the server.
	shutdown bool

	// dead is closed when the input loop terminates.
	dead chan struct{}

	// inputLoopError holds the error that caused the input loop to
	// terminate prematurely.  It is set before dead is closed.
	inputLoopError error
}

// NewConn creates a new connection that uses the given codec for
// transport, but it does not start it. Conn.Start must be called before
// any requests are sent or received.
func NewConn(codec Codec) *Conn {
	return &Conn{
		codec:         codec,
		clientPending: make(map[uint64]*Call),
	}
}

// Start starts the RPC connection running.  It has no effect if it has
// already been called.  By default, a connection serves no methods.
// See Conn.Serve for a description of how to serve methods on a Conn.
func (conn *Conn) Start() {
	conn.mutex.Lock()
	defer conn.mutex.Unlock()
	if conn.dead == nil {
		conn.dead = make(chan struct{})
		go conn.input()
	}
}

// Serve serves RPC requests on the connection by invoking methods on
// rootValue. Note that it does not start the connection running,
// though it may be called once the connection is already started.
//
// The server executes each client request by calling a method on root
// to obtain an object to act on; then it invokes an method on that
// object with the request parameters, possibly returning some result.
//
// Methods on the root value are of the form:
//
//      M(id string) (O, error)
//
// where M is an exported name, conventionally naming the object type,
// id is some identifier for the object and O is the type of the
// returned object.
//
// Methods defined on O may defined in one of the following forms, where
// T and R must be struct types.
//
//	Method()
//	Method() R
//	Method() (R, error)
//	Method() error
//	Method(T)
//	Method(T) R
//	Method(T) (R, error)
//	Method(T) error
//
// If transformErrors is non-nil, it will be called on all returned
// non-nil errors, for example to transform the errors into ServerErrors
// with specified codes.  There will be a panic if transformErrors
// returns nil.
//
// It is an error if the connection is already serving requests
// or if the root value implements no RPC methods.
func (conn *Conn) Serve(root interface{}, transformErrors func(error) error) error {
	if transformErrors == nil {
		transformErrors = func(err error) error { return err }
	}
	rootValue := reflect.ValueOf(root)
	// Check that rootValue is ok to use as an RPC server type.
	if _, err := methods(rootValue.Type()); err != nil {
		return err
	}
	conn.mutex.Lock()
	defer conn.mutex.Unlock()
	if conn.transformErrors != nil {
		return errors.New("RPC connection is already serving requests")
	}
	conn.rootValue = rootValue
	conn.transformErrors = transformErrors
	return nil
}

// Dead returns a channel that is closed when the connection
// has been closed or the underlying transport has received
// an error. There may still be outstanding requests.
func (conn *Conn) Dead() <-chan struct{} {
	return conn.dead
}

// Close closes the connection and its underlying codec; it returns when
// all requests have been terminated.
//
// If the connection is serving requests, and the root value implements
// the Killer interface, its Kill method will be called.  The codec will
// then be closed only when all its outstanding server calls have
// completed.
func (conn *Conn) Close() error {
	conn.mutex.Lock()
	if conn.closing {
		conn.mutex.Unlock()
		return errors.New("already closed")
	}
	conn.closing = true
	// Kill server requests if appropriate.  Client requests will be
	// terminated when the input loop finishes.
	if conn.rootValue.IsValid() {
		if killer, ok := conn.rootValue.Interface().(Killer); ok {
			killer.Kill()
		}
	}
	conn.mutex.Unlock()

	// Wait for any outstanding server requests to complete
	// and write their replies before closing the codec.
	conn.srvPending.Wait()

	// Closing the codec should cause the input loop to terminate.
	if err := conn.codec.Close(); err != nil {
		log.Infof("rpc: error closing codec: %v", err)
	}
	<-conn.dead
	return conn.inputLoopError
}

// ErrorCoder represents an any error that has an associated
// error code. An error code is a short string that represents the
// kind of an error.
type ErrorCoder interface {
	ErrorCode() string
}

// Killer represents a type that can be asked to abort any outstanding
// requests.  The Kill method should return immediately.
type Killer interface {
	Kill()
}

// input reads messages from the connection and handles them
// appropriately.
func (conn *Conn) input() {
	err := conn.loop()
	conn.sending.Lock()
	defer conn.sending.Unlock()
	conn.mutex.Lock()
	defer conn.mutex.Unlock()

	if conn.closing || err == io.EOF {
		err = ErrShutdown
	} else {
		// Make the error available for Conn.Close to see.
		conn.inputLoopError = err
	}
	// Terminate all client requests.
	for _, call := range conn.clientPending {
		call.Error = err
		call.done()
	}
	conn.clientPending = nil
	conn.shutdown = true
	close(conn.dead)
}

// loop implements the looping part of Conn.input.
func (conn *Conn) loop() error {
	var hdr Header
	for {
		hdr = Header{}
		err := conn.codec.ReadHeader(&hdr)
		if err != nil {
			return err
		}
		if hdr.IsRequest() {
			err = conn.handleRequest(&hdr)
		} else {
			err = conn.handleResponse(&hdr)
		}
		if err != nil {
			return err
		}
	}
	panic("unreachable")
}

func (conn *Conn) readBody(resp interface{}, isRequest bool) error {
	if resp == nil {
		resp = &struct{}{}
	}
	return conn.codec.ReadBody(resp, isRequest)
}

func (conn *Conn) handleRequest(hdr *Header) error {
	obtain, act, err := conn.findRequest(hdr)
	if err != nil {
		if err := conn.readBody(nil, true); err != nil {
			return err
		}
		return conn.writeErrorResponse(hdr.RequestId, err)
	}
	var argp interface{}
	var arg reflect.Value
	if act.arg != nil {
		v := reflect.New(act.arg)
		arg = v.Elem()
		argp = v.Interface()
	}
	if err := conn.readBody(argp, true); err != nil {
		// If we get EOF, we know the connection is a
		// goner, so don't try to respond.
		if err == io.EOF || err == io.ErrUnexpectedEOF {
			return err
		}
		// An error reading the body often indicates bad
		// request parameters rather than an issue with
		// the connection itself, so we reply with an
		// error rather than tearing down the connection
		// unless it's obviously a connection issue.  If
		// the error is actually a framing or syntax
		// problem, then the next ReadHeader should pick
		// up the problem and abort.
		return conn.writeErrorResponse(hdr.RequestId, err)
	}
	conn.mutex.Lock()
	closing := conn.closing
	if !closing {
		conn.srvPending.Add(1)
		go conn.runRequest(hdr.RequestId, hdr.Id, obtain, act, arg)
	}
	conn.mutex.Unlock()
	if closing {
		// We're closing down - no new requests may be initiated.
		return conn.writeErrorResponse(hdr.RequestId, ErrShutdown)
	}
	return nil
}

func (conn *Conn) writeErrorResponse(reqId uint64, err error) error {
	conn.sending.Lock()
	defer conn.sending.Unlock()
	hdr := &Header{
		RequestId: reqId,
	}
	err = conn.transformErrors(err)
	if err, ok := err.(ErrorCoder); ok {
		hdr.ErrorCode = err.ErrorCode()
	} else {
		hdr.ErrorCode = ""
	}
	hdr.Error = err.Error()
	if err := conn.codec.WriteMessage(hdr, struct{}{}); err != nil {
		return err
	}
	return nil
}

func (conn *Conn) isServing() bool {
	conn.mutex.Lock()
	defer conn.mutex.Unlock()
	return conn.transformErrors != nil
}

func (conn *Conn) findRequest(hdr *Header) (*obtainer, *action, error) {
	if !conn.isServing() {
		return nil, nil, fmt.Errorf("no service")
	}
	m, err := methods(conn.rootValue.Type())
	if err != nil {
		panic("failed to get methods")
	}
	o := m.obtain[hdr.Type]
	if o == nil {
		return nil, nil, fmt.Errorf("unknown object type %q", hdr.Type)
	}
	a := m.action[o.ret][hdr.Request]
	if a == nil {
		return nil, nil, fmt.Errorf("no such request %q on %s", hdr.Request, hdr.Type)
	}
	return o, a, nil
}

// runRequest runs the given request and sends the reply.
func (conn *Conn) runRequest(reqId uint64, objId string, obtain *obtainer, act *action, arg reflect.Value) {
	defer conn.srvPending.Done()
	rv, err := conn.runRequest0(reqId, objId, obtain, act, arg)
	if err != nil {
		err = conn.writeErrorResponse(reqId, err)
	} else {
		var rvi interface{}
		hdr := &Header{
			RequestId: reqId,
		}
		conn.sending.Lock()
		defer conn.sending.Unlock()
		if rv.IsValid() {
			rvi = rv.Interface()
		} else {
			rvi = struct{}{}
		}
		err = conn.codec.WriteMessage(hdr, rvi)
	}
	if err != nil {
		log.Errorf("rpc: error writing response: %v", err)
	}
}

func (conn *Conn) runRequest0(reqId uint64, objId string, obtain *obtainer, act *action, arg reflect.Value) (reflect.Value, error) {
	obj, err := obtain.call(conn.rootValue, objId)
	if err != nil {
		return reflect.Value{}, err
	}
	return act.call(obj, arg)
}
<commit_msg>rpc: allow dynamic call to Serve<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package rpc

import (
	"errors"
	"fmt"
	"io"
	"launchpad.net/juju-core/log"
	"reflect"
	"sync"
)

// A Codec implements reading and writing of messages in an RPC
// session.  The RPC code calls WriteMessage to write a message to the
// connection and calls ReadHeader and ReadBody in pairs to read
// messages.
type Codec interface {
	// ReadHeader reads a message header into hdr.
	ReadHeader(hdr *Header) error

	// ReadBody reads a message body into the given body value.  The
	// isRequest parameter specifies whether the message being read
	// is a request; if not, it's a response.  The body value will
	// be a non-nil struct pointer, or nil to signify that the body
	// should be read and discarded.
	ReadBody(body interface{}, isRequest bool) error

	// WriteMessage writes a message with the given header and body.
	// The body will always be a struct.
	WriteMessage(hdr *Header, body interface{}) error

	// Close closes the codec. It may be called concurrently
	// and should cause the Read methods to unblock.
	Close() error
}

// Header is a header written before every RPC call.  Since RPC requests
// can be initiated from either side, the header may represent a request
// from the other side or a response to an outstanding request.
type Header struct {
	// RequestId holds the sequence number of the request.
	RequestId uint64

	// Type holds the type of object to act on.
	Type string

	// Id holds the id of the object to act on.
	Id string

	// Request holds the action to invoke on the remote object.
	Request string

	// Error holds the error, if any.
	Error string

	// ErrorCode holds the code of the error, if any.
	ErrorCode string
}

// IsRequest returns whether the header represents an RPC request.  If
// it is not a request, it is a response.
func (hdr *Header) IsRequest() bool {
	return hdr.Type != "" || hdr.Request != ""
}

// Note that we use "client request" and "server request" to name
// requests initiated locally and remotely respectively.

// Conn represents an RPC endpoint.  It can both initiate and receive
// RPC requests.  There may be multiple outstanding Calls associated
// with a single Client, and a Client may be used by multiple goroutines
// simultaneously.
type Conn struct {
	// codec holds the underlying RPC connection.
	codec Codec

	// srvPending represents the current server requests.
	srvPending sync.WaitGroup

	// sending guards the write side of the codec - it ensures
	// that codec.WriteMessage is not called concurrently.
	// It also guards shutdown.
	sending sync.Mutex

	// mutex guards the following values.
	mutex sync.Mutex

	// rootValue holds the value to use to serve RPC requests, if any.
	rootValue reflect.Value

	// transformErrors is used to transform returned errors.
	transformErrors func(error) error

	// reqId holds the latest client request id.
	reqId uint64

	// clientPending holds all pending client requests.
	clientPending map[uint64]*Call

	// closing is set when the connection is shutting down via
	// Close.  When this is set, no more client or server requests
	// will be initiated.
	closing bool

	// shutdown is set when the input loop terminates. When this
	// is set, no more client requests will be sent to the server.
	shutdown bool

	// dead is closed when the input loop terminates.
	dead chan struct{}

	// inputLoopError holds the error that caused the input loop to
	// terminate prematurely.  It is set before dead is closed.
	inputLoopError error
}

// NewConn creates a new connection that uses the given codec for
// transport, but it does not start it. Conn.Start must be called before
// any requests are sent or received.
func NewConn(codec Codec) *Conn {
	return &Conn{
		codec:         codec,
		clientPending: make(map[uint64]*Call),
	}
}

// Start starts the RPC connection running.  It must be called at least
// one for any RPC connection (client or server side) It has no effect
// if it has already been called.  By default, a connection serves no
// methods.  See Conn.Serve for a description of how to serve methods on
// a Conn.
func (conn *Conn) Start() {
	conn.mutex.Lock()
	defer conn.mutex.Unlock()
	if conn.dead == nil {
		conn.dead = make(chan struct{})
		go conn.input()
	}
}

// Serve serves RPC requests on the connection by invoking methods on
// root. Note that it does not start the connection running,
// though it may be called once the connection is already started.
//
// The server executes each client request by calling a method on root
// to obtain an object to act on; then it invokes an method on that
// object with the request parameters, possibly returning some result.
//
// Methods on the root value are of the form:
//
//      M(id string) (O, error)
//
// where M is an exported name, conventionally naming the object type,
// id is some identifier for the object and O is the type of the
// returned object.
//
// Methods defined on O may defined in one of the following forms, where
// T and R must be struct types.
//
//	Method()
//	Method() R
//	Method() (R, error)
//	Method() error
//	Method(T)
//	Method(T) R
//	Method(T) (R, error)
//	Method(T) error
//
// If transformErrors is non-nil, it will be called on all returned
// non-nil errors, for example to transform the errors into ServerErrors
// with specified codes.  There will be a panic if transformErrors
// returns nil.
//
// It is an error if if the root value implements no RPC methods.
//
// Serve may be called at any time on a connection to change the
// set of methods being served by the connection. This will have
// no effect on calls that are currently being services.
// If root is nil, the connection will serve no methods.
func (conn *Conn) Serve(root interface{}, transformErrors func(error) error) error {
	rootValue := reflect.ValueOf(root)
	if root != nil {
		if transformErrors == nil {
			transformErrors = func(err error) error { return err }
		}
		// Check that rootValue is ok to use as an RPC server type.
		if _, err := methods(rootValue.Type()); err != nil {
			return err
		}
	}
	conn.mutex.Lock()
	defer conn.mutex.Unlock()
	conn.rootValue = rootValue
	conn.transformErrors = transformErrors
	return nil
}

// Dead returns a channel that is closed when the connection
// has been closed or the underlying transport has received
// an error. There may still be outstanding requests.
func (conn *Conn) Dead() <-chan struct{} {
	return conn.dead
}

// Close closes the connection and its underlying codec; it returns when
// all requests have been terminated.
//
// If the connection is serving requests, and the root value implements
// the Killer interface, its Kill method will be called.  The codec will
// then be closed only when all its outstanding server calls have
// completed.
func (conn *Conn) Close() error {
	conn.mutex.Lock()
	if conn.closing {
		conn.mutex.Unlock()
		return errors.New("already closed")
	}
	conn.closing = true
	// Kill server requests if appropriate.  Client requests will be
	// terminated when the input loop finishes.
	if conn.rootValue.IsValid() {
		if killer, ok := conn.rootValue.Interface().(Killer); ok {
			killer.Kill()
		}
	}
	conn.mutex.Unlock()

	// Wait for any outstanding server requests to complete
	// and write their replies before closing the codec.
	conn.srvPending.Wait()

	// Closing the codec should cause the input loop to terminate.
	if err := conn.codec.Close(); err != nil {
		log.Infof("rpc: error closing codec: %v", err)
	}
	<-conn.dead
	return conn.inputLoopError
}

// ErrorCoder represents an any error that has an associated
// error code. An error code is a short string that represents the
// kind of an error.
type ErrorCoder interface {
	ErrorCode() string
}

// Killer represents a type that can be asked to abort any outstanding
// requests.  The Kill method should return immediately.
type Killer interface {
	Kill()
}

// input reads messages from the connection and handles them
// appropriately.
func (conn *Conn) input() {
	err := conn.loop()
	conn.sending.Lock()
	defer conn.sending.Unlock()
	conn.mutex.Lock()
	defer conn.mutex.Unlock()

	if conn.closing || err == io.EOF {
		err = ErrShutdown
	} else {
		// Make the error available for Conn.Close to see.
		conn.inputLoopError = err
	}
	// Terminate all client requests.
	for _, call := range conn.clientPending {
		call.Error = err
		call.done()
	}
	conn.clientPending = nil
	conn.shutdown = true
	close(conn.dead)
}

// loop implements the looping part of Conn.input.
func (conn *Conn) loop() error {
	var hdr Header
	for {
		hdr = Header{}
		err := conn.codec.ReadHeader(&hdr)
		if err != nil {
			return err
		}
		if hdr.IsRequest() {
			err = conn.handleRequest(&hdr)
		} else {
			err = conn.handleResponse(&hdr)
		}
		if err != nil {
			return err
		}
	}
	panic("unreachable")
}

func (conn *Conn) readBody(resp interface{}, isRequest bool) error {
	if resp == nil {
		resp = &struct{}{}
	}
	return conn.codec.ReadBody(resp, isRequest)
}

func (conn *Conn) handleRequest(hdr *Header) error {
	reqInfo, err := conn.findRequest(hdr)
	if err != nil {
		if err := conn.readBody(nil, true); err != nil {
			return err
		}
		// We don't transform the error because there
		// may be no transformErrors function available.
		return conn.writeErrorResponse(hdr.RequestId, err)
	}
	var argp interface{}
	var arg reflect.Value
	if reqInfo.action.arg != nil {
		v := reflect.New(reqInfo.action.arg)
		arg = v.Elem()
		argp = v.Interface()
	}
	if err := conn.readBody(argp, true); err != nil {
		// If we get EOF, we know the connection is a
		// goner, so don't try to respond.
		if err == io.EOF || err == io.ErrUnexpectedEOF {
			return err
		}
		// An error reading the body often indicates bad
		// request parameters rather than an issue with
		// the connection itself, so we reply with an
		// error rather than tearing down the connection
		// unless it's obviously a connection issue.  If
		// the error is actually a framing or syntax
		// problem, then the next ReadHeader should pick
		// up the problem and abort.
		return conn.writeErrorResponse(hdr.RequestId, reqInfo.transformErrors(err))
	}
	conn.mutex.Lock()
	closing := conn.closing
	if !closing {
		conn.srvPending.Add(1)
		go conn.runRequest(hdr.RequestId, hdr.Id, reqInfo, arg)
	}
	conn.mutex.Unlock()
	if closing {
		// We're closing down - no new requests may be initiated.
		return conn.writeErrorResponse(hdr.RequestId, reqInfo.transformErrors(ErrShutdown))
	}
	return nil
}

func (conn *Conn) writeErrorResponse(reqId uint64, err error) error {
	conn.sending.Lock()
	defer conn.sending.Unlock()
	hdr := &Header{
		RequestId: reqId,
	}
	if err, ok := err.(ErrorCoder); ok {
		hdr.ErrorCode = err.ErrorCode()
	} else {
		hdr.ErrorCode = ""
	}
	hdr.Error = err.Error()
	if err := conn.codec.WriteMessage(hdr, struct{}{}); err != nil {
		return err
	}
	return nil
}

type requestInfo struct {
	obtain          *obtainer
	action          *action
	transformErrors func(error) error
}

func (conn *Conn) findRequest(hdr *Header) (requestInfo, error) {
	conn.mutex.Lock()
	rootValue := conn.rootValue
	transformErrors := conn.transformErrors
	conn.mutex.Unlock()

	if !rootValue.IsValid() {
		return requestInfo{}, fmt.Errorf("no service")
	}
	m, err := methods(rootValue.Type())
	if err != nil {
		panic("failed to get methods")
	}
	o := m.obtain[hdr.Type]
	if o == nil {
		return requestInfo{}, fmt.Errorf("unknown object type %q", hdr.Type)
	}
	a := m.action[o.ret][hdr.Request]
	if a == nil {
		return requestInfo{}, fmt.Errorf("no such request %q on %s", hdr.Request, hdr.Type)
	}
	info := requestInfo{
		obtain:          o,
		action:          a,
		transformErrors: transformErrors,
	}
	return info, nil
}

// runRequest runs the given request and sends the reply.
func (conn *Conn) runRequest(reqId uint64, objId string, reqInfo requestInfo, arg reflect.Value) {
	defer conn.srvPending.Done()
	rv, err := conn.runRequest0(reqId, objId, reqInfo.obtain, reqInfo.action, arg)
	if err != nil {
		err = conn.writeErrorResponse(reqId, reqInfo.transformErrors(err))
	} else {
		var rvi interface{}
		hdr := &Header{
			RequestId: reqId,
		}
		conn.sending.Lock()
		defer conn.sending.Unlock()
		if rv.IsValid() {
			rvi = rv.Interface()
		} else {
			rvi = struct{}{}
		}
		err = conn.codec.WriteMessage(hdr, rvi)
	}
	if err != nil {
		log.Errorf("rpc: error writing response: %v", err)
	}
}

func (conn *Conn) runRequest0(reqId uint64, objId string, obtain *obtainer, act *action, arg reflect.Value) (reflect.Value, error) {
	obj, err := obtain.call(conn.rootValue, objId)
	if err != nil {
		return reflect.Value{}, err
	}
	return act.call(obj, arg)
}
<|endoftext|>


<commit_before>package draw

import (
	"fmt"
	"image"
	_ "image/gif"  // Support loading of GIF
	_ "image/jpeg" // Support loading of JPEG
	_ "image/png"  // Support loading of PNG
	"io"
	"net/http"
	"sync"
	"unsafe"

	"github.com/richardwilkes/toolbox/errs"
	"github.com/richardwilkes/toolbox/xmath/geom"
	"github.com/richardwilkes/ui/color"
	"github.com/richardwilkes/ui/object"

	// #cgo pkg-config: pangocairo
	// #include <pango/pangocairo.h>
	"C"
)

type imgRef struct {
	img   *Image
	count int
}

type fsKey struct {
	fs   http.FileSystem
	path string
}

// Image represents a set of pixels that can be drawn to a graphics.Context.
type Image struct {
	object.Base
	disabledID uint64
	width      int
	height     int
	surface    *C.cairo_surface_t
	key        interface{}
}

var (
	imageRegistryLock sync.Mutex
	imageRegistry     = make(map[interface{}]*imgRef)
)

func loadFromStream(key interface{}, stream io.ReadCloser) (ref *imgRef, err error) {
	defer func() {
		if serr := stream.Close(); serr != nil && err == nil {
			err = errs.Wrap(serr)
		}
	}()
	var simg image.Image
	if simg, _, err = image.Decode(stream); err != nil {
		return nil, errs.Wrap(err)
	}
	bounds := simg.Bounds()
	surface := C.cairo_image_surface_create(C.CAIRO_FORMAT_ARGB32, C.int(bounds.Dx()), C.int(bounds.Dy()))
	stride := int(C.cairo_image_surface_get_stride(surface)) / 4
	pixels := (*[1 << 30]color.Color)(unsafe.Pointer(C.cairo_image_surface_get_data(surface)))
	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			r, g, b, a := simg.At(x, y).RGBA()
			pixels[(y-bounds.Min.Y)*stride+(x-bounds.Min.X)] = color.RGBA(int(r>>8), int(g>>8), int(b>>8), float64(a>>8)/255)
		}
	}
	C.cairo_surface_mark_dirty(surface)
	img := &Image{width: bounds.Dx(), height: bounds.Dy(), surface: surface, key: key}
	img.InitTypeAndID(img)
	return &imgRef{img: img}, nil
}

// AcquireImageFromFile attempts to load an image from the file system.
func AcquireImageFromFile(fs http.FileSystem, path string) (img *Image, err error) {
	imageRegistryLock.Lock()
	defer imageRegistryLock.Unlock()
	var ref *imgRef
	var ok bool
	key := fsKey{fs: fs, path: path}
	if ref, ok = imageRegistry[key]; !ok {
		var file http.File
		if file, err = fs.Open(path); err != nil {
			return nil, errs.Wrap(err)
		}
		if ref, err = loadFromStream(key, file); err != nil {
			return nil, err
		}
		imageRegistry[key] = ref
	}
	ref.count++
	return ref.img, nil
}

// AcquireImageFromURL attempts to load an image from a URL.
func AcquireImageFromURL(url string) (img *Image, err error) {
	imageRegistryLock.Lock()
	defer imageRegistryLock.Unlock()
	var ref *imgRef
	var ok bool
	if ref, ok = imageRegistry[url]; !ok {
		var resp *http.Response
		if resp, err = http.Get(url); err != nil {
			return nil, errs.Wrap(err)
		}
		if ref, err = loadFromStream(url, resp.Body); err != nil {
			return nil, err
		}
		imageRegistry[url] = ref
	}
	ref.count++
	return ref.img, nil
}

// AcquireImageFromID attempts to find an already loaded image by its ID and return it. Returns nil
// if it cannot be found.
func AcquireImageFromID(id uint64) *Image {
	imageRegistryLock.Lock()
	defer imageRegistryLock.Unlock()
	if r, ok := imageRegistry[id]; ok {
		r.count++
		return r.img
	}
	return nil
}

// AcquireImageFromData creates a new image from the specified data.
func AcquireImageFromData(data *ImageData) (img *Image, err error) {
	surface := C.cairo_image_surface_create(C.CAIRO_FORMAT_ARGB32, C.int(data.Width), C.int(data.Height))
	stride := int(C.cairo_image_surface_get_stride(surface)) / 4
	pixels := (*[1 << 30]color.Color)(unsafe.Pointer(C.cairo_image_surface_get_data(surface)))
	for y := 0; y < data.Height; y++ {
		for x := 0; x < data.Width; x++ {
			pixels[y*stride+x] = data.Pixels[y*data.Width+x].Premultiply()
		}
	}
	C.cairo_surface_mark_dirty(surface)
	imageRegistryLock.Lock()
	defer imageRegistryLock.Unlock()
	img = &Image{width: data.Width, height: data.Height, surface: surface}
	img.InitTypeAndID(img)
	img.key = img.ID()
	ref := &imgRef{img: img, count: 1}
	imageRegistry[img.ID()] = ref
	return img, nil
}

// AcquireImageArea creates a new image from an area within this image.
func (img *Image) AcquireImageArea(bounds geom.Rect) (image *Image, err error) {
	return AcquireImageFromData(img.DataFromArea(bounds))
}

// AcquireDisabled returns an image based on this image which is desaturated and ghosted to
// represent a disabled state.
func (img *Image) AcquireDisabled() (image *Image, e error) {
	image = AcquireImageFromID(img.disabledID)
	if image != nil {
		return image, nil
	}
	data := img.Data()
	for i := range data.Pixels {
		p := data.Pixels[i]
		v := int((p.Luminance() * 255) + 0.5)
		data.Pixels[i] = color.RGBA(v, v, v, p.AlphaIntensity()*0.4)
	}
	if image, e = AcquireImageFromData(data); e == nil {
		img.disabledID = image.ID()
	}
	return image, e
}

// Size returns the size of the image.
func (img *Image) Size() geom.Size {
	return geom.Size{Width: float64(img.width), Height: float64(img.height)}
}

// Data extracts the raw image data.
func (img *Image) Data() *ImageData {
	data := &ImageData{Width: img.width, Height: img.height, Pixels: make([]color.Color, img.width*img.height)}
	stride := int(C.cairo_image_surface_get_stride(img.surface)) / 4
	pixels := (*[1 << 30]color.Color)(unsafe.Pointer(C.cairo_image_surface_get_data(img.surface)))
	for y := 0; y < img.height; y++ {
		for x := 0; x < img.width; x++ {
			data.Pixels[y*img.width+x] = pixels[y*stride+x].Unpremultiply()
		}
	}
	return data
}

// DataFromArea extracts the raw image data from an area within an image.
func (img *Image) DataFromArea(bounds geom.Rect) *ImageData {
	width := int(bounds.Width)
	height := int(bounds.Height)
	data := &ImageData{Width: width, Height: height, Pixels: make([]color.Color, width*height)}
	stride := int(C.cairo_image_surface_get_stride(img.surface)) / 4
	pixels := (*[1 << 30]color.Color)(unsafe.Pointer(C.cairo_image_surface_get_data(img.surface)))
	baseX := int(bounds.X)
	baseY := int(bounds.Y)
	outsidePixel := color.RGBA(0, 0, 0, 0)
	for y := 0; y < height; y++ {
		yy := y + baseY
		for x := 0; x < width; x++ {
			var pixel color.Color
			xx := x + baseX
			if xx < 0 || xx >= img.width || yy < 0 || yy >= img.height {
				pixel = outsidePixel
			} else {
				pixel = pixels[yy*stride+xx].Unpremultiply()
			}
			data.Pixels[y*width+x] = pixel
		}
	}
	return data
}

// Release releases the image. If no other client is using the image, then the underlying OS
// resources for the image will be disposed of.
func (img *Image) Release() {
	imageRegistryLock.Lock()
	defer imageRegistryLock.Unlock()
	if ref, ok := imageRegistry[img.key]; ok {
		ref.count--
		if ref.count > 0 {
			return
		}
		delete(imageRegistry, img.key)
	}
	if img.surface != nil {
		C.cairo_surface_destroy(img.surface)
		img.surface = nil
	}
}

func (img *Image) String() string {
	return fmt.Sprintf("Image #%d", img.ID())
}
<commit_msg>Add NewImage() and NewCairoContext(); Cleanup<commit_after>package draw

import (
	"fmt"
	"image"
	_ "image/gif"  // Support loading of GIF
	_ "image/jpeg" // Support loading of JPEG
	_ "image/png"  // Support loading of PNG
	"io"
	"net/http"
	"sync"
	"unsafe"

	"github.com/richardwilkes/toolbox/errs"
	"github.com/richardwilkes/toolbox/xmath/geom"
	"github.com/richardwilkes/ui/color"
	"github.com/richardwilkes/ui/object"

	// #cgo pkg-config: pangocairo
	// #include <pango/pangocairo.h>
	"C"
)

type imgRef struct {
	img   *Image
	count int
}

type fsKey struct {
	fs   http.FileSystem
	path string
}

// Image represents a set of pixels that can be drawn to a graphics.Context.
type Image struct {
	object.Base
	disabledID uint64
	width      int
	height     int
	surface    *C.cairo_surface_t
	key        interface{}
}

var (
	imageRegistryLock sync.Mutex
	imageRegistry     = make(map[interface{}]*imgRef)
)

func loadFromStream(key interface{}, stream io.ReadCloser) (ref *imgRef, err error) {
	defer func() {
		if serr := stream.Close(); serr != nil && err == nil {
			err = errs.Wrap(serr)
		}
	}()
	var simg image.Image
	if simg, _, err = image.Decode(stream); err != nil {
		return nil, errs.Wrap(err)
	}
	bounds := simg.Bounds()
	surface := C.cairo_image_surface_create(C.CAIRO_FORMAT_ARGB32, C.int(bounds.Dx()), C.int(bounds.Dy()))
	stride := int(C.cairo_image_surface_get_stride(surface)) / 4
	pixels := (*[1 << 30]color.Color)(unsafe.Pointer(C.cairo_image_surface_get_data(surface)))
	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			r, g, b, a := simg.At(x, y).RGBA()
			pixels[(y-bounds.Min.Y)*stride+(x-bounds.Min.X)] = color.RGBA(int(r>>8), int(g>>8), int(b>>8), float64(a>>8)/255)
		}
	}
	C.cairo_surface_mark_dirty(surface)
	img := &Image{width: bounds.Dx(), height: bounds.Dy(), surface: surface, key: key}
	img.InitTypeAndID(img)
	return &imgRef{img: img}, nil
}

// AcquireImageFromFile attempts to load an image from the file system.
func AcquireImageFromFile(fs http.FileSystem, path string) (img *Image, err error) {
	imageRegistryLock.Lock()
	defer imageRegistryLock.Unlock()
	var ref *imgRef
	var ok bool
	key := fsKey{fs: fs, path: path}
	if ref, ok = imageRegistry[key]; !ok {
		var file http.File
		if file, err = fs.Open(path); err != nil {
			return nil, errs.Wrap(err)
		}
		if ref, err = loadFromStream(key, file); err != nil {
			return nil, err
		}
		imageRegistry[key] = ref
	}
	ref.count++
	return ref.img, nil
}

// AcquireImageFromURL attempts to load an image from a URL.
func AcquireImageFromURL(url string) (img *Image, err error) {
	imageRegistryLock.Lock()
	defer imageRegistryLock.Unlock()
	var ref *imgRef
	var ok bool
	if ref, ok = imageRegistry[url]; !ok {
		var resp *http.Response
		if resp, err = http.Get(url); err != nil {
			return nil, errs.Wrap(err)
		}
		if ref, err = loadFromStream(url, resp.Body); err != nil {
			return nil, err
		}
		imageRegistry[url] = ref
	}
	ref.count++
	return ref.img, nil
}

// AcquireImageFromID attempts to find an already loaded image by its ID and return it. Returns nil
// if it cannot be found.
func AcquireImageFromID(id uint64) *Image {
	imageRegistryLock.Lock()
	defer imageRegistryLock.Unlock()
	if r, ok := imageRegistry[id]; ok {
		r.count++
		return r.img
	}
	return nil
}

// AcquireImageFromData creates a new image from the specified data.
func AcquireImageFromData(data *ImageData) *Image {
	surface := C.cairo_image_surface_create(C.CAIRO_FORMAT_ARGB32, C.int(data.Width), C.int(data.Height))
	stride := int(C.cairo_image_surface_get_stride(surface)) / 4
	pixels := (*[1 << 30]color.Color)(unsafe.Pointer(C.cairo_image_surface_get_data(surface)))
	for y := 0; y < data.Height; y++ {
		for x := 0; x < data.Width; x++ {
			pixels[y*stride+x] = data.Pixels[y*data.Width+x].Premultiply()
		}
	}
	C.cairo_surface_mark_dirty(surface)
	img := &Image{width: data.Width, height: data.Height, surface: surface}
	img.InitTypeAndID(img)
	img.key = img.ID()
	ref := &imgRef{img: img, count: 1}
	imageRegistryLock.Lock()
	imageRegistry[img.ID()] = ref
	imageRegistryLock.Unlock()
	return img
}

// NewImage creates a new image.
func NewImage(width, height int) *Image {
	img := &Image{width: width, height: height, surface: C.cairo_image_surface_create(C.CAIRO_FORMAT_ARGB32, C.int(width), C.int(height))}
	img.InitTypeAndID(img)
	img.key = img.ID()
	ref := &imgRef{img: img, count: 1}
	imageRegistryLock.Lock()
	imageRegistry[img.ID()] = ref
	imageRegistryLock.Unlock()
	return img
}

// AcquireImageArea creates a new image from an area within this image.
func (img *Image) AcquireImageArea(bounds geom.Rect) *Image {
	return AcquireImageFromData(img.DataFromArea(bounds))
}

// AcquireDisabled returns an image based on this image which is desaturated and ghosted to
// represent a disabled state.
func (img *Image) AcquireDisabled() *Image {
	image := AcquireImageFromID(img.disabledID)
	if image != nil {
		return image
	}
	data := img.Data()
	for i := range data.Pixels {
		p := data.Pixels[i]
		v := int((p.Luminance() * 255) + 0.5)
		data.Pixels[i] = color.RGBA(v, v, v, p.AlphaIntensity()*0.4)
	}
	image = AcquireImageFromData(data)
	img.disabledID = image.ID()
	return image
}

// Size returns the size of the image.
func (img *Image) Size() geom.Size {
	return geom.Size{Width: float64(img.width), Height: float64(img.height)}
}

// Data extracts the raw image data.
func (img *Image) Data() *ImageData {
	data := &ImageData{Width: img.width, Height: img.height, Pixels: make([]color.Color, img.width*img.height)}
	stride := int(C.cairo_image_surface_get_stride(img.surface)) / 4
	pixels := (*[1 << 30]color.Color)(unsafe.Pointer(C.cairo_image_surface_get_data(img.surface)))
	for y := 0; y < img.height; y++ {
		for x := 0; x < img.width; x++ {
			data.Pixels[y*img.width+x] = pixels[y*stride+x].Unpremultiply()
		}
	}
	return data
}

// DataFromArea extracts the raw image data from an area within an image.
func (img *Image) DataFromArea(bounds geom.Rect) *ImageData {
	width := int(bounds.Width)
	height := int(bounds.Height)
	data := &ImageData{Width: width, Height: height, Pixels: make([]color.Color, width*height)}
	stride := int(C.cairo_image_surface_get_stride(img.surface)) / 4
	pixels := (*[1 << 30]color.Color)(unsafe.Pointer(C.cairo_image_surface_get_data(img.surface)))
	baseX := int(bounds.X)
	baseY := int(bounds.Y)
	outsidePixel := color.RGBA(0, 0, 0, 0)
	for y := 0; y < height; y++ {
		yy := y + baseY
		for x := 0; x < width; x++ {
			var pixel color.Color
			xx := x + baseX
			if xx < 0 || xx >= img.width || yy < 0 || yy >= img.height {
				pixel = outsidePixel
			} else {
				pixel = pixels[yy*stride+xx].Unpremultiply()
			}
			data.Pixels[y*width+x] = pixel
		}
	}
	return data
}

// NewCairoContext creates a new CairoContext.
func (img *Image) NewCairoContext() CairoContext {
	return CairoContext(C.cairo_create(img.surface))
}

// Release releases the image. If no other client is using the image, then the underlying OS
// resources for the image will be disposed of.
func (img *Image) Release() {
	imageRegistryLock.Lock()
	defer imageRegistryLock.Unlock()
	if ref, ok := imageRegistry[img.key]; ok {
		ref.count--
		if ref.count > 0 {
			return
		}
		delete(imageRegistry, img.key)
	}
	if img.surface != nil {
		C.cairo_surface_destroy(img.surface)
		img.surface = nil
	}
}

func (img *Image) String() string {
	return fmt.Sprintf("Image #%d", img.ID())
}
<|endoftext|>


<commit_before>/*
The dummy task is designed for regresion test of meritop framework.
This works with
*/
package meritop

import (
	"log"
	"os"
)

// DummyData is used to carry parameter and gradient;
type DummyData struct {
	fromTaskID, toTaskID, epochID, uuID uint64
	value                               float32
	data                                [10]float32
}

func (d DummyData) EpochID() uint64 {
	return d.epochID
}

func (d DummyData) ToTaskID() uint64 {
	return d.toTaskID
}

func (d DummyData) FromTaskID() uint64 {
	return d.fromTaskID
}

func (d DummyData) UUID() uint64 {
	return d.uuID
}

// DummyMaster is prototype of parameter server, for now it does not
// carry out optimization yet. But it should be easy to add support when
// this full tests out.
// Note: in theory, since there should be no parent of this, so we should
// add error checing in the right places. We will skip these test for now.
type DummyMaster struct {
	framework       Framework
	epochID, taskID uint64
	logger          *log.Logger

	param, gradient DummyData
	fromChildren    map[uint64]DummyData
}

// This is useful to bring the task up to speed from scratch or if it recovers.
func (t DummyMaster) Init(taskID uint64, framework Framework, config Config) {
	t.taskID = taskID
	t.framework = framework
	t.logger = log.New(os.Stdout, "DummyMaster:", log.Ldate|log.Ltime|log.Lshortfile)

	// Jump start the taskgraph
	t.framework.SetEpoch(0)
}

// Task need to finish up for exit, last chance to save work?
func (t DummyMaster) Exit() {}

// These are called by framework implementation so that task implementation can
// reacts to parent or children restart.
func (t DummyMaster) ParentRestart(parentID uint64) {}
func (t DummyMaster) ChildRestart(childID uint64)   {}

func (t DummyMaster) ParentDie(parentID uint64) {}
func (t DummyMaster) ChildDie(childID uint64)   {}

// Ideally, we should also have the following:
func (t DummyMaster) ParentMetaReady(taskID uint64, meta Metadata) {}
func (t DummyMaster) ChildMetaReady(taskID uint64, meta Metadata) {
	// Get data from child. When all the data is back, starts the next epoch.
	t.framework.DataRequest(taskID, meta)
}

// This give the task an opportunity to cleanup and regroup.
func (t DummyMaster) SetEpoch(epochID uint64) {
	t.epochID = epochID
	for i := 0; i < 10; i++ {
		t.param.data[i] = float32(t.epochID)
	}

	// Make sure we have a clean slate.
	t.fromChildren = make(map[uint64]DummyData)
	t.framework.FlagChildMetaReady(t.param)
}

// These are payload rpc for application purpose.
func (t DummyMaster) ServeAsParent(req Metadata) Metadata { return t.param }
func (t DummyMaster) ServeAsChild(reg Metadata) Metadata  { return nil }

func (t DummyMaster) ParentDataReady(req, response Metadata) {}
func (t DummyMaster) ChildDataReady(req, response Metadata) {
	if req.EpochID() != t.epochID {
		return
	}

	data, ok := req.(DummyData)
	if !ok {
		t.logger.Fatal("Can't interpret request")
	}
	t.fromChildren[data.FromTaskID()] = data

	// This is a weak form of checking. We can also check the task ids.
	// But this really means that we get all the events from children, we
	// should go into the next epoch now.
	if len(t.fromChildren) == len(t.framework.GetTopology().GetChildren(t.epochID)) {
		// In real ML, we modify the gradient first. But here it is noop.
		t.framework.SetEpoch(t.epochID + 1)
	}
}

// DummySlave is an prototype for data shard in machine learning applications.
// It mainly does to things, pass on parameters to its children, and collect
// gradient back then add them together before make it available to its parent.
type DummySlave struct {
	framework       Framework
	epochID, taskID uint64
	logger          *log.Logger

	param, gradient DummyData
	fromChildren    map[uint64]DummyData
}

// This is useful to bring the task up to speed from scratch or if it recovers.
func (t DummySlave) Init(taskID uint64, framework Framework, config Config) {
	t.taskID = taskID
	t.framework = framework
	t.logger = log.New(os.Stdout, "DummySlave:", log.Ldate|log.Ltime|log.Lshortfile)
}

// Task need to finish up for exit, last chance to save work?
func (t DummySlave) Exit() {}

// These are called by framework implementation so that task implementation can
// reacts to parent or children restart.
func (t DummySlave) ParentRestart(parentID uint64) {}
func (t DummySlave) ChildRestart(childID uint64)   {}

func (t DummySlave) ParentDie(parentID uint64) {}
func (t DummySlave) ChildDie(childID uint64)   {}

// Ideally, we should also have the following:
func (t DummySlave) ParentMetaReady(taskID uint64, meta Metadata) {
	t.framework.DataRequest(taskID, meta)
}

func (t DummySlave) ChildMetaReady(taskID uint64, meta Metadata) {
	t.framework.DataRequest(taskID, meta)
}

// This give the task an opportunity to cleanup and regroup.
func (t DummySlave) SetEpoch(epochID uint64) {
	t.epochID = epochID

	// Make sure we have a clean slate.
	t.fromChildren = make(map[uint64]DummyData)
}

// These are payload rpc for application purpose.
func (t DummySlave) ServeAsParent(req Metadata) Metadata {
	return t.param
}
func (t DummySlave) ServeAsChild(reg Metadata) Metadata {
	return t.gradient
}

func (t DummySlave) ParentDataReady(req, response Metadata) {
	if req.EpochID() != t.epochID {
		return
	}

	data, ok := req.(DummyData)
	if !ok {
		t.logger.Fatal("Can't interpret request")
	}
	t.param = data

	// We need to carry out local compuation.
	for i := 0; i < 10; i++ {
		t.gradient.data[i] = float32(t.framework.GetTaskID())
	}

	// If this task has children, flag meta so that children can start pull
	// parameter.
	if t.framework.HasChildren() {
		t.framework.FlagChildMetaReady(t.param)
	} else {
		// On leaf node, we can immediately return by and flag parent
		// that this node is ready.
		t.framework.FlagParentMetaReady(t.gradient)
	}
}

func (t DummySlave) ChildDataReady(req, response Metadata) {
	if req.EpochID() != t.epochID {
		return
	}
	data, ok := req.(DummyData)
	if !ok {
		t.logger.Fatal("Can't interpret request")
	}
	t.fromChildren[data.FromTaskID()] = data

	// This is a weak form of checking. We can also check the task ids.
	// But this really means that we get all the events from children, we
	// should go into the next epoch now.
	if len(t.fromChildren) == len(t.framework.GetTopology().GetChildren(t.epochID)) {
		// In real ML, we add the gradient first.
		for _, g := range t.fromChildren {
			for i := 0; i < 10; i++ {
				t.gradient.data[i] += g.data[i]
			}
		}

		t.framework.FlagParentMetaReady(t.gradient)
	}
}

type simpleTaskBuilder struct{}

// This method is called once by framework implementation to get the
// right task implementation for the node/task. It requires the taskID
// for current node, and also a global array of tasks.
func (tc simpleTaskBuilder) GetTask(taskID uint64) Task {
	if taskID == 0 {
		return DummyMaster{}
	} else {
		return DummySlave{}
	}
}

// This is used to show how to drive the network.
func drive() {
	var framework Bootstrapper
	var taskBuilder simpleTaskBuilder
	framework.SetTaskBuilder(taskBuilder)
	framework.SetTopology(NewTreeTopology(2, 127))
	framework.Start()
}
<commit_msg>lowercase dummy*<commit_after>/*
The dummy task is designed for regresion test of meritop framework.
This works with
*/
package meritop

import (
	"log"
	"os"
)

// dummyData is used to carry parameter and gradient;
type dummyData struct {
	fromTaskID, toTaskID, epochID, uuID uint64
	value                               float32
	data                                [10]float32
}

func (d dummyData) EpochID() uint64 {
	return d.epochID
}

func (d dummyData) ToTaskID() uint64 {
	return d.toTaskID
}

func (d dummyData) FromTaskID() uint64 {
	return d.fromTaskID
}

func (d dummyData) UUID() uint64 {
	return d.uuID
}

// dummyMaster is prototype of parameter server, for now it does not
// carry out optimization yet. But it should be easy to add support when
// this full tests out.
// Note: in theory, since there should be no parent of this, so we should
// add error checing in the right places. We will skip these test for now.
type dummyMaster struct {
	framework       Framework
	epochID, taskID uint64
	logger          *log.Logger

	param, gradient dummyData
	fromChildren    map[uint64]dummyData
}

// This is useful to bring the task up to speed from scratch or if it recovers.
func (t dummyMaster) Init(taskID uint64, framework Framework, config Config) {
	t.taskID = taskID
	t.framework = framework
	t.logger = log.New(os.Stdout, "dummyMaster:", log.Ldate|log.Ltime|log.Lshortfile)

	// Jump start the taskgraph
	t.framework.SetEpoch(0)
}

// Task need to finish up for exit, last chance to save work?
func (t dummyMaster) Exit() {}

// These are called by framework implementation so that task implementation can
// reacts to parent or children restart.
func (t dummyMaster) ParentRestart(parentID uint64) {}
func (t dummyMaster) ChildRestart(childID uint64)   {}

func (t dummyMaster) ParentDie(parentID uint64) {}
func (t dummyMaster) ChildDie(childID uint64)   {}

// Ideally, we should also have the following:
func (t dummyMaster) ParentMetaReady(taskID uint64, meta Metadata) {}
func (t dummyMaster) ChildMetaReady(taskID uint64, meta Metadata) {
	// Get data from child. When all the data is back, starts the next epoch.
	t.framework.DataRequest(taskID, meta)
}

// This give the task an opportunity to cleanup and regroup.
func (t dummyMaster) SetEpoch(epochID uint64) {
	t.epochID = epochID
	for i := 0; i < 10; i++ {
		t.param.data[i] = float32(t.epochID)
	}

	// Make sure we have a clean slate.
	t.fromChildren = make(map[uint64]dummyData)
	t.framework.FlagChildMetaReady(t.param)
}

// These are payload rpc for application purpose.
func (t dummyMaster) ServeAsParent(req Metadata) Metadata { return t.param }
func (t dummyMaster) ServeAsChild(reg Metadata) Metadata  { return nil }

func (t dummyMaster) ParentDataReady(req, response Metadata) {}
func (t dummyMaster) ChildDataReady(req, response Metadata) {
	if req.EpochID() != t.epochID {
		return
	}

	data, ok := req.(dummyData)
	if !ok {
		t.logger.Fatal("Can't interpret request")
	}
	t.fromChildren[data.FromTaskID()] = data

	// This is a weak form of checking. We can also check the task ids.
	// But this really means that we get all the events from children, we
	// should go into the next epoch now.
	if len(t.fromChildren) == len(t.framework.GetTopology().GetChildren(t.epochID)) {
		// In real ML, we modify the gradient first. But here it is noop.
		t.framework.SetEpoch(t.epochID + 1)
	}
}

// dummySlave is an prototype for data shard in machine learning applications.
// It mainly does to things, pass on parameters to its children, and collect
// gradient back then add them together before make it available to its parent.
type dummySlave struct {
	framework       Framework
	epochID, taskID uint64
	logger          *log.Logger

	param, gradient dummyData
	fromChildren    map[uint64]dummyData
}

// This is useful to bring the task up to speed from scratch or if it recovers.
func (t dummySlave) Init(taskID uint64, framework Framework, config Config) {
	t.taskID = taskID
	t.framework = framework
	t.logger = log.New(os.Stdout, "dummySlave:", log.Ldate|log.Ltime|log.Lshortfile)
}

// Task need to finish up for exit, last chance to save work?
func (t dummySlave) Exit() {}

// These are called by framework implementation so that task implementation can
// reacts to parent or children restart.
func (t dummySlave) ParentRestart(parentID uint64) {}
func (t dummySlave) ChildRestart(childID uint64)   {}

func (t dummySlave) ParentDie(parentID uint64) {}
func (t dummySlave) ChildDie(childID uint64)   {}

// Ideally, we should also have the following:
func (t dummySlave) ParentMetaReady(taskID uint64, meta Metadata) {
	t.framework.DataRequest(taskID, meta)
}

func (t dummySlave) ChildMetaReady(taskID uint64, meta Metadata) {
	t.framework.DataRequest(taskID, meta)
}

// This give the task an opportunity to cleanup and regroup.
func (t dummySlave) SetEpoch(epochID uint64) {
	t.epochID = epochID

	// Make sure we have a clean slate.
	t.fromChildren = make(map[uint64]dummyData)
}

// These are payload rpc for application purpose.
func (t dummySlave) ServeAsParent(req Metadata) Metadata {
	return t.param
}
func (t dummySlave) ServeAsChild(reg Metadata) Metadata {
	return t.gradient
}

func (t dummySlave) ParentDataReady(req, response Metadata) {
	if req.EpochID() != t.epochID {
		return
	}

	data, ok := req.(dummyData)
	if !ok {
		t.logger.Fatal("Can't interpret request")
	}
	t.param = data

	// We need to carry out local compuation.
	for i := 0; i < 10; i++ {
		t.gradient.data[i] = float32(t.framework.GetTaskID())
	}

	// If this task has children, flag meta so that children can start pull
	// parameter.
	if t.framework.HasChildren() {
		t.framework.FlagChildMetaReady(t.param)
	} else {
		// On leaf node, we can immediately return by and flag parent
		// that this node is ready.
		t.framework.FlagParentMetaReady(t.gradient)
	}
}

func (t dummySlave) ChildDataReady(req, response Metadata) {
	if req.EpochID() != t.epochID {
		return
	}
	data, ok := req.(dummyData)
	if !ok {
		t.logger.Fatal("Can't interpret request")
	}
	t.fromChildren[data.FromTaskID()] = data

	// This is a weak form of checking. We can also check the task ids.
	// But this really means that we get all the events from children, we
	// should go into the next epoch now.
	if len(t.fromChildren) == len(t.framework.GetTopology().GetChildren(t.epochID)) {
		// In real ML, we add the gradient first.
		for _, g := range t.fromChildren {
			for i := 0; i < 10; i++ {
				t.gradient.data[i] += g.data[i]
			}
		}

		t.framework.FlagParentMetaReady(t.gradient)
	}
}

type simpleTaskBuilder struct{}

// This method is called once by framework implementation to get the
// right task implementation for the node/task. It requires the taskID
// for current node, and also a global array of tasks.
func (tc simpleTaskBuilder) GetTask(taskID uint64) Task {
	if taskID == 0 {
		return dummyMaster{}
	} else {
		return dummySlave{}
	}
}

// This is used to show how to drive the network.
func drive() {
	var framework Bootstrapper
	var taskBuilder simpleTaskBuilder
	framework.SetTaskBuilder(taskBuilder)
	framework.SetTopology(NewTreeTopology(2, 127))
	framework.Start()
}
<|endoftext|>


<commit_before>package perfdata

import (
	"github.com/anchor/dataframe"
	"strconv"
	"time"
)

// Return a struct of key/value pairs in the form of a slice of pointers
// to dataframe.DataFrame_Tags.
func getDataSource(datum RenderedPerfDatumValue, unit UOM) []*dataframe.DataFrame_Tag {
	source := make([]*dataframe.DataFrame_Tag, 0)
	source = append(source, dataframe.NewDataFrameTag("hostname", datum.Hostname))
	source = append(source, dataframe.NewDataFrameTag("service_name", datum.ServiceName))
	source = append(source, dataframe.NewDataFrameTag("metric", datum.Metric))
	source = append(source, dataframe.NewDataFrameTag("uom", unit.String()))
	return source
}

// Take a RenderedPerfDatumValue and return a DataFrame suitable for
// serialisation. This includes parsing the value of the metric (which
// is still a raw string in a RenderedPerfDatumValue) into its correct
// units.
func NewDataFrame(datum RenderedPerfDatumValue) (*dataframe.DataFrame, error) {
	frame := new(dataframe.DataFrame)
	// Need a variable to get an address
	frameTime := time.Unix(int64(datum.Timestamp), 0)
	timestamp := uint64(frameTime.UnixNano())
	rawValue, suffix := splitValueUOM(datum.Value)
	uom, err := ParseUOM(suffix)
	if err != nil {
		return nil, err
	}
	dataSource := getDataSource(datum, uom)
	switch {
	case isMicrosecondUOM(uom):
		value, err := parseMicroseconds(rawValue, uom)
		if err != nil {
			return nil, err
		}
		payload := dataframe.DataFrame_REAL
		frame.Payload = &payload
		frame.ValueMeasurement = &value
	case isByteUOM(uom):
		value, err := parseBytes(rawValue, uom)
		if err != nil {
			return nil, err
		}
		payload := dataframe.DataFrame_NUMBER
		frame.Payload = &payload
		frame.ValueNumeric = &value
	case uom == Counter:
		payload := dataframe.DataFrame_EMPTY
		frame.Payload = &payload
	// If we're given an empty string as UOM, we assume
	// it's an unadorned float and parse it accordingly. If that
	// fails, we treat it as empty/'counter'.
	case uom == NullUnit:
		value, err := strconv.ParseFloat(rawValue, 64)
		if err != nil {
			payload := dataframe.DataFrame_EMPTY
			frame.Payload = &payload
		} else {
			payload := dataframe.DataFrame_REAL
			frame.Payload = &payload
			frame.ValueMeasurement = &value
		}
	}
	frame.Source = dataSource
	frame.Timestamp = &timestamp
	return frame, nil
}
<commit_msg>Fix bug in UOM parsing<commit_after>package perfdata

import (
	"github.com/anchor/dataframe"
	"strconv"
	"time"
)

// Return a struct of key/value pairs in the form of a slice of pointers
// to dataframe.DataFrame_Tags.
func getDataSource(datum RenderedPerfDatumValue, unit UOM) []*dataframe.DataFrame_Tag {
	source := make([]*dataframe.DataFrame_Tag, 0)
	source = append(source, dataframe.NewDataFrameTag("hostname", datum.Hostname))
	source = append(source, dataframe.NewDataFrameTag("service_name", datum.ServiceName))
	source = append(source, dataframe.NewDataFrameTag("metric", datum.Metric))
	source = append(source, dataframe.NewDataFrameTag("uom", unit.String()))
	return source
}

// Take a RenderedPerfDatumValue and return a DataFrame suitable for
// serialisation. This includes parsing the value of the metric (which
// is still a raw string in a RenderedPerfDatumValue) into its correct
// units.
func NewDataFrame(datum RenderedPerfDatumValue) (*dataframe.DataFrame, error) {
	frame := new(dataframe.DataFrame)
	// Need a variable to get an address
	frameTime := time.Unix(int64(datum.Timestamp), 0)
	timestamp := uint64(frameTime.UnixNano())
	rawValue, suffix := splitValueUOM(datum.Value)
	uom, err := ParseUOM(suffix)
	if err != nil {
		return nil, err
	}
	switch {
	case isMicrosecondUOM(uom):
		value, err := parseMicroseconds(rawValue, uom)
		if err != nil {
			return nil, err
		}
		payload := dataframe.DataFrame_REAL
		frame.Payload = &payload
		frame.ValueMeasurement = &value
		uom = Microseconds
	case isByteUOM(uom):
		value, err := parseBytes(rawValue, uom)
		if err != nil {
			return nil, err
		}
		payload := dataframe.DataFrame_NUMBER
		frame.Payload = &payload
		frame.ValueNumeric = &value
		uom = Bytes
	case uom == Counter:
		payload := dataframe.DataFrame_EMPTY
		frame.Payload = &payload
	// If we're given an empty string as UOM, we assume
	// it's an unadorned float and parse it accordingly. If that
	// fails, we treat it as empty/'counter'.
	case uom == NullUnit:
		value, err := strconv.ParseFloat(rawValue, 64)
		if err != nil {
			payload := dataframe.DataFrame_EMPTY
			frame.Payload = &payload
		} else {
			payload := dataframe.DataFrame_REAL
			frame.Payload = &payload
			frame.ValueMeasurement = &value
		}
	}
	dataSource := getDataSource(datum, uom)
	frame.Source = dataSource
	frame.Timestamp = &timestamp
	return frame, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package embed

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"path"

	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/api/v2http"
	"github.com/coreos/etcd/pkg/cors"
	runtimeutil "github.com/coreos/etcd/pkg/runtime"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/rafthttp"
	"github.com/coreos/pkg/capnslog"
)

var plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "embed")

const (
	// internal fd usage includes disk usage and transport usage.
	// To read/write snapshot, snap pkg needs 1. In normal case, wal pkg needs
	// at most 2 to read/lock/write WALs. One case that it needs to 2 is to
	// read all logs after some snapshot index, which locates at the end of
	// the second last and the head of the last. For purging, it needs to read
	// directory, so it needs 1. For fd monitor, it needs 1.
	// For transport, rafthttp builds two long-polling connections and at most
	// four temporary connections with each member. There are at most 9 members
	// in a cluster, so it should reserve 96.
	// For the safety, we set the total reserved number to 150.
	reservedInternalFDNum = 150
)

// Etcd contains a running etcd server and its listeners.
type Etcd struct {
	Peers   []net.Listener
	Clients []net.Listener
	Server  *etcdserver.EtcdServer

	cfg   Config
	errc  chan error
	sctxs map[string]*serveCtx
}

// StartEtcd launches the etcd server and HTTP handlers for client/server communication.
func StartEtcd(inCfg *Config) (e *Etcd, err error) {
	if err = inCfg.Validate(); err != nil {
		return nil, err
	}
	e = &Etcd{cfg: *inCfg}
	cfg := &e.cfg
	defer func() {
		if e != nil && err != nil {
			e.Close()
			e = nil
		}
	}()

	if e.Peers, err = startPeerListeners(cfg); err != nil {
		return
	}
	if e.sctxs, err = startClientListeners(cfg); err != nil {
		return
	}
	for _, sctx := range e.sctxs {
		e.Clients = append(e.Clients, sctx.l)
	}

	var (
		urlsmap types.URLsMap
		token   string
	)

	if !isMemberInitialized(cfg) {
		urlsmap, token, err = cfg.PeerURLsMapAndToken("etcd")
		if err != nil {
			return e, fmt.Errorf("error setting up initial cluster: %v", err)
		}
	}

	srvcfg := &etcdserver.ServerConfig{
		Name:                    cfg.Name,
		ClientURLs:              cfg.ACUrls,
		PeerURLs:                cfg.APUrls,
		DataDir:                 cfg.Dir,
		DedicatedWALDir:         cfg.WalDir,
		SnapCount:               cfg.SnapCount,
		MaxSnapFiles:            cfg.MaxSnapFiles,
		MaxWALFiles:             cfg.MaxWalFiles,
		InitialPeerURLsMap:      urlsmap,
		InitialClusterToken:     token,
		DiscoveryURL:            cfg.Durl,
		DiscoveryProxy:          cfg.Dproxy,
		NewCluster:              cfg.IsNewCluster(),
		ForceNewCluster:         cfg.ForceNewCluster,
		PeerTLSInfo:             cfg.PeerTLSInfo,
		TickMs:                  cfg.TickMs,
		ElectionTicks:           cfg.ElectionTicks(),
		AutoCompactionRetention: cfg.AutoCompactionRetention,
		QuotaBackendBytes:       cfg.QuotaBackendBytes,
		StrictReconfigCheck:     cfg.StrictReconfigCheck,
		EnablePprof:             cfg.EnablePprof,
	}

	if e.Server, err = etcdserver.NewServer(srvcfg); err != nil {
		return
	}

	// buffer channel so goroutines on closed connections won't wait forever
	e.errc = make(chan error, len(e.Peers)+len(e.Clients)+2*len(e.sctxs))

	e.Server.Start()
	e.serve()
	<-e.Server.ReadyNotify()
	return
}

func (e *Etcd) Close() {
	for _, sctx := range e.sctxs {
		sctx.cancel()
	}
	for i := range e.Peers {
		if e.Peers[i] != nil {
			e.Peers[i].Close()
		}
	}
	for i := range e.Clients {
		if e.Clients[i] != nil {
			e.Clients[i].Close()
		}
	}
	if e.Server != nil {
		e.Server.Stop()
	}
}

func (e *Etcd) Err() <-chan error { return e.errc }

func startPeerListeners(cfg *Config) (plns []net.Listener, err error) {
	if cfg.PeerAutoTLS && cfg.PeerTLSInfo.Empty() {
		phosts := make([]string, len(cfg.LPUrls))
		for i, u := range cfg.LPUrls {
			phosts[i] = u.Host
		}
		cfg.PeerTLSInfo, err = transport.SelfCert(path.Join(cfg.Dir, "fixtures/peer"), phosts)
		if err != nil {
			plog.Fatalf("could not get certs (%v)", err)
		}
	} else if cfg.PeerAutoTLS {
		plog.Warningf("ignoring peer auto TLS since certs given")
	}

	if !cfg.PeerTLSInfo.Empty() {
		plog.Infof("peerTLS: %s", cfg.PeerTLSInfo)
	}

	plns = make([]net.Listener, len(cfg.LPUrls))
	defer func() {
		if err == nil {
			return
		}
		for i := range plns {
			if plns[i] == nil {
				continue
			}
			plns[i].Close()
			plog.Info("stopping listening for peers on ", cfg.LPUrls[i].String())
		}
	}()

	for i, u := range cfg.LPUrls {
		var tlscfg *tls.Config
		if u.Scheme == "http" {
			if !cfg.PeerTLSInfo.Empty() {
				plog.Warningf("The scheme of peer url %s is HTTP while peer key/cert files are presented. Ignored peer key/cert files.", u.String())
			}
			if cfg.PeerTLSInfo.ClientCertAuth {
				plog.Warningf("The scheme of peer url %s is HTTP while client cert auth (--peer-client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String())
			}
		}
		if !cfg.PeerTLSInfo.Empty() {
			if tlscfg, err = cfg.PeerTLSInfo.ServerConfig(); err != nil {
				return nil, err
			}
		}
		if plns[i], err = rafthttp.NewListener(u, tlscfg); err != nil {
			return nil, err
		}
		plog.Info("listening for peers on ", u.String())
	}
	return plns, nil
}

func startClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err error) {
	if cfg.ClientAutoTLS && cfg.ClientTLSInfo.Empty() {
		chosts := make([]string, len(cfg.LCUrls))
		for i, u := range cfg.LCUrls {
			chosts[i] = u.Host
		}
		cfg.ClientTLSInfo, err = transport.SelfCert(path.Join(cfg.Dir, "fixtures/client"), chosts)
		if err != nil {
			plog.Fatalf("could not get certs (%v)", err)
		}
	} else if cfg.ClientAutoTLS {
		plog.Warningf("ignoring client auto TLS since certs given")
	}

	sctxs = make(map[string]*serveCtx)
	for _, u := range cfg.LCUrls {
		sctx := newServeCtx()

		if u.Scheme == "http" || u.Scheme == "unix" {
			if !cfg.ClientTLSInfo.Empty() {
				plog.Warningf("The scheme of client url %s is HTTP while peer key/cert files are presented. Ignored key/cert files.", u.String())
			}
			if cfg.ClientTLSInfo.ClientCertAuth {
				plog.Warningf("The scheme of client url %s is HTTP while client cert auth (--client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String())
			}
		}
		if (u.Scheme == "https" || u.Scheme == "unixs") && cfg.ClientTLSInfo.Empty() {
			return nil, fmt.Errorf("TLS key/cert (--cert-file, --key-file) must be provided for client url %s with HTTPs scheme", u.String())
		}

		proto := "tcp"
		if u.Scheme == "unix" || u.Scheme == "unixs" {
			proto = "unix"
		}

		sctx.secure = u.Scheme == "https" || u.Scheme == "unixs"
		sctx.insecure = !sctx.secure
		if oldctx := sctxs[u.Host]; oldctx != nil {
			oldctx.secure = oldctx.secure || sctx.secure
			oldctx.insecure = oldctx.insecure || sctx.insecure
			continue
		}

		if sctx.l, err = net.Listen(proto, u.Host); err != nil {
			return nil, err
		}

		if fdLimit, fderr := runtimeutil.FDLimit(); fderr == nil {
			if fdLimit <= reservedInternalFDNum {
				plog.Fatalf("file descriptor limit[%d] of etcd process is too low, and should be set higher than %d to ensure internal usage", fdLimit, reservedInternalFDNum)
			}
			sctx.l = transport.LimitListener(sctx.l, int(fdLimit-reservedInternalFDNum))
		}

		if proto == "tcp" {
			if sctx.l, err = transport.NewKeepAliveListener(sctx.l, "tcp", nil); err != nil {
				return nil, err
			}
		}

		plog.Info("listening for client requests on ", u.Host)
		defer func() {
			if err != nil {
				sctx.l.Close()
				plog.Info("stopping listening for client requests on ", u.Host)
			}
		}()
		sctxs[u.Host] = sctx
	}
	return sctxs, nil
}

func (e *Etcd) serve() (err error) {
	var ctlscfg *tls.Config
	if !e.cfg.ClientTLSInfo.Empty() {
		plog.Infof("ClientTLS: %s", e.cfg.ClientTLSInfo)
		if ctlscfg, err = e.cfg.ClientTLSInfo.ServerConfig(); err != nil {
			return err
		}
	}

	if e.cfg.CorsInfo.String() != "" {
		plog.Infof("cors = %s", e.cfg.CorsInfo)
	}

	// Start the peer server in a goroutine
	ph := v2http.NewPeerHandler(e.Server)
	for _, l := range e.Peers {
		go func(l net.Listener) {
			e.errc <- servePeerHTTP(l, ph)
		}(l)
	}

	// Start a client server goroutine for each listen address
	ch := http.Handler(&cors.CORSHandler{
		Handler: v2http.NewClientHandler(e.Server, e.Server.Cfg.ReqTimeout()),
		Info:    e.cfg.CorsInfo,
	})
	for _, sctx := range e.sctxs {
		// read timeout does not work with http close notify
		// TODO: https://github.com/golang/go/issues/9524
		go func(s *serveCtx) {
			e.errc <- s.serve(e.Server, ctlscfg, ch, e.errc)
		}(sctx)
	}
	return nil
}
<commit_msg>embed: set 'ClientCertAuthEnabled'<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package embed

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"path"

	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/api/v2http"
	"github.com/coreos/etcd/pkg/cors"
	runtimeutil "github.com/coreos/etcd/pkg/runtime"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/rafthttp"
	"github.com/coreos/pkg/capnslog"
)

var plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "embed")

const (
	// internal fd usage includes disk usage and transport usage.
	// To read/write snapshot, snap pkg needs 1. In normal case, wal pkg needs
	// at most 2 to read/lock/write WALs. One case that it needs to 2 is to
	// read all logs after some snapshot index, which locates at the end of
	// the second last and the head of the last. For purging, it needs to read
	// directory, so it needs 1. For fd monitor, it needs 1.
	// For transport, rafthttp builds two long-polling connections and at most
	// four temporary connections with each member. There are at most 9 members
	// in a cluster, so it should reserve 96.
	// For the safety, we set the total reserved number to 150.
	reservedInternalFDNum = 150
)

// Etcd contains a running etcd server and its listeners.
type Etcd struct {
	Peers   []net.Listener
	Clients []net.Listener
	Server  *etcdserver.EtcdServer

	cfg   Config
	errc  chan error
	sctxs map[string]*serveCtx
}

// StartEtcd launches the etcd server and HTTP handlers for client/server communication.
func StartEtcd(inCfg *Config) (e *Etcd, err error) {
	if err = inCfg.Validate(); err != nil {
		return nil, err
	}
	e = &Etcd{cfg: *inCfg}
	cfg := &e.cfg
	defer func() {
		if e != nil && err != nil {
			e.Close()
			e = nil
		}
	}()

	if e.Peers, err = startPeerListeners(cfg); err != nil {
		return
	}
	if e.sctxs, err = startClientListeners(cfg); err != nil {
		return
	}
	for _, sctx := range e.sctxs {
		e.Clients = append(e.Clients, sctx.l)
	}

	var (
		urlsmap types.URLsMap
		token   string
	)

	if !isMemberInitialized(cfg) {
		urlsmap, token, err = cfg.PeerURLsMapAndToken("etcd")
		if err != nil {
			return e, fmt.Errorf("error setting up initial cluster: %v", err)
		}
	}

	srvcfg := &etcdserver.ServerConfig{
		Name:                    cfg.Name,
		ClientURLs:              cfg.ACUrls,
		PeerURLs:                cfg.APUrls,
		DataDir:                 cfg.Dir,
		DedicatedWALDir:         cfg.WalDir,
		SnapCount:               cfg.SnapCount,
		MaxSnapFiles:            cfg.MaxSnapFiles,
		MaxWALFiles:             cfg.MaxWalFiles,
		InitialPeerURLsMap:      urlsmap,
		InitialClusterToken:     token,
		DiscoveryURL:            cfg.Durl,
		DiscoveryProxy:          cfg.Dproxy,
		NewCluster:              cfg.IsNewCluster(),
		ForceNewCluster:         cfg.ForceNewCluster,
		PeerTLSInfo:             cfg.PeerTLSInfo,
		TickMs:                  cfg.TickMs,
		ElectionTicks:           cfg.ElectionTicks(),
		AutoCompactionRetention: cfg.AutoCompactionRetention,
		QuotaBackendBytes:       cfg.QuotaBackendBytes,
		StrictReconfigCheck:     cfg.StrictReconfigCheck,
		EnablePprof:             cfg.EnablePprof,
		ClientCertAuthEnabled:   cfg.ClientTLSInfo.ClientCertAuth,
	}

	if e.Server, err = etcdserver.NewServer(srvcfg); err != nil {
		return
	}

	// buffer channel so goroutines on closed connections won't wait forever
	e.errc = make(chan error, len(e.Peers)+len(e.Clients)+2*len(e.sctxs))

	e.Server.Start()
	e.serve()
	<-e.Server.ReadyNotify()
	return
}

func (e *Etcd) Close() {
	for _, sctx := range e.sctxs {
		sctx.cancel()
	}
	for i := range e.Peers {
		if e.Peers[i] != nil {
			e.Peers[i].Close()
		}
	}
	for i := range e.Clients {
		if e.Clients[i] != nil {
			e.Clients[i].Close()
		}
	}
	if e.Server != nil {
		e.Server.Stop()
	}
}

func (e *Etcd) Err() <-chan error { return e.errc }

func startPeerListeners(cfg *Config) (plns []net.Listener, err error) {
	if cfg.PeerAutoTLS && cfg.PeerTLSInfo.Empty() {
		phosts := make([]string, len(cfg.LPUrls))
		for i, u := range cfg.LPUrls {
			phosts[i] = u.Host
		}
		cfg.PeerTLSInfo, err = transport.SelfCert(path.Join(cfg.Dir, "fixtures/peer"), phosts)
		if err != nil {
			plog.Fatalf("could not get certs (%v)", err)
		}
	} else if cfg.PeerAutoTLS {
		plog.Warningf("ignoring peer auto TLS since certs given")
	}

	if !cfg.PeerTLSInfo.Empty() {
		plog.Infof("peerTLS: %s", cfg.PeerTLSInfo)
	}

	plns = make([]net.Listener, len(cfg.LPUrls))
	defer func() {
		if err == nil {
			return
		}
		for i := range plns {
			if plns[i] == nil {
				continue
			}
			plns[i].Close()
			plog.Info("stopping listening for peers on ", cfg.LPUrls[i].String())
		}
	}()

	for i, u := range cfg.LPUrls {
		var tlscfg *tls.Config
		if u.Scheme == "http" {
			if !cfg.PeerTLSInfo.Empty() {
				plog.Warningf("The scheme of peer url %s is HTTP while peer key/cert files are presented. Ignored peer key/cert files.", u.String())
			}
			if cfg.PeerTLSInfo.ClientCertAuth {
				plog.Warningf("The scheme of peer url %s is HTTP while client cert auth (--peer-client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String())
			}
		}
		if !cfg.PeerTLSInfo.Empty() {
			if tlscfg, err = cfg.PeerTLSInfo.ServerConfig(); err != nil {
				return nil, err
			}
		}
		if plns[i], err = rafthttp.NewListener(u, tlscfg); err != nil {
			return nil, err
		}
		plog.Info("listening for peers on ", u.String())
	}
	return plns, nil
}

func startClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err error) {
	if cfg.ClientAutoTLS && cfg.ClientTLSInfo.Empty() {
		chosts := make([]string, len(cfg.LCUrls))
		for i, u := range cfg.LCUrls {
			chosts[i] = u.Host
		}
		cfg.ClientTLSInfo, err = transport.SelfCert(path.Join(cfg.Dir, "fixtures/client"), chosts)
		if err != nil {
			plog.Fatalf("could not get certs (%v)", err)
		}
	} else if cfg.ClientAutoTLS {
		plog.Warningf("ignoring client auto TLS since certs given")
	}

	sctxs = make(map[string]*serveCtx)
	for _, u := range cfg.LCUrls {
		sctx := newServeCtx()

		if u.Scheme == "http" || u.Scheme == "unix" {
			if !cfg.ClientTLSInfo.Empty() {
				plog.Warningf("The scheme of client url %s is HTTP while peer key/cert files are presented. Ignored key/cert files.", u.String())
			}
			if cfg.ClientTLSInfo.ClientCertAuth {
				plog.Warningf("The scheme of client url %s is HTTP while client cert auth (--client-cert-auth) is enabled. Ignored client cert auth for this url.", u.String())
			}
		}
		if (u.Scheme == "https" || u.Scheme == "unixs") && cfg.ClientTLSInfo.Empty() {
			return nil, fmt.Errorf("TLS key/cert (--cert-file, --key-file) must be provided for client url %s with HTTPs scheme", u.String())
		}

		proto := "tcp"
		if u.Scheme == "unix" || u.Scheme == "unixs" {
			proto = "unix"
		}

		sctx.secure = u.Scheme == "https" || u.Scheme == "unixs"
		sctx.insecure = !sctx.secure
		if oldctx := sctxs[u.Host]; oldctx != nil {
			oldctx.secure = oldctx.secure || sctx.secure
			oldctx.insecure = oldctx.insecure || sctx.insecure
			continue
		}

		if sctx.l, err = net.Listen(proto, u.Host); err != nil {
			return nil, err
		}

		if fdLimit, fderr := runtimeutil.FDLimit(); fderr == nil {
			if fdLimit <= reservedInternalFDNum {
				plog.Fatalf("file descriptor limit[%d] of etcd process is too low, and should be set higher than %d to ensure internal usage", fdLimit, reservedInternalFDNum)
			}
			sctx.l = transport.LimitListener(sctx.l, int(fdLimit-reservedInternalFDNum))
		}

		if proto == "tcp" {
			if sctx.l, err = transport.NewKeepAliveListener(sctx.l, "tcp", nil); err != nil {
				return nil, err
			}
		}

		plog.Info("listening for client requests on ", u.Host)
		defer func() {
			if err != nil {
				sctx.l.Close()
				plog.Info("stopping listening for client requests on ", u.Host)
			}
		}()
		sctxs[u.Host] = sctx
	}
	return sctxs, nil
}

func (e *Etcd) serve() (err error) {
	var ctlscfg *tls.Config
	if !e.cfg.ClientTLSInfo.Empty() {
		plog.Infof("ClientTLS: %s", e.cfg.ClientTLSInfo)
		if ctlscfg, err = e.cfg.ClientTLSInfo.ServerConfig(); err != nil {
			return err
		}
	}

	if e.cfg.CorsInfo.String() != "" {
		plog.Infof("cors = %s", e.cfg.CorsInfo)
	}

	// Start the peer server in a goroutine
	ph := v2http.NewPeerHandler(e.Server)
	for _, l := range e.Peers {
		go func(l net.Listener) {
			e.errc <- servePeerHTTP(l, ph)
		}(l)
	}

	// Start a client server goroutine for each listen address
	ch := http.Handler(&cors.CORSHandler{
		Handler: v2http.NewClientHandler(e.Server, e.Server.Cfg.ReqTimeout()),
		Info:    e.cfg.CorsInfo,
	})
	for _, sctx := range e.sctxs {
		// read timeout does not work with http close notify
		// TODO: https://github.com/golang/go/issues/9524
		go func(s *serveCtx) {
			e.errc <- s.serve(e.Server, ctlscfg, ch, e.errc)
		}(sctx)
	}
	return nil
}
<|endoftext|>