<commit_before>package main

import (
	"flag"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/dustin/frames"
	"github.com/dustin/frames/http"
)

const (
	frameConnectTimeout = time.Second * 5
	frameCheckFreq      = time.Second * 5
	frameMaxIdle        = time.Minute * 5
)

var framesBind = flag.String("frameBind", ":8423",
	"Binding for frames protocol.")

type frameClient struct {
	client   *http.Client
	checker  *time.Timer
	lastUsed time.Time
}

var frameClients = map[string]*frameClient{}
var frameClientsLock sync.Mutex

func findExistingFrameClient(addr string) *frameClient {
	frameClientsLock.Lock()
	defer frameClientsLock.Unlock()
	return frameClients[addr]
}

func destroyFrameClient(addr string) {
	frameClientsLock.Lock()
	defer frameClientsLock.Unlock()
	fc := frameClients[addr]
	if fc == nil {
		return
	}
	framesweb.CloseFramesClient(fc.client)
	delete(frameClients, addr)
}

func checkFrameClient(addr string) {
	fc := findExistingFrameClient(addr)
	if fc == nil {
		return
	}
	if time.Since(fc.lastUsed) > frameMaxIdle {
		log.Printf("Client was last used %v (%v), closing",
			fc.lastUsed, time.Since(fc.lastUsed))
		destroyFrameClient(addr)
		return
	}
	res, err := fc.client.Get("http://" + addr + "/.cbfs/ping/")
	if err != nil || res.StatusCode != 204 {
		status := "<none>"
		if err == nil {
			status = res.Status
		}
		log.Printf("Found error checking frame client, killing: %v/%v",
			err, status)
		destroyFrameClient(addr)
	}
}

func connectNewFramesClient(addr string) *frameClient {
	c, err := net.DialTimeout("tcp", addr, frameConnectTimeout)
	if err != nil {
		log.Printf("Error connecting to %v: %v", addr, err)
		return nil
	}
	frt := &framesweb.FramesRoundTripper{
		Dialer: frames.NewClient(c),
	}
	hc := &http.Client{Transport: frt}
	frameClientsLock.Lock()
	defer frameClientsLock.Unlock()

	fc := &frameClient{
		client: hc,
		checker: time.AfterFunc(frameCheckFreq, func() {
			checkFrameClient(addr)
		}),
	}
	frameClients[addr] = fc
	return fc
}

func getFrameClient(addr string) *http.Client {
	fc := findExistingFrameClient(addr)
	if fc == nil {
		fc = connectNewFramesClient(addr)
	}
	if fc == nil {
		log.Printf("Failed to find or get frames client for %v", addr)
		return http.DefaultClient
	} else {
		fc.lastUsed = time.Now()
	}
	return fc.client
}

func serveFrame() {
	if *framesBind == "" {
		return
	}

	l, err := net.Listen("tcp4", *framesBind)
	if err != nil {
		log.Fatalf("Error setting up frames listener.")
	}

	ll, err := frames.ListenerListener(l)
	if err != nil {
		log.Fatalf("Error listen listening: %v", err)
	}

	s := &http.Server{
		Handler: http.HandlerFunc(httpHandler),
	}

	log.Fatal(s.Serve(ll))
}
<commit_msg>Handle frames checker scheduler timing stuff.<commit_after>package main

import (
	"flag"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/dustin/frames"
	"github.com/dustin/frames/http"
)

const (
	frameConnectTimeout = time.Second * 5
	frameCheckFreq      = time.Second * 5
	frameMaxIdle        = time.Minute * 5
)

var framesBind = flag.String("frameBind", ":8423",
	"Binding for frames protocol.")

type frameClient struct {
	client   *http.Client
	checker  *time.Timer
	lastUsed time.Time
}

var frameClients = map[string]*frameClient{}
var frameClientsLock sync.Mutex

func findExistingFrameClient(addr string) *frameClient {
	frameClientsLock.Lock()
	defer frameClientsLock.Unlock()
	return frameClients[addr]
}

func destroyFrameClient(addr string) {
	frameClientsLock.Lock()
	defer frameClientsLock.Unlock()
	fc := frameClients[addr]
	if fc == nil {
		return
	}
	fc.checker.Stop()
	framesweb.CloseFramesClient(fc.client)
	delete(frameClients, addr)
}

func checkFrameClient(addr string) {
	fc := findExistingFrameClient(addr)
	if fc == nil {
		return
	}
	if time.Since(fc.lastUsed) > frameMaxIdle {
		log.Printf("Client was last used %v (%v), closing",
			fc.lastUsed, time.Since(fc.lastUsed))
		destroyFrameClient(addr)
		return
	}
	res, err := fc.client.Get("http://" + addr + "/.cbfs/ping/")
	if err != nil || res.StatusCode != 204 {
		status := "<none>"
		if err == nil {
			status = res.Status
		}
		log.Printf("Found error checking frame client, killing: %v/%v",
			err, status)
		destroyFrameClient(addr)
		return
	}
	fc.checker = time.AfterFunc(frameCheckFreq, func() {
		checkFrameClient(addr)
	})
}

func connectNewFramesClient(addr string) *frameClient {
	c, err := net.DialTimeout("tcp", addr, frameConnectTimeout)
	if err != nil {
		log.Printf("Error connecting to %v: %v", addr, err)
		return nil
	}
	frt := &framesweb.FramesRoundTripper{
		Dialer: frames.NewClient(c),
	}
	hc := &http.Client{Transport: frt}
	frameClientsLock.Lock()
	defer frameClientsLock.Unlock()

	fc := &frameClient{
		client: hc,
		checker: time.AfterFunc(frameCheckFreq, func() {
			checkFrameClient(addr)
		}),
	}
	frameClients[addr] = fc
	return fc
}

func getFrameClient(addr string) *http.Client {
	fc := findExistingFrameClient(addr)
	if fc == nil {
		fc = connectNewFramesClient(addr)
	}
	if fc == nil {
		log.Printf("Failed to find or get frames client for %v", addr)
		return http.DefaultClient
	} else {
		fc.lastUsed = time.Now()
	}
	return fc.client
}

func serveFrame() {
	if *framesBind == "" {
		return
	}

	l, err := net.Listen("tcp4", *framesBind)
	if err != nil {
		log.Fatalf("Error setting up frames listener.")
	}

	ll, err := frames.ListenerListener(l)
	if err != nil {
		log.Fatalf("Error listen listening: %v", err)
	}

	s := &http.Server{
		Handler: http.HandlerFunc(httpHandler),
	}

	log.Fatal(s.Serve(ll))
}
<|endoftext|>


<commit_before>package kite

import (
	"errors"
	"fmt"
	"strings"

	"github.com/dgrijalva/jwt-go"
	"github.com/koding/kite/dnode"
	"github.com/koding/kite/kitekey"
)

// Request contains information about the incoming request.
type Request struct {
	Method         string
	Args           *dnode.Partial
	LocalKite      *Kite
	Client         *Client
	Username       string
	Authentication *Authentication
}

// Response is the type of the object that is returned from request handlers
// and the type of only argument that is passed to callback functions.
type Response struct {
	Error  *Error      `json:"error" dnode:"-"`
	Result interface{} `json:"result"`
}

// runMethod is called when a method is received from remote Kite.
func (c *Client) runMethod(method *Method, args *dnode.Partial) {
	var (
		callFunc func(interface{}, *Error)
		request  *Request
	)

	// Recover dnode argument errors and send them back. The caller can use
	// functions like MustString(), MustSlice()... without the fear of panic.
	defer func() {
		if r := recover(); r != nil {
			callFunc(nil, createError(r))
		}
	}()

	// The request that will be constructed from incoming dnode message.
	request, callFunc = c.newRequest(method.name, args)

	if method.authenticate {
		if err := request.authenticate(); err != nil {
			callFunc(nil, err)
			return
		}
	}

	// Call the handler function.
	result, err := method.handler.ServeKite(request)

	callFunc(result, createError(err))
}

// runCallback is called when a callback method call is received from remote Kite.
func (c *Client) runCallback(callback func(*dnode.Partial), args *dnode.Partial) {
	// Do not panic no matter what.
	defer func() {
		if err := recover(); err != nil {
			c.LocalKite.Log.Warning("Error in calling the callback function : %v", err)
		}
	}()

	// Call the callback function.
	callback(args)
}

// newRequest returns a new *Request from the method and arguments passed.
func (c *Client) newRequest(method string, args *dnode.Partial) (*Request, func(interface{}, *Error)) {
	// Parse dnode method arguments: [options]
	var options callOptions
	args.One().MustUnmarshal(&options)

	// Notify the handlers registered with Kite.OnFirstRequest().
	if c.RemoteAddr() != "" {
		c.firstRequestHandlersNotified.Do(func() {
			c.Kite = options.Kite
			c.LocalKite.callOnFirstRequestHandlers(c)
		})
	}

	request := &Request{
		Method:         method,
		Args:           options.WithArgs,
		LocalKite:      c.LocalKite,
		Client:         c,
		Authentication: options.Authentication,
	}

	// Call response callback function, send back our response
	callFunc := func(result interface{}, err *Error) {
		if options.ResponseCallback.Caller == nil {
			return
		}

		// Only argument to the callback.
		response := Response{
			Result: result,
			Error:  err,
		}

		if err := options.ResponseCallback.Call(response); err != nil {
			c.LocalKite.Log.Error(err.Error())
		}
	}

	return request, callFunc
}

// authenticate tries to authenticate the user by selecting appropriate
// authenticator function.
func (r *Request) authenticate() *Error {
	// Trust the Kite if we have initiated the connection.
	// RemoteAddr() returns "" if this is an outgoing connection.
	if r.Client.RemoteAddr() == "" {
		return nil
	}

	if r.Authentication == nil {
		return &Error{
			Type:    "authenticationError",
			Message: "No authentication information is provided",
		}
	}

	// Select authenticator function.
	f := r.LocalKite.Authenticators[r.Authentication.Type]
	if f == nil {
		return &Error{
			Type:    "authenticationError",
			Message: fmt.Sprintf("Unknown authentication type: %s", r.Authentication.Type),
		}
	}

	// Call authenticator function. It sets the Request.Username field.
	err := f(r)
	if err != nil {
		return &Error{
			Type:    "authenticationError",
			Message: err.Error(),
		}
	}

	// Fix username of the remote Kite if it is invalid.
	// This prevents a Kite to impersonate someone else's Kite.
	r.Client.Kite.Username = r.Username
	return nil
}

// AuthenticateFromToken is the default Authenticator for Kite.
func (k *Kite) AuthenticateFromToken(r *Request) error {
	token, err := jwt.Parse(r.Authentication.Key, r.LocalKite.RSAKey)
	if err != nil {
		return err
	}

	if !token.Valid {
		return errors.New("Invalid signature in token")
	}

	if audience, ok := token.Claims["aud"].(string); !ok || !strings.HasPrefix(k.Kite().String(), audience) {
		return fmt.Errorf("Invalid audience in token: %s", audience)
	}

	// We don't check for exp and nbf claims here because jwt-go package already checks them.

	if username, ok := token.Claims["sub"].(string); !ok {
		return errors.New("Username is not present in token")
	} else {
		r.Username = username
	}

	return nil
}

// AuthenticateFromKiteKey authenticates user from kite key.
func (k *Kite) AuthenticateFromKiteKey(r *Request) error {
	token, err := jwt.Parse(r.Authentication.Key, kitekey.GetKontrolKey)
	if err != nil {
		return err
	}

	if !token.Valid {
		return errors.New("Invalid signature in token")
	}

	if username, ok := token.Claims["sub"].(string); !ok {
		return errors.New("Username is not present in token")
	} else {
		r.Username = username
	}

	return nil
}
<commit_msg>kites: changes from github.com<commit_after>package kite

import (
	"errors"
	"fmt"
	"strings"

	"github.com/dgrijalva/jwt-go"
	"github.com/koding/kite/dnode"
	"github.com/koding/kite/kitekey"
)

// Request contains information about the incoming request.
type Request struct {
	Method         string
	Args           *dnode.Partial
	LocalKite      *Kite
	Client         *Client
	Username       string
	Authentication *Authentication
}

// Response is the type of the object that is returned from request handlers
// and the type of only argument that is passed to callback functions.
type Response struct {
	Error  *Error      `json:"error" dnode:"-"`
	Result interface{} `json:"result"`
}

// runMethod is called when a method is received from remote Kite.
func (c *Client) runMethod(method *Method, args *dnode.Partial) {
	var (
		callFunc func(interface{}, *Error)
		request  *Request
	)

	// Recover dnode argument errors and send them back. The caller can use
	// functions like MustString(), MustSlice()... without the fear of panic.
	defer func() {
		if r := recover(); r != nil {
			callFunc(nil, createError(r))
		}
	}()

	// The request that will be constructed from incoming dnode message.
	request, callFunc = c.newRequest(method.name, args)
	if method.authenticate {
		if err := request.authenticate(); err != nil {
			callFunc(nil, err)
			return
		}
	} else {
		// if not valided accept any username it sends, also useful for test
		// cases.
		request.Username = request.Client.Kite.Username
	}

	// Call the handler function.
	result, err := method.handler.ServeKite(request)

	callFunc(result, createError(err))
}

// runCallback is called when a callback method call is received from remote Kite.
func (c *Client) runCallback(callback func(*dnode.Partial), args *dnode.Partial) {
	// Do not panic no matter what.
	defer func() {
		if err := recover(); err != nil {
			c.LocalKite.Log.Warning("Error in calling the callback function : %v", err)
		}
	}()

	// Call the callback function.
	callback(args)
}

// newRequest returns a new *Request from the method and arguments passed.
func (c *Client) newRequest(method string, args *dnode.Partial) (*Request, func(interface{}, *Error)) {
	// Parse dnode method arguments: [options]
	var options callOptions
	args.One().MustUnmarshal(&options)

	// Notify the handlers registered with Kite.OnFirstRequest().
	if c.RemoteAddr() != "" {
		c.firstRequestHandlersNotified.Do(func() {
			c.Kite = options.Kite
			c.LocalKite.callOnFirstRequestHandlers(c)
		})
	}

	request := &Request{
		Method:         method,
		Args:           options.WithArgs,
		LocalKite:      c.LocalKite,
		Client:         c,
		Authentication: options.Authentication,
	}

	// Call response callback function, send back our response
	callFunc := func(result interface{}, err *Error) {
		if options.ResponseCallback.Caller == nil {
			return
		}

		// Only argument to the callback.
		response := Response{
			Result: result,
			Error:  err,
		}

		if err := options.ResponseCallback.Call(response); err != nil {
			c.LocalKite.Log.Error(err.Error())
		}
	}

	return request, callFunc
}

// authenticate tries to authenticate the user by selecting appropriate
// authenticator function.
func (r *Request) authenticate() *Error {
	// Trust the Kite if we have initiated the connection.
	// RemoteAddr() returns "" if this is an outgoing connection.
	if r.Client.RemoteAddr() == "" {
		return nil
	}

	if r.Authentication == nil {
		return &Error{
			Type:    "authenticationError",
			Message: "No authentication information is provided",
		}
	}

	// Select authenticator function.
	f := r.LocalKite.Authenticators[r.Authentication.Type]
	if f == nil {
		return &Error{
			Type:    "authenticationError",
			Message: fmt.Sprintf("Unknown authentication type: %s", r.Authentication.Type),
		}
	}

	// Call authenticator function. It sets the Request.Username field.
	err := f(r)
	if err != nil {
		return &Error{
			Type:    "authenticationError",
			Message: err.Error(),
		}
	}

	// Fix username of the remote Kite if it is invalid.
	// This prevents a Kite to impersonate someone else's Kite.
	r.Client.Kite.Username = r.Username
	return nil
}

// AuthenticateFromToken is the default Authenticator for Kite.
func (k *Kite) AuthenticateFromToken(r *Request) error {
	token, err := jwt.Parse(r.Authentication.Key, r.LocalKite.RSAKey)
	if err != nil {
		return err
	}

	if !token.Valid {
		return errors.New("Invalid signature in token")
	}

	if audience, ok := token.Claims["aud"].(string); !ok || !strings.HasPrefix(k.Kite().String(), audience) {
		return fmt.Errorf("Invalid audience in token: %s", audience)
	}

	// We don't check for exp and nbf claims here because jwt-go package already checks them.

	if username, ok := token.Claims["sub"].(string); !ok {
		return errors.New("Username is not present in token")
	} else {
		r.Username = username
	}

	return nil
}

// AuthenticateFromKiteKey authenticates user from kite key.
func (k *Kite) AuthenticateFromKiteKey(r *Request) error {
	token, err := jwt.Parse(r.Authentication.Key, kitekey.GetKontrolKey)
	if err != nil {
		return err
	}

	if !token.Valid {
		return errors.New("Invalid signature in token")
	}

	if username, ok := token.Claims["sub"].(string); !ok {
		return errors.New("Username is not present in token")
	} else {
		r.Username = username
	}

	return nil
}
<|endoftext|>


<commit_before>package kite

import (
	"archive/tar"
	"compress/gzip"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"koding/newKite/kd/util"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

type Install struct{}

func NewInstall() *Install {
	return &Install{}
}

func (*Install) Definition() string {
	return "Install kite from Koding repository"
}

const s3URL = "http://koding-kites.s3.amazonaws.com/"

func (*Install) Exec() error {
	flag.Parse()
	if flag.NArg() != 1 {
		return errors.New("You should give a kite name")
	}

	// Generate download URL
	kiteName := flag.Arg(0)
	kiteURL := s3URL + kiteName + ".kite.tar.gz"
	log.Println(kiteURL)

	// Make download request
	fmt.Println("Downloading...")
	res, err := http.Get(kiteURL)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return fmt.Errorf("Unexpected response from server: %d", res.StatusCode)
	}

	// Extract gzip
	gz, err := gzip.NewReader(res.Body)
	if err != nil {
		return err
	}
	defer gz.Close()

	// Extract tar
	tempKitePath, err := ioutil.TempDir("", "koding-kite-")
	log.Println("Created temp dir:", tempKitePath)
	if err != nil {
		return err
	}
	defer os.RemoveAll(tempKitePath)
	err = extractTar(gz, tempKitePath)
	if err != nil {
		return err
	}

	// Move kite from tmp to kites folder (~/.kd/kites)
	tempKitePath = filepath.Join(tempKitePath, kiteName+".kite")
	kitesPath := filepath.Join(util.GetKdPath(), "kites")
	os.MkdirAll(kitesPath, 0700)
	kitePath := filepath.Join(kitesPath, kiteName+".kite")
	log.Println("Moving from:", tempKitePath, "to:", kitePath)
	err = os.Rename(tempKitePath, kitePath)
	if err != nil {
		return err
	}

	fmt.Println("Done.")
	return nil
}

// extractTar reads from the io.Reader and writes the files into the directory.
func extractTar(r io.Reader, dir string) error {
	tr := tar.NewReader(r)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			return err
		}

		fi := hdr.FileInfo()
		name := fi.Name()
		path := filepath.Join(dir, name)

		// TODO make the binary under /bin executable
		// TODO assert contents of the tar file, it must contain online one directory named kitename-0.0.1.kite

		if fi.IsDir() {
			os.MkdirAll(path, 0700)
		} else {
			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
			if err != nil {
				return err
			}

			if _, err := io.Copy(f, tr); err != nil {
				return err
			}
		}
	}
	return nil
}

// splitVersion takes a name like "asdf-1.2.3" and
// returns the name "asdf" and version "1.2.3" seperately.
func splitVersion(fullname string) (name, version string, err error) {
	notFound := errors.New("name does not contain a version number")

	parts := strings.Split(fullname, "-")
	n := len(parts)
	if n < 2 {
		return "", "", notFound
	}

	version = parts[n-1]
	versionParts := strings.Split(version, ".")
	for _, v := range versionParts {
		if _, err := strconv.ParseUint(v, 10, 64); err != nil {
			return "", "", notFound
		}
	}

	name = strings.Join(parts[:n-2], "-")
	return name, version, nil
}
<commit_msg>kd kite install respects versions<commit_after>package kite

import (
	"archive/tar"
	"compress/gzip"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"koding/newKite/kd/util"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

type Install struct{}

func NewInstall() *Install {
	return &Install{}
}

func (*Install) Definition() string {
	return "Install kite from Koding repository"
}

const s3URL = "http://koding-kites.s3.amazonaws.com/"

func (*Install) Exec() error {
	flag.Parse()
	if flag.NArg() != 1 {
		return errors.New("You should give a kite name")
	}

	// Generate download URL
	kiteFullName := flag.Arg(0)
	kiteName, kiteVersion, err := splitVersion(kiteFullName, true)
	if err != nil {
		kiteName, kiteVersion = kiteFullName, "latest"
	}
	kiteURL := s3URL + kiteName + "-" + kiteVersion + ".kite.tar.gz"
	log.Println(kiteURL)

	// Make download request
	fmt.Println("Downloading...")
	res, err := http.Get(kiteURL)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode == 404 {
		return errors.New("Package is not found on the server.")
	}
	if res.StatusCode != 200 {
		return fmt.Errorf("Unexpected response from server: %d", res.StatusCode)
	}

	// Extract gzip
	gz, err := gzip.NewReader(res.Body)
	if err != nil {
		return err
	}
	defer gz.Close()

	// Extract tar
	tempKitePath, err := ioutil.TempDir("", "kd-kite-install-")
	log.Println("Created temp dir:", tempKitePath)
	if err != nil {
		return err
	}
	defer os.RemoveAll(tempKitePath)
	err = extractTar(gz, tempKitePath)
	if err != nil {
		return err
	}

	// Move kite from tmp to kites folder (~/.kd/kites)
	dirs, err := ioutil.ReadDir(tempKitePath)
	if err != nil {
		return err
	}
	if len(dirs) != 1 {
		return errors.New("Invalid package: Package must contain only one directory.")
	}
	// found prefix means we got it from extracted tar.
	// We should assert that they are expected.
	foundKiteBundleName := dirs[0].Name() // Example: asdf-1.2.3.kite
	if !strings.HasSuffix(foundKiteBundleName, ".kite") {
		return errors.New("Invalid package: Direcory name must end with \".kite\".")
	}
	foundKiteFullName := strings.TrimSuffix(foundKiteBundleName, ".kite") // Example: asdf-1.2.3
	foundKiteName, foundKiteVersion, err := splitVersion(foundKiteFullName, false)
	if err != nil {
		return errors.New("Invalid package: No version number in Kite bundle")
	}
	tempKitePath = filepath.Join(tempKitePath, foundKiteBundleName)
	kitesPath := filepath.Join(util.GetKdPath(), "kites")
	os.MkdirAll(kitesPath, 0700)
	kitePath := filepath.Join(kitesPath, foundKiteBundleName)
	log.Println("Moving from:", tempKitePath, "to:", kitePath)
	err = os.Rename(tempKitePath, kitePath)
	if err != nil {
		return err
	}

	fmt.Println("Installed successfully:", foundKiteName+"-"+foundKiteVersion)
	return nil
}

// extractTar reads from the io.Reader and writes the files into the directory.
func extractTar(r io.Reader, dir string) error {
	tr := tar.NewReader(r)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			return err
		}

		fi := hdr.FileInfo()
		name := fi.Name()
		path := filepath.Join(dir, name)

		// TODO make the binary under /bin executable
		// TODO assert contents of the tar file, it must contain online one directory named kitename-0.0.1.kite

		if fi.IsDir() {
			os.MkdirAll(path, 0700)
		} else {
			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
			if err != nil {
				return err
			}

			if _, err := io.Copy(f, tr); err != nil {
				return err
			}
		}
	}
	return nil
}

// splitVersion takes a name like "asdf-1.2.3" and
// returns the name "asdf" and version "1.2.3" seperately.
// If allowLatest is true, then the version must not be numeric and can be "latest".
func splitVersion(fullname string, allowLatest bool) (name, version string, err error) {
	notFound := errors.New("name does not contain a version number")

	parts := strings.Split(fullname, "-")
	n := len(parts)
	if n < 2 {
		return "", "", notFound
	}

	name = strings.Join(parts[:n-1], "-")
	version = parts[n-1]

	if allowLatest && version == "latest" {
		return name, version, nil
	}

	versionParts := strings.Split(version, ".")
	for _, v := range versionParts {
		if _, err := strconv.ParseUint(v, 10, 64); err != nil {
			return "", "", notFound
		}
	}

	return name, version, nil
}
<|endoftext|>


<commit_before>package virtualbox

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"strings"

	log "github.com/Sirupsen/logrus"
)

var (
	reVMNameUUID      = regexp.MustCompile(`"(.+)" {([0-9a-f-]+)}`)
	reVMInfoLine      = regexp.MustCompile(`(?:"(.+)"|(.+))=(?:"(.*)"|(.*))`)
	reColonLine       = regexp.MustCompile(`(.+):\s+(.*)`)
	reMachineNotFound = regexp.MustCompile(`Could not find a registered machine named '(.+)'`)
)

var (
	ErrMachineExist    = errors.New("machine already exists")
	ErrMachineNotExist = errors.New("machine does not exist")
	ErrVBMNotFound     = errors.New("VBoxManage not found")
	vboxManageCmd      = "VBoxManage"
)

func vbm(args ...string) error {
	cmd := exec.Command(vboxManageCmd, args...)
	if os.Getenv("DEBUG") != "" {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}
	log.Debugf("executing: %v %v", vboxManageCmd, strings.Join(args, " "))
	if err := cmd.Run(); err != nil {
		if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound {
			return ErrVBMNotFound
		}
		return fmt.Errorf("%v %v failed: %v", vboxManageCmd, strings.Join(args, " "), err)
	}
	return nil
}

func vbmOut(args ...string) (string, error) {
	cmd := exec.Command(vboxManageCmd, args...)
	if os.Getenv("DEBUG") != "" {
		cmd.Stderr = os.Stderr
	}
	log.Debugf("executing: %v %v", vboxManageCmd, strings.Join(args, " "))

	b, err := cmd.Output()
	if err != nil {
		if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound {
			err = ErrVBMNotFound
		}
	}
	return string(b), err
}

func vbmOutErr(args ...string) (string, string, error) {
	cmd := exec.Command(vboxManageCmd, args...)
	log.Debugf("executing: %v %v", vboxManageCmd, strings.Join(args, " "))
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	if err != nil {
		if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound {
			err = ErrVBMNotFound
		}
	}
	return stdout.String(), stderr.String(), err
}
<commit_msg>Detect the location of VBoxManage.exe - it is often not in the PATH<commit_after>package virtualbox

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	log "github.com/Sirupsen/logrus"
)

var (
	reVMNameUUID      = regexp.MustCompile(`"(.+)" {([0-9a-f-]+)}`)
	reVMInfoLine      = regexp.MustCompile(`(?:"(.+)"|(.+))=(?:"(.*)"|(.*))`)
	reColonLine       = regexp.MustCompile(`(.+):\s+(.*)`)
	reMachineNotFound = regexp.MustCompile(`Could not find a registered machine named '(.+)'`)
)

var (
	ErrMachineExist    = errors.New("machine already exists")
	ErrMachineNotExist = errors.New("machine does not exist")
	ErrVBMNotFound     = errors.New("VBoxManage not found")
	vboxManageCmd      = setVBoxManageCmd()
)

// detect the VBoxManage cmd's path if needed
func setVBoxManageCmd() string {
	cmd := "VBoxManage"
	if path, err := exec.LookPath(cmd); err == nil {
		return path
	}
	if runtime.GOOS == "windows" {
		if p := os.Getenv("VBOX_INSTALL_PATH"); p != "" {
			if path, err := exec.LookPath(filepath.Join(p, cmd)); err == nil {
				return path
			}
		}
		if p := os.Getenv("VBOX_MSI_INSTALL_PATH"); p != "" {
			if path, err := exec.LookPath(filepath.Join(p, cmd)); err == nil {
				return path
			}
		}
		// look at HKEY_LOCAL_MACHINE\SOFTWARE\Oracle\VirtualBox\InstallDir
		p := "C:\\Program Files\\Oracle\\VirtualBox"
		if path, err := exec.LookPath(filepath.Join(p, cmd)); err == nil {
			return path
		}
	}
	return cmd
}

func vbm(args ...string) error {
	cmd := exec.Command(vboxManageCmd, args...)
	if os.Getenv("DEBUG") != "" {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}
	log.Debugf("executing: %v %v", vboxManageCmd, strings.Join(args, " "))
	if err := cmd.Run(); err != nil {
		if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound {
			return ErrVBMNotFound
		}
		return fmt.Errorf("%v %v failed: %v", vboxManageCmd, strings.Join(args, " "), err)
	}
	return nil
}

func vbmOut(args ...string) (string, error) {
	cmd := exec.Command(vboxManageCmd, args...)
	if os.Getenv("DEBUG") != "" {
		cmd.Stderr = os.Stderr
	}
	log.Debugf("executing: %v %v", vboxManageCmd, strings.Join(args, " "))

	b, err := cmd.Output()
	if err != nil {
		if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound {
			err = ErrVBMNotFound
		}
	}
	return string(b), err
}

func vbmOutErr(args ...string) (string, string, error) {
	cmd := exec.Command(vboxManageCmd, args...)
	log.Debugf("executing: %v %v", vboxManageCmd, strings.Join(args, " "))
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	if err != nil {
		if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound {
			err = ErrVBMNotFound
		}
	}
	return stdout.String(), stderr.String(), err
}
<|endoftext|>


<commit_before>package weavedns

import (
	"github.com/miekg/dns"
	"log"
	"math"
	"net"
	"time"
)

// Portions of this code taken from github.com/armon/mdns

const (
	ipv4mdns    = "224.0.0.251" // link-local multicast address
	mdnsPort    = 5353          // mDNS assigned port
	mDNSTimeout = time.Second
	MaxDuration = time.Duration(math.MaxInt64)
)

var (
	ipv4Addr = &net.UDPAddr{
		IP:   net.ParseIP(ipv4mdns),
		Port: mdnsPort,
	}
)

type ResponseA struct {
	Name string
	Addr net.IP
}

type responseInfo struct {
	timeout time.Time // if no answer by this time, give up
	ch      chan<- *ResponseA
}

// Represents one query that we have sent for one name.
// If we, internally, get several requests for the same name while we have
// a query in flight, then we don't want to send more queries out.
type inflightQuery struct {
	name          string
	Id            uint16 // the DNS message ID
	responseInfos []*responseInfo
}

type MDNSClient struct {
	conn      *net.UDPConn
	addr      *net.UDPAddr
	inflight  map[string]*inflightQuery
	queryChan chan<- *MDNSInteraction
}

type mDNSQueryInfo struct {
	name       string
	querytype  uint16
	responseCh chan<- *ResponseA
}

func NewMDNSClient() (*MDNSClient, error) {
	conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
	if err != nil {
		return nil, err
	}
	retval := &MDNSClient{
		conn:     conn,
		addr:     ipv4Addr,
		inflight: make(map[string]*inflightQuery)}
	return retval, nil
}

func (c *MDNSClient) Start(ifi *net.Interface) error {
	multicast, err := LinkLocalMulticastListener(ifi)
	if err != nil {
		return err
	}

	handleMDNS := func(w dns.ResponseWriter, r *dns.Msg) {
		//log.Println("client received:", r)
		// Only handle responses here
		if len(r.Answer) > 0 {
			c.ResponseCallback(r)
		}
	}

	go dns.ActivateAndServe(nil, multicast, dns.HandlerFunc(handleMDNS))

	queryChan := make(chan *MDNSInteraction, 4)
	c.queryChan = queryChan
	go c.queryLoop(queryChan)

	return nil
}

func LinkLocalMulticastListener(ifi *net.Interface) (net.PacketConn, error) {
	conn, err := net.ListenMulticastUDP("udp", ifi, ipv4Addr)
	return conn, err
}

// ACTOR client API

const (
	CSendQuery       = iota
	CShutdown        = iota
	CMessageReceived = iota
)

type MDNSInteraction struct {
	code       int
	resultChan chan<- interface{}
	payload    interface{}
}

// Async
func (c *MDNSClient) Shutdown() {
	c.queryChan <- &MDNSInteraction{code: CShutdown}
}

// Async
func (c *MDNSClient) SendQuery(name string, querytype uint16, responseCh chan<- *ResponseA) {
	c.queryChan <- &MDNSInteraction{
		code:    CSendQuery,
		payload: mDNSQueryInfo{name, querytype, responseCh},
	}
}

// Async - called from dns library multiplexer
func (c *MDNSClient) ResponseCallback(r *dns.Msg) {
	c.queryChan <- &MDNSInteraction{code: CMessageReceived, payload: r}
}

// ACTOR server

func (c *MDNSClient) queryLoop(queryChan <-chan *MDNSInteraction) {
	timer := time.NewTimer(MaxDuration)
	run := func() {
		now := time.Now()
		after := MaxDuration
		for name, query := range c.inflight {
			// Count down from end of slice to beginning
			length := len(query.responseInfos)
			for i := length - 1; i >= 0; i-- {
				item := query.responseInfos[i]
				switch duration := item.timeout.Sub(now); {
				case duration <= 0: // timed out
					close(item.ch)
					// Swap item from the end of the slice
					length--
					if i < length {
						query.responseInfos[i] = query.responseInfos[length]
					}
				case duration < after:
					after = duration
				}
			}
			query.responseInfos = query.responseInfos[:length]
			if length == 0 {
				delete(c.inflight, name)
			}
		}
		timer.Reset(after)
	}

	var err error
	terminate := false
	for !terminate {
		if err != nil {
			log.Printf("encountered error", err)
			break
		}
		select {
		case query, ok := <-queryChan:
			if !ok {
				break
			}
			switch query.code {
			case CShutdown:
				terminate = true
			case CSendQuery:
				err = c.handleSendQuery(query.payload.(mDNSQueryInfo))
				run()
			case CMessageReceived:
				c.handleResponse(query.payload.(*dns.Msg))
				run()
			}
		case <-timer.C:
			run()
		}
	}
	// handle shutdown here
}

func (c *MDNSClient) handleSendQuery(q mDNSQueryInfo) (err error) {
	query, found := c.inflight[q.name]
	if !found {
		m := new(dns.Msg)
		m.SetQuestion(q.name, q.querytype)
		m.RecursionDesired = false
		buf, err := m.Pack()
		if err != nil {
			return err
		}
		query = &inflightQuery{
			name: q.name,
			Id:   m.Id,
		}
		c.inflight[q.name] = query
		_, err = c.conn.WriteTo(buf, c.addr)
	}
	info := &responseInfo{
		ch:      q.responseCh,
		timeout: time.Now().Add(mDNSTimeout),
	}
	query.responseInfos = append(query.responseInfos, info)

	return err
}

func (c *MDNSClient) handleResponse(r *dns.Msg) {
	for _, answer := range r.Answer {
		switch rr := answer.(type) {
		case *dns.A:
			if query, found := c.inflight[rr.Hdr.Name]; found {
				for _, resp := range query.responseInfos {
					resp.ch <- &ResponseA{Name: rr.Hdr.Name, Addr: rr.A}
				}
			} else {
				// We've received a response that didn't match a query
				// Do we want to cache it?
			}
		}
	}
}
<commit_msg>Shut down multicast listener when client is shut down<commit_after>package weavedns

import (
	"github.com/miekg/dns"
	"log"
	"math"
	"net"
	"time"
)

// Portions of this code taken from github.com/armon/mdns

const (
	ipv4mdns    = "224.0.0.251" // link-local multicast address
	mdnsPort    = 5353          // mDNS assigned port
	mDNSTimeout = time.Second
	MaxDuration = time.Duration(math.MaxInt64)
)

var (
	ipv4Addr = &net.UDPAddr{
		IP:   net.ParseIP(ipv4mdns),
		Port: mdnsPort,
	}
)

type ResponseA struct {
	Name string
	Addr net.IP
}

type responseInfo struct {
	timeout time.Time // if no answer by this time, give up
	ch      chan<- *ResponseA
}

// Represents one query that we have sent for one name.
// If we, internally, get several requests for the same name while we have
// a query in flight, then we don't want to send more queries out.
type inflightQuery struct {
	name          string
	Id            uint16 // the DNS message ID
	responseInfos []*responseInfo
}

type MDNSClient struct {
	server    *dns.Server
	conn      *net.UDPConn
	addr      *net.UDPAddr
	inflight  map[string]*inflightQuery
	queryChan chan<- *MDNSInteraction
}

type mDNSQueryInfo struct {
	name       string
	querytype  uint16
	responseCh chan<- *ResponseA
}

func NewMDNSClient() (*MDNSClient, error) {
	conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
	if err != nil {
		return nil, err
	}
	retval := &MDNSClient{
		conn:     conn,
		addr:     ipv4Addr,
		inflight: make(map[string]*inflightQuery)}
	return retval, nil
}

func (c *MDNSClient) Start(ifi *net.Interface) error {
	multicast, err := LinkLocalMulticastListener(ifi)
	if err != nil {
		return err
	}

	handleMDNS := func(w dns.ResponseWriter, r *dns.Msg) {
		//log.Println("client received:", r)
		// Only handle responses here
		if len(r.Answer) > 0 {
			c.ResponseCallback(r)
		}
	}

	c.server = &dns.Server{Listener: nil, PacketConn: multicast, Handler: dns.HandlerFunc(handleMDNS)}
	go c.server.ActivateAndServe()

	queryChan := make(chan *MDNSInteraction, 4)
	c.queryChan = queryChan
	go c.queryLoop(queryChan)

	return nil
}

func LinkLocalMulticastListener(ifi *net.Interface) (net.PacketConn, error) {
	conn, err := net.ListenMulticastUDP("udp", ifi, ipv4Addr)
	return conn, err
}

// ACTOR client API

const (
	CSendQuery       = iota
	CShutdown        = iota
	CMessageReceived = iota
)

type MDNSInteraction struct {
	code       int
	resultChan chan<- interface{}
	payload    interface{}
}

// Async
func (c *MDNSClient) Shutdown() {
	c.queryChan <- &MDNSInteraction{code: CShutdown}
}

// Async
func (c *MDNSClient) SendQuery(name string, querytype uint16, responseCh chan<- *ResponseA) {
	c.queryChan <- &MDNSInteraction{
		code:    CSendQuery,
		payload: mDNSQueryInfo{name, querytype, responseCh},
	}
}

// Async - called from dns library multiplexer
func (c *MDNSClient) ResponseCallback(r *dns.Msg) {
	c.queryChan <- &MDNSInteraction{code: CMessageReceived, payload: r}
}

// ACTOR server

func (c *MDNSClient) queryLoop(queryChan <-chan *MDNSInteraction) {
	timer := time.NewTimer(MaxDuration)
	run := func() {
		now := time.Now()
		after := MaxDuration
		for name, query := range c.inflight {
			// Count down from end of slice to beginning
			length := len(query.responseInfos)
			for i := length - 1; i >= 0; i-- {
				item := query.responseInfos[i]
				switch duration := item.timeout.Sub(now); {
				case duration <= 0: // timed out
					close(item.ch)
					// Swap item from the end of the slice
					length--
					if i < length {
						query.responseInfos[i] = query.responseInfos[length]
					}
				case duration < after:
					after = duration
				}
			}
			query.responseInfos = query.responseInfos[:length]
			if length == 0 {
				delete(c.inflight, name)
			}
		}
		timer.Reset(after)
	}

	var err error
	terminate := false
	for !terminate {
		if err != nil {
			log.Printf("encountered error", err)
			break
		}
		select {
		case query, ok := <-queryChan:
			if !ok {
				break
			}
			switch query.code {
			case CShutdown:
				c.server.Shutdown()
				terminate = true
			case CSendQuery:
				err = c.handleSendQuery(query.payload.(mDNSQueryInfo))
				run()
			case CMessageReceived:
				c.handleResponse(query.payload.(*dns.Msg))
				run()
			}
		case <-timer.C:
			run()
		}
	}
}

func (c *MDNSClient) handleSendQuery(q mDNSQueryInfo) (err error) {
	query, found := c.inflight[q.name]
	if !found {
		m := new(dns.Msg)
		m.SetQuestion(q.name, q.querytype)
		m.RecursionDesired = false
		buf, err := m.Pack()
		if err != nil {
			return err
		}
		query = &inflightQuery{
			name: q.name,
			Id:   m.Id,
		}
		c.inflight[q.name] = query
		_, err = c.conn.WriteTo(buf, c.addr)
	}
	info := &responseInfo{
		ch:      q.responseCh,
		timeout: time.Now().Add(mDNSTimeout),
	}
	query.responseInfos = append(query.responseInfos, info)

	return err
}

func (c *MDNSClient) handleResponse(r *dns.Msg) {
	for _, answer := range r.Answer {
		switch rr := answer.(type) {
		case *dns.A:
			if query, found := c.inflight[rr.Hdr.Name]; found {
				for _, resp := range query.responseInfos {
					resp.ch <- &ResponseA{Name: rr.Hdr.Name, Addr: rr.A}
				}
			} else {
				// We've received a response that didn't match a query
				// Do we want to cache it?
			}
		}
	}
}
<|endoftext|>


<commit_before>package zoo_test

import (
	. "github.com/gotgo/fw/zoo"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("TopicConsumer", func() {

	It("should create the correct path", func() {
		t := &TopicConsumer{
			Root:  "a",
			Topic: "mytopic",
			App:   "myapp",
		}

		path := t.PartitionsPath()
		Expect(path).To(Equal("/a/mytopic/apps/myapp/partitions"))
	})
})
<commit_msg>compile fixes<commit_after>package zoo_test

import (
	. "github.com/gotgo/fw/zoo"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("TopicConsumer", func() {

	It("should create the correct path", func() {
		t := &TopicConsumer{
			Root:        "a",
			Topic:       "mytopic",
			ConsumerApp: "myapp",
		}

		path := t.PartitionsPath()
		Expect(path).To(Equal("/a/mytopic/apps/myapp/partitions"))
	})
})
<|endoftext|>


<commit_before>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not
//  use this file except in compliance with the License. You may obtain a copy
//  of the License at http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
//  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
//  License for the specific language governing permissions and limitations
//  under the License.

// Package collatejson supplies Encoding and Decoding function to transform
// JSON text into binary representation without loosing information. That is,
//
// * binary representation should preserve the sort order such that, sorting
//   binary encoded json documents much match sorting by functions that parse
//   and compare JSON documents.
//
// * it must be possible to get back the original document, in semantically
//   correct form, from its binary representation.
//
// Notes:
//
// * items in a property object are sorted by its property name before they
//   are compared with property's value.
package collatejson

import (
	"bytes"
	"encoding/json"
	"errors"
	"sort"
	"strconv"
)

// error codes
var ErrorNumberType = errors.New("collatejson.numberType")

// Length is an internal type used for prefixing length
// of arrays and properties.
type Length int64

// While encoding JSON data-element, both basic and composite, encoded string
// is prefixed with a type-byte. `Terminator` terminates encoded datum.
const (
	Terminator byte = iota
	TypeMissing
	TypeNull
	TypeFalse
	TypeTrue
	TypeNumber
	TypeString
	TypeLength
	TypeArray
	TypeObj
)

// Codec structure
type Codec struct {
	arrayLenPrefix    bool        // if true, first sort arrays based on its length.
	propertyLenPrefix bool        // if true, first sort properties based on length.
	numberType        interface{} // "float64" | "int64" | "decimal"
	//-- unicode
	//backwards        bool
	//hiraganaQ        bool
	//caseLevel        bool
	//numeric          bool
	//nfkd              bool
	//utf8              bool
	//strength          colltab.Level
	//alternate         collate.AlternateHandling
	//language          language.Tag
}

// NewCodec creates a new codec object and returns a reference to it.
func NewCodec(propSize int) *Codec {
	return &Codec{
		arrayLenPrefix:    false,
		propertyLenPrefix: true,
		numberType:        float64(0.0),
	}
}

// SortbyArrayLen sorts array by length before sorting by array elements. Use
// `false` to sort only by array elements. Default is `true`.
func (codec *Codec) SortbyArrayLen(what bool) {
	codec.arrayLenPrefix = what
}

// SortbyPropertyLen sorts property by length before sorting by property items.
// Use `false` to sort only by proprety items. Default is `true`.
func (codec *Codec) SortbyPropertyLen(what bool) {
	codec.propertyLenPrefix = what
}

// NumberType chooses type of encoding / decoding for JSON numbers. Can be
// "float64", "int64", "decimal". Default is "float64"
func (codec *Codec) NumberType(what string) {
	switch what {
	case "float64":
		codec.numberType = float64(0.0)
	case "int64":
		codec.numberType = int64(0)
	case "decimal":
		codec.numberType = "0"
	}
}

// Encode json documents to order preserving binary representation.
func (codec *Codec) Encode(text, code []byte) ([]byte, error) {
	var m interface{}
	if err := json.Unmarshal(text, &m); err != nil {
		return nil, err
	}
	return codec.json2code(m, code)
}

// Decode a slice of byte into json string and return them as slice of byte.
func (codec *Codec) Decode(code, text []byte) ([]byte, error) {
	text, _, err := codec.code2json(code, text)
	return text, err
}

// local function that encodes basic json types to binary representation.
// composite types recursively call this function.
func (codec *Codec) json2code(val interface{}, code []byte) ([]byte, error) {
	if val == nil {
		code = append(code, TypeNull, Terminator)
	}

	var cs []byte
	var err error

	switch value := val.(type) {
	case bool:
		if value {
			code = append(code, TypeTrue, Terminator)
		} else {
			code = append(code, TypeFalse, Terminator)
		}

	case float64:
		code = append(code, TypeNumber)
		cs, err = codec.normalizeFloat(value, code[1:])
		if err == nil {
			code = code[:len(code)+len(cs)]
			code = append(code, Terminator)
		}

	case int:
		code = append(code, TypeNumber)
		cs = EncodeInt([]byte(strconv.Itoa(value)), code[1:])
		code = code[:len(code)+len(cs)]
		code = append(code, Terminator)

	case Length:
		code = append(code, TypeLength)
		cs = EncodeInt([]byte(strconv.Itoa(int(value))), code[1:])
		code = code[:len(code)+len(cs)]
		code = append(code, Terminator)

	case string:
		code = append(code, TypeString)
		cs = suffixEncodeString([]byte(value), code[1:])
		code = code[:len(code)+len(cs)]
		code = append(code, Terminator)

	case []interface{}:
		code = append(code, TypeArray)
		if codec.arrayLenPrefix {
			arrlen := Length(len(value))
			if cs, err = codec.json2code(arrlen, code[1:]); err == nil {
				code = code[:len(code)+len(cs)]
			}
		}
		if err == nil {
			for _, val := range value {
				l := len(code)
				cs, err = codec.json2code(val, code[l:])
				if err == nil {
					code = code[:l+len(cs)]
					continue
				}
				break
			}
			code = append(code, Terminator)
		}

	case map[string]interface{}:
		code = append(code, TypeObj)
		if codec.propertyLenPrefix {
			proplen := Length(len(value))
			if cs, err = codec.json2code(proplen, code[1:]); err == nil {
				code = code[:len(code)+len(cs)]
			}
		}

		if err == nil {
			keys := codec.sortProps(value)
			for _, key := range keys {
				l := len(code)
				// encode key
				if cs, err = codec.json2code(key, code[l:]); err != nil {
					break
				}
				code = code[:l+len(cs)]
				l = len(code)
				// encode value
				if cs, err = codec.json2code(value[key], code[l:]); err != nil {
					break
				}
				code = code[:l+len(cs)]
			}
			code = append(code, Terminator)
		}
	}
	return code, err
}

var null = []byte("null")
var boolTrue = []byte("true")
var boolFalse = []byte("false")

func (codec *Codec) code2json(code, text []byte) ([]byte, []byte, error) {
	if len(code) == 0 {
		return text, code, nil
	}

	var ts, remaining, datum []byte
	var err error

	switch code[0] {
	case Terminator:
		remaining = code

	case TypeNull:
		datum, remaining = getDatum(code)
		text = append(text, null...)

	case TypeTrue:
		datum, remaining = getDatum(code)
		text = append(text, boolTrue...)

	case TypeFalse:
		datum, remaining = getDatum(code)
		text = append(text, boolFalse...)

	case TypeLength:
		datum, remaining = getDatum(code)
		_, ts = DecodeInt(datum[1:], text)
		text = text[:len(text)+len(ts)]

	case TypeNumber:
		datum, remaining = getDatum(code)
		ts = DecodeFloat(datum[1:], text)
		ts, err = codec.denormalizeFloat(ts)
		ts = bytes.TrimLeft(ts, "+")
		text = append(text, ts...)

	case TypeString:
		text = append(text, '"')
		ts, remaining, err = suffixDecodeString(code[1:], text[1:])
		if err == nil {
			text = text[:len(text)+len(ts)]
			text = append(text, '"')
		}

	case TypeArray:
		var l int
		text = append(text, '[', ' ')
		if codec.arrayLenPrefix {
			datum, code = getDatum(code[1:])
			_, ts := DecodeInt(datum[1:], text[2:])
			l, err = strconv.Atoi(string(ts))
			if err == nil {
				for ; l > 0; l-- {
					ln := len(text)
					ts, code, err = codec.code2json(code, text[ln:])
					if err != nil {
						break
					}
					text = text[:ln+len(ts)]
					if l > 1 {
						text = append(text, ',', ' ')
					}
				}
			}
		} else {
			comma := false
			code = code[1:]
			for code[0] != Terminator {
				if comma {
					text = append(text, ',', ' ')
				}
				ln := len(text)
				ts, code, err = codec.code2json(code, text[ln:])
				if err != nil {
					break
				}
				text = text[:ln+len(ts)]
				comma = true
			}
		}
		remaining = code[1:] // remove Terminator
		text = append(text, ' ', ']')

	case TypeObj:
		var l int
		var key, value []byte
		text = append(text, '{', ' ')
		if codec.propertyLenPrefix {
			datum, code = getDatum(code[1:])
			_, ts := DecodeInt(datum[1:], text[2:])
			l, err = strconv.Atoi(string(ts))
			if err == nil {
				for ; l > 0; l-- {
					// decode key
					ln := len(text)
					key, code, err = codec.code2json(code, text[ln:])
					if err != nil {
						break
					}
					text = text[:ln+len(key)]
					text = append(text, ' ', ':', ' ')
					// decode value
					ln = len(text)
					key, code, err = codec.code2json(code, text[ln:])
					if err != nil {
						break
					}
					text = text[:ln+len(key)]
					if l > 1 {
						text = append(text, ',', ' ')
					}
				}
			}
		} else {
			comma := false
			code = code[1:]
			for code[0] != Terminator {
				if comma {
					text = append(text, ',', ' ')
				}
				// decode key
				ln := len(text)
				key, code, err = codec.code2json(code, text[ln:])
				if err != nil {
					break
				}
				text = text[:ln+len(key)]
				text = append(text, ' ', ':', ' ')
				// decode value
				ln = len(text)
				value, code, err = codec.code2json(code, text[ln:])
				if err != nil {
					break
				}
				text = text[:ln+len(value)]
				comma = true
			}
		}
		remaining = code[1:] // remove Terminator
		text = append(text, ' ', '}')
	}
	return text, remaining, err
}

// local function that sorts JSON property objects based on property names.
func (codec *Codec) sortProps(props map[string]interface{}) []string {
	keys := make([]string, 0, len(props))
	for k := range props {
		keys = append(keys, k)
	}
	ss := sort.StringSlice(keys)
	ss.Sort()
	return keys
}

// get the encoded datum (basic JSON datatype) based on Terminator and return a
// tuple of, `encoded-datum`, `remaining-code`, where remaining-code starts
// after the Terminator
func getDatum(code []byte) (datum []byte, remaining []byte) {
	var i int
	var b byte
	for i, b = range code {
		if b == Terminator {
			break
		}
	}
	return code[:i], code[i+1:]
}

func (codec *Codec) normalizeFloat(value float64, code []byte) ([]byte, error) {
	switch codec.numberType.(type) {
	case float64:
		cs := EncodeFloat([]byte(strconv.FormatFloat(value, 'e', -1, 64)), code)
		return cs, nil

	case int64:
		return EncodeInt([]byte(strconv.Itoa(int(value))), code), nil

	case string:
		cs := EncodeFloat([]byte(strconv.FormatFloat(value, 'e', -1, 64)), code)
		return cs, nil
	}
	return nil, ErrorNumberType
}

func (codec *Codec) denormalizeFloat(text []byte) ([]byte, error) {
	var err error
	var f float64

	switch codec.numberType.(type) {
	case float64:
		return text, nil

	case int64:
		f, err = strconv.ParseFloat(string(text), 64)
		return []byte(strconv.Itoa(int(f))), nil

	case string:
		f, err = strconv.ParseFloat(string(text), 64)
		if err == nil {
			return []byte(strconv.FormatFloat(f, 'f', -1, 64)), nil
		}

	default:
		return text, nil
	}
	return nil, ErrorNumberType
}
<commit_msg>collatejson: TypeMissing base on special literal.<commit_after>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not
//  use this file except in compliance with the License. You may obtain a copy
//  of the License at http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
//  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
//  License for the specific language governing permissions and limitations
//  under the License.

// Package collatejson supplies Encoding and Decoding function to transform
// JSON text into binary representation without loosing information. That is,
//
// * binary representation should preserve the sort order such that, sorting
//   binary encoded json documents much match sorting by functions that parse
//   and compare JSON documents.
//
// * it must be possible to get back the original document, in semantically
//   correct form, from its binary representation.
//
// Notes:
//
// * items in a property object are sorted by its property name before they
//   are compared with property's value.
package collatejson

import (
	"bytes"
	"encoding/json"
	"errors"
	"sort"
	"strconv"
)

// error codes
var ErrorNumberType = errors.New("collatejson.numberType")

// Length is an internal type used for prefixing length
// of arrays and properties.
type Length int64

// Missing denotes a special type for an item that evaluates
// to _nothing_.
type Missing string

// MissingLiteral is special string to denote missing item.
// IMPORTANT: we are assuming that MissingLiteral will not
// occur in the keyspace.
var MissingLiteral = Missing("~[]{}falsenilNA~")

// While encoding JSON data-element, both basic and composite, encoded string
// is prefixed with a type-byte. `Terminator` terminates encoded datum.
const (
	Terminator byte = iota
	TypeMissing
	TypeNull
	TypeFalse
	TypeTrue
	TypeNumber
	TypeString
	TypeLength
	TypeArray
	TypeObj
)

// Codec structure
type Codec struct {
	arrayLenPrefix    bool        // if true, first sort arrays based on its length
	propertyLenPrefix bool        // if true, first sort properties based on length
	doMissing         bool        // if true, handle missing values (for N1QL)
	numberType        interface{} // "float64" | "int64" | "decimal"
	//-- unicode
	//backwards        bool
	//hiraganaQ        bool
	//caseLevel        bool
	//numeric          bool
	//nfkd              bool
	//utf8              bool
	//strength          colltab.Level
	//alternate         collate.AlternateHandling
	//language          language.Tag
}

// NewCodec creates a new codec object and returns a reference to it.
func NewCodec(propSize int) *Codec {
	return &Codec{
		arrayLenPrefix:    false,
		propertyLenPrefix: true,
		doMissing:         true,
		numberType:        float64(0.0),
	}
}

// SortbyArrayLen sorts array by length before sorting by array elements. Use
// `false` to sort only by array elements. Default is `true`.
func (codec *Codec) SortbyArrayLen(what bool) {
	codec.arrayLenPrefix = what
}

// SortbyPropertyLen sorts property by length before sorting by property items.
// Use `false` to sort only by proprety items. Default is `true`.
func (codec *Codec) SortbyPropertyLen(what bool) {
	codec.propertyLenPrefix = what
}

// UseMissing will interpret special string MissingLiteral and encode them
// as TypeMissing. Default is `true`.
func (codec *Codec) UseMissing(what bool) {
	codec.doMissing = what
}

// NumberType chooses type of encoding / decoding for JSON numbers. Can be
// "float64", "int64", "decimal". Default is "float64"
func (codec *Codec) NumberType(what string) {
	switch what {
	case "float64":
		codec.numberType = float64(0.0)
	case "int64":
		codec.numberType = int64(0)
	case "decimal":
		codec.numberType = "0"
	}
}

// Encode json documents to order preserving binary representation.
func (codec *Codec) Encode(text, code []byte) ([]byte, error) {
	var m interface{}
	if err := json.Unmarshal(text, &m); err != nil {
		return nil, err
	}
	return codec.json2code(m, code)
}

// Decode a slice of byte into json string and return them as slice of byte.
func (codec *Codec) Decode(code, text []byte) ([]byte, error) {
	text, _, err := codec.code2json(code, text)
	return text, err
}

// local function that encodes basic json types to binary representation.
// composite types recursively call this function.
func (codec *Codec) json2code(val interface{}, code []byte) ([]byte, error) {
	if val == nil {
		code = append(code, TypeNull, Terminator)
	}

	var cs []byte
	var err error

	switch value := val.(type) {
	case bool:
		if value {
			code = append(code, TypeTrue, Terminator)
		} else {
			code = append(code, TypeFalse, Terminator)
		}

	case float64:
		code = append(code, TypeNumber)
		cs, err = codec.normalizeFloat(value, code[1:])
		if err == nil {
			code = code[:len(code)+len(cs)]
			code = append(code, Terminator)
		}

	case int:
		code = append(code, TypeNumber)
		cs = EncodeInt([]byte(strconv.Itoa(value)), code[1:])
		code = code[:len(code)+len(cs)]
		code = append(code, Terminator)

	case Length:
		code = append(code, TypeLength)
		cs = EncodeInt([]byte(strconv.Itoa(int(value))), code[1:])
		code = code[:len(code)+len(cs)]
		code = append(code, Terminator)

	case string:
		if codec.doMissing && MissingLiteral.Equal(value) {
			code = append(code, TypeMissing)
			code = append(code, Terminator)
		} else {
			code = append(code, TypeString)
			cs = suffixEncodeString([]byte(value), code[1:])
			code = code[:len(code)+len(cs)]
			code = append(code, Terminator)
		}

	case []interface{}:
		code = append(code, TypeArray)
		if codec.arrayLenPrefix {
			arrlen := Length(len(value))
			if cs, err = codec.json2code(arrlen, code[1:]); err == nil {
				code = code[:len(code)+len(cs)]
			}
		}
		if err == nil {
			for _, val := range value {
				l := len(code)
				cs, err = codec.json2code(val, code[l:])
				if err == nil {
					code = code[:l+len(cs)]
					continue
				}
				break
			}
			code = append(code, Terminator)
		}

	case map[string]interface{}:
		code = append(code, TypeObj)
		if codec.propertyLenPrefix {
			proplen := Length(len(value))
			if cs, err = codec.json2code(proplen, code[1:]); err == nil {
				code = code[:len(code)+len(cs)]
			}
		}

		if err == nil {
			keys := codec.sortProps(value)
			for _, key := range keys {
				l := len(code)
				// encode key
				if cs, err = codec.json2code(key, code[l:]); err != nil {
					break
				}
				code = code[:l+len(cs)]
				l = len(code)
				// encode value
				if cs, err = codec.json2code(value[key], code[l:]); err != nil {
					break
				}
				code = code[:l+len(cs)]
			}
			code = append(code, Terminator)
		}
	}
	return code, err
}

var null = []byte("null")
var boolTrue = []byte("true")
var boolFalse = []byte("false")

func (codec *Codec) code2json(code, text []byte) ([]byte, []byte, error) {
	if len(code) == 0 {
		return text, code, nil
	}

	var ts, remaining, datum []byte
	var err error

	switch code[0] {
	case Terminator:
		remaining = code

	case TypeMissing:
		datum, remaining = getDatum(code)
		text = append(text, []byte(MissingLiteral)...)

	case TypeNull:
		datum, remaining = getDatum(code)
		text = append(text, null...)

	case TypeTrue:
		datum, remaining = getDatum(code)
		text = append(text, boolTrue...)

	case TypeFalse:
		datum, remaining = getDatum(code)
		text = append(text, boolFalse...)

	case TypeLength:
		datum, remaining = getDatum(code)
		_, ts = DecodeInt(datum[1:], text)
		text = text[:len(text)+len(ts)]

	case TypeNumber:
		datum, remaining = getDatum(code)
		ts = DecodeFloat(datum[1:], text)
		ts, err = codec.denormalizeFloat(ts)
		ts = bytes.TrimLeft(ts, "+")
		text = append(text, ts...)

	case TypeString:
		text = append(text, '"')
		ts, remaining, err = suffixDecodeString(code[1:], text[1:])
		if err == nil {
			text = text[:len(text)+len(ts)]
			text = append(text, '"')
		}

	case TypeArray:
		var l int
		text = append(text, '[', ' ')
		if codec.arrayLenPrefix {
			datum, code = getDatum(code[1:])
			_, ts := DecodeInt(datum[1:], text[2:])
			l, err = strconv.Atoi(string(ts))
			if err == nil {
				for ; l > 0; l-- {
					ln := len(text)
					ts, code, err = codec.code2json(code, text[ln:])
					if err != nil {
						break
					}
					text = text[:ln+len(ts)]
					if l > 1 {
						text = append(text, ',', ' ')
					}
				}
			}
		} else {
			comma := false
			code = code[1:]
			for code[0] != Terminator {
				if comma {
					text = append(text, ',', ' ')
				}
				ln := len(text)
				ts, code, err = codec.code2json(code, text[ln:])
				if err != nil {
					break
				}
				text = text[:ln+len(ts)]
				comma = true
			}
		}
		remaining = code[1:] // remove Terminator
		text = append(text, ' ', ']')

	case TypeObj:
		var l int
		var key, value []byte
		text = append(text, '{', ' ')
		if codec.propertyLenPrefix {
			datum, code = getDatum(code[1:])
			_, ts := DecodeInt(datum[1:], text[2:])
			l, err = strconv.Atoi(string(ts))
			if err == nil {
				for ; l > 0; l-- {
					// decode key
					ln := len(text)
					key, code, err = codec.code2json(code, text[ln:])
					if err != nil {
						break
					}
					text = text[:ln+len(key)]
					text = append(text, ' ', ':', ' ')
					// decode value
					ln = len(text)
					key, code, err = codec.code2json(code, text[ln:])
					if err != nil {
						break
					}
					text = text[:ln+len(key)]
					if l > 1 {
						text = append(text, ',', ' ')
					}
				}
			}
		} else {
			comma := false
			code = code[1:]
			for code[0] != Terminator {
				if comma {
					text = append(text, ',', ' ')
				}
				// decode key
				ln := len(text)
				key, code, err = codec.code2json(code, text[ln:])
				if err != nil {
					break
				}
				text = text[:ln+len(key)]
				text = append(text, ' ', ':', ' ')
				// decode value
				ln = len(text)
				value, code, err = codec.code2json(code, text[ln:])
				if err != nil {
					break
				}
				text = text[:ln+len(value)]
				comma = true
			}
		}
		remaining = code[1:] // remove Terminator
		text = append(text, ' ', '}')
	}
	return text, remaining, err
}

// local function that sorts JSON property objects based on property names.
func (codec *Codec) sortProps(props map[string]interface{}) []string {
	keys := make([]string, 0, len(props))
	for k := range props {
		keys = append(keys, k)
	}
	ss := sort.StringSlice(keys)
	ss.Sort()
	return keys
}

// get the encoded datum (basic JSON datatype) based on Terminator and return a
// tuple of, `encoded-datum`, `remaining-code`, where remaining-code starts
// after the Terminator
func getDatum(code []byte) (datum []byte, remaining []byte) {
	var i int
	var b byte
	for i, b = range code {
		if b == Terminator {
			break
		}
	}
	return code[:i], code[i+1:]
}

func (codec *Codec) normalizeFloat(value float64, code []byte) ([]byte, error) {
	switch codec.numberType.(type) {
	case float64:
		cs := EncodeFloat([]byte(strconv.FormatFloat(value, 'e', -1, 64)), code)
		return cs, nil

	case int64:
		return EncodeInt([]byte(strconv.Itoa(int(value))), code), nil

	case string:
		cs := EncodeFloat([]byte(strconv.FormatFloat(value, 'e', -1, 64)), code)
		return cs, nil
	}
	return nil, ErrorNumberType
}

func (codec *Codec) denormalizeFloat(text []byte) ([]byte, error) {
	var err error
	var f float64

	switch codec.numberType.(type) {
	case float64:
		return text, nil

	case int64:
		f, err = strconv.ParseFloat(string(text), 64)
		return []byte(strconv.Itoa(int(f))), nil

	case string:
		f, err = strconv.ParseFloat(string(text), 64)
		if err == nil {
			return []byte(strconv.FormatFloat(f, 'f', -1, 64)), nil
		}

	default:
		return text, nil
	}
	return nil, ErrorNumberType
}

// Equal checks wether n is MissingLiteral
func (m Missing) Equal(n string) bool {
	s := string(m)
	if len(n) == len(s) && n[0] == '~' && n[1] == '[' {
		return s == n
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fs

import (
	"container/list"
	"errors"
	"fmt"
	"log"
	"os"
	"path"
	"reflect"
	"sort"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
)

const dirSeparator = '/'

// Implementation detail, do not touch.
//
// How long we cache the most recent listing for a particular directory from
// GCS before regarding it as stale.
//
// Intended to paper over performance issues caused by quick follow-up calls;
// for example when the fuse VFS performs a readdir followed quickly by a
// lookup for each child. The drawback is that this increases the time before a
// write by a foreign machine within a recently-listed directory will be seen
// locally.
//
// TODO(jacobsa): Set this according to real-world performance issues when the
// kernel does e.g. ReadDir followed by Lookup. Can probably be set quite
// small.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ListingCacheTTL = 10 * time.Second

// Implementation detail, do not touch.
//
// How long we remember that we took some action on the contents of a directory
// (linking or unlinking), and pretend the action is reflected in the listing
// even if it is not.
//
// Intended to paper over the fact that GCS doesn't offer list-your-own-writes
// consistency: it may be an arbitrarily long time before you see the creation
// or deletion of an object in a subsequent listing, and even if you see it in
// one listing you may not in the next. The drawback is that modifications to
// recently-modified directories by foreign machines will not be reflected
// locally for awhile.
//
// TODO(jacobsa): Set this according to information about listing staleness
// distributions from the GCS team.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ChildActionMemoryTTL = 5 * time.Minute

// See the childModifications field of dir.
type childModification struct {
	time time.Time
	name string
	node fusefs.Node
}

// A "directory" in GCS, defined by an object name prefix.
//
// For example, if the bucket contains objects "foo/bar" and "foo/baz", this
// implicitly defines the directory "foo/". No matter what the contents of the
// bucket, there is an implicit root directory "".
type dir struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// INVARIANT: objectPrefix is "" (representing the root directory) or ends
	// with dirSeparator.
	objectPrefix string

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// Our current best understanding of the contents of the directory in GCS,
	// formed by listing the bucket and then patching according to child addition
	// and removal records at the time, and patched since then by subsequent
	// additions and removals.
	//
	// The time after which this should be generated anew from a new listing is
	// also stored. This is set to the time at which the listing completed plus
	// ListingCacheTTL.
	//
	// INVARIANT: contents != nil
	// INVARIANT: All nodes are of type *dir or *file.
	// INVARIANT: All nodes are indexed by names that agree with node contents.
	contents           map[string]fusefs.Node // GUARDED_BY(mu)
	contentsExpiration time.Time              // GUARDED_BY(mu)

	// A collection of children that have recently been added or removed locally
	// and the time at which it happened, ordered by the sequence in which it
	// happened. Elements M with M.node == nil are removals; all others are
	// additions.
	//
	// For a record M in this list with M's age less than ChildActionMemoryTTL,
	// any listing from the bucket should be augmented by pretending M just
	// happened.
	//
	// TODO(jacobsa): Make sure to test link followed by unlink, and unlink
	// followed by link.
	//
	// TODO(jacobsa): Make sure to test that these expire eventually, i.e. that
	// foreign overwrites, deletes, and recreates are reflected eventually.
	//
	// INVARIANT: All elements are of type childModification.
	// INVARIANT: Contains no duplicate names.
	// INVARIANT: For each M with M.node == nil, contents does not contain M.name.
	// INVARIANT: For each M with M.node != nil, contents[M.name] == M.node.
	childModifications list.List // GUARDED_BY(mu)

	// An index of childModifications by name.
	//
	// INVARIANT: childModificationsIndex != nil
	// INVARIANT: For all names N in the map, the indexed modification has name N.
	// INVARIANT: Contains exactly the set of names in childModifications.
	childModificationsIndex map[string]*list.Element // GUARDED_BY(mu)
}

// Make sure dir implements the interfaces we think it does.
var (
	// TODO(jacobsa): I think we want to embed fusefs.NdoeRef in all of our
	// fusefs.Node types, so that we better benefit from fusefs.Server node
	// caching.
	_ fusefs.Node               = &dir{}
	_ fusefs.NodeCreater        = &dir{}
	_ fusefs.NodeMknoder        = &dir{}
	_ fusefs.NodeStringLookuper = &dir{}

	_ fusefs.Handle             = &dir{}
	_ fusefs.HandleReadDirAller = &dir{}
)

func newDir(
	logger *log.Logger,
	clock timeutil.Clock,
	bucket gcs.Bucket,
	objectPrefix string) *dir {
	// Check the object prefix.
	if objectPrefix != "" && objectPrefix[len(objectPrefix)-1] != dirSeparator {
		panic("Unexpected object prefix: " + objectPrefix)
	}

	// Create the dir struct.
	d := &dir{
		logger:                  logger,
		clock:                   clock,
		bucket:                  bucket,
		objectPrefix:            objectPrefix,
		contents:                make(map[string]fusefs.Node),
		childModificationsIndex: make(map[string]*list.Element),
	}

	d.mu = syncutil.NewInvariantMutex(func() { d.checkInvariants() })

	return d
}

func (d *dir) checkInvariants() {
	// Check that maps are non-nil.
	if d.contents == nil || d.childModificationsIndex == nil {
		panic("Expected contents and childModificationsIndex to be non-nil.")
	}

	// Check each element of the contents map.
	for name, node := range d.contents {
		if node == nil {
			panic("nil node for name: " + name)
		}

		d, okDir := node.(*dir)
		f, okFile := node.(*file)
		if !okDir && !okFile {
			panic(fmt.Sprintf("Unexpected node type: %v", reflect.TypeOf(node)))
		}

		if okDir && name != path.Base(d.objectPrefix) {
			panic(fmt.Sprintf("Name mismatch: %s vs. %s", name, d.objectPrefix))
		}

		if okFile && name != path.Base(f.objectName) {
			panic(fmt.Sprintf("Name mismatch: %s vs. %s", name, f.objectName))
		}
	}

	// Check each child modification. Build a list of names we've seen while
	// doing so.
	var listNames sort.StringSlice
	for e := d.childModifications.Front(); e != nil; e = e.Next() {
		m := e.Value.(childModification)
		listNames = append(listNames, m.name)

		if m.node == nil {
			if n, ok := d.contents[m.name]; ok {
				panic(fmt.Sprintf("d.contents[%s] == %v for removal", m.name, n))
			}
		} else {
			if n := d.contents[m.name]; n != m.node {
				panic(fmt.Sprintf("d.contents[%s] == %v, not %v", m.name, n, m.node))
			}
		}
	}

	sort.Sort(listNames)

	// Check that there were no duplicate names.
	for i, name := range listNames {
		if i == 0 {
			continue
		}

		if name == listNames[i-1] {
			panic("Duplicated name in childModifications: " + name)
		}
	}

	// Check the index. Build a list of names it contains While doing so.
	var indexNames sort.StringSlice
	for name, e := range d.childModificationsIndex {
		indexNames = append(indexNames, name)

		m := e.Value.(childModification)
		if m.name != name {
			panic(fmt.Sprintf("Index name mismatch: %s vs. %s", m.name, name))
		}
	}

	sort.Sort(indexNames)

	// Check that the index contains the same set of names.
	if !reflect.DeepEqual(listNames, indexNames) {
		panic(fmt.Sprintf("Names mismatch:\n%v\n%v", listNames, indexNames))
	}
}

// Ensure that d.contents is fresh and usable. Must be called before using
// d.contents.
//
// TODO(jacobsa): If contents hasn't expired, return immediately. Otherwise
// list parasitically while holding the lock (why not, we can make this more
// subtle later if we must) and modify the result appropriately.
//
// EXCLUSIVE_LOCKS_REQUIRED(d.mu)
func (d *dir) ensureContents(ctx context.Context) error {
	// Are the contents already fresh?
	if d.clock.Now().Before(d.contentsExpiration) {
		return nil
	}

	// Grab a listing.
}

func (d *dir) Attr() fuse.Attr {
	return fuse.Attr{
		// TODO(jacobsa): Reflect that we allow writes now. Make sure to test.
		// TODO(jacobsa): Expose ACLs from GCS?
		Mode: os.ModeDir | 0500,
	}
}

// LOCKS_EXCLUDED(d.mu)
func (d *dir) ReadDirAll(ctx context.Context) (ents []fuse.Dirent, err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("ReadDirAll: [%s]/%s", d.bucket.Name(), d.objectPrefix)

	// Ensure that we can use d.contents.
	if err = d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Read out entries from the contents map.
	for name, node := range d.contents {
		ent := fuse.Dirent{
			Name: name,
		}

		if _, ok := node.(*dir); ok {
			ent.Type = fuse.DT_Dir
		} else {
			ent.Type = fuse.DT_File
		}

		ents = append(ents, ent)
	}

	return
}

// LOCKS_EXCLUDED(d.mu)
func (d *dir) Lookup(
	ctx context.Context,
	name string) (n fusefs.Node, err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("Lookup: ([%s]/%s) %s", d.bucket.Name(), d.objectPrefix, name)

	// Ensure that we can use d.contents.
	if err = d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Find the object within the map.
	var ok bool
	if n, ok = d.contents[name]; ok {
		return
	}

	err = fuse.ENOENT
	return
}

func (d *dir) Create(
	ctx context.Context,
	req *fuse.CreateRequest,
	resp *fuse.CreateResponse) (
	fusefs.Node,
	fusefs.Handle,
	error) {
	// Tell fuse to use Mknod followed by Open, rather than re-implementing much
	// of Open here.
	return nil, nil, fuse.ENOSYS
}

func (d *dir) Mknod(
	ctx context.Context,
	req *fuse.MknodRequest) (fusefs.Node, error) {
	return nil, errors.New("TODO(jacobsa): Support Mknod.")
}
<commit_msg>Finished most of dir.ensureContents.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fs

import (
	"container/list"
	"errors"
	"fmt"
	"log"
	"os"
	"path"
	"reflect"
	"sort"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
)

const dirSeparator = '/'

// Implementation detail, do not touch.
//
// How long we cache the most recent listing for a particular directory from
// GCS before regarding it as stale.
//
// Intended to paper over performance issues caused by quick follow-up calls;
// for example when the fuse VFS performs a readdir followed quickly by a
// lookup for each child. The drawback is that this increases the time before a
// write by a foreign machine within a recently-listed directory will be seen
// locally.
//
// TODO(jacobsa): Set this according to real-world performance issues when the
// kernel does e.g. ReadDir followed by Lookup. Can probably be set quite
// small.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ListingCacheTTL = 10 * time.Second

// Implementation detail, do not touch.
//
// How long we remember that we took some action on the contents of a directory
// (linking or unlinking), and pretend the action is reflected in the listing
// even if it is not.
//
// Intended to paper over the fact that GCS doesn't offer list-your-own-writes
// consistency: it may be an arbitrarily long time before you see the creation
// or deletion of an object in a subsequent listing, and even if you see it in
// one listing you may not in the next. The drawback is that modifications to
// recently-modified directories by foreign machines will not be reflected
// locally for awhile.
//
// TODO(jacobsa): Set this according to information about listing staleness
// distributions from the GCS team.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ChildActionMemoryTTL = 5 * time.Minute

// See the childModifications field of dir.
type childModification struct {
	time time.Time
	name string
	node fusefs.Node
}

// A "directory" in GCS, defined by an object name prefix.
//
// For example, if the bucket contains objects "foo/bar" and "foo/baz", this
// implicitly defines the directory "foo/". No matter what the contents of the
// bucket, there is an implicit root directory "".
type dir struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// INVARIANT: objectPrefix is "" (representing the root directory) or ends
	// with dirSeparator.
	objectPrefix string

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// Our current best understanding of the contents of the directory in GCS,
	// formed by listing the bucket and then patching according to child addition
	// and removal records at the time, and patched since then by subsequent
	// additions and removals.
	//
	// The time after which this should be generated anew from a new listing is
	// also stored. This is set to the time at which the listing completed plus
	// ListingCacheTTL.
	//
	// INVARIANT: contents != nil
	// INVARIANT: All nodes are of type *dir or *file.
	// INVARIANT: All nodes are indexed by names that agree with node contents.
	contents           map[string]fusefs.Node // GUARDED_BY(mu)
	contentsExpiration time.Time              // GUARDED_BY(mu)

	// A collection of children that have recently been added or removed locally
	// and the time at which it happened, ordered by the sequence in which it
	// happened. Elements M with M.node == nil are removals; all others are
	// additions.
	//
	// For a record M in this list with M's age less than ChildActionMemoryTTL,
	// any listing from the bucket should be augmented by pretending M just
	// happened.
	//
	// TODO(jacobsa): Make sure to test link followed by unlink, and unlink
	// followed by link.
	//
	// TODO(jacobsa): Make sure to test that these expire eventually, i.e. that
	// foreign overwrites, deletes, and recreates are reflected eventually.
	//
	// INVARIANT: All elements are of type childModification.
	// INVARIANT: Contains no duplicate names.
	// INVARIANT: For each M with M.node == nil, contents does not contain M.name.
	// INVARIANT: For each M with M.node != nil, contents[M.name] == M.node.
	childModifications list.List // GUARDED_BY(mu)

	// An index of childModifications by name.
	//
	// INVARIANT: childModificationsIndex != nil
	// INVARIANT: For all names N in the map, the indexed modification has name N.
	// INVARIANT: Contains exactly the set of names in childModifications.
	childModificationsIndex map[string]*list.Element // GUARDED_BY(mu)
}

// Make sure dir implements the interfaces we think it does.
var (
	// TODO(jacobsa): I think we want to embed fusefs.NdoeRef in all of our
	// fusefs.Node types, so that we better benefit from fusefs.Server node
	// caching.
	_ fusefs.Node               = &dir{}
	_ fusefs.NodeCreater        = &dir{}
	_ fusefs.NodeMknoder        = &dir{}
	_ fusefs.NodeStringLookuper = &dir{}

	_ fusefs.Handle             = &dir{}
	_ fusefs.HandleReadDirAller = &dir{}
)

func newDir(
	logger *log.Logger,
	clock timeutil.Clock,
	bucket gcs.Bucket,
	objectPrefix string) *dir {
	// Check the object prefix.
	if objectPrefix != "" && objectPrefix[len(objectPrefix)-1] != dirSeparator {
		panic("Unexpected object prefix: " + objectPrefix)
	}

	// Create the dir struct.
	d := &dir{
		logger:                  logger,
		clock:                   clock,
		bucket:                  bucket,
		objectPrefix:            objectPrefix,
		contents:                make(map[string]fusefs.Node),
		childModificationsIndex: make(map[string]*list.Element),
	}

	d.mu = syncutil.NewInvariantMutex(func() { d.checkInvariants() })

	return d
}

func (d *dir) checkInvariants() {
	// Check that maps are non-nil.
	if d.contents == nil || d.childModificationsIndex == nil {
		panic("Expected contents and childModificationsIndex to be non-nil.")
	}

	// Check each element of the contents map.
	for name, node := range d.contents {
		if node == nil {
			panic("nil node for name: " + name)
		}

		d, okDir := node.(*dir)
		f, okFile := node.(*file)
		if !okDir && !okFile {
			panic(fmt.Sprintf("Unexpected node type: %v", reflect.TypeOf(node)))
		}

		if okDir && name != path.Base(d.objectPrefix) {
			panic(fmt.Sprintf("Name mismatch: %s vs. %s", name, d.objectPrefix))
		}

		if okFile && name != path.Base(f.objectName) {
			panic(fmt.Sprintf("Name mismatch: %s vs. %s", name, f.objectName))
		}
	}

	// Check each child modification. Build a list of names we've seen while
	// doing so.
	var listNames sort.StringSlice
	for e := d.childModifications.Front(); e != nil; e = e.Next() {
		m := e.Value.(childModification)
		listNames = append(listNames, m.name)

		if m.node == nil {
			if n, ok := d.contents[m.name]; ok {
				panic(fmt.Sprintf("d.contents[%s] == %v for removal", m.name, n))
			}
		} else {
			if n := d.contents[m.name]; n != m.node {
				panic(fmt.Sprintf("d.contents[%s] == %v, not %v", m.name, n, m.node))
			}
		}
	}

	sort.Sort(listNames)

	// Check that there were no duplicate names.
	for i, name := range listNames {
		if i == 0 {
			continue
		}

		if name == listNames[i-1] {
			panic("Duplicated name in childModifications: " + name)
		}
	}

	// Check the index. Build a list of names it contains While doing so.
	var indexNames sort.StringSlice
	for name, e := range d.childModificationsIndex {
		indexNames = append(indexNames, name)

		m := e.Value.(childModification)
		if m.name != name {
			panic(fmt.Sprintf("Index name mismatch: %s vs. %s", m.name, name))
		}
	}

	sort.Sort(indexNames)

	// Check that the index contains the same set of names.
	if !reflect.DeepEqual(listNames, indexNames) {
		panic(fmt.Sprintf("Names mismatch:\n%v\n%v", listNames, indexNames))
	}
}

// Ensure that d.contents is fresh and usable. Must be called before using
// d.contents.
//
// TODO(jacobsa): If contents hasn't expired, return immediately. Otherwise
// list parasitically while holding the lock (why not, we can make this more
// subtle later if we must) and modify the result appropriately.
//
// EXCLUSIVE_LOCKS_REQUIRED(d.mu)
func (d *dir) ensureContents(ctx context.Context) error {
	// Are the contents already fresh?
	if d.clock.Now().Before(d.contentsExpiration) {
		return nil
	}

	// Grab a listing.
	query := &storage.Query{
		Delimiter: string(dirSeparator),
		Prefix:    d.objectPrefix,
	}

	objects, prefixes, err := gcsutil.List(ctx, d.bucket, query)
	if err != nil {
		return fmt.Errorf("gcsutil.List: %v", err)
	}

	// Convert the listing into a contents map.
	d.contents = make(map[string]fusefs.Node)

	for _, o := range objects {
		d.contents[path.Base(o.Name)] =
			newFile(
				d.logger,
				d.bucket,
				o.Name,
				uint64(o.Size))
	}

	for _, prefix := range prefixes {
		d.contents[path.Base(prefix)] =
			newDir(
				d.logger,
				d.clock,
				d.bucket,
				prefix)
	}

	// TODO(jacobsa): Apply child modifications.
	return nil
}

func (d *dir) Attr() fuse.Attr {
	return fuse.Attr{
		// TODO(jacobsa): Reflect that we allow writes now. Make sure to test.
		// TODO(jacobsa): Expose ACLs from GCS?
		Mode: os.ModeDir | 0500,
	}
}

// LOCKS_EXCLUDED(d.mu)
func (d *dir) ReadDirAll(ctx context.Context) (ents []fuse.Dirent, err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("ReadDirAll: [%s]/%s", d.bucket.Name(), d.objectPrefix)

	// Ensure that we can use d.contents.
	if err = d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Read out entries from the contents map.
	for name, node := range d.contents {
		ent := fuse.Dirent{
			Name: name,
		}

		if _, ok := node.(*dir); ok {
			ent.Type = fuse.DT_Dir
		} else {
			ent.Type = fuse.DT_File
		}

		ents = append(ents, ent)
	}

	return
}

// LOCKS_EXCLUDED(d.mu)
func (d *dir) Lookup(
	ctx context.Context,
	name string) (n fusefs.Node, err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("Lookup: ([%s]/%s) %s", d.bucket.Name(), d.objectPrefix, name)

	// Ensure that we can use d.contents.
	if err = d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Find the object within the map.
	var ok bool
	if n, ok = d.contents[name]; ok {
		return
	}

	err = fuse.ENOENT
	return
}

func (d *dir) Create(
	ctx context.Context,
	req *fuse.CreateRequest,
	resp *fuse.CreateResponse) (
	fusefs.Node,
	fusefs.Handle,
	error) {
	// Tell fuse to use Mknod followed by Open, rather than re-implementing much
	// of Open here.
	return nil, nil, fuse.ENOSYS
}

func (d *dir) Mknod(
	ctx context.Context,
	req *fuse.MknodRequest) (fusefs.Node, error) {
	return nil, errors.New("TODO(jacobsa): Support Mknod.")
}
<|endoftext|>


<commit_before>// +build linux

package main

import (
	"errors"
	"net"
	"os"
	"path/filepath"
	"time"

	"github.com/Symantec/Dominator/lib/fsutil"
	"github.com/Symantec/Dominator/lib/json"
	"github.com/Symantec/Dominator/lib/log"
	libnet "github.com/Symantec/Dominator/lib/net"
	"github.com/Symantec/Dominator/lib/net/configurator"
	fm_proto "github.com/Symantec/Dominator/proto/fleetmanager"
	hyper_proto "github.com/Symantec/Dominator/proto/hypervisor"
	"github.com/d2g/dhcp4"
	"github.com/d2g/dhcp4client"
	"github.com/pin/tftp"
)

var (
	tftpFiles = []string{
		"config.json",
		"imagename",
		"imageserver",
		"storage-layout.json",
	}
	zeroIP = net.IP(make([]byte, 4))
)

func configureLocalNetwork(logger log.DebugLogger) (
	*fm_proto.GetMachineInfoResponse, map[string]net.Interface, error) {
	if err := run("ifconfig", "", logger, "lo", "up"); err != nil {
		return nil, nil, err
	}
	_, interfaces, err := libnet.ListBroadcastInterfaces(
		libnet.InterfaceTypeEtherNet, logger)
	if err != nil {
		return nil, nil, err
	}
	// Raise interfaces so that by the time the OS is installed link status
	// should be stable. This is how we discover connected interfaces.
	if err := raiseInterfaces(interfaces, logger); err != nil {
		return nil, nil, err
	}
	machineInfo, err := getConfiguration(interfaces, logger)
	if err != nil {
		return nil, nil, err
	}
	return machineInfo, interfaces, nil
}

func dhcpRequest(interfaces map[string]net.Interface,
	logger log.DebugLogger) (string, dhcp4.Packet, error) {
	clients := make(map[string]*dhcp4client.Client, len(interfaces))
	for name, iface := range interfaces {
		packetSocket, err := dhcp4client.NewPacketSock(iface.Index)
		if err != nil {
			return "", nil, err
		}
		defer packetSocket.Close()
		client, err := dhcp4client.New(
			dhcp4client.HardwareAddr(iface.HardwareAddr),
			dhcp4client.Connection(packetSocket),
			dhcp4client.Timeout(time.Second*5))
		if err != nil {
			return "", nil, err
		}
		defer client.Close()
		clients[name] = client
	}
	stopTime := time.Now().Add(time.Minute * 5)
	for ; time.Until(stopTime) > 0; time.Sleep(time.Second) {
		for name, client := range clients {
			if libnet.TestCarrier(name) {
				logger.Debugf(1, "%s: DHCP attempt\n", name)
				if ok, packet, err := client.Request(); err != nil {
					logger.Debugf(1, "%s: DHCP failed: %s\n", name, err)
				} else if ok {
					return name, packet, nil
				}
			}
		}
	}
	return "", nil, errors.New("timed out waiting for DHCP")
}

func findInterfaceToConfigure(interfaces map[string]net.Interface,
	machineInfo fm_proto.GetMachineInfoResponse, logger log.DebugLogger) (
	net.Interface, net.IP, *hyper_proto.Subnet, error) {
	networkEntries := configurator.GetNetworkEntries(machineInfo)
	hwAddrToInterface := make(map[string]net.Interface, len(interfaces))
	for _, iface := range interfaces {
		hwAddrToInterface[iface.HardwareAddr.String()] = iface
	}
	for _, networkEntry := range networkEntries {
		if len(networkEntry.HostIpAddress) < 1 {
			continue
		}
		iface, ok := hwAddrToInterface[networkEntry.HostMacAddress.String()]
		if !ok {
			continue
		}
		subnet := configurator.FindMatchingSubnet(machineInfo.Subnets,
			networkEntry.HostIpAddress)
		if subnet == nil {
			logger.Printf("no matching subnet for ip=%s\n",
				networkEntry.HostIpAddress)
			continue
		}
		return iface, networkEntry.HostIpAddress, subnet, nil
	}
	return net.Interface{}, nil, nil,
		errors.New("no network interfaces match injected configuration")
}

func getConfiguration(interfaces map[string]net.Interface,
	logger log.DebugLogger) (*fm_proto.GetMachineInfoResponse, error) {
	var machineInfo fm_proto.GetMachineInfoResponse
	err := json.ReadFromFile(filepath.Join(*tftpDirectory, "config.json"),
		&machineInfo)
	if err == nil { // Configuration was injected.
		err := setupNetworkFromConfig(interfaces, machineInfo, logger)
		if err != nil {
			return nil, err
		}
		return &machineInfo, nil
	}
	if !os.IsNotExist(err) {
		return nil, err
	}
	if err := setupNetworkFromDhcp(interfaces, logger); err != nil {
		return nil, err
	}
	err = json.ReadFromFile(filepath.Join(*tftpDirectory, "config.json"),
		&machineInfo)
	if err != nil {
		return nil, err
	}
	return &machineInfo, nil
}

func loadTftpFiles(tftpServer net.IP, logger log.DebugLogger) error {
	client, err := tftp.NewClient(tftpServer.String() + ":69")
	if err != nil {
		return err
	}
	if err := os.MkdirAll(*tftpDirectory, fsutil.DirPerms); err != nil {
		return err
	}
	for _, name := range tftpFiles {
		logger.Debugf(1, "downloading: %s\n", name)
		if wt, err := client.Receive(name, "octet"); err != nil {
			return err
		} else {
			filename := filepath.Join(*tftpDirectory, name)
			if file, err := create(filename); err != nil {
				return err
			} else {
				defer file.Close()
				if _, err := wt.WriteTo(file); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func raiseInterfaces(interfaces map[string]net.Interface,
	logger log.DebugLogger) error {
	for name := range interfaces {
		if err := run("ifconfig", "", logger, name, "up"); err != nil {
			return err
		}
	}
	return nil
}

func setupNetwork(ifName string, ipAddr net.IP, subnet *hyper_proto.Subnet,
	logger log.DebugLogger) error {
	err := run("ifconfig", "", logger, ifName, ipAddr.String(), "netmask",
		subnet.IpMask.String(), "up")
	if err != nil {
		return err
	}
	err = run("route", "", logger, "add", "default", "gw",
		subnet.IpGateway.String())
	if err != nil {
		e := run("route", "", logger, "del", "default", "gw",
			subnet.IpGateway.String())
		if e != nil {
			return err
		}
		err = run("route", "", logger, "add", "default", "gw",
			subnet.IpGateway.String())
		if err != nil {
			return err
		}
	}
	if !*dryRun {
		if err := configurator.WriteResolvConf("", subnet); err != nil {
			return err
		}
	}
	return nil
}

func setupNetworkFromConfig(interfaces map[string]net.Interface,
	machineInfo fm_proto.GetMachineInfoResponse, logger log.DebugLogger) error {
	iface, ipAddr, subnet, err := findInterfaceToConfigure(interfaces,
		machineInfo, logger)
	if err != nil {
		return err
	}
	return setupNetwork(iface.Name, ipAddr, subnet, logger)
}

func setupNetworkFromDhcp(interfaces map[string]net.Interface,
	logger log.DebugLogger) error {
	ifName, packet, err := dhcpRequest(interfaces, logger)
	if err != nil {
		return err
	}
	ipAddr := packet.YIAddr()
	options := packet.ParseOptions()
	subnet := hyper_proto.Subnet{
		IpGateway: net.IP(options[dhcp4.OptionRouter]),
		IpMask:    net.IP(options[dhcp4.OptionSubnetMask]),
	}
	dnsServersBuffer := options[dhcp4.OptionDomainNameServer]
	for len(dnsServersBuffer) > 0 {
		if len(dnsServersBuffer) >= 4 {
			subnet.DomainNameServers = append(subnet.DomainNameServers,
				net.IP(dnsServersBuffer[:4]))
			dnsServersBuffer = dnsServersBuffer[4:]
		} else {
			return errors.New("truncated DNS server address")
		}
	}
	if err := setupNetwork(ifName, ipAddr, &subnet, logger); err != nil {
		return err
	}
	tftpServer := packet.SIAddr()
	if tftpServer.Equal(zeroIP) {
		tftpServer = net.IP(options[dhcp4.OptionTFTPServerName])
		if tftpServer.Equal(zeroIP) {
			return errors.New("no TFTP server given")
		}
		logger.Printf("tftpServer from OptionTFTPServerName: %s\n", tftpServer)
	} else {
		logger.Printf("tftpServer from SIAddr: %s\n", tftpServer)
	}
	return loadTftpFiles(tftpServer, logger)
}
<commit_msg>installer: support injecting random seed data for /dev/urandom.<commit_after>// +build linux

package main

import (
	"bytes"
	"errors"
	"io"
	"net"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/Symantec/Dominator/lib/fsutil"
	"github.com/Symantec/Dominator/lib/json"
	"github.com/Symantec/Dominator/lib/log"
	libnet "github.com/Symantec/Dominator/lib/net"
	"github.com/Symantec/Dominator/lib/net/configurator"
	fm_proto "github.com/Symantec/Dominator/proto/fleetmanager"
	hyper_proto "github.com/Symantec/Dominator/proto/hypervisor"
	"github.com/d2g/dhcp4"
	"github.com/d2g/dhcp4client"
	"github.com/pin/tftp"
)

var (
	tftpFiles = []string{
		"config.json",
		"imagename",
		"imageserver",
		"storage-layout.json",
	}
	zeroIP = net.IP(make([]byte, 4))
)

func configureLocalNetwork(logger log.DebugLogger) (
	*fm_proto.GetMachineInfoResponse, map[string]net.Interface, error) {
	if err := run("ifconfig", "", logger, "lo", "up"); err != nil {
		return nil, nil, err
	}
	_, interfaces, err := libnet.ListBroadcastInterfaces(
		libnet.InterfaceTypeEtherNet, logger)
	if err != nil {
		return nil, nil, err
	}
	// Raise interfaces so that by the time the OS is installed link status
	// should be stable. This is how we discover connected interfaces.
	if err := raiseInterfaces(interfaces, logger); err != nil {
		return nil, nil, err
	}
	machineInfo, err := getConfiguration(interfaces, logger)
	if err != nil {
		return nil, nil, err
	}
	return machineInfo, interfaces, nil
}

func dhcpRequest(interfaces map[string]net.Interface,
	logger log.DebugLogger) (string, dhcp4.Packet, error) {
	clients := make(map[string]*dhcp4client.Client, len(interfaces))
	for name, iface := range interfaces {
		packetSocket, err := dhcp4client.NewPacketSock(iface.Index)
		if err != nil {
			return "", nil, err
		}
		defer packetSocket.Close()
		client, err := dhcp4client.New(
			dhcp4client.HardwareAddr(iface.HardwareAddr),
			dhcp4client.Connection(packetSocket),
			dhcp4client.Timeout(time.Second*5))
		if err != nil {
			return "", nil, err
		}
		defer client.Close()
		clients[name] = client
	}
	stopTime := time.Now().Add(time.Minute * 5)
	for ; time.Until(stopTime) > 0; time.Sleep(time.Second) {
		for name, client := range clients {
			if libnet.TestCarrier(name) {
				logger.Debugf(1, "%s: DHCP attempt\n", name)
				if ok, packet, err := client.Request(); err != nil {
					logger.Debugf(1, "%s: DHCP failed: %s\n", name, err)
				} else if ok {
					return name, packet, nil
				}
			}
		}
	}
	return "", nil, errors.New("timed out waiting for DHCP")
}

func findInterfaceToConfigure(interfaces map[string]net.Interface,
	machineInfo fm_proto.GetMachineInfoResponse, logger log.DebugLogger) (
	net.Interface, net.IP, *hyper_proto.Subnet, error) {
	networkEntries := configurator.GetNetworkEntries(machineInfo)
	hwAddrToInterface := make(map[string]net.Interface, len(interfaces))
	for _, iface := range interfaces {
		hwAddrToInterface[iface.HardwareAddr.String()] = iface
	}
	for _, networkEntry := range networkEntries {
		if len(networkEntry.HostIpAddress) < 1 {
			continue
		}
		iface, ok := hwAddrToInterface[networkEntry.HostMacAddress.String()]
		if !ok {
			continue
		}
		subnet := configurator.FindMatchingSubnet(machineInfo.Subnets,
			networkEntry.HostIpAddress)
		if subnet == nil {
			logger.Printf("no matching subnet for ip=%s\n",
				networkEntry.HostIpAddress)
			continue
		}
		return iface, networkEntry.HostIpAddress, subnet, nil
	}
	return net.Interface{}, nil, nil,
		errors.New("no network interfaces match injected configuration")
}

func getConfiguration(interfaces map[string]net.Interface,
	logger log.DebugLogger) (*fm_proto.GetMachineInfoResponse, error) {
	var machineInfo fm_proto.GetMachineInfoResponse
	err := json.ReadFromFile(filepath.Join(*tftpDirectory, "config.json"),
		&machineInfo)
	if err == nil { // Configuration was injected.
		err := setupNetworkFromConfig(interfaces, machineInfo, logger)
		if err != nil {
			return nil, err
		}
		return &machineInfo, nil
	}
	if !os.IsNotExist(err) {
		return nil, err
	}
	if err := setupNetworkFromDhcp(interfaces, logger); err != nil {
		return nil, err
	}
	err = json.ReadFromFile(filepath.Join(*tftpDirectory, "config.json"),
		&machineInfo)
	if err != nil {
		return nil, err
	}
	return &machineInfo, nil
}

func injectRandomSeed(client *tftp.Client, logger log.DebugLogger) error {
	randomSeed := &bytes.Buffer{}
	if wt, err := client.Receive("random-seed", "octet"); err != nil {
		if strings.Contains(err.Error(), os.ErrNotExist.Error()) {
			return nil
		}
		return err
	} else if _, err := wt.WriteTo(randomSeed); err != nil {
		return err
	}
	if file, err := os.OpenFile("/dev/urandom", os.O_WRONLY, 0); err != nil {
		return err
	} else {
		defer file.Close()
		if nCopied, err := io.Copy(file, randomSeed); err != nil {
			return err
		} else {
			logger.Printf("copied %d bytes of random data\n", nCopied)
		}
	}
	return nil
}

func loadTftpFiles(tftpServer net.IP, logger log.DebugLogger) error {
	client, err := tftp.NewClient(tftpServer.String() + ":69")
	if err != nil {
		return err
	}
	if err := os.MkdirAll(*tftpDirectory, fsutil.DirPerms); err != nil {
		return err
	}
	for _, name := range tftpFiles {
		logger.Debugf(1, "downloading: %s\n", name)
		if wt, err := client.Receive(name, "octet"); err != nil {
			return err
		} else {
			filename := filepath.Join(*tftpDirectory, name)
			if file, err := create(filename); err != nil {
				return err
			} else {
				defer file.Close()
				if _, err := wt.WriteTo(file); err != nil {
					return err
				}
			}
		}
	}
	return injectRandomSeed(client, logger)
}

func raiseInterfaces(interfaces map[string]net.Interface,
	logger log.DebugLogger) error {
	for name := range interfaces {
		if err := run("ifconfig", "", logger, name, "up"); err != nil {
			return err
		}
	}
	return nil
}

func setupNetwork(ifName string, ipAddr net.IP, subnet *hyper_proto.Subnet,
	logger log.DebugLogger) error {
	err := run("ifconfig", "", logger, ifName, ipAddr.String(), "netmask",
		subnet.IpMask.String(), "up")
	if err != nil {
		return err
	}
	err = run("route", "", logger, "add", "default", "gw",
		subnet.IpGateway.String())
	if err != nil {
		e := run("route", "", logger, "del", "default", "gw",
			subnet.IpGateway.String())
		if e != nil {
			return err
		}
		err = run("route", "", logger, "add", "default", "gw",
			subnet.IpGateway.String())
		if err != nil {
			return err
		}
	}
	if !*dryRun {
		if err := configurator.WriteResolvConf("", subnet); err != nil {
			return err
		}
	}
	return nil
}

func setupNetworkFromConfig(interfaces map[string]net.Interface,
	machineInfo fm_proto.GetMachineInfoResponse, logger log.DebugLogger) error {
	iface, ipAddr, subnet, err := findInterfaceToConfigure(interfaces,
		machineInfo, logger)
	if err != nil {
		return err
	}
	return setupNetwork(iface.Name, ipAddr, subnet, logger)
}

func setupNetworkFromDhcp(interfaces map[string]net.Interface,
	logger log.DebugLogger) error {
	ifName, packet, err := dhcpRequest(interfaces, logger)
	if err != nil {
		return err
	}
	ipAddr := packet.YIAddr()
	options := packet.ParseOptions()
	subnet := hyper_proto.Subnet{
		IpGateway: net.IP(options[dhcp4.OptionRouter]),
		IpMask:    net.IP(options[dhcp4.OptionSubnetMask]),
	}
	dnsServersBuffer := options[dhcp4.OptionDomainNameServer]
	for len(dnsServersBuffer) > 0 {
		if len(dnsServersBuffer) >= 4 {
			subnet.DomainNameServers = append(subnet.DomainNameServers,
				net.IP(dnsServersBuffer[:4]))
			dnsServersBuffer = dnsServersBuffer[4:]
		} else {
			return errors.New("truncated DNS server address")
		}
	}
	if err := setupNetwork(ifName, ipAddr, &subnet, logger); err != nil {
		return err
	}
	tftpServer := packet.SIAddr()
	if tftpServer.Equal(zeroIP) {
		tftpServer = net.IP(options[dhcp4.OptionTFTPServerName])
		if tftpServer.Equal(zeroIP) {
			return errors.New("no TFTP server given")
		}
		logger.Printf("tftpServer from OptionTFTPServerName: %s\n", tftpServer)
	} else {
		logger.Printf("tftpServer from SIAddr: %s\n", tftpServer)
	}
	return loadTftpFiles(tftpServer, logger)
}
<|endoftext|>