<commit_before>package vxlan

import (
	"encoding/json"
	"fmt"
	"net"
	"sync"
	"time"

	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/vishvananda/netlink"

	"github.com/coreos/flannel/backend"
	"github.com/coreos/flannel/subnet"
	"github.com/coreos/flannel/pkg/ip"
	"github.com/coreos/flannel/pkg/task"
)

const (
	defaultVNI = 1
)

type VXLANBackend struct {
	sm     *subnet.SubnetManager
	rawCfg json.RawMessage
	cfg    struct {
		VNI  int
		Port int
	}
	dev    *vxlanDevice
	stop   chan bool
	wg     sync.WaitGroup
	rts    routes
}

func New(sm *subnet.SubnetManager, config json.RawMessage) backend.Backend {
	vb := &VXLANBackend{
		sm:     sm,
		rawCfg: config,
		stop:   make(chan bool),
	}
	vb.cfg.VNI = defaultVNI

	return vb
}

func newSubnetAttrs(pubIP net.IP, mac net.HardwareAddr) (*subnet.LeaseAttrs, error) {
	data, err := json.Marshal(&vxlanLeaseAttrs{hardwareAddr(mac)})
	if err != nil {
		return nil, err
	}

	return &subnet.LeaseAttrs{
		PublicIP:    ip.FromIP(pubIP),
		BackendType: "vxlan",
		BackendData: json.RawMessage(data),
	}, nil
}

func (vb *VXLANBackend) Init(extIface *net.Interface, extIP net.IP) (*backend.SubnetDef, error) {
	// Parse our configuration
	if len(vb.rawCfg) > 0 {
		if err := json.Unmarshal(vb.rawCfg, &vb.cfg); err != nil {
			return nil, fmt.Errorf("error decoding UDP backend config: %v", err)
		}
	}

	devAttrs := vxlanDeviceAttrs{
		vni:       uint32(vb.cfg.VNI),
		name:      fmt.Sprintf("flannel.%v", vb.cfg.VNI),
		vtepIndex: extIface.Index,
		vtepAddr:  extIP,
		vtepPort:  vb.cfg.Port,
	}

	var err error
	for {
		vb.dev, err = newVXLANDevice(&devAttrs)
		if err == nil {
			break
		} else {
			log.Warning("Failed to create VXLAN interface, retrying...")

			// wait 1 sec before retrying
			time.Sleep(1*time.Second)
		}
	}

	sa, err := newSubnetAttrs(extIP, vb.dev.MACAddr())
	if err != nil {
		return nil, err
	}

	sn, err := vb.sm.AcquireLease(sa, vb.stop)
	if err != nil {
		if err == task.ErrCanceled {
			return nil, err
		} else {
			return nil, fmt.Errorf("failed to acquire lease: %v", err)
		}
	}

	// vxlan's subnet is that of the whole overlay network (e.g. /16)
	// and not that of the individual host (e.g. /24)
	vxlanNet := ip.IP4Net{
		IP:        sn.IP,
		PrefixLen: vb.sm.GetConfig().Network.PrefixLen,
	}
	if err = vb.dev.Configure(vxlanNet); err != nil {
		return nil, err
	}

	return &backend.SubnetDef{
		Net: sn,
		MTU: vb.dev.MTU(),
	}, nil
}

func (vb *VXLANBackend) Run() {
	vb.wg.Add(1)
	go func() {
		vb.sm.LeaseRenewer(vb.stop)
		vb.wg.Done()
	}()

	log.Info("Watching for L2/L3 misses")
	misses := make(chan *netlink.Neigh, 100)
	// Unfortunately MonitorMisses does not take a cancel channel
	// as there's no wait to interrupt netlink socket recv
	go vb.dev.MonitorMisses(misses)

	log.Info("Watching for new subnet leases")
	evts := make(chan subnet.EventBatch)
	vb.wg.Add(1)
	go func() {
		vb.sm.WatchLeases(evts, vb.stop)
		vb.wg.Done()
	}()

	defer vb.wg.Wait()

	for {
		select {
		case miss := <-misses:
			vb.handleMiss(miss)

		case evtBatch := <-evts:
			vb.handleSubnetEvents(evtBatch)

		case <-vb.stop:
			return
		}
	}
}

func (vb *VXLANBackend) Stop() {
	close(vb.stop)
}

func (vb *VXLANBackend) Name() string {
	return "VXLAN"
}

// So we can make it JSON (un)marshalable
type hardwareAddr net.HardwareAddr

func (hw hardwareAddr) MarshalJSON() ([]byte, error) {
	return []byte(fmt.Sprintf("%q", net.HardwareAddr(hw))), nil
}

func (hw *hardwareAddr) UnmarshalJSON(b []byte) error {
	if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' {
		return fmt.Errorf("error parsing hardware addr")
	}

	b = b[1:len(b)-1]

	mac, err := net.ParseMAC(string(b))
	if err != nil {
		return err
	}

	*hw = hardwareAddr(mac)
	return nil
}

type vxlanLeaseAttrs struct {
	VtepMAC hardwareAddr
}

func (vb *VXLANBackend) handleSubnetEvents(batch subnet.EventBatch) {
	for _, evt := range batch {
		switch evt.Type {
		case subnet.SubnetAdded:
			log.Info("Subnet added: ", evt.Lease.Network)

			if evt.Lease.Attrs.BackendType != "vxlan" {
				log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType)
				continue
			}

			var attrs vxlanLeaseAttrs
			if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &attrs); err != nil {
				log.Error("Error decoding subnet lease JSON: ", err)
				continue
			}

			vb.rts.set(evt.Lease.Network, evt.Lease.Attrs.PublicIP.ToIP(), net.HardwareAddr(attrs.VtepMAC))

		case subnet.SubnetRemoved:
			log.Info("Subnet removed: ", evt.Lease.Network)

			vb.rts.remove(evt.Lease.Network)

			if evt.Lease.Attrs.BackendType != "vxlan" {
				log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType)
				continue
			}

			var attrs vxlanLeaseAttrs
			if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &attrs); err != nil {
				log.Error("Error decoding subnet lease JSON: ", err)
				continue
			}

			if len(attrs.VtepMAC) > 0 {
				vb.dev.DelL2(net.HardwareAddr(attrs.VtepMAC), evt.Lease.Attrs.PublicIP.ToIP())
			}

		default:
			log.Error("Internal error: unknown event type: ", int(evt.Type))
		}
	}
}

func (vb *VXLANBackend) handleMiss(miss *netlink.Neigh) {
	switch {
	case len(miss.IP) == 0 && len(miss.HardwareAddr) == 0:
		log.Info("Ignoring nil miss")

	case len(miss.IP) == 0:
		vb.handleL2Miss(miss)

	case len(miss.HardwareAddr) == 0:
		vb.handleL3Miss(miss)

	default:
		log.Infof("Ignoring not a miss: %v, %v", miss.HardwareAddr, miss.IP)
	}
}

func (vb *VXLANBackend) handleL2Miss(miss *netlink.Neigh) {
	log.Infof("L2 miss: %v", miss.HardwareAddr)

	rt := vb.rts.findByVtepMAC(miss.HardwareAddr)
	if rt == nil {
		log.Infof("Route for %v not found", miss.HardwareAddr)
		return
	}

	if err := vb.dev.AddL2(miss.HardwareAddr, rt.vtepIP); err != nil {
		log.Errorf("AddL2 failed: %v", err)
	} else {
		log.Info("AddL2 succeeded")
	}
}

func (vb *VXLANBackend) handleL3Miss(miss *netlink.Neigh) {
	log.Infof("L3 miss: %v", miss.IP)

	rt := vb.rts.findByNetwork(ip.FromIP(miss.IP))
	if rt == nil {
		log.Infof("Route for %v not found", miss.IP)
		return
	}

	if err := vb.dev.AddL3(miss.IP, rt.vtepMAC); err != nil {
		log.Errorf("AddL3 failed: %v", err)
	} else {
		log.Info("AddL3 succeeded")
	}
}
<commit_msg>Clarify the logging when VXLAN interface creation fails<commit_after>package vxlan

import (
	"encoding/json"
	"fmt"
	"net"
	"sync"
	"time"

	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/vishvananda/netlink"

	"github.com/coreos/flannel/backend"
	"github.com/coreos/flannel/subnet"
	"github.com/coreos/flannel/pkg/ip"
	"github.com/coreos/flannel/pkg/task"
)

const (
	defaultVNI = 1
)

type VXLANBackend struct {
	sm     *subnet.SubnetManager
	rawCfg json.RawMessage
	cfg    struct {
		VNI  int
		Port int
	}
	dev    *vxlanDevice
	stop   chan bool
	wg     sync.WaitGroup
	rts    routes
}

func New(sm *subnet.SubnetManager, config json.RawMessage) backend.Backend {
	vb := &VXLANBackend{
		sm:     sm,
		rawCfg: config,
		stop:   make(chan bool),
	}
	vb.cfg.VNI = defaultVNI

	return vb
}

func newSubnetAttrs(pubIP net.IP, mac net.HardwareAddr) (*subnet.LeaseAttrs, error) {
	data, err := json.Marshal(&vxlanLeaseAttrs{hardwareAddr(mac)})
	if err != nil {
		return nil, err
	}

	return &subnet.LeaseAttrs{
		PublicIP:    ip.FromIP(pubIP),
		BackendType: "vxlan",
		BackendData: json.RawMessage(data),
	}, nil
}

func (vb *VXLANBackend) Init(extIface *net.Interface, extIP net.IP) (*backend.SubnetDef, error) {
	// Parse our configuration
	if len(vb.rawCfg) > 0 {
		if err := json.Unmarshal(vb.rawCfg, &vb.cfg); err != nil {
			return nil, fmt.Errorf("error decoding UDP backend config: %v", err)
		}
	}

	devAttrs := vxlanDeviceAttrs{
		vni:       uint32(vb.cfg.VNI),
		name:      fmt.Sprintf("flannel.%v", vb.cfg.VNI),
		vtepIndex: extIface.Index,
		vtepAddr:  extIP,
		vtepPort:  vb.cfg.Port,
	}

	var err error
	for {
		vb.dev, err = newVXLANDevice(&devAttrs)
		if err == nil {
			break
		} else {
			log.Error("VXLAN init: ", err)
			log.Info("Retrying in 1 second...")

			// wait 1 sec before retrying
			time.Sleep(1*time.Second)
		}
	}

	sa, err := newSubnetAttrs(extIP, vb.dev.MACAddr())
	if err != nil {
		return nil, err
	}

	sn, err := vb.sm.AcquireLease(sa, vb.stop)
	if err != nil {
		if err == task.ErrCanceled {
			return nil, err
		} else {
			return nil, fmt.Errorf("failed to acquire lease: %v", err)
		}
	}

	// vxlan's subnet is that of the whole overlay network (e.g. /16)
	// and not that of the individual host (e.g. /24)
	vxlanNet := ip.IP4Net{
		IP:        sn.IP,
		PrefixLen: vb.sm.GetConfig().Network.PrefixLen,
	}
	if err = vb.dev.Configure(vxlanNet); err != nil {
		return nil, err
	}

	return &backend.SubnetDef{
		Net: sn,
		MTU: vb.dev.MTU(),
	}, nil
}

func (vb *VXLANBackend) Run() {
	vb.wg.Add(1)
	go func() {
		vb.sm.LeaseRenewer(vb.stop)
		vb.wg.Done()
	}()

	log.Info("Watching for L2/L3 misses")
	misses := make(chan *netlink.Neigh, 100)
	// Unfortunately MonitorMisses does not take a cancel channel
	// as there's no wait to interrupt netlink socket recv
	go vb.dev.MonitorMisses(misses)

	log.Info("Watching for new subnet leases")
	evts := make(chan subnet.EventBatch)
	vb.wg.Add(1)
	go func() {
		vb.sm.WatchLeases(evts, vb.stop)
		vb.wg.Done()
	}()

	defer vb.wg.Wait()

	for {
		select {
		case miss := <-misses:
			vb.handleMiss(miss)

		case evtBatch := <-evts:
			vb.handleSubnetEvents(evtBatch)

		case <-vb.stop:
			return
		}
	}
}

func (vb *VXLANBackend) Stop() {
	close(vb.stop)
}

func (vb *VXLANBackend) Name() string {
	return "VXLAN"
}

// So we can make it JSON (un)marshalable
type hardwareAddr net.HardwareAddr

func (hw hardwareAddr) MarshalJSON() ([]byte, error) {
	return []byte(fmt.Sprintf("%q", net.HardwareAddr(hw))), nil
}

func (hw *hardwareAddr) UnmarshalJSON(b []byte) error {
	if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' {
		return fmt.Errorf("error parsing hardware addr")
	}

	b = b[1:len(b)-1]

	mac, err := net.ParseMAC(string(b))
	if err != nil {
		return err
	}

	*hw = hardwareAddr(mac)
	return nil
}

type vxlanLeaseAttrs struct {
	VtepMAC hardwareAddr
}

func (vb *VXLANBackend) handleSubnetEvents(batch subnet.EventBatch) {
	for _, evt := range batch {
		switch evt.Type {
		case subnet.SubnetAdded:
			log.Info("Subnet added: ", evt.Lease.Network)

			if evt.Lease.Attrs.BackendType != "vxlan" {
				log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType)
				continue
			}

			var attrs vxlanLeaseAttrs
			if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &attrs); err != nil {
				log.Error("Error decoding subnet lease JSON: ", err)
				continue
			}

			vb.rts.set(evt.Lease.Network, evt.Lease.Attrs.PublicIP.ToIP(), net.HardwareAddr(attrs.VtepMAC))

		case subnet.SubnetRemoved:
			log.Info("Subnet removed: ", evt.Lease.Network)

			vb.rts.remove(evt.Lease.Network)

			if evt.Lease.Attrs.BackendType != "vxlan" {
				log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType)
				continue
			}

			var attrs vxlanLeaseAttrs
			if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &attrs); err != nil {
				log.Error("Error decoding subnet lease JSON: ", err)
				continue
			}

			if len(attrs.VtepMAC) > 0 {
				vb.dev.DelL2(net.HardwareAddr(attrs.VtepMAC), evt.Lease.Attrs.PublicIP.ToIP())
			}

		default:
			log.Error("Internal error: unknown event type: ", int(evt.Type))
		}
	}
}

func (vb *VXLANBackend) handleMiss(miss *netlink.Neigh) {
	switch {
	case len(miss.IP) == 0 && len(miss.HardwareAddr) == 0:
		log.Info("Ignoring nil miss")

	case len(miss.IP) == 0:
		vb.handleL2Miss(miss)

	case len(miss.HardwareAddr) == 0:
		vb.handleL3Miss(miss)

	default:
		log.Infof("Ignoring not a miss: %v, %v", miss.HardwareAddr, miss.IP)
	}
}

func (vb *VXLANBackend) handleL2Miss(miss *netlink.Neigh) {
	log.Infof("L2 miss: %v", miss.HardwareAddr)

	rt := vb.rts.findByVtepMAC(miss.HardwareAddr)
	if rt == nil {
		log.Infof("Route for %v not found", miss.HardwareAddr)
		return
	}

	if err := vb.dev.AddL2(miss.HardwareAddr, rt.vtepIP); err != nil {
		log.Errorf("AddL2 failed: %v", err)
	} else {
		log.Info("AddL2 succeeded")
	}
}

func (vb *VXLANBackend) handleL3Miss(miss *netlink.Neigh) {
	log.Infof("L3 miss: %v", miss.IP)

	rt := vb.rts.findByNetwork(ip.FromIP(miss.IP))
	if rt == nil {
		log.Infof("Route for %v not found", miss.IP)
		return
	}

	if err := vb.dev.AddL3(miss.IP, rt.vtepMAC); err != nil {
		log.Errorf("AddL3 failed: %v", err)
	} else {
		log.Info("AddL3 succeeded")
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vxlan

import (
	"encoding/json"
	"fmt"
	"net"
	"sync"
	"time"

	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/vishvananda/netlink"
	"github.com/coreos/flannel/Godeps/_workspace/src/golang.org/x/net/context"
	"github.com/coreos/flannel/backend"
	"github.com/coreos/flannel/pkg/ip"
	"github.com/coreos/flannel/subnet"
)

const (
	defaultVNI = 1
)

type VXLANBackend struct {
	sm      subnet.Manager
	network string
	config  *subnet.Config
	cfg     struct {
		VNI  int
		Port int
	}
	lease  *subnet.Lease
	dev    *vxlanDevice
	ctx    context.Context
	cancel context.CancelFunc
	wg     sync.WaitGroup
	rts    routes
}

func New(sm subnet.Manager, network string, config *subnet.Config) backend.Backend {
	ctx, cancel := context.WithCancel(context.Background())

	vb := &VXLANBackend{
		sm:      sm,
		network: network,
		config:  config,
		ctx:     ctx,
		cancel:  cancel,
	}
	vb.cfg.VNI = defaultVNI

	return vb
}

func newSubnetAttrs(pubIP net.IP, mac net.HardwareAddr) (*subnet.LeaseAttrs, error) {
	data, err := json.Marshal(&vxlanLeaseAttrs{hardwareAddr(mac)})
	if err != nil {
		return nil, err
	}

	return &subnet.LeaseAttrs{
		PublicIP:    ip.FromIP(pubIP),
		BackendType: "vxlan",
		BackendData: json.RawMessage(data),
	}, nil
}

func (vb *VXLANBackend) Init(extIface *net.Interface, extIP net.IP) (*backend.SubnetDef, error) {
	// Parse our configuration
	if len(vb.config.Backend) > 0 {
		if err := json.Unmarshal(vb.config.Backend, &vb.cfg); err != nil {
			return nil, fmt.Errorf("error decoding UDP backend config: %v", err)
		}
	}

	devAttrs := vxlanDeviceAttrs{
		vni:       uint32(vb.cfg.VNI),
		name:      fmt.Sprintf("flannel.%v", vb.cfg.VNI),
		vtepIndex: extIface.Index,
		vtepAddr:  extIP,
		vtepPort:  vb.cfg.Port,
	}

	var err error
	for {
		vb.dev, err = newVXLANDevice(&devAttrs)
		if err == nil {
			break
		} else {
			log.Error("VXLAN init: ", err)
			log.Info("Retrying in 1 second...")

			// wait 1 sec before retrying
			time.Sleep(1 * time.Second)
		}
	}

	sa, err := newSubnetAttrs(extIP, vb.dev.MACAddr())
	if err != nil {
		return nil, err
	}

	l, err := vb.sm.AcquireLease(vb.ctx, vb.network, sa)
	switch err {
	case nil:
		vb.lease = l

	case context.Canceled, context.DeadlineExceeded:
		return nil, err

	default:
		return nil, fmt.Errorf("failed to acquire lease: %v", err)
	}

	// vxlan's subnet is that of the whole overlay network (e.g. /16)
	// and not that of the individual host (e.g. /24)
	vxlanNet := ip.IP4Net{
		IP:        l.Subnet.IP,
		PrefixLen: vb.config.Network.PrefixLen,
	}
	if err = vb.dev.Configure(vxlanNet); err != nil {
		return nil, err
	}

	return &backend.SubnetDef{
		Net: l.Subnet,
		MTU: vb.dev.MTU(),
	}, nil
}

func (vb *VXLANBackend) Run() {
	vb.wg.Add(1)
	go func() {
		subnet.LeaseRenewer(vb.ctx, vb.sm, vb.network, vb.lease)
		log.Info("LeaseRenewer exited")
		vb.wg.Done()
	}()

	log.Info("Watching for L2/L3 misses")
	misses := make(chan *netlink.Neigh, 100)
	// Unfortunately MonitorMisses does not take a cancel channel
	// as there's no wait to interrupt netlink socket recv
	go vb.dev.MonitorMisses(misses)

	log.Info("Watching for new subnet leases")
	evts := make(chan []subnet.Event)
	vb.wg.Add(1)
	go func() {
		subnet.WatchLeases(vb.ctx, vb.sm, vb.network, evts)
		log.Info("WatchLeases exited")
		vb.wg.Done()
	}()

	defer vb.wg.Wait()

	for {
		select {
		case miss := <-misses:
			vb.handleMiss(miss)

		case evtBatch := <-evts:
			vb.handleSubnetEvents(evtBatch)

		case <-vb.ctx.Done():
			return
		}
	}
}

func (vb *VXLANBackend) Stop() {
	vb.cancel()
}

func (vb *VXLANBackend) Name() string {
	return "VXLAN"
}

// So we can make it JSON (un)marshalable
type hardwareAddr net.HardwareAddr

func (hw hardwareAddr) MarshalJSON() ([]byte, error) {
	return []byte(fmt.Sprintf("%q", net.HardwareAddr(hw))), nil
}

func (hw *hardwareAddr) UnmarshalJSON(b []byte) error {
	if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' {
		return fmt.Errorf("error parsing hardware addr")
	}

	b = b[1 : len(b)-1]

	mac, err := net.ParseMAC(string(b))
	if err != nil {
		return err
	}

	*hw = hardwareAddr(mac)
	return nil
}

type vxlanLeaseAttrs struct {
	VtepMAC hardwareAddr
}

func (vb *VXLANBackend) handleSubnetEvents(batch []subnet.Event) {
	for _, evt := range batch {
		switch evt.Type {
		case subnet.SubnetAdded:
			log.Info("Subnet added: ", evt.Lease.Subnet)

			if evt.Lease.Attrs.BackendType != "vxlan" {
				log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType)
				continue
			}

			var attrs vxlanLeaseAttrs
			if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &attrs); err != nil {
				log.Error("Error decoding subnet lease JSON: ", err)
				continue
			}

			vb.rts.set(evt.Lease.Subnet, evt.Lease.Attrs.PublicIP.ToIP(), net.HardwareAddr(attrs.VtepMAC))

		case subnet.SubnetRemoved:
			log.Info("Subnet removed: ", evt.Lease.Subnet)

			vb.rts.remove(evt.Lease.Subnet)

			if evt.Lease.Attrs.BackendType != "vxlan" {
				log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType)
				continue
			}

			var attrs vxlanLeaseAttrs
			if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &attrs); err != nil {
				log.Error("Error decoding subnet lease JSON: ", err)
				continue
			}

			if len(attrs.VtepMAC) > 0 {
				vb.dev.DelL2(net.HardwareAddr(attrs.VtepMAC), evt.Lease.Attrs.PublicIP.ToIP())
			}

		default:
			log.Error("Internal error: unknown event type: ", int(evt.Type))
		}
	}
}

func (vb *VXLANBackend) handleMiss(miss *netlink.Neigh) {
	switch {
	case len(miss.IP) == 0 && len(miss.HardwareAddr) == 0:
		log.Info("Ignoring nil miss")

	case len(miss.IP) == 0:
		vb.handleL2Miss(miss)

	case len(miss.HardwareAddr) == 0:
		vb.handleL3Miss(miss)

	default:
		log.Infof("Ignoring not a miss: %v, %v", miss.HardwareAddr, miss.IP)
	}
}

func (vb *VXLANBackend) handleL2Miss(miss *netlink.Neigh) {
	log.Infof("L2 miss: %v", miss.HardwareAddr)

	rt := vb.rts.findByVtepMAC(miss.HardwareAddr)
	if rt == nil {
		log.Infof("Route for %v not found", miss.HardwareAddr)
		return
	}

	if err := vb.dev.AddL2(miss.HardwareAddr, rt.vtepIP); err != nil {
		log.Errorf("AddL2 failed: %v", err)
	} else {
		log.Info("AddL2 succeeded")
	}
}

func (vb *VXLANBackend) handleL3Miss(miss *netlink.Neigh) {
	log.Infof("L3 miss: %v", miss.IP)

	rt := vb.rts.findByNetwork(ip.FromIP(miss.IP))
	if rt == nil {
		log.Infof("Route for %v not found", miss.IP)
		return
	}

	if err := vb.dev.AddL3(miss.IP, rt.vtepMAC); err != nil {
		log.Errorf("AddL3 failed: %v", err)
	} else {
		log.Info("AddL3 succeeded")
	}
}
<commit_msg>fix type where error claimed it was the UDP backend<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vxlan

import (
	"encoding/json"
	"fmt"
	"net"
	"sync"
	"time"

	log "github.com/coreos/flannel/Godeps/_workspace/src/github.com/golang/glog"
	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/vishvananda/netlink"
	"github.com/coreos/flannel/Godeps/_workspace/src/golang.org/x/net/context"
	"github.com/coreos/flannel/backend"
	"github.com/coreos/flannel/pkg/ip"
	"github.com/coreos/flannel/subnet"
)

const (
	defaultVNI = 1
)

type VXLANBackend struct {
	sm      subnet.Manager
	network string
	config  *subnet.Config
	cfg     struct {
		VNI  int
		Port int
	}
	lease  *subnet.Lease
	dev    *vxlanDevice
	ctx    context.Context
	cancel context.CancelFunc
	wg     sync.WaitGroup
	rts    routes
}

func New(sm subnet.Manager, network string, config *subnet.Config) backend.Backend {
	ctx, cancel := context.WithCancel(context.Background())

	vb := &VXLANBackend{
		sm:      sm,
		network: network,
		config:  config,
		ctx:     ctx,
		cancel:  cancel,
	}
	vb.cfg.VNI = defaultVNI

	return vb
}

func newSubnetAttrs(pubIP net.IP, mac net.HardwareAddr) (*subnet.LeaseAttrs, error) {
	data, err := json.Marshal(&vxlanLeaseAttrs{hardwareAddr(mac)})
	if err != nil {
		return nil, err
	}

	return &subnet.LeaseAttrs{
		PublicIP:    ip.FromIP(pubIP),
		BackendType: "vxlan",
		BackendData: json.RawMessage(data),
	}, nil
}

func (vb *VXLANBackend) Init(extIface *net.Interface, extIP net.IP) (*backend.SubnetDef, error) {
	// Parse our configuration
	if len(vb.config.Backend) > 0 {
		if err := json.Unmarshal(vb.config.Backend, &vb.cfg); err != nil {
			return nil, fmt.Errorf("error decoding VXLAN backend config: %v", err)
		}
	}

	devAttrs := vxlanDeviceAttrs{
		vni:       uint32(vb.cfg.VNI),
		name:      fmt.Sprintf("flannel.%v", vb.cfg.VNI),
		vtepIndex: extIface.Index,
		vtepAddr:  extIP,
		vtepPort:  vb.cfg.Port,
	}

	var err error
	for {
		vb.dev, err = newVXLANDevice(&devAttrs)
		if err == nil {
			break
		} else {
			log.Error("VXLAN init: ", err)
			log.Info("Retrying in 1 second...")

			// wait 1 sec before retrying
			time.Sleep(1 * time.Second)
		}
	}

	sa, err := newSubnetAttrs(extIP, vb.dev.MACAddr())
	if err != nil {
		return nil, err
	}

	l, err := vb.sm.AcquireLease(vb.ctx, vb.network, sa)
	switch err {
	case nil:
		vb.lease = l

	case context.Canceled, context.DeadlineExceeded:
		return nil, err

	default:
		return nil, fmt.Errorf("failed to acquire lease: %v", err)
	}

	// vxlan's subnet is that of the whole overlay network (e.g. /16)
	// and not that of the individual host (e.g. /24)
	vxlanNet := ip.IP4Net{
		IP:        l.Subnet.IP,
		PrefixLen: vb.config.Network.PrefixLen,
	}
	if err = vb.dev.Configure(vxlanNet); err != nil {
		return nil, err
	}

	return &backend.SubnetDef{
		Net: l.Subnet,
		MTU: vb.dev.MTU(),
	}, nil
}

func (vb *VXLANBackend) Run() {
	vb.wg.Add(1)
	go func() {
		subnet.LeaseRenewer(vb.ctx, vb.sm, vb.network, vb.lease)
		log.Info("LeaseRenewer exited")
		vb.wg.Done()
	}()

	log.Info("Watching for L2/L3 misses")
	misses := make(chan *netlink.Neigh, 100)
	// Unfortunately MonitorMisses does not take a cancel channel
	// as there's no wait to interrupt netlink socket recv
	go vb.dev.MonitorMisses(misses)

	log.Info("Watching for new subnet leases")
	evts := make(chan []subnet.Event)
	vb.wg.Add(1)
	go func() {
		subnet.WatchLeases(vb.ctx, vb.sm, vb.network, evts)
		log.Info("WatchLeases exited")
		vb.wg.Done()
	}()

	defer vb.wg.Wait()

	for {
		select {
		case miss := <-misses:
			vb.handleMiss(miss)

		case evtBatch := <-evts:
			vb.handleSubnetEvents(evtBatch)

		case <-vb.ctx.Done():
			return
		}
	}
}

func (vb *VXLANBackend) Stop() {
	vb.cancel()
}

func (vb *VXLANBackend) Name() string {
	return "VXLAN"
}

// So we can make it JSON (un)marshalable
type hardwareAddr net.HardwareAddr

func (hw hardwareAddr) MarshalJSON() ([]byte, error) {
	return []byte(fmt.Sprintf("%q", net.HardwareAddr(hw))), nil
}

func (hw *hardwareAddr) UnmarshalJSON(b []byte) error {
	if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' {
		return fmt.Errorf("error parsing hardware addr")
	}

	b = b[1 : len(b)-1]

	mac, err := net.ParseMAC(string(b))
	if err != nil {
		return err
	}

	*hw = hardwareAddr(mac)
	return nil
}

type vxlanLeaseAttrs struct {
	VtepMAC hardwareAddr
}

func (vb *VXLANBackend) handleSubnetEvents(batch []subnet.Event) {
	for _, evt := range batch {
		switch evt.Type {
		case subnet.SubnetAdded:
			log.Info("Subnet added: ", evt.Lease.Subnet)

			if evt.Lease.Attrs.BackendType != "vxlan" {
				log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType)
				continue
			}

			var attrs vxlanLeaseAttrs
			if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &attrs); err != nil {
				log.Error("Error decoding subnet lease JSON: ", err)
				continue
			}

			vb.rts.set(evt.Lease.Subnet, evt.Lease.Attrs.PublicIP.ToIP(), net.HardwareAddr(attrs.VtepMAC))

		case subnet.SubnetRemoved:
			log.Info("Subnet removed: ", evt.Lease.Subnet)

			vb.rts.remove(evt.Lease.Subnet)

			if evt.Lease.Attrs.BackendType != "vxlan" {
				log.Warningf("Ignoring non-vxlan subnet: type=%v", evt.Lease.Attrs.BackendType)
				continue
			}

			var attrs vxlanLeaseAttrs
			if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &attrs); err != nil {
				log.Error("Error decoding subnet lease JSON: ", err)
				continue
			}

			if len(attrs.VtepMAC) > 0 {
				vb.dev.DelL2(net.HardwareAddr(attrs.VtepMAC), evt.Lease.Attrs.PublicIP.ToIP())
			}

		default:
			log.Error("Internal error: unknown event type: ", int(evt.Type))
		}
	}
}

func (vb *VXLANBackend) handleMiss(miss *netlink.Neigh) {
	switch {
	case len(miss.IP) == 0 && len(miss.HardwareAddr) == 0:
		log.Info("Ignoring nil miss")

	case len(miss.IP) == 0:
		vb.handleL2Miss(miss)

	case len(miss.HardwareAddr) == 0:
		vb.handleL3Miss(miss)

	default:
		log.Infof("Ignoring not a miss: %v, %v", miss.HardwareAddr, miss.IP)
	}
}

func (vb *VXLANBackend) handleL2Miss(miss *netlink.Neigh) {
	log.Infof("L2 miss: %v", miss.HardwareAddr)

	rt := vb.rts.findByVtepMAC(miss.HardwareAddr)
	if rt == nil {
		log.Infof("Route for %v not found", miss.HardwareAddr)
		return
	}

	if err := vb.dev.AddL2(miss.HardwareAddr, rt.vtepIP); err != nil {
		log.Errorf("AddL2 failed: %v", err)
	} else {
		log.Info("AddL2 succeeded")
	}
}

func (vb *VXLANBackend) handleL3Miss(miss *netlink.Neigh) {
	log.Infof("L3 miss: %v", miss.IP)

	rt := vb.rts.findByNetwork(ip.FromIP(miss.IP))
	if rt == nil {
		log.Infof("Route for %v not found", miss.IP)
		return
	}

	if err := vb.dev.AddL3(miss.IP, rt.vtepMAC); err != nil {
		log.Errorf("AddL3 failed: %v", err)
	} else {
		log.Info("AddL3 succeeded")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"strings"
	"text/template"
)

func init() {
	Generators["yash"] = GenerateYashCompletion
}

func escapeYashString(s string) string {
	return strings.Replace(s, `'`, `'"'"'`, -1)
}

func toYashOpt(f *Flag) string {
	var opts []string
	for _, short := range f.Short {
		opt := escapeYashString(short)
		if f.Arg != "" {
			opt += ":"
		}
		opts = append(opts, opt)
	}
	for _, long := range f.Long {
		opt := "--" + escapeYashString(long)
		if f.Arg != "" {
			opt += ":"
		}
		opts = append(opts, opt)
	}

	description := escapeYashString(f.Description)

	return fmt.Sprintf("'%s; %s'",
		strings.Join(opts, " "), description)
}

type Yash struct {
	Name string
	Opts []string
}

func NewYash(c *Command) (y *Yash, err error) {
	y = new(Yash)
	y.Name = c.Name
	for _, flag := range c.Flags {
		y.Opts = append(y.Opts, toYashOpt(flag))
	}
	return y, nil
}

var YashCompletionTemplateText = `
function completion/{{.Name}} {
	typeset OPTIONS ARGOPT PREFIX
	OPTIONS=({{range .Opts}}
		{{.}}{{end}}
	)
	command -f completion//parseoptions -es
	case $ARGOPT in
	(-)
		command -f completion//completeoptions
		;;
	(*)
		complete -f
		;;
	esac
}
# vim: set ft=sh ts=8 sts=8 sw=8 noet:
`[1:]

func GenerateYashCompletion(w io.Writer, c *Command) error {
	tmpl, err := template.New("yash").Parse(YashCompletionTemplateText)
	if err != nil {
		return err
	}
	y, err := NewYash(c)
	if err != nil {
		return err
	}
	return tmpl.Execute(w, y)
}
<commit_msg>Remove too many indents for OPTIONS in yash-completion<commit_after>package main

import (
	"fmt"
	"io"
	"strings"
	"text/template"
)

func init() {
	Generators["yash"] = GenerateYashCompletion
}

func escapeYashString(s string) string {
	return strings.Replace(s, `'`, `'"'"'`, -1)
}

func toYashOpt(f *Flag) string {
	var opts []string
	for _, short := range f.Short {
		opt := escapeYashString(short)
		if f.Arg != "" {
			opt += ":"
		}
		opts = append(opts, opt)
	}
	for _, long := range f.Long {
		opt := "--" + escapeYashString(long)
		if f.Arg != "" {
			opt += ":"
		}
		opts = append(opts, opt)
	}

	description := escapeYashString(f.Description)

	return fmt.Sprintf("'%s; %s'",
		strings.Join(opts, " "), description)
}

type Yash struct {
	Name string
	Opts []string
}

func NewYash(c *Command) (y *Yash, err error) {
	y = new(Yash)
	y.Name = c.Name
	for _, flag := range c.Flags {
		y.Opts = append(y.Opts, toYashOpt(flag))
	}
	return y, nil
}

var YashCompletionTemplateText = `
function completion/{{.Name}} {
	typeset OPTIONS ARGOPT PREFIX
	OPTIONS=({{range .Opts}}
	{{.}}{{end}}
	)
	command -f completion//parseoptions -es
	case $ARGOPT in
	(-)
		command -f completion//completeoptions
		;;
	(*)
		complete -f
		;;
	esac
}
# vim: set ft=sh ts=8 sts=8 sw=8 noet:
`[1:]

func GenerateYashCompletion(w io.Writer, c *Command) error {
	tmpl, err := template.New("yash").Parse(YashCompletionTemplateText)
	if err != nil {
		return err
	}
	y, err := NewYash(c)
	if err != nil {
		return err
	}
	return tmpl.Execute(w, y)
}
<|endoftext|>


<commit_before>// Package ztex manages ZTEX USB FPGA modules.
package ztex

import (
	"errors"
	"fmt"
	"time"

	"github.com/google/gousb"
)

const (
	// VendorID is the ZTEX USB vendor ID (VID).
	VendorID = gousb.ID(0x221A)
	// ProductID is the standard ZTEX USB product ID (PID)
	ProductID = gousb.ID(0x0100)

	// ControlTimeout is the default timeout for control transfers.
	ControlTimeout = 1000 * time.Millisecond
)

// FirmwareVersion indicates which version of the ZTEX firmware is
// present.  Currently only FX1 and FX2 are supported.
type FirmwareVersion uint8

// String returns a human-readable description of a firmware version.
func (f FirmwareVersion) String() string {
	switch f {
	case 2:
		return "FX2"
	case 3:
		return "FX3"
	case 255:
		fallthrough
	default:
		return "Unknown"
	}
}

// Number returns the raw representation of a firmware version.
func (f FirmwareVersion) Number() uint8 { return uint8(f) }

// BoardSeries indicates an entire generation of boards.  Currently only
// Series 1 and Series 2 boards are supported.
type BoardSeries uint8

// String returns a human-readable description of a board series.
func (b BoardSeries) String() string {
	switch b {
	case 1:
		return "1"
	case 2:
		return "2"
	default:
		return "Unknown"
	}
}

// Number returns the raw representation of a board series.
func (b BoardSeries) Number() uint8 { return uint8(b) }

// BoardNumber indicates a board in a series.
type BoardNumber uint8

// String returns a human-readable description of a board number.
func (b BoardNumber) String() string {
	if b == 255 {
		return "Unknown"
	}
	return fmt.Sprintf("%d", uint8(b))
}

// Number returns the raw representation of a board number.
func (b BoardNumber) Number() uint8 { return uint8(b) }

// BoardVariant indicates a variation on a board series and number.
type BoardVariant []byte

// String returns a human-readable description of a board variant.
func (b BoardVariant) String() string { return string(b) }

// Number returns the raw representation of a board variant.
func (b BoardVariant) Bytes() []byte { return []byte(b) }

// BoardVersion comprises the series, number, and variant of a ZTEX USB
// FPGA module.  For example, a ZTEX USB FPGA module 2.14b device would be
// represented by
//
//   BoardVersion{
//     Series: BoardSeries(2),
//     Number: BoardNumber(14),
//     Variant: BoardVariant("b"),
//   }
//
// as a BoardVersion structure.
type BoardVersion struct {
	Series  BoardSeries
	Number  BoardNumber
	Variant BoardVariant
}

// String returns a human-readable representation of a board version.
func (b BoardVersion) String() string {
	return b.Series.String() + "." + b.Number.String() + b.Variant.String()
}

// Device represents a ZTEX USB FPGA module.
type Device struct {
	*gousb.Device

	Firmware FirmwareVersion
	Board    BoardVersion

	// Endpoint for fast FPGA configuration.  (0 = Unsupported)
	FastConfigurationEndpoint byte
	// Interface for fast FPGA configuration.  (0 = Unsupported)
	FastConfigurationInterface byte
	// Default interface major version number.  (0 = Not Available)
	DefaultInterfaceMajorVersion byte
	// Default interface minor version number.  (0 = Not Available)
	DefaultInterfaceMinorVersion byte
	// Output endpoint of default interface.  (255 = Not Available)
	DefaultInterfaceOutputEndpoint byte
	// Input endpoint of default interface.  (255 = Not Available)
	DefaultInterfaceInputEndpoint byte
}

// OpenDevice opens the
func OpenDevice(ctx *gousb.Context) (*Device, error) {
	d := &Device{}
	if dev, err := ctx.OpenDeviceWithVIDPID(VendorID, ProductID); err != nil {
		return nil, err
	} else if dev == nil {
		return nil, errors.New("no device")
	} else {
		d.Device = dev
		d.Device.ControlTimeout = ControlTimeout
	}

	buf := make([]byte, 128)

	// VR 0x3b: MAC EEPROM support: Read from MAC EEPROM
	if nbt, err := d.Control(0xc0, 0x3b, 0, 0, buf); err != nil {
		return nil, err
	} else if nbt == 128 {
		d.Firmware = FirmwareVersion(buf[3])
		d.Board = BoardVersion{
			Series:  BoardSeries(buf[4]),
			Number:  BoardNumber(buf[5]),
			Variant: BoardVariant(buf[6:8]),
		}
	} else {
		d.Firmware = FirmwareVersion(255)
		d.Board = BoardVersion{
			Series:  BoardSeries(255),
			Number:  BoardNumber(255),
			Variant: BoardVariant([]byte{255, 255}),
		}
	}

	// VR 0x33: High speed FPGA configuration support: Read Endpoint settings
	if nbt, err := d.Control(0xc0, 0x33, 0, 0, buf); err != nil {
		return nil, err
	} else if nbt == 2 {
		d.FastConfigurationEndpoint = buf[0]
		d.FastConfigurationInterface = buf[1]
	} else {
		return nil, errors.New("internal error")
	}

	// VR 0x64: Default firmware interface: Return Default Interface information
	if nbt, err := d.Control(0xc0, 0x64, 0, 0, buf); err != nil {
		return nil, err
	} else if nbt == 4 {
		d.DefaultInterfaceMajorVersion = buf[0]
		d.DefaultInterfaceMinorVersion = buf[3]
		d.DefaultInterfaceOutputEndpoint = buf[1] & 127
		d.DefaultInterfaceInputEndpoint = buf[2] | 128
	} else if nbt == 3 {
		d.DefaultInterfaceMajorVersion = buf[0]
		d.DefaultInterfaceOutputEndpoint = buf[1] & 127
		d.DefaultInterfaceInputEndpoint = buf[2] | 128
	} else {
		d.DefaultInterfaceOutputEndpoint = 255
		d.DefaultInterfaceInputEndpoint = 255
	}

	return d, nil
}
<commit_msg>Minor updates to human-readable device representation.<commit_after>// Package ztex manages ZTEX USB FPGA modules.
package ztex

import (
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/google/gousb"
)

const (
	// VendorID is the ZTEX USB vendor ID (VID).
	VendorID = gousb.ID(0x221A)
	// ProductID is the standard ZTEX USB product ID (PID)
	ProductID = gousb.ID(0x0100)

	// ControlTimeout is the default timeout for control transfers.
	ControlTimeout = 1000 * time.Millisecond
)

// FirmwareVersion indicates which version of the ZTEX firmware is
// present.  Currently only FX1 and FX2 are supported.
type FirmwareVersion uint8

// String returns a human-readable description of a firmware version.
func (f FirmwareVersion) String() string {
	switch f {
	case 2:
		return "FX2"
	case 3:
		return "FX3"
	case 255:
		fallthrough
	default:
		return "Unknown"
	}
}

// Number returns the raw representation of a firmware version.
func (f FirmwareVersion) Number() uint8 { return uint8(f) }

// BoardSeries indicates an entire generation of boards.  Currently only
// Series 1 and Series 2 boards are supported.
type BoardSeries uint8

// String returns a human-readable description of a board series.
func (b BoardSeries) String() string {
	switch b {
	case 1:
		return "1"
	case 2:
		return "2"
	default:
		return "Unknown"
	}
}

// Number returns the raw representation of a board series.
func (b BoardSeries) Number() uint8 { return uint8(b) }

// BoardNumber indicates a board in a series.
type BoardNumber uint8

// String returns a human-readable description of a board number.
func (b BoardNumber) String() string {
	if b == 255 {
		return "Unknown"
	}
	return fmt.Sprintf("%d", uint8(b))
}

// Number returns the raw representation of a board number.
func (b BoardNumber) Number() uint8 { return uint8(b) }

// BoardVariant indicates a variation on a board series and number.
type BoardVariant []byte

// String returns a human-readable description of a board variant.
func (b BoardVariant) String() string { return string(b) }

// Number returns the raw representation of a board variant.
func (b BoardVariant) Bytes() []byte { return []byte(b) }

// BoardVersion comprises the series, number, and variant of a ZTEX USB
// FPGA module.  For example, a ZTEX USB FPGA module 2.14b device would be
// represented by
//
//   BoardVersion{
//     Series: BoardSeries(2),
//     Number: BoardNumber(14),
//     Variant: BoardVariant("b"),
//   }
//
// as a BoardVersion structure.
type BoardVersion struct {
	Series  BoardSeries
	Number  BoardNumber
	Variant BoardVariant
}

// String returns a human-readable representation of a board version.
func (b BoardVersion) String() string {
	return b.Series.String() + "." + b.Number.String() + b.Variant.String()
}

// Device represents a ZTEX USB FPGA module.
type Device struct {
	*gousb.Device

	Firmware FirmwareVersion
	Board    BoardVersion

	// Endpoint for fast FPGA configuration.  (0 = Unsupported)
	FastConfigurationEndpoint byte
	// Interface for fast FPGA configuration.  (0 = Unsupported)
	FastConfigurationInterface byte
	// Default interface major version number.  (0 = Not Available)
	DefaultInterfaceMajorVersion byte
	// Default interface minor version number.  (0 = Not Available)
	DefaultInterfaceMinorVersion byte
	// Output endpoint of default interface.  (255 = Not Available)
	DefaultInterfaceOutputEndpoint byte
	// Input endpoint of default interface.  (255 = Not Available)
	DefaultInterfaceInputEndpoint byte
}

// String returns a human-readable representation of the device.
func (d *Device) String() string {
	mfr, _ := d.Manufacturer()
	prd, _ := d.Product()
	snr, _ := d.SerialNumber()

	lines := []string{}
	lines = append(lines, fmt.Sprintf("Manufacturer: %v", mfr))
	lines = append(lines, fmt.Sprintf("Product: %v", prd))
	lines = append(lines, fmt.Sprintf("Serial Number: %v", snr))
	lines = append(lines, fmt.Sprintf("Firmware Version: %v", d.Firmware))
	lines = append(lines, fmt.Sprintf("Board Version: %v", d.Board))

	return strings.Join(lines, "\n")
}

// OpenDevice opens a ZTEX USB FPGA module and returns its device handle.
// If there are multiple modules present, then one is chosen arbitrarily.
func OpenDevice(ctx *gousb.Context) (*Device, error) {
	d := &Device{}
	if dev, err := ctx.OpenDeviceWithVIDPID(VendorID, ProductID); err != nil {
		return nil, err
	} else if dev == nil {
		return nil, errors.New("no device")
	} else {
		d.Device = dev
		d.Device.ControlTimeout = ControlTimeout
	}

	buf := make([]byte, 128)

	// VR 0x3b: MAC EEPROM support: Read from MAC EEPROM
	if nbt, err := d.Control(0xc0, 0x3b, 0, 0, buf); err != nil {
		return nil, err
	} else if nbt == 128 {
		d.Firmware = FirmwareVersion(buf[3])
		d.Board = BoardVersion{
			Series:  BoardSeries(buf[4]),
			Number:  BoardNumber(buf[5]),
			Variant: BoardVariant(buf[6:8]),
		}
	} else {
		d.Firmware = FirmwareVersion(255)
		d.Board = BoardVersion{
			Series:  BoardSeries(255),
			Number:  BoardNumber(255),
			Variant: BoardVariant([]byte{255, 255}),
		}
	}

	// VR 0x33: High speed FPGA configuration support: Read Endpoint settings
	if nbt, err := d.Control(0xc0, 0x33, 0, 0, buf); err != nil {
		return nil, err
	} else if nbt == 2 {
		d.FastConfigurationEndpoint = buf[0]
		d.FastConfigurationInterface = buf[1]
	} else {
		return nil, errors.New("internal error")
	}

	// VR 0x64: Default firmware interface: Return Default Interface information
	if nbt, err := d.Control(0xc0, 0x64, 0, 0, buf); err != nil {
		return nil, err
	} else if nbt == 4 {
		d.DefaultInterfaceMajorVersion = buf[0]
		d.DefaultInterfaceMinorVersion = buf[3]
		d.DefaultInterfaceOutputEndpoint = buf[1] & 127
		d.DefaultInterfaceInputEndpoint = buf[2] | 128
	} else if nbt == 3 {
		d.DefaultInterfaceMajorVersion = buf[0]
		d.DefaultInterfaceMinorVersion = 0
		d.DefaultInterfaceOutputEndpoint = buf[1] & 127
		d.DefaultInterfaceInputEndpoint = buf[2] | 128
	} else {
		d.DefaultInterfaceOutputEndpoint = 255
		d.DefaultInterfaceInputEndpoint = 255
	}

	return d, nil
}
<|endoftext|>


<commit_before>package feeds

// rss support
// validation done according to spec here:
//    http://cyber.law.harvard.edu/rss/rss.html

import (
	"encoding/xml"
	"fmt"
	"time"
)

// private wrapper around the RssFeed which gives us the <rss>..</rss> xml
type rssFeedXml struct {
	XMLName xml.Name `xml:"rss"`
	Version string   `xml:"version,attr"`
	Channel *RssFeed
}

type RssImage struct {
	XMLName xml.Name `xml:"image"`
	Url     string   `xml:"url"`
	Title   string   `xml:"title"`
	Link    string   `xml:"link"`
	Width   int      `xml:"width,omitempty"`
	Height  int      `xml:"height,omitempty"`
}

type RssTextInput struct {
	XMLName     xml.Name `xml:"textInput"`
	Title       string   `xml:"title"`
	Description string   `xml:"description"`
	Name        string   `xml:"name"`
	Link        string   `xml:"link"`
}

type RssFeed struct {
	XMLName        xml.Name `xml:"channel"`
	Title          string   `xml:"title"`       // required
	Link           string   `xml:"link"`        // required
	Description    string   `xml:"description"` // required
	Language       string   `xml:"language,omitempty"`
	Copyright      string   `xml:"copyright,omitempty"`
	ManagingEditor string   `xml:"managingEditor,omitempty"` // Author used
	WebMaster      string   `xml:"webMaster,omitempty"`
	PubDate        string   `xml:"pubDate,omitempty"`       // created or updated
	LastBuildDate  string   `xml:"lastBuildDate,omitempty"` // updated used
	Category       string   `xml:"category,omitempty"`
	Generator      string   `xml:"generator,omitempty"`
	Docs           string   `xml:"docs,omitempty"`
	Cloud          string   `xml:"cloud,omitempty"`
	Ttl            int      `xml:"ttl,omitempty"`
	Rating         string   `xml:"rating,omitempty"`
	SkipHours      string   `xml:"skipHours,omitempty"`
	SkipDays       string   `xml:"skipDays,omitempty"`
	Image          *RssImage
	TextInput      *RssTextInput
	Items          []*RssItem
}

type RssItem struct {
	XMLName     xml.Name `xml:"item"`
	Title       string   `xml:"title"`       // required
	Link        string   `xml:"link"`        // required
	Description string   `xml:"description"` // required
	Author      string   `xml:"author,omitempty"`
	Category    string   `xml:"category,omitempty"`
	Comments    string   `xml:"comments,omitempty"`
	Enclosure   *RssEnclosure
	Guid        string `xml:"guid,omitempty"`    // Id used
	PubDate     string `xml:"pubDate,omitempty"` // created or updated
	Source      string `xml:"source,omitempty"`
}

type RssEnclosure struct {
	XMLName xml.Name `xml:"enclosure"`
	Url     string   `xml:"url,attr"`
	Length  string   `xml:"length,attr"`
	Type    string   `xml:"type,attr"`
}

type Rss struct {
	*Feed
}

// create a new RssItem with a generic Item struct's data
func newRssItem(i *Item) *RssItem {
	item := &RssItem{
		Title:       i.Title,
		Link:        i.Link.Href,
		Description: i.Description,
		Guid:        i.Id,
		PubDate:     anyTimeFormat(time.RFC822, i.Created, i.Updated),
	}
	if i.Author != nil {
		item.Author = i.Author.Name
	}
	return item
}

// create a new RssFeed with a generic Feed struct's data
func (r *Rss) RssFeed() *RssFeed {
	pub := anyTimeFormat(time.RFC822, r.Created, r.Updated)
	build := anyTimeFormat(time.RFC822, r.Updated)
	author := r.Author.Email
	if len(r.Author.Name) > 0 {
		author = fmt.Sprintf("%s (%s)", r.Author.Email, r.Author.Name)
	}

	channel := &RssFeed{
		Title:          r.Title,
		Link:           r.Link.Href,
		Description:    r.Description,
		ManagingEditor: author,
		PubDate:        pub,
		LastBuildDate:  build,
		Copyright:      r.Copyright,
	}
	for _, i := range r.Items {
		channel.Items = append(channel.Items, newRssItem(i))
	}
	return channel
}

// return an XML-Ready object for an Rss object
func (r *Rss) FeedXml() interface{} {
	// only generate version 2.0 feeds for now
	return r.RssFeed().FeedXml()

}

// return an XML-ready object for an RssFeed object
func (r *RssFeed) FeedXml() interface{} {
	return &rssFeedXml{Version: "2.0", Channel: r}
}
<commit_msg>author can be nil<commit_after>package feeds

// rss support
// validation done according to spec here:
//    http://cyber.law.harvard.edu/rss/rss.html

import (
	"encoding/xml"
	"fmt"
	"time"
)

// private wrapper around the RssFeed which gives us the <rss>..</rss> xml
type rssFeedXml struct {
	XMLName xml.Name `xml:"rss"`
	Version string   `xml:"version,attr"`
	Channel *RssFeed
}

type RssImage struct {
	XMLName xml.Name `xml:"image"`
	Url     string   `xml:"url"`
	Title   string   `xml:"title"`
	Link    string   `xml:"link"`
	Width   int      `xml:"width,omitempty"`
	Height  int      `xml:"height,omitempty"`
}

type RssTextInput struct {
	XMLName     xml.Name `xml:"textInput"`
	Title       string   `xml:"title"`
	Description string   `xml:"description"`
	Name        string   `xml:"name"`
	Link        string   `xml:"link"`
}

type RssFeed struct {
	XMLName        xml.Name `xml:"channel"`
	Title          string   `xml:"title"`       // required
	Link           string   `xml:"link"`        // required
	Description    string   `xml:"description"` // required
	Language       string   `xml:"language,omitempty"`
	Copyright      string   `xml:"copyright,omitempty"`
	ManagingEditor string   `xml:"managingEditor,omitempty"` // Author used
	WebMaster      string   `xml:"webMaster,omitempty"`
	PubDate        string   `xml:"pubDate,omitempty"`       // created or updated
	LastBuildDate  string   `xml:"lastBuildDate,omitempty"` // updated used
	Category       string   `xml:"category,omitempty"`
	Generator      string   `xml:"generator,omitempty"`
	Docs           string   `xml:"docs,omitempty"`
	Cloud          string   `xml:"cloud,omitempty"`
	Ttl            int      `xml:"ttl,omitempty"`
	Rating         string   `xml:"rating,omitempty"`
	SkipHours      string   `xml:"skipHours,omitempty"`
	SkipDays       string   `xml:"skipDays,omitempty"`
	Image          *RssImage
	TextInput      *RssTextInput
	Items          []*RssItem
}

type RssItem struct {
	XMLName     xml.Name `xml:"item"`
	Title       string   `xml:"title"`       // required
	Link        string   `xml:"link"`        // required
	Description string   `xml:"description"` // required
	Author      string   `xml:"author,omitempty"`
	Category    string   `xml:"category,omitempty"`
	Comments    string   `xml:"comments,omitempty"`
	Enclosure   *RssEnclosure
	Guid        string `xml:"guid,omitempty"`    // Id used
	PubDate     string `xml:"pubDate,omitempty"` // created or updated
	Source      string `xml:"source,omitempty"`
}

type RssEnclosure struct {
	XMLName xml.Name `xml:"enclosure"`
	Url     string   `xml:"url,attr"`
	Length  string   `xml:"length,attr"`
	Type    string   `xml:"type,attr"`
}

type Rss struct {
	*Feed
}

// create a new RssItem with a generic Item struct's data
func newRssItem(i *Item) *RssItem {
	item := &RssItem{
		Title:       i.Title,
		Link:        i.Link.Href,
		Description: i.Description,
		Guid:        i.Id,
		PubDate:     anyTimeFormat(time.RFC822, i.Created, i.Updated),
	}
	if i.Author != nil {
		item.Author = i.Author.Name
	}
	return item
}

// create a new RssFeed with a generic Feed struct's data
func (r *Rss) RssFeed() *RssFeed {
	pub := anyTimeFormat(time.RFC822, r.Created, r.Updated)
	build := anyTimeFormat(time.RFC822, r.Updated)
	author := ""
	if r.Author != nil {
		author = r.Author.Email
		if len(r.Author.Name) > 0 {
			author = fmt.Sprintf("%s (%s)", r.Author.Email, r.Author.Name)
		}
	}

	channel := &RssFeed{
		Title:          r.Title,
		Link:           r.Link.Href,
		Description:    r.Description,
		ManagingEditor: author,
		PubDate:        pub,
		LastBuildDate:  build,
		Copyright:      r.Copyright,
	}
	for _, i := range r.Items {
		channel.Items = append(channel.Items, newRssItem(i))
	}
	return channel
}

// return an XML-Ready object for an Rss object
func (r *Rss) FeedXml() interface{} {
	// only generate version 2.0 feeds for now
	return r.RssFeed().FeedXml()

}

// return an XML-ready object for an RssFeed object
func (r *RssFeed) FeedXml() interface{} {
	return &rssFeedXml{Version: "2.0", Channel: r}
}
<|endoftext|>


<commit_before>// 22-24 august 2012
package main

import (
	"fmt"
	"sort"
)

type SortOrder int
const (
	SortByRegion SortOrder = iota
	SortByBoxState
	SortByMediaState
)

type GameScan struct {
	Name		string
	HasNoScans	bool			// for whole games
	Region		string
	BoxState		ScanState
	MediaState	ScanState
	Error			error
}

type ScanSet []*GameScan

func GetConsoleScans(console string) (ScanSet, error) {
	return Run(console + " games", console)
}

func GetAlbumScans() (ScanSet, error) {
	return Run("Albums", "CD")
}

func Run(category string, console string) (ScanSet, error) {
	var gameScans ScanSet

	games, err := GetGameList(category)
	if err != nil {
		return nil, fmt.Errorf("error getting %s list: %v", category, err)
	}
	for _, game := range games {
//fmt.Println(game)
		scans, err := GetScans(game, console)
		if err == ErrGameNoScans {		// omit games for this console that will not have scans
			continue
		}
		if err != nil {
			gameScans = append(gameScans, &GameScan{
				Name:	game,
				Error:	err,
			})
			continue
		}
		if len(scans) == 0 {				// there are no scans at all
			gameScans = append(gameScans, &GameScan{
				Name:		game,
				HasNoScans:	true,
			})
			continue
		}
		nScans := 0
		for _, scan := range scans {
			var mediaState ScanState

			nScans++
			boxState := scan.BoxScanState()
			mediaState = scan.MediaScanState()
			gameScans = append(gameScans, &GameScan{
				Name:		game,
				Region:		scan.Region,
				BoxState:		boxState,
				MediaState:	mediaState,
			})
		}
		if nScans == 0 {					// there are no scans for the specified console
			gameScans = append(gameScans, &GameScan{
				Name:		game,
				HasNoScans:	true,
			})
			continue
		}
	}
	return gameScans, nil
}

type sorter struct {
	scans		ScanSet
	sortOrder		SortOrder
}

// for sort.Interface
func (s sorter) Len() int {
	return len(s.scans)
}

func (s sorter) Less(i, j int) bool {
	scans := s.scans
	// the sort orders make no sense if there is either an error or no scans for a game, so handle those cases first
	if scans[i].Error != nil && scans[j].Error != nil {		// errors go first
		return scans[i].Name < scans[j].Name		// by title if they both error
	}
	if scans[i].Error != nil {
		return true
	}
	if scans[j].Error != nil {
		return false
	}
	if scans[i].HasNoScans && scans[j].HasNoScans {	// then lack of scans
		return scans[i].Name < scans[j].Name		// by title if they both lack scans
	}
	if scans[i].HasNoScans {
		return true
	}
	if scans[j].HasNoScans {
		return false
	}
	switch s.sortOrder {
	case SortByRegion:		// sort by region, then by name
		if scans[i].Region == scans[j].Region {					// then if they have the same region, alphabetically
			return scans[i].Name < scans[j].Name
		}
		return scans[i].Region < scans[j].Region					// finally
	case SortByBoxState:
		if scans[i].BoxState.State == scans[j].BoxState.State {		// then if they have the same region, alphabetically
			return scans[i].Name < scans[j].Name
		}
		return scans[i].BoxState.State < scans[j].BoxState.State		// finally
	case SortByMediaState:
		if scans[i].MediaState.State == scans[j].MediaState.State {		// then if they have the same region, alphabetically
			return scans[i].Name < scans[j].Name
		}
		return scans[i].MediaState.State < scans[j].MediaState.State	// finally
	}
	panic(fmt.Sprintf("invalid sort order %d", int(s.sortOrder)))
}

func (s sorter) Swap(i, j int) {
	s.scans[i], s.scans[j] = s.scans[j], s.scans[i]
}

func (scans ScanSet) Sort(so SortOrder) {
	sort.Sort(sorter{
		scans:		scans,
		sortOrder:		so,
	})
}
<commit_msg>Changed the category for albums from CD to Album. This allows scanboxes to also monitor cassettes and vinyl releases. I may need to make drastic changes to how categories are determined if I want to get rid of all this special casing... right now I like that game categories are autodetermined but :/<commit_after>// 22-24 august 2012
package main

import (
	"fmt"
	"sort"
)

type SortOrder int
const (
	SortByRegion SortOrder = iota
	SortByBoxState
	SortByMediaState
)

type GameScan struct {
	Name		string
	HasNoScans	bool			// for whole games
	Region		string
	BoxState		ScanState
	MediaState	ScanState
	Error			error
}

type ScanSet []*GameScan

func GetConsoleScans(console string) (ScanSet, error) {
	return Run(console + " games", console)
}

func GetAlbumScans() (ScanSet, error) {
	return Run("Albums", "Album")
}

func Run(category string, console string) (ScanSet, error) {
	var gameScans ScanSet

	games, err := GetGameList(category)
	if err != nil {
		return nil, fmt.Errorf("error getting %s list: %v", category, err)
	}
	for _, game := range games {
//fmt.Println(game)
		scans, err := GetScans(game, console)
		if err == ErrGameNoScans {		// omit games for this console that will not have scans
			continue
		}
		if err != nil {
			gameScans = append(gameScans, &GameScan{
				Name:	game,
				Error:	err,
			})
			continue
		}
		if len(scans) == 0 {				// there are no scans at all
			gameScans = append(gameScans, &GameScan{
				Name:		game,
				HasNoScans:	true,
			})
			continue
		}
		nScans := 0
		for _, scan := range scans {
			var mediaState ScanState

			nScans++
			boxState := scan.BoxScanState()
			mediaState = scan.MediaScanState()
			gameScans = append(gameScans, &GameScan{
				Name:		game,
				Region:		scan.Region,
				BoxState:		boxState,
				MediaState:	mediaState,
			})
		}
		if nScans == 0 {					// there are no scans for the specified console
			gameScans = append(gameScans, &GameScan{
				Name:		game,
				HasNoScans:	true,
			})
			continue
		}
	}
	return gameScans, nil
}

type sorter struct {
	scans		ScanSet
	sortOrder		SortOrder
}

// for sort.Interface
func (s sorter) Len() int {
	return len(s.scans)
}

func (s sorter) Less(i, j int) bool {
	scans := s.scans
	// the sort orders make no sense if there is either an error or no scans for a game, so handle those cases first
	if scans[i].Error != nil && scans[j].Error != nil {		// errors go first
		return scans[i].Name < scans[j].Name		// by title if they both error
	}
	if scans[i].Error != nil {
		return true
	}
	if scans[j].Error != nil {
		return false
	}
	if scans[i].HasNoScans && scans[j].HasNoScans {	// then lack of scans
		return scans[i].Name < scans[j].Name		// by title if they both lack scans
	}
	if scans[i].HasNoScans {
		return true
	}
	if scans[j].HasNoScans {
		return false
	}
	switch s.sortOrder {
	case SortByRegion:		// sort by region, then by name
		if scans[i].Region == scans[j].Region {					// then if they have the same region, alphabetically
			return scans[i].Name < scans[j].Name
		}
		return scans[i].Region < scans[j].Region					// finally
	case SortByBoxState:
		if scans[i].BoxState.State == scans[j].BoxState.State {		// then if they have the same region, alphabetically
			return scans[i].Name < scans[j].Name
		}
		return scans[i].BoxState.State < scans[j].BoxState.State		// finally
	case SortByMediaState:
		if scans[i].MediaState.State == scans[j].MediaState.State {		// then if they have the same region, alphabetically
			return scans[i].Name < scans[j].Name
		}
		return scans[i].MediaState.State < scans[j].MediaState.State	// finally
	}
	panic(fmt.Sprintf("invalid sort order %d", int(s.sortOrder)))
}

func (s sorter) Swap(i, j int) {
	s.scans[i], s.scans[j] = s.scans[j], s.scans[i]
}

func (scans ScanSet) Sort(so SortOrder) {
	sort.Sort(sorter{
		scans:		scans,
		sortOrder:		so,
	})
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"time"
)

const (
	StateStopped = iota
	StateLoading
	StateRunning
	StateShuttingDown
)

const BufferSize = 1024 * 512

var (
	saveCmd = []byte{'\r', '\n', 's', 'a', 'v', 'e', 'a', 'l', 'l', ' ', '\r', '\n'}
	stopCmd = []byte{'\r', '\n', 's', 't', 'o', 'p', ' ', '\r', '\n'}
)

type Controller struct {
	c       *Config
	running map[int]*running
}

func (c *Controller) Start(sID int) error {
	c.c.mu.Lock()
	defer c.c.mu.Unlock()
	s, ok := c.c.Servers[sID]
	if !ok {
		return ErrNoServer
	}
	if s.State != StateStopped {
		return ErrServerRunning
	}
	if s.Map == -1 {
		return ErrNoMap
	}
	m, ok := c.c.Maps[s.Map]
	if !ok {
		return ErrNoMap // Shouldn't happen, different error?
	}
	mapPath := path.Join(s.Path, "world")
	if !path.IsAbs(mapPath) {
		pwd, err := os.Getwd()
		if err != nil {
			return err
		}
		mapPath = path.Join(pwd, mapPath)
	}
	os.Remove(mapPath)
	err := os.Symlink(mapPath, path.Join(s.Path, "world"))
	if err != nil {
		return err
	}
	ps, err := os.Open(path.Join(s.Path, "properties.server"))
	if err != nil {
		return err
	}
	defer ps.Close()
	pm, err := os.Open(path.Join(m.Path, "properties.map"))
	if err != nil {
		return err
	}
	defer pm.Close()
	f, err := os.Create(path.Join(s.Path, "server.properties"))
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = io.Copy(f, ps)
	if err != nil {
		return err
	}
	_, err = io.Copy(f, pm)
	if err != nil {
		return err
	}

	s.State = StateLoading
	//c.c.Servers[sID] = s
	sc := make(chan struct{})
	r := &running{shutdown: sc}
	c.running[s.ID] = r
	go c.run(s, r)
	return nil
}

func (c *Controller) Stop(sID int) error {
	c.c.mu.Lock()
	defer c.c.mu.Unlock()
	r, ok := c.running[sID]
	if !ok {
		return ErrServerNotRunning
	}
	close(r.shutdown)
	delete(c.running, sID)
	return nil
}

// runs in its own goroutine
func (c *Controller) run(s *Server, r *running) {
	cmd := exec.Command("java", append(s.Args, "-jar", "server.jar", "nogui")...)
	cmd.Dir = s.Path
	r.Writer, _ = cmd.StdinPipe()

	err := cmd.Start()
	if err != nil {
		fmt.Println(err)
		// Write to Stderr
	} else {

		c.c.mu.Lock()
		s.State = StateRunning
		//c.c.Servers[s.ID] = s
		c.c.mu.Unlock()

		died := make(chan struct{})
		go func() {
			select {
			case <-r.shutdown:
				c.c.mu.Lock()
				s.State = StateShuttingDown
				//c.c.Servers[s.ID] = s
				c.c.mu.Unlock()
				t := time.NewTimer(time.Second * 10)
				defer t.Stop()
				for {
					r.Write(stopCmd)
					select {
					case <-died:
						return
					case <-t.C:
					}
				}
			case <-died:
				c.c.mu.Lock()
				delete(c.running, s.ID)
				c.c.mu.Unlock()
			}
		}()

		cmd.Wait()
		r.shutdown = nil
		close(died)
	}
	c.c.mu.Lock()
	s.State = StateStopped
	//c.c.Servers[s.ID] = s
	c.c.mu.Unlock()
}

func (c *Controller) CmdWriter(sID int) io.Writer {
	c.c.mu.Lock()
	defer c.c.mu.Unlock()
	return c.running[sID]
}

type running struct {
	shutdown chan struct{}
	io.Writer
}

// Errors
var (
	ErrServerRunning    = errors.New("server already running")
	ErrServerNotRunning = errors.New("server not running")
)
<commit_msg>Corrected map/server paths in symlinking<commit_after>package main

import (
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"time"
)

const (
	StateStopped = iota
	StateLoading
	StateRunning
	StateShuttingDown
)

const BufferSize = 1024 * 512

var (
	saveCmd = []byte{'\r', '\n', 's', 'a', 'v', 'e', 'a', 'l', 'l', ' ', '\r', '\n'}
	stopCmd = []byte{'\r', '\n', 's', 't', 'o', 'p', ' ', '\r', '\n'}
)

type Controller struct {
	c       *Config
	running map[int]*running
}

func (c *Controller) Start(sID int) error {
	c.c.mu.Lock()
	defer c.c.mu.Unlock()
	s, ok := c.c.Servers[sID]
	if !ok {
		return ErrNoServer
	}
	if s.State != StateStopped {
		return ErrServerRunning
	}
	if s.Map == -1 {
		return ErrNoMap
	}
	m, ok := c.c.Maps[s.Map]
	if !ok {
		return ErrNoMap // Shouldn't happen, different error?
	}
	mapPath := m.Path
	if !path.IsAbs(mapPath) {
		pwd, err := os.Getwd()
		if err != nil {
			return err
		}
		mapPath = path.Join(pwd, mapPath)
	}
	serverMapPath := path.Join(s.Path, "world")
	if err := os.Remove(serverMapPath); err != nil && !os.IsNotExist(err) {
		return err
	}
	if err := os.Symlink(mapPath, serverMapPath); err != nil {
		return err
	}
	ps, err := os.Open(path.Join(s.Path, "properties.server"))
	if err != nil {
		return err
	}
	defer ps.Close()
	pm, err := os.Open(path.Join(m.Path, "properties.map"))
	if err != nil {
		return err
	}
	defer pm.Close()
	f, err := os.Create(path.Join(s.Path, "server.properties"))
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = io.Copy(f, ps)
	if err != nil {
		return err
	}
	_, err = io.Copy(f, pm)
	if err != nil {
		return err
	}

	s.State = StateLoading
	//c.c.Servers[sID] = s
	sc := make(chan struct{})
	r := &running{shutdown: sc}
	c.running[s.ID] = r
	go c.run(s, r)
	return nil
}

func (c *Controller) Stop(sID int) error {
	c.c.mu.Lock()
	defer c.c.mu.Unlock()
	r, ok := c.running[sID]
	if !ok {
		return ErrServerNotRunning
	}
	close(r.shutdown)
	delete(c.running, sID)
	return nil
}

// runs in its own goroutine
func (c *Controller) run(s *Server, r *running) {
	cmd := exec.Command("java", append(s.Args, "-jar", "server.jar", "nogui")...)
	cmd.Dir = s.Path
	r.Writer, _ = cmd.StdinPipe()

	err := cmd.Start()
	if err != nil {
		fmt.Println(err)
		// Write to Stderr
	} else {

		c.c.mu.Lock()
		s.State = StateRunning
		//c.c.Servers[s.ID] = s
		c.c.mu.Unlock()

		died := make(chan struct{})
		go func() {
			select {
			case <-r.shutdown:
				c.c.mu.Lock()
				s.State = StateShuttingDown
				//c.c.Servers[s.ID] = s
				c.c.mu.Unlock()
				t := time.NewTimer(time.Second * 10)
				defer t.Stop()
				for {
					r.Write(stopCmd)
					select {
					case <-died:
						return
					case <-t.C:
					}
				}
			case <-died:
				c.c.mu.Lock()
				delete(c.running, s.ID)
				c.c.mu.Unlock()
			}
		}()

		cmd.Wait()
		r.shutdown = nil
		close(died)
	}
	c.c.mu.Lock()
	s.State = StateStopped
	//c.c.Servers[s.ID] = s
	c.c.mu.Unlock()
}

func (c *Controller) CmdWriter(sID int) io.Writer {
	c.c.mu.Lock()
	defer c.c.mu.Unlock()
	return c.running[sID]
}

type running struct {
	shutdown chan struct{}
	io.Writer
}

// Errors
var (
	ErrServerRunning    = errors.New("server already running")
	ErrServerNotRunning = errors.New("server not running")
)
<|endoftext|>


<commit_before>package main

import (
  "encoding/json"
  "errors"
  "flag"
  "fmt"
  "io/ioutil"
  "log"
  "os"
  "os/exec"
  "path"
  "path/filepath"
  "runtime"
  "strings"
  "syscall"
)

// The current version number of Run. Run versions will be tagged in the git
// repo, so this is merely provided as a convenience.
const version = "0.1.0"

// The collection of supported commands. This is represented as a map of file
// extensions (strings) to commands (strings).
var commands = map[string]string {
  "c": "gcc % && ./a.out",
  "clj": "clj %",
  "coffee": "coffee %",
  "go": "go run %",
  "hs": "runhaskell %",
  "jar": "java -jar %",
  "lua": "lua %",
  "ml": "ocaml %",
  "py": "python %",
  "rb": "ruby %",
  "sh": "sh %",
  "zsh": "zsh %",
}

// callerDir returns the directory of this source code file in Run's
// implementation. Similar to __dir__ in Ruby.
func callerDir() string {
  _, callerFile, _, _ := runtime.Caller(1)
  return path.Dir(callerFile)
}

// getCommands gets a collection of commands from a JSON config file. This is
// represented as a map of file extensions (strings) to commands (strings).
func getCommands(file string) (map[string]string, error) {
  // Load the commands from the data file to a slice of bytes.
  var commands map[string]string
  jsonStream, fileErr := ioutil.ReadFile(file)
  if fileErr != nil {
    return commands, fileErr
  }

  // Parse the byte slice to get a map of type commands.
  jsonErr := json.Unmarshal(jsonStream, &commands)
  return commands, jsonErr
}

// commandForFile returns the command that should be used to run the given file.
// The beginning of the command depends on the extension of the file, while the
// file path portion(s) of the command will automatically be substituted with
// the given file path.
func commandForFile(path string) (string, error) {
  extension := strings.Replace(filepath.Ext(path), ".", "", -1)

  // Fill out the command template.
  if command, success := commands[extension]; success {
    return strings.Replace(command, "%", path, -1), nil
  }
  return "", fmt.Errorf("run %s: could not determine how to run the file because \"%s\" is not a known extension", path, extension)
}

// runCommand finds the appropriate command to run a file and executes it,
// replacing the current process. If the command fails, an error will be
// returned.
func runCommand(command string) error {
  // Separate the command into arguments for syscall.Exec.
  args := strings.Split(command, " ")

  // Get the path to the binary.
  binary, err := exec.LookPath(args[0])
  if err != nil {
    return err
  }

  // Execute the command, replacing the current process with it.
  return syscall.Exec(binary, args, os.Environ())
}

// start takes the command line args given to Run. It a filename is given as the
// first argument, the command to run it is returned. Otherwise, it returns an
// error. This mostly exists for testing purposes so that the args for main
// won't need to be mocked.
func start(args ...string) (string, error) {
  if len(args) <= 1 {
    return "", errors.New("run: no files given to run")
  }
  return commandForFile(args[1])
}

// merge merges the contents of map2 into map1, using map1 as the default
// values.
func merge(map1 map[string]string, map2 map[string]string) {
  for key, value := range map2 {
    map1[key] = value
  }
}

// main runs start and executes the resulting command if it succeeds. Otherwise,
// it returns an error.
func main() {
  verbosePtr := flag.Bool("verbose", false, "displays information on all commands that are run, whether or not they are successful")
  dryRunPtr := flag.Bool("dry-run", false, "don't actually run the file, just show any error messages and verbose messages from Run")
  flag.Parse()

  args := append([]string{os.Args[0]}, flag.Args()...)
  command, err := start(args...)
  if err != nil {
    log.Fatal(err)
  }

  if *verbosePtr {
    fmt.Println(command)
  }
  if !*dryRunPtr {
    runCommand(command)
  }
}
<commit_msg>Add a -list flag for listing supported commands<commit_after>package main

import (
  "encoding/json"
  "errors"
  "flag"
  "fmt"
  "io/ioutil"
  "log"
  "os"
  "os/exec"
  "path"
  "path/filepath"
  "runtime"
  "strings"
  "syscall"
)

// The current version number of Run. Run versions will be tagged in the git
// repo, so this is merely provided as a convenience.
const version = "0.1.0"

// The collection of supported commands. This is represented as a map of file
// extensions (strings) to commands (strings).
var commands = map[string]string {
  "c": "gcc % && ./a.out",
  "clj": "clj %",
  "coffee": "coffee %",
  "go": "go run %",
  "hs": "runhaskell %",
  "jar": "java -jar %",
  "lua": "lua %",
  "ml": "ocaml %",
  "py": "python %",
  "rb": "ruby %",
  "sh": "sh %",
  "zsh": "zsh %",
}

// callerDir returns the directory of this source code file in Run's
// implementation. Similar to __dir__ in Ruby.
func callerDir() string {
  _, callerFile, _, _ := runtime.Caller(1)
  return path.Dir(callerFile)
}

// getCommands gets a collection of commands from a JSON config file. This is
// represented as a map of file extensions (strings) to commands (strings).
func getCommands(file string) (map[string]string, error) {
  // Load the commands from the data file to a slice of bytes.
  var commands map[string]string
  jsonStream, fileErr := ioutil.ReadFile(file)
  if fileErr != nil {
    return commands, fileErr
  }

  // Parse the byte slice to get a map of type commands.
  jsonErr := json.Unmarshal(jsonStream, &commands)
  return commands, jsonErr
}

// commandForFile returns the command that should be used to run the given file.
// The beginning of the command depends on the extension of the file, while the
// file path portion(s) of the command will automatically be substituted with
// the given file path.
func commandForFile(path string) (string, error) {
  extension := strings.Replace(filepath.Ext(path), ".", "", -1)

  // Fill out the command template.
  if command, success := commands[extension]; success {
    return strings.Replace(command, "%", path, -1), nil
  }
  return "", fmt.Errorf("run %s: could not determine how to run the file because \"%s\" is not a known extension", path, extension)
}

// runCommand finds the appropriate command to run a file and executes it,
// replacing the current process. If the command fails, an error will be
// returned.
func runCommand(command string) error {
  // Separate the command into arguments for syscall.Exec.
  args := strings.Split(command, " ")

  // Get the path to the binary.
  binary, err := exec.LookPath(args[0])
  if err != nil {
    return err
  }

  // Execute the command, replacing the current process with it.
  return syscall.Exec(binary, args, os.Environ())
}

// start takes the command line args given to Run. It a filename is given as the
// first argument, the command to run it is returned. Otherwise, it returns an
// error. This mostly exists for testing purposes so that the args for main
// won't need to be mocked.
func start(args ...string) (string, error) {
  if len(args) <= 1 {
    return "", errors.New("run: no files given to run")
  }
  return commandForFile(args[1])
}

// merge merges the contents of map2 into map1, using map1 as the default
// values.
func merge(map1 map[string]string, map2 map[string]string) {
  for key, value := range map2 {
    map1[key] = value
  }
}

// printCommands prints the given map of commands in the following format.
//
//   extension: command
//   extension: command
func printCommands(commands map[string]string) {
  for extension, command := range commands {
    fmt.Printf("%s: %s\n", extension, command)
  }
}

// main runs start and executes the resulting command if it succeeds. Otherwise,
// it returns an error.
func main() {
  verbosePtr := flag.Bool("verbose", false, "displays information on all commands that are run, whether or not they are successful")
  dryRunPtr := flag.Bool("dry-run", false, "don't actually run the file, just show any error messages and verbose messages from Run")
  listPtr := flag.Bool("list", false, "list all supported extensions and commands and stop")
  flag.Parse()

  if *listPtr {
    fmt.Println("Note that every % will be replaced with the given filename.")
    fmt.Println()
    printCommands(commands)
    os.Exit(0)
  }

  args := append([]string{os.Args[0]}, flag.Args()...)
  command, err := start(args...)
  if err != nil {
    log.Fatal(err)
  }

  if *verbosePtr {
    fmt.Println(command)
  }
  if !*dryRunPtr {
    runCommand(command)
  }
}
<|endoftext|>


<commit_before>package s3

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/aws/session"
	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3iface"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
	"github.com/chrislusf/seaweedfs/weed/remote_storage"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"reflect"
)

func init() {
	remote_storage.RemoteStorageClientMakers["s3"] = new(s3RemoteStorageMaker)
}

type s3RemoteStorageMaker struct{}

func (s s3RemoteStorageMaker) HasBucket() bool {
	return true
}

func (s s3RemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
	client := &s3RemoteStorageClient{
		conf: conf,
	}
	config := &aws.Config{
		Region:                        aws.String(conf.S3Region),
		Endpoint:                      aws.String(conf.S3Endpoint),
		S3ForcePathStyle:              aws.Bool(conf.S3ForcePathStyle),
		S3DisableContentMD5Validation: aws.Bool(true),
	}
	if conf.S3AccessKey != "" && conf.S3SecretKey != "" {
		config.Credentials = credentials.NewStaticCredentials(conf.S3AccessKey, conf.S3SecretKey, "")
	}

	sess, err := session.NewSession(config)
	if err != nil {
		return nil, fmt.Errorf("create aws session: %v", err)
	}
	if conf.S3V4Signature {
		sess.Handlers.Sign.PushBackNamed(v4.SignRequestHandler)
	}
	sess.Handlers.Build.PushBack(func(r *request.Request) {
		r.HTTPRequest.Header.Set("User-Agent", "SeaweedFS/"+util.VERSION_NUMBER)
	})
	sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
	client.conn = s3.New(sess)
	return client, nil
}

type s3RemoteStorageClient struct {
	conf *remote_pb.RemoteConf
	conn s3iface.S3API
}

var _ = remote_storage.RemoteStorageClient(&s3RemoteStorageClient{})

func (s *s3RemoteStorageClient) Traverse(remote *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {

	pathKey := remote.Path[1:]

	listInput := &s3.ListObjectsV2Input{
		Bucket:              aws.String(remote.Bucket),
		ContinuationToken:   nil,
		Delimiter:           nil, // not aws.String("/"), iterate through all entries
		EncodingType:        nil,
		ExpectedBucketOwner: nil,
		FetchOwner:          nil,
		MaxKeys:             nil, // aws.Int64(1000),
		Prefix:              aws.String(pathKey),
		RequestPayer:        nil,
		StartAfter:          nil,
	}
	isLastPage := false
	for !isLastPage && err == nil {
		listErr := s.conn.ListObjectsV2Pages(listInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
			for _, content := range page.Contents {
				key := *content.Key
				key = "/" + key
				dir, name := util.FullPath(key).DirAndName()
				if err := visitFn(dir, name, false, &filer_pb.RemoteEntry{
					RemoteMtime: (*content.LastModified).Unix(),
					RemoteSize:  *content.Size,
					RemoteETag:  *content.ETag,
					StorageName: s.conf.Name,
				}); err != nil {
					return false
				}
			}
			listInput.ContinuationToken = page.NextContinuationToken
			isLastPage = lastPage
			return true
		})
		if listErr != nil {
			err = fmt.Errorf("list %v: %v", remote, listErr)
		}
	}
	return
}
func (s *s3RemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
	downloader := s3manager.NewDownloaderWithClient(s.conn, func(u *s3manager.Downloader) {
		u.PartSize = int64(4 * 1024 * 1024)
		u.Concurrency = 1
	})

	dataSlice := make([]byte, int(size))
	writerAt := aws.NewWriteAtBuffer(dataSlice)

	_, err = downloader.Download(writerAt, &s3.GetObjectInput{
		Bucket: aws.String(loc.Bucket),
		Key:    aws.String(loc.Path[1:]),
		Range:  aws.String(fmt.Sprintf("bytes=%d-%d", offset, offset+size-1)),
	})
	if err != nil {
		return nil, fmt.Errorf("failed to download file %s%s: %v", loc.Bucket, loc.Path, err)
	}

	return writerAt.Bytes(), nil
}

func (s *s3RemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) {
	return nil
}

func (s *s3RemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) {
	return nil
}

func (s *s3RemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) {

	fileSize := int64(filer.FileSize(entry))

	partSize := int64(8 * 1024 * 1024) // The minimum/default allowed part size is 5MB
	for partSize*1000 < fileSize {
		partSize *= 4
	}

	// Create an uploader with the session and custom options
	uploader := s3manager.NewUploaderWithClient(s.conn, func(u *s3manager.Uploader) {
		u.PartSize = partSize
		u.Concurrency = 1
	})

	// process tagging
	tags := ""
	for k, v := range entry.Extended {
		if len(tags) > 0 {
			tags = tags + "&"
		}
		tags = tags + k + "=" + string(v)
	}

	// Upload the file to S3.
	_, err = uploader.Upload(&s3manager.UploadInput{
		Bucket:       aws.String(loc.Bucket),
		Key:          aws.String(loc.Path[1:]),
		Body:         reader,
		Tagging:      aws.String(tags),
		StorageClass: aws.String(s.conf.S3StorageClass),
	})

	//in case it fails to upload
	if err != nil {
		return nil, fmt.Errorf("upload to %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
	}

	// read back the remote entry
	return s.readFileRemoteEntry(loc)

}

func toTagging(attributes map[string][]byte) *s3.Tagging {
	tagging := &s3.Tagging{}
	for k, v := range attributes {
		tagging.TagSet = append(tagging.TagSet, &s3.Tag{
			Key:   aws.String(k),
			Value: aws.String(string(v)),
		})
	}
	return tagging
}

func (s *s3RemoteStorageClient) readFileRemoteEntry(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
	resp, err := s.conn.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(loc.Bucket),
		Key:    aws.String(loc.Path[1:]),
	})
	if err != nil {
		return nil, err
	}

	return &filer_pb.RemoteEntry{
		RemoteMtime: resp.LastModified.Unix(),
		RemoteSize:  *resp.ContentLength,
		RemoteETag:  *resp.ETag,
		StorageName: s.conf.Name,
	}, nil

}

func (s *s3RemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error) {
	if reflect.DeepEqual(oldEntry.Extended, newEntry.Extended) {
		return nil
	}
	tagging := toTagging(newEntry.Extended)
	if len(tagging.TagSet) > 0 {
		_, err = s.conn.PutObjectTagging(&s3.PutObjectTaggingInput{
			Bucket:  aws.String(loc.Bucket),
			Key:     aws.String(loc.Path[1:]),
			Tagging: toTagging(newEntry.Extended),
		})
	} else {
		_, err = s.conn.DeleteObjectTagging(&s3.DeleteObjectTaggingInput{
			Bucket: aws.String(loc.Bucket),
			Key:    aws.String(loc.Path[1:]),
		})
	}
	return
}
func (s *s3RemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) {
	_, err = s.conn.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(loc.Bucket),
		Key:    aws.String(loc.Path[1:]),
	})
	return
}

func (s *s3RemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) {
	resp, err := s.conn.ListBuckets(&s3.ListBucketsInput{})
	if err != nil {
		return nil, fmt.Errorf("list buckets: %v", err)
	}
	for _, b := range resp.Buckets {
		buckets = append(buckets, &remote_storage.Bucket{
			Name:      *b.Name,
			CreatedAt: *b.CreationDate,
		})
	}
	return
}

func (s *s3RemoteStorageClient) CreateBucket(name string) (err error) {
	_, err = s.conn.CreateBucket(&s3.CreateBucketInput{
		ACL:                        nil,
		Bucket:                     aws.String(name),
		CreateBucketConfiguration:  nil,
		GrantFullControl:           nil,
		GrantRead:                  nil,
		GrantReadACP:               nil,
		GrantWrite:                 nil,
		GrantWriteACP:              nil,
		ObjectLockEnabledForBucket: nil,
	})
	if err != nil {
		return fmt.Errorf("%s create bucket %s: %v", s.conf.Name, name, err)
	}
	return
}

func (s *s3RemoteStorageClient) DeleteBucket(name string) (err error) {
	_, err = s.conn.DeleteBucket(&s3.DeleteBucketInput{
		Bucket: aws.String(name),
	})
	if err != nil {
		return fmt.Errorf("delete bucket %s: %v", name, err)
	}
	return
}
<commit_msg>remote.mount: print out metadata sync errors<commit_after>package s3

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/aws/session"
	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3iface"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
	"github.com/chrislusf/seaweedfs/weed/remote_storage"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"reflect"
)

func init() {
	remote_storage.RemoteStorageClientMakers["s3"] = new(s3RemoteStorageMaker)
}

type s3RemoteStorageMaker struct{}

func (s s3RemoteStorageMaker) HasBucket() bool {
	return true
}

func (s s3RemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
	client := &s3RemoteStorageClient{
		conf: conf,
	}
	config := &aws.Config{
		Region:                        aws.String(conf.S3Region),
		Endpoint:                      aws.String(conf.S3Endpoint),
		S3ForcePathStyle:              aws.Bool(conf.S3ForcePathStyle),
		S3DisableContentMD5Validation: aws.Bool(true),
	}
	if conf.S3AccessKey != "" && conf.S3SecretKey != "" {
		config.Credentials = credentials.NewStaticCredentials(conf.S3AccessKey, conf.S3SecretKey, "")
	}

	sess, err := session.NewSession(config)
	if err != nil {
		return nil, fmt.Errorf("create aws session: %v", err)
	}
	if conf.S3V4Signature {
		sess.Handlers.Sign.PushBackNamed(v4.SignRequestHandler)
	}
	sess.Handlers.Build.PushBack(func(r *request.Request) {
		r.HTTPRequest.Header.Set("User-Agent", "SeaweedFS/"+util.VERSION_NUMBER)
	})
	sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
	client.conn = s3.New(sess)
	return client, nil
}

type s3RemoteStorageClient struct {
	conf *remote_pb.RemoteConf
	conn s3iface.S3API
}

var _ = remote_storage.RemoteStorageClient(&s3RemoteStorageClient{})

func (s *s3RemoteStorageClient) Traverse(remote *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {

	pathKey := remote.Path[1:]

	listInput := &s3.ListObjectsV2Input{
		Bucket:              aws.String(remote.Bucket),
		ContinuationToken:   nil,
		Delimiter:           nil, // not aws.String("/"), iterate through all entries
		EncodingType:        nil,
		ExpectedBucketOwner: nil,
		FetchOwner:          nil,
		MaxKeys:             nil, // aws.Int64(1000),
		Prefix:              aws.String(pathKey),
		RequestPayer:        nil,
		StartAfter:          nil,
	}
	isLastPage := false
	for !isLastPage && err == nil {
		var localErr error
		listErr := s.conn.ListObjectsV2Pages(listInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
			for _, content := range page.Contents {
				key := *content.Key
				key = "/" + key
				dir, name := util.FullPath(key).DirAndName()
				if err := visitFn(dir, name, false, &filer_pb.RemoteEntry{
					RemoteMtime: (*content.LastModified).Unix(),
					RemoteSize:  *content.Size,
					RemoteETag:  *content.ETag,
					StorageName: s.conf.Name,
				}); err != nil {
					localErr = err
					return false
				}
			}
			listInput.ContinuationToken = page.NextContinuationToken
			isLastPage = lastPage
			return true
		})
		if listErr != nil {
			err = fmt.Errorf("list %v: %v", remote, listErr)
		}
		if localErr != nil {
			err = fmt.Errorf("process %v: %v", remote, localErr)
		}
	}
	return
}
func (s *s3RemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
	downloader := s3manager.NewDownloaderWithClient(s.conn, func(u *s3manager.Downloader) {
		u.PartSize = int64(4 * 1024 * 1024)
		u.Concurrency = 1
	})

	dataSlice := make([]byte, int(size))
	writerAt := aws.NewWriteAtBuffer(dataSlice)

	_, err = downloader.Download(writerAt, &s3.GetObjectInput{
		Bucket: aws.String(loc.Bucket),
		Key:    aws.String(loc.Path[1:]),
		Range:  aws.String(fmt.Sprintf("bytes=%d-%d", offset, offset+size-1)),
	})
	if err != nil {
		return nil, fmt.Errorf("failed to download file %s%s: %v", loc.Bucket, loc.Path, err)
	}

	return writerAt.Bytes(), nil
}

func (s *s3RemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) {
	return nil
}

func (s *s3RemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) {
	return nil
}

func (s *s3RemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) {

	fileSize := int64(filer.FileSize(entry))

	partSize := int64(8 * 1024 * 1024) // The minimum/default allowed part size is 5MB
	for partSize*1000 < fileSize {
		partSize *= 4
	}

	// Create an uploader with the session and custom options
	uploader := s3manager.NewUploaderWithClient(s.conn, func(u *s3manager.Uploader) {
		u.PartSize = partSize
		u.Concurrency = 1
	})

	// process tagging
	tags := ""
	for k, v := range entry.Extended {
		if len(tags) > 0 {
			tags = tags + "&"
		}
		tags = tags + k + "=" + string(v)
	}

	// Upload the file to S3.
	_, err = uploader.Upload(&s3manager.UploadInput{
		Bucket:       aws.String(loc.Bucket),
		Key:          aws.String(loc.Path[1:]),
		Body:         reader,
		Tagging:      aws.String(tags),
		StorageClass: aws.String(s.conf.S3StorageClass),
	})

	//in case it fails to upload
	if err != nil {
		return nil, fmt.Errorf("upload to %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
	}

	// read back the remote entry
	return s.readFileRemoteEntry(loc)

}

func toTagging(attributes map[string][]byte) *s3.Tagging {
	tagging := &s3.Tagging{}
	for k, v := range attributes {
		tagging.TagSet = append(tagging.TagSet, &s3.Tag{
			Key:   aws.String(k),
			Value: aws.String(string(v)),
		})
	}
	return tagging
}

func (s *s3RemoteStorageClient) readFileRemoteEntry(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
	resp, err := s.conn.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(loc.Bucket),
		Key:    aws.String(loc.Path[1:]),
	})
	if err != nil {
		return nil, err
	}

	return &filer_pb.RemoteEntry{
		RemoteMtime: resp.LastModified.Unix(),
		RemoteSize:  *resp.ContentLength,
		RemoteETag:  *resp.ETag,
		StorageName: s.conf.Name,
	}, nil

}

func (s *s3RemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error) {
	if reflect.DeepEqual(oldEntry.Extended, newEntry.Extended) {
		return nil
	}
	tagging := toTagging(newEntry.Extended)
	if len(tagging.TagSet) > 0 {
		_, err = s.conn.PutObjectTagging(&s3.PutObjectTaggingInput{
			Bucket:  aws.String(loc.Bucket),
			Key:     aws.String(loc.Path[1:]),
			Tagging: toTagging(newEntry.Extended),
		})
	} else {
		_, err = s.conn.DeleteObjectTagging(&s3.DeleteObjectTaggingInput{
			Bucket: aws.String(loc.Bucket),
			Key:    aws.String(loc.Path[1:]),
		})
	}
	return
}
func (s *s3RemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) {
	_, err = s.conn.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(loc.Bucket),
		Key:    aws.String(loc.Path[1:]),
	})
	return
}

func (s *s3RemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) {
	resp, err := s.conn.ListBuckets(&s3.ListBucketsInput{})
	if err != nil {
		return nil, fmt.Errorf("list buckets: %v", err)
	}
	for _, b := range resp.Buckets {
		buckets = append(buckets, &remote_storage.Bucket{
			Name:      *b.Name,
			CreatedAt: *b.CreationDate,
		})
	}
	return
}

func (s *s3RemoteStorageClient) CreateBucket(name string) (err error) {
	_, err = s.conn.CreateBucket(&s3.CreateBucketInput{
		ACL:                        nil,
		Bucket:                     aws.String(name),
		CreateBucketConfiguration:  nil,
		GrantFullControl:           nil,
		GrantRead:                  nil,
		GrantReadACP:               nil,
		GrantWrite:                 nil,
		GrantWriteACP:              nil,
		ObjectLockEnabledForBucket: nil,
	})
	if err != nil {
		return fmt.Errorf("%s create bucket %s: %v", s.conf.Name, name, err)
	}
	return
}

func (s *s3RemoteStorageClient) DeleteBucket(name string) (err error) {
	_, err = s.conn.DeleteBucket(&s3.DeleteBucketInput{
		Bucket: aws.String(name),
	})
	if err != nil {
		return fmt.Errorf("delete bucket %s: %v", name, err)
	}
	return
}
<|endoftext|>


<commit_before>package rye

import (
	"context"
	"encoding/json"
	"errors"
	"net/http"
	"reflect"
	"runtime"
	"strconv"
	"strings"
	"time"

	//log "github.com/Sirupsen/logrus"
	"github.com/cactus/go-statsd-client/statsd"
)

//go:generate counterfeiter -o fakes/statsdfakes/fake_statter.go $GOPATH/src/github.com/cactus/go-statsd-client/statsd/client.go Statter
//go:generate perl -pi -e 's/$GOPATH\/src\///g' fakes/statsdfakes/fake_statter.go

// MWHandler struct is used to configure and access rye's basic functionality.
type MWHandler struct {
	Config         Config
	beforeHandlers []Handler
}

// Config struct allows you to set a reference to a statsd.Statter and include it's stats rate.
type Config struct {
	Statter  statsd.Statter
	StatRate float32
}

// JSONStatus is a simple container used for conveying status messages.
type JSONStatus struct {
	Message string `json:"message"`
	Status  string `json:"status"`
}

// Response struct is utilized by middlewares as a way to share state;
// ie. a middleware can return a *Response as a way to indicate
// that further middleware execution should stop (without an error) or return a
// a hard error by setting `Err` + `StatusCode`.
type Response struct {
	Err           error
	StatusCode    int
	StopExecution bool
	Context       context.Context
}

// Error bubbles a response error providing an implementation of the Error interface.
// It returns the error as a string.
func (r *Response) Error() string {
	return r.Err.Error()
}

// Handler is the primary type that any rye middleware must implement to be called in the Handle() function.
// In order to use this you must return a *rye.Response.
type Handler func(w http.ResponseWriter, r *http.Request) *Response

// Constructor for new instantiating new rye instances
// It returns a constructed *MWHandler instance.
func NewMWHandler(config Config) *MWHandler {
	return &MWHandler{
		Config: config,
	}
}

// Use adds a handler to every request. All handlers set up with use
// are fired first and then any route specific handlers are called
func (m *MWHandler) Use(handler Handler) {
	m.beforeHandlers = append(m.beforeHandlers, handler)
}

// The Handle function is the primary way to set up your chain of middlewares to be called by rye.
// It returns a http.HandlerFunc from net/http that can be set as a route in your http server.
func (m *MWHandler) Handle(handlers []Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		handlers = append(m.beforeHandlers, handlers...)
		for _, handler := range handlers {
			var resp *Response

			// Record handler runtime
			func() {
				statusCode := "2xx"
				startTime := time.Now()

				if resp = handler(w, r); resp != nil {
					func() {
						// Stop execution if it's passed
						if resp.StopExecution {
							return
						}

						// If a context is returned, we will
						// replace the current request with a new request
						if resp.Context != nil {
							r = r.WithContext(resp.Context)
							return
						}

						// If there's no error but we have a response
						if resp.Err == nil {
							resp.Err = errors.New("Problem with middleware; neither Err or StopExecution is set")
							resp.StatusCode = http.StatusInternalServerError
						}

						// Now assume we have an error.
						if m.Config.Statter != nil && resp.StatusCode >= 500 {
							go m.Config.Statter.Inc("errors", 1, m.Config.StatRate)
						}

						// Write the error out
						statusCode = strconv.Itoa(resp.StatusCode)
						WriteJSONStatus(w, "error", resp.Error(), resp.StatusCode)
					}()
				}

				handlerName := getFuncName(handler)

				if m.Config.Statter != nil {
					// Record runtime metric
					go m.Config.Statter.TimingDuration(
						"handlers."+handlerName+".runtime",
						time.Since(startTime), // delta
						m.Config.StatRate,
					)

					// Record status code metric (default 2xx)
					go m.Config.Statter.Inc(
						"handlers."+handlerName+"."+statusCode,
						1,
						m.Config.StatRate,
					)
				}
			}()

			// stop executing rest of the
			// handlers if we encounter an error
			if resp != nil && (resp.StopExecution || resp.Err != nil) {
				return
			}
		}
	})
}

// WriteJSONStatus is a wrapper for WriteJSONResponse that returns a marshalled JSONStatus blob
func WriteJSONStatus(rw http.ResponseWriter, status, message string, statusCode int) {
	jsonData, _ := json.Marshal(&JSONStatus{
		Message: message,
		Status:  status,
	})

	WriteJSONResponse(rw, statusCode, jsonData)
}

// WriteJSONResponse writes data and status code to the ResponseWriter
func WriteJSONResponse(rw http.ResponseWriter, statusCode int, content []byte) {
	rw.Header().Set("Content-Type", "application/json")
	rw.WriteHeader(statusCode)
	rw.Write(content)
}

// getFuncName uses reflection to determine a given function name
// It returns a string version of the function name (and performs string cleanup)
func getFuncName(i interface{}) string {
	fullName := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
	ns := strings.Split(fullName, ".")

	// when we get a method (not a raw function) it comes attached to whatever struct is in its
	// method receiver via a function closure, this is not precisely the same as that method itself
	// so the compiler appends "-fm" so the name of the closure does not conflict with the actual function
	// http://grokbase.com/t/gg/golang-nuts/153jyb5b7p/go-nuts-fm-suffix-in-function-name-what-does-it-mean#20150318ssinqqzrmhx2ep45wjkxsa4rua
	return strings.TrimSuffix(ns[len(ns)-1], ")-fm")
}
<commit_msg>Fixing issue where .Use duplicates handlers<commit_after>package rye

import (
	"context"
	"encoding/json"
	"errors"
	"net/http"
	"reflect"
	"runtime"
	"strconv"
	"strings"
	"time"

	//log "github.com/Sirupsen/logrus"
	"github.com/cactus/go-statsd-client/statsd"
)

//go:generate counterfeiter -o fakes/statsdfakes/fake_statter.go $GOPATH/src/github.com/cactus/go-statsd-client/statsd/client.go Statter
//go:generate perl -pi -e 's/$GOPATH\/src\///g' fakes/statsdfakes/fake_statter.go

// MWHandler struct is used to configure and access rye's basic functionality.
type MWHandler struct {
	Config         Config
	beforeHandlers []Handler
}

// Config struct allows you to set a reference to a statsd.Statter and include it's stats rate.
type Config struct {
	Statter  statsd.Statter
	StatRate float32
}

// JSONStatus is a simple container used for conveying status messages.
type JSONStatus struct {
	Message string `json:"message"`
	Status  string `json:"status"`
}

// Response struct is utilized by middlewares as a way to share state;
// ie. a middleware can return a *Response as a way to indicate
// that further middleware execution should stop (without an error) or return a
// a hard error by setting `Err` + `StatusCode`.
type Response struct {
	Err           error
	StatusCode    int
	StopExecution bool
	Context       context.Context
}

// Error bubbles a response error providing an implementation of the Error interface.
// It returns the error as a string.
func (r *Response) Error() string {
	return r.Err.Error()
}

// Handler is the primary type that any rye middleware must implement to be called in the Handle() function.
// In order to use this you must return a *rye.Response.
type Handler func(w http.ResponseWriter, r *http.Request) *Response

// Constructor for new instantiating new rye instances
// It returns a constructed *MWHandler instance.
func NewMWHandler(config Config) *MWHandler {
	return &MWHandler{
		Config: config,
	}
}

// Use adds a handler to every request. All handlers set up with use
// are fired first and then any route specific handlers are called
func (m *MWHandler) Use(handler Handler) {
	m.beforeHandlers = append(m.beforeHandlers, handler)
}

// The Handle function is the primary way to set up your chain of middlewares to be called by rye.
// It returns a http.HandlerFunc from net/http that can be set as a route in your http server.
func (m *MWHandler) Handle(handlers []Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		handlers := append(m.beforeHandlers, handlers...)
		for _, handler := range handlers {
			var resp *Response

			// Record handler runtime
			func() {
				statusCode := "2xx"
				startTime := time.Now()

				if resp = handler(w, r); resp != nil {
					func() {
						// Stop execution if it's passed
						if resp.StopExecution {
							return
						}

						// If a context is returned, we will
						// replace the current request with a new request
						if resp.Context != nil {
							r = r.WithContext(resp.Context)
							return
						}

						// If there's no error but we have a response
						if resp.Err == nil {
							resp.Err = errors.New("Problem with middleware; neither Err or StopExecution is set")
							resp.StatusCode = http.StatusInternalServerError
						}

						// Now assume we have an error.
						if m.Config.Statter != nil && resp.StatusCode >= 500 {
							go m.Config.Statter.Inc("errors", 1, m.Config.StatRate)
						}

						// Write the error out
						statusCode = strconv.Itoa(resp.StatusCode)
						WriteJSONStatus(w, "error", resp.Error(), resp.StatusCode)
					}()
				}

				handlerName := getFuncName(handler)

				if m.Config.Statter != nil {
					// Record runtime metric
					go m.Config.Statter.TimingDuration(
						"handlers."+handlerName+".runtime",
						time.Since(startTime), // delta
						m.Config.StatRate,
					)

					// Record status code metric (default 2xx)
					go m.Config.Statter.Inc(
						"handlers."+handlerName+"."+statusCode,
						1,
						m.Config.StatRate,
					)
				}
			}()

			// stop executing rest of the
			// handlers if we encounter an error
			if resp != nil && (resp.StopExecution || resp.Err != nil) {
				return
			}
		}
	})
}

// WriteJSONStatus is a wrapper for WriteJSONResponse that returns a marshalled JSONStatus blob
func WriteJSONStatus(rw http.ResponseWriter, status, message string, statusCode int) {
	jsonData, _ := json.Marshal(&JSONStatus{
		Message: message,
		Status:  status,
	})

	WriteJSONResponse(rw, statusCode, jsonData)
}

// WriteJSONResponse writes data and status code to the ResponseWriter
func WriteJSONResponse(rw http.ResponseWriter, statusCode int, content []byte) {
	rw.Header().Set("Content-Type", "application/json")
	rw.WriteHeader(statusCode)
	rw.Write(content)
}

// getFuncName uses reflection to determine a given function name
// It returns a string version of the function name (and performs string cleanup)
func getFuncName(i interface{}) string {
	fullName := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
	ns := strings.Split(fullName, ".")

	// when we get a method (not a raw function) it comes attached to whatever struct is in its
	// method receiver via a function closure, this is not precisely the same as that method itself
	// so the compiler appends "-fm" so the name of the closure does not conflict with the actual function
	// http://grokbase.com/t/gg/golang-nuts/153jyb5b7p/go-nuts-fm-suffix-in-function-name-what-does-it-mean#20150318ssinqqzrmhx2ep45wjkxsa4rua
	return strings.TrimSuffix(ns[len(ns)-1], ")-fm")
}
<|endoftext|>