<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate protoc -I ./model/cni --go_out=plugins=grpc:./model/cni ./model/cni/cni.proto
//go:generate protoc -I ./model/nodeID --go_out=plugins=grpc:./model/nodeID ./model/nodeID/nodeID.proto

package contiv

import (
	"context"
	"fmt"
	"net"

	"git.fd.io/govpp.git/api"
	"github.com/contiv/vpp/plugins/contiv/containeridx"
	"github.com/contiv/vpp/plugins/contiv/ipam"
	"github.com/contiv/vpp/plugins/contiv/model/cni"
	"github.com/contiv/vpp/plugins/kvdbproxy"
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/datasync/resync"
	"github.com/ligato/cn-infra/db/keyval/etcdv3"
	"github.com/ligato/cn-infra/flavors/local"
	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/cn-infra/rpc/grpc"
	"github.com/ligato/cn-infra/utils/safeclose"
	"github.com/ligato/vpp-agent/clientv1/linux"
	linuxlocalclient "github.com/ligato/vpp-agent/clientv1/linux/localclient"
	"github.com/ligato/vpp-agent/plugins/defaultplugins"
	"github.com/ligato/vpp-agent/plugins/govppmux"
)

// Plugin transforms GRPC requests into configuration for the VPP in order
// to connect a container into the network.
type Plugin struct {
	Deps
	govppCh *api.Channel

	configuredContainers *containeridx.ConfigIndex
	cniServer            *remoteCNIserver

	nodeIDAllocator   *idAllocator
	nodeIDsresyncChan chan datasync.ResyncEvent
	nodeIDSchangeChan chan datasync.ChangeEvent
	nodeIDwatchReg    datasync.WatchRegistration

	ctx           context.Context
	ctxCancelFunc context.CancelFunc

	Config *Config
}

// Deps groups the dependencies of the Plugin.
type Deps struct {
	local.PluginInfraDeps
	GRPC    grpc.Server
	Proxy   *kvdbproxy.Plugin
	VPP     *defaultplugins.Plugin
	GoVPP   govppmux.API
	Resync  resync.Subscriber
	ETCD    *etcdv3.Plugin
	Watcher datasync.KeyValProtoWatcher
}

// Config represents configuration for Contiv plugin.
// It can be injected or loaded from external config file. Injection has priority to external config. To use external
// config file, add `-contiv-config="<path to config>` argument when running the contiv-agent.
type Config struct {
	TCPChecksumOffloadDisabled bool
	TCPstackDisabled           bool
	UseTAPInterfaces           bool
	TAPInterfaceVersion        uint8
	TAPv2RxRingSize            uint16
	TAPv2TxRingSize            uint16
	IPAMConfig                 ipam.Config
	NodeConfig                 []OneNodeConfig
}

// OneNodeConfig represents configuration for one node. It contains only settings specific to given node.
type OneNodeConfig struct {
	NodeName             string
	MainVppInterfaceName string
	OtherVPPInterfaces   []InterfaceWithIP // other configured interfaces get only ip address assigned in vpp
}

// InterfaceWithIP binds interface name with ip for configuration purposes.
type InterfaceWithIP struct {
	InterfaceName string
	IP            string
}

// Init initializes the grpc server handling the request from the CNI.
func (plugin *Plugin) Init() error {
	plugin.configuredContainers = containeridx.NewConfigIndex(plugin.Log, plugin.PluginName, "containers")

	plugin.ctx, plugin.ctxCancelFunc = context.WithCancel(context.Background())
	if plugin.Config == nil {
		if err := plugin.applyExternalConfig(); err != nil {
			return err
		}
	}

	var err error
	plugin.govppCh, err = plugin.GoVPP.NewAPIChannel()
	if err != nil {
		return err
	}

	plugin.nodeIDAllocator = newIDAllocator(plugin.ETCD, plugin.ServiceLabel.GetAgentLabel())
	nodeID, err := plugin.nodeIDAllocator.getID()
	if err != nil {
		return err
	}
	plugin.Log.Infof("ID of the node is %v", nodeID)

	plugin.nodeIDsresyncChan = make(chan datasync.ResyncEvent)
	plugin.nodeIDSchangeChan = make(chan datasync.ChangeEvent)

	plugin.nodeIDwatchReg, err = plugin.Watcher.Watch("contiv-plugin", plugin.nodeIDSchangeChan, plugin.nodeIDsresyncChan, allocatedIDsKeyPrefix)
	if err != nil {
		return err
	}

	plugin.cniServer, err = newRemoteCNIServer(plugin.Log,
		func() linux.DataChangeDSL {
			return linuxlocalclient.DataChangeRequest(plugin.PluginName)
		},
		plugin.Proxy,
		plugin.configuredContainers,
		plugin.govppCh,
		plugin.VPP.GetSwIfIndexes(),
		plugin.ServiceLabel.GetAgentLabel(),
		plugin.Config,
		nodeID)
	if err != nil {
		return fmt.Errorf("Can't create new remote CNI server due to error: %v ", err)
	}
	cni.RegisterRemoteCNIServer(plugin.GRPC.Server(), plugin.cniServer)

	go plugin.cniServer.handleNodeEvents(plugin.ctx, plugin.nodeIDsresyncChan, plugin.nodeIDSchangeChan)

	return nil
}

func (plugin *Plugin) applyExternalConfig() error {
	externalCfg := &Config{}
	found, err := plugin.PluginConfig.GetValue(externalCfg) // It tries to lookup `PluginName + "-config"` in go run command flags.
	if err != nil {
		return fmt.Errorf("External Contiv plugin configuration could not load or other problem happened: %v", err)
	}
	if !found {
		return fmt.Errorf("External Contiv plugin configuration was not found")
	}
	plugin.Config = externalCfg
	return nil
}

// AfterInit registers to the ResyncOrchestrator. The registration is done in this phase
// in order to trigger the resync for this plugin once the resync of VPP plugins is finished.
func (plugin *Plugin) AfterInit() error {
	if plugin.Resync != nil {
		reg := plugin.Resync.Register(string(plugin.PluginName))
		go plugin.handleResync(reg.StatusChan())
	}
	return nil
}

// Close cleans up the resources allocated by the plugin
func (plugin *Plugin) Close() error {
	plugin.ctxCancelFunc()
	plugin.cniServer.close()
	plugin.nodeIDAllocator.releaseID()
	_, err := safeclose.CloseAll(plugin.govppCh, plugin.nodeIDwatchReg)
	return err
}

// getContainerConfig return the configuration of the container associated
// with the given pod.
func (plugin *Plugin) getContainerConfig(podNamespace string, podName string) *containeridx.Config {
	podNamesMatch := plugin.configuredContainers.LookupPodName(podName)
	podNamespacesMatch := plugin.configuredContainers.LookupPodNamespace(podNamespace)

	for _, pod1 := range podNamespacesMatch {
		for _, pod2 := range podNamesMatch {
			if pod1 == pod2 {
				data, found := plugin.configuredContainers.LookupContainer(pod1)
				if found {
					return data
				}
			}
		}
	}

	return nil
}

// GetIfName looks up logical interface name that corresponds to the interface associated with the given pod.
func (plugin *Plugin) GetIfName(podNamespace string, podName string) (name string, exists bool) {
	config := plugin.getContainerConfig(podNamespace, podName)
	if config != nil && config.VppIf != nil {
		return config.VppIf.Name, true
	}
	plugin.Log.WithFields(logging.Fields{"podNamespace": podNamespace, "podName": podName}).Warn("No matching result found")
	return "", false
}

// GetNsIndex returns the index of the VPP session namespace associated
// with the given pod.
func (plugin *Plugin) GetNsIndex(podNamespace string, podName string) (nsIndex uint32, exists bool) {
	config := plugin.getContainerConfig(podNamespace, podName)
	if config != nil {
		nsIndex, _, exists = plugin.VPP.GetAppNsIndexes().LookupIdx(config.AppNamespace.NamespaceId)
		return nsIndex, exists
	}
	plugin.Log.WithFields(logging.Fields{"podNamespace": podNamespace, "podName": podName}).Warn("No matching result found")
	return 0, false
}

// GetPodNetwork provides subnet used for allocating pod IP addresses on this host node.
func (plugin *Plugin) GetPodNetwork() *net.IPNet {
	return plugin.cniServer.ipam.PodNetwork()
}

// IsTCPstackDisabled returns true if the tcp stack is disabled and only VETHs/TAPs are configured
func (plugin *Plugin) IsTCPstackDisabled() bool {
	return plugin.Config.TCPstackDisabled
}

func (plugin *Plugin) handleResync(resyncChan chan resync.StatusEvent) {
	for {
		select {
		case ev := <-resyncChan:
			status := ev.ResyncStatus()
			if status == resync.Started {
				err := plugin.cniServer.resync()
				if err != nil {
					plugin.Log.Error(err)
				}
			}
			ev.Ack()
		case <-plugin.ctx.Done():
			return
		}
	}
}
<commit_msg>fixed go:generate rules<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate protoc -I ./model/cni --go_out=plugins=grpc:./model/cni ./model/cni/cni.proto
//go:generate protoc -I ./model/uid --go_out=plugins=grpc:./model/uid ./model/uid/uid.proto

package contiv

import (
	"context"
	"fmt"
	"net"

	"git.fd.io/govpp.git/api"
	"github.com/contiv/vpp/plugins/contiv/containeridx"
	"github.com/contiv/vpp/plugins/contiv/ipam"
	"github.com/contiv/vpp/plugins/contiv/model/cni"
	"github.com/contiv/vpp/plugins/kvdbproxy"
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/datasync/resync"
	"github.com/ligato/cn-infra/db/keyval/etcdv3"
	"github.com/ligato/cn-infra/flavors/local"
	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/cn-infra/rpc/grpc"
	"github.com/ligato/cn-infra/utils/safeclose"
	"github.com/ligato/vpp-agent/clientv1/linux"
	linuxlocalclient "github.com/ligato/vpp-agent/clientv1/linux/localclient"
	"github.com/ligato/vpp-agent/plugins/defaultplugins"
	"github.com/ligato/vpp-agent/plugins/govppmux"
)

// Plugin transforms GRPC requests into configuration for the VPP in order
// to connect a container into the network.
type Plugin struct {
	Deps
	govppCh *api.Channel

	configuredContainers *containeridx.ConfigIndex
	cniServer            *remoteCNIserver

	nodeIDAllocator   *idAllocator
	nodeIDsresyncChan chan datasync.ResyncEvent
	nodeIDSchangeChan chan datasync.ChangeEvent
	nodeIDwatchReg    datasync.WatchRegistration

	ctx           context.Context
	ctxCancelFunc context.CancelFunc

	Config *Config
}

// Deps groups the dependencies of the Plugin.
type Deps struct {
	local.PluginInfraDeps
	GRPC    grpc.Server
	Proxy   *kvdbproxy.Plugin
	VPP     *defaultplugins.Plugin
	GoVPP   govppmux.API
	Resync  resync.Subscriber
	ETCD    *etcdv3.Plugin
	Watcher datasync.KeyValProtoWatcher
}

// Config represents configuration for Contiv plugin.
// It can be injected or loaded from external config file. Injection has priority to external config. To use external
// config file, add `-contiv-config="<path to config>` argument when running the contiv-agent.
type Config struct {
	TCPChecksumOffloadDisabled bool
	TCPstackDisabled           bool
	UseTAPInterfaces           bool
	TAPInterfaceVersion        uint8
	TAPv2RxRingSize            uint16
	TAPv2TxRingSize            uint16
	IPAMConfig                 ipam.Config
	NodeConfig                 []OneNodeConfig
}

// OneNodeConfig represents configuration for one node. It contains only settings specific to given node.
type OneNodeConfig struct {
	NodeName             string
	MainVppInterfaceName string
	OtherVPPInterfaces   []InterfaceWithIP // other configured interfaces get only ip address assigned in vpp
}

// InterfaceWithIP binds interface name with ip for configuration purposes.
type InterfaceWithIP struct {
	InterfaceName string
	IP            string
}

// Init initializes the grpc server handling the request from the CNI.
func (plugin *Plugin) Init() error {
	plugin.configuredContainers = containeridx.NewConfigIndex(plugin.Log, plugin.PluginName, "containers")

	plugin.ctx, plugin.ctxCancelFunc = context.WithCancel(context.Background())
	if plugin.Config == nil {
		if err := plugin.applyExternalConfig(); err != nil {
			return err
		}
	}

	var err error
	plugin.govppCh, err = plugin.GoVPP.NewAPIChannel()
	if err != nil {
		return err
	}

	plugin.nodeIDAllocator = newIDAllocator(plugin.ETCD, plugin.ServiceLabel.GetAgentLabel())
	nodeID, err := plugin.nodeIDAllocator.getID()
	if err != nil {
		return err
	}
	plugin.Log.Infof("ID of the node is %v", nodeID)

	plugin.nodeIDsresyncChan = make(chan datasync.ResyncEvent)
	plugin.nodeIDSchangeChan = make(chan datasync.ChangeEvent)

	plugin.nodeIDwatchReg, err = plugin.Watcher.Watch("contiv-plugin", plugin.nodeIDSchangeChan, plugin.nodeIDsresyncChan, allocatedIDsKeyPrefix)
	if err != nil {
		return err
	}

	plugin.cniServer, err = newRemoteCNIServer(plugin.Log,
		func() linux.DataChangeDSL {
			return linuxlocalclient.DataChangeRequest(plugin.PluginName)
		},
		plugin.Proxy,
		plugin.configuredContainers,
		plugin.govppCh,
		plugin.VPP.GetSwIfIndexes(),
		plugin.ServiceLabel.GetAgentLabel(),
		plugin.Config,
		nodeID)
	if err != nil {
		return fmt.Errorf("Can't create new remote CNI server due to error: %v ", err)
	}
	cni.RegisterRemoteCNIServer(plugin.GRPC.Server(), plugin.cniServer)

	go plugin.cniServer.handleNodeEvents(plugin.ctx, plugin.nodeIDsresyncChan, plugin.nodeIDSchangeChan)

	return nil
}

func (plugin *Plugin) applyExternalConfig() error {
	externalCfg := &Config{}
	found, err := plugin.PluginConfig.GetValue(externalCfg) // It tries to lookup `PluginName + "-config"` in go run command flags.
	if err != nil {
		return fmt.Errorf("External Contiv plugin configuration could not load or other problem happened: %v", err)
	}
	if !found {
		return fmt.Errorf("External Contiv plugin configuration was not found")
	}
	plugin.Config = externalCfg
	return nil
}

// AfterInit registers to the ResyncOrchestrator. The registration is done in this phase
// in order to trigger the resync for this plugin once the resync of VPP plugins is finished.
func (plugin *Plugin) AfterInit() error {
	if plugin.Resync != nil {
		reg := plugin.Resync.Register(string(plugin.PluginName))
		go plugin.handleResync(reg.StatusChan())
	}
	return nil
}

// Close cleans up the resources allocated by the plugin
func (plugin *Plugin) Close() error {
	plugin.ctxCancelFunc()
	plugin.cniServer.close()
	plugin.nodeIDAllocator.releaseID()
	_, err := safeclose.CloseAll(plugin.govppCh, plugin.nodeIDwatchReg)
	return err
}

// getContainerConfig return the configuration of the container associated
// with the given pod.
func (plugin *Plugin) getContainerConfig(podNamespace string, podName string) *containeridx.Config {
	podNamesMatch := plugin.configuredContainers.LookupPodName(podName)
	podNamespacesMatch := plugin.configuredContainers.LookupPodNamespace(podNamespace)

	for _, pod1 := range podNamespacesMatch {
		for _, pod2 := range podNamesMatch {
			if pod1 == pod2 {
				data, found := plugin.configuredContainers.LookupContainer(pod1)
				if found {
					return data
				}
			}
		}
	}

	return nil
}

// GetIfName looks up logical interface name that corresponds to the interface associated with the given pod.
func (plugin *Plugin) GetIfName(podNamespace string, podName string) (name string, exists bool) {
	config := plugin.getContainerConfig(podNamespace, podName)
	if config != nil && config.VppIf != nil {
		return config.VppIf.Name, true
	}
	plugin.Log.WithFields(logging.Fields{"podNamespace": podNamespace, "podName": podName}).Warn("No matching result found")
	return "", false
}

// GetNsIndex returns the index of the VPP session namespace associated
// with the given pod.
func (plugin *Plugin) GetNsIndex(podNamespace string, podName string) (nsIndex uint32, exists bool) {
	config := plugin.getContainerConfig(podNamespace, podName)
	if config != nil {
		nsIndex, _, exists = plugin.VPP.GetAppNsIndexes().LookupIdx(config.AppNamespace.NamespaceId)
		return nsIndex, exists
	}
	plugin.Log.WithFields(logging.Fields{"podNamespace": podNamespace, "podName": podName}).Warn("No matching result found")
	return 0, false
}

// GetPodNetwork provides subnet used for allocating pod IP addresses on this host node.
func (plugin *Plugin) GetPodNetwork() *net.IPNet {
	return plugin.cniServer.ipam.PodNetwork()
}

// IsTCPstackDisabled returns true if the tcp stack is disabled and only VETHs/TAPs are configured
func (plugin *Plugin) IsTCPstackDisabled() bool {
	return plugin.Config.TCPstackDisabled
}

func (plugin *Plugin) handleResync(resyncChan chan resync.StatusEvent) {
	for {
		select {
		case ev := <-resyncChan:
			status := ev.ResyncStatus()
			if status == resync.Started {
				err := plugin.cniServer.resync()
				if err != nil {
					plugin.Log.Error(err)
				}
			}
			ev.Ack()
		case <-plugin.ctx.Done():
			return
		}
	}
}
<|endoftext|>


<commit_before>package myslave

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/funkygao/dbus/model"
	log "github.com/funkygao/log4go"
	uuid "github.com/satori/go.uuid"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
)

// TODO graceful shutdown
// TODO GTID
func (m *MySlave) StartReplication(ready chan struct{}) {
	m.rowsEvent = make(chan *model.RowsEvent, m.c.Int("event_buffer_len", 100))
	m.errors = make(chan error, 1)

	m.r = replication.NewBinlogSyncer(&replication.BinlogSyncerConfig{
		ServerID:        uint32(m.c.Int("server_id", 137)), // 137 unique enough?
		Flavor:          "mysql",                           // currently mariadb not implemented
		Host:            m.host,
		Port:            m.port,
		User:            m.c.String("user", "root"),
		Password:        m.c.String("password", ""),
		SemiSyncEnabled: false,
	})

	file, offset, err := m.p.Committed()
	if err != nil {
		close(ready)
		m.emitFatalError(err)
		return
	}

	var syncer *replication.BinlogStreamer
	if m.GTID {
		// SELECT @@gtid_mode
		// SHOW GLOBAL VARIABLES LIKE 'SERVER_UUID'
		var masterUuid uuid.UUID
		// 07c93cd7-a7d3-12a5-94e1-a0369a7c3790:1-313225133
		set, _ := mysql.ParseMysqlGTIDSet(fmt.Sprintf("%s:%d-%d", masterUuid.String(), 1, 2))
		syncer, err = m.r.StartSyncGTID(set)
		panic("not implemented") // TODO
	} else {
		syncer, err = m.r.StartSync(mysql.Position{
			Name: file,
			Pos:  offset,
		})
	}

	if err != nil {
		// unrecoverable err encountered
		// e,g.
		// ERROR 1045 (28000): Access denied for user 'xx'@'1.1.1.1'
		close(ready)
		m.emitFatalError(err)
		return
	}

	close(ready)
	log.Trace("[%s] ready to receive binlog stream", m.masterAddr)

	timeout := time.Second
	maxTimeout := time.Minute
	for {
		// what if the conn broken?
		// 2017/02/08 08:31:39 binlogsyncer.go:529: [info] receive EOF packet, retry ReadPacket
		// 2017/02/08 08:31:39 binlogsyncer.go:484: [error] connection was bad
		// 2017/02/08 08:31:40 binlogsyncer.go:446: [info] begin to re-sync from (mysql.000003, 2492)
		// 2017/02/08 08:31:40 binlogsyncer.go:130: [info] register slave for master server localhost:3306
		// 2017/02/08 08:31:40 binlogsyncer.go:502: [error] retry sync err: dial tcp [::1]:3306: getsockopt: connection refused, wait 1s and retry again
		// 2017/02/08 08:31:42 binlogsyncer.go:446: [info] begin to re-sync from (mysql.000003, 2492)
		// 2017/02/08 08:31:42 binlogsyncer.go:130: [info] register slave for master server localhost:3306
		// 2017/02/08 08:31:42 binlogsyncer.go:502: [error] retry sync err: dial tcp [::1]:3306: getsockopt: connection refused, wait 1s and retry again
		//
		// or invalid position sent?
		// 2017/02/08 08:42:09 binlogstreamer.go:47: [error] close sync with err: ERROR 1236 (HY000): Client requested master to start replication from position > file size; the first event 'mysql.000004' at 2274, the last event read from '/usr/local/var/mysql.000004' at 4, the last byte read from '/usr/local/var/mysql.000004' at 4.
		//
		// [02/08/17 17:39:20] [EROR] ( mysqlbinlog.go:64) backoff 5s: invalid table id 2413, no correspond table map event
		ctx, cancel := context.WithTimeout(context.Background(), timeout)
		ev, err := syncer.GetEvent(ctx)
		cancel()

		if err != nil {
			if err == context.DeadlineExceeded {
				if timeout < maxTimeout {
					// backoff
					timeout *= 2
				}

				continue
			}

			if strings.HasPrefix(err.Error(), "invalid table id") {
				log.Error("[%s] %s", m.masterAddr, err)
				// TODO how to handle this?
				continue
			}

			// TODO try out all the cases
			m.emitFatalError(err)
			return
		}

		m.m.Events.Mark(1)

		log.Debug("-> %T", ev.Event)
		switch e := ev.Event.(type) {
		case *replication.RotateEvent:
			// e,g.
			// Position: 4
			// Next log name: mysql.000002
			file = string(e.NextLogName)
			log.Trace("[%s] rotate to (%s, %d)", m.masterAddr, file, e.Position)

		case *replication.RowsEvent:
			m.m.TPS.Mark(1)
			m.m.Lag.Update(time.Now().Unix() - int64(ev.Header.Timestamp))
			m.handleRowsEvent(file, ev.Header, e)

		case *replication.QueryEvent:
			// DDL comes this way
			// e,g. create table y(id int)
			// e,g. BEGIN

		case *replication.XIDEvent:
			// e,g. xid: 1293

		case *replication.FormatDescriptionEvent:
			// Version: 4
			// Server version: 5.6.23-log
			// Checksum algorithm: 1

		case *replication.TableMapEvent:
			// e,g.
			// TableID: 170
			// TableID size: 6
			// Flags: 1
			// Schema: test
			// Table: y
			// Column count: 1
			// Column type:
			// 00000000  03
			// NULL bitmap:
			// 00000000  01

		case *replication.GenericEvent:
			// known misc types of event

		case *replication.GTIDEvent:
			if m.GTID {
				// TODO
			}

		default:
			log.Warn("[%s] unexpected event: %+v", m.masterAddr, e)
		}
	}

}

func (m *MySlave) emitFatalError(err error) {
	m.errors <- err
	if m.r != nil {
		m.r.Close()
	}
}
<commit_msg>dig into binlog events<commit_after>package myslave

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/funkygao/dbus/model"
	log "github.com/funkygao/log4go"
	uuid "github.com/satori/go.uuid"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
)

// TODO graceful shutdown
// TODO GTID
func (m *MySlave) StartReplication(ready chan struct{}) {
	m.rowsEvent = make(chan *model.RowsEvent, m.c.Int("event_buffer_len", 100))
	m.errors = make(chan error, 1)

	m.r = replication.NewBinlogSyncer(&replication.BinlogSyncerConfig{
		ServerID:        uint32(m.c.Int("server_id", 137)), // 137 unique enough?
		Flavor:          "mysql",                           // currently mariadb not implemented
		Host:            m.host,
		Port:            m.port,
		User:            m.c.String("user", "root"),
		Password:        m.c.String("password", ""),
		SemiSyncEnabled: false,
	})

	file, offset, err := m.p.Committed()
	if err != nil {
		close(ready)
		m.emitFatalError(err)
		return
	}

	var syncer *replication.BinlogStreamer
	if m.GTID {
		// SELECT @@gtid_mode
		// SHOW GLOBAL VARIABLES LIKE 'SERVER_UUID'
		var masterUuid uuid.UUID
		// 07c93cd7-a7d3-12a5-94e1-a0369a7c3790:1-313225133
		set, _ := mysql.ParseMysqlGTIDSet(fmt.Sprintf("%s:%d-%d", masterUuid.String(), 1, 2))
		syncer, err = m.r.StartSyncGTID(set)
		panic("not implemented") // TODO
	} else {
		syncer, err = m.r.StartSync(mysql.Position{
			Name: file,
			Pos:  offset,
		})
	}

	if err != nil {
		// unrecoverable err encountered
		// e,g.
		// ERROR 1045 (28000): Access denied for user 'xx'@'1.1.1.1'
		close(ready)
		m.emitFatalError(err)
		return
	}

	close(ready)
	log.Trace("[%s] ready to receive binlog stream", m.masterAddr)

	timeout := time.Second
	maxTimeout := time.Minute
	for {
		// what if the conn broken?
		// 2017/02/08 08:31:39 binlogsyncer.go:529: [info] receive EOF packet, retry ReadPacket
		// 2017/02/08 08:31:39 binlogsyncer.go:484: [error] connection was bad
		// 2017/02/08 08:31:40 binlogsyncer.go:446: [info] begin to re-sync from (mysql.000003, 2492)
		// 2017/02/08 08:31:40 binlogsyncer.go:130: [info] register slave for master server localhost:3306
		// 2017/02/08 08:31:40 binlogsyncer.go:502: [error] retry sync err: dial tcp [::1]:3306: getsockopt: connection refused, wait 1s and retry again
		// 2017/02/08 08:31:42 binlogsyncer.go:446: [info] begin to re-sync from (mysql.000003, 2492)
		// 2017/02/08 08:31:42 binlogsyncer.go:130: [info] register slave for master server localhost:3306
		// 2017/02/08 08:31:42 binlogsyncer.go:502: [error] retry sync err: dial tcp [::1]:3306: getsockopt: connection refused, wait 1s and retry again
		//
		// or invalid position sent?
		// 2017/02/08 08:42:09 binlogstreamer.go:47: [error] close sync with err: ERROR 1236 (HY000): Client requested master to start replication from position > file size; the first event 'mysql.000004' at 2274, the last event read from '/usr/local/var/mysql.000004' at 4, the last byte read from '/usr/local/var/mysql.000004' at 4.
		//
		// [02/08/17 17:39:20] [EROR] ( mysqlbinlog.go:64) backoff 5s: invalid table id 2413, no correspond table map event
		ctx, cancel := context.WithTimeout(context.Background(), timeout)
		ev, err := syncer.GetEvent(ctx)
		cancel()

		if err != nil {
			if err == context.DeadlineExceeded {
				if timeout < maxTimeout {
					// backoff
					timeout *= 2
				}

				continue
			}

			if strings.HasPrefix(err.Error(), "invalid table id") {
				log.Error("[%s] %s", m.masterAddr, err)
				// TODO how to handle this?
				continue
			}

			// TODO try out all the cases
			m.emitFatalError(err)
			return
		}

		m.m.Events.Mark(1)

		log.Debug("-> %T", ev.Event)
		// insert into tbtest values(1) will trigger the following events:
		// QueryEvent    BEGIN, Log position: 4800
		// TableMapEvent Schema: mydb, Table: tbtest, TableID: 81, Log position: 4844
		// RowsEvent     Values: 0:1, TableID: 81, Log position: 4884
		// XIDEvent      XID: 356, Log position: 4915 // COMMMIT
		switch e := ev.Event.(type) {
		case *replication.RotateEvent:
			// e,g.
			// Position: 4
			// Next log name: mysql.000002
			file = string(e.NextLogName)
			log.Trace("[%s] rotate to (%s, %d)", m.masterAddr, file, e.Position)

		case *replication.RowsEvent:
			m.m.TPS.Mark(1)
			m.m.Lag.Update(time.Now().Unix() - int64(ev.Header.Timestamp))
			m.handleRowsEvent(file, ev.Header, e)

		case *replication.QueryEvent:
			// DDL comes this way
			// e,g. create table y(id int)
			// e,g. BEGIN
			// e,g. flush tables

		case *replication.XIDEvent:
			// e,g. xid: 1293

		case *replication.FormatDescriptionEvent:
			// Version: 4
			// Server version: 5.6.23-log
			// Checksum algorithm: 1

		case *replication.TableMapEvent:
			// e,g.
			// TableID: 170
			// TableID size: 6
			// Flags: 1
			// Schema: test
			// Table: y
			// Column count: 1
			// Column type:
			// 00000000  03
			// NULL bitmap:
			// 00000000  01

		case *replication.GenericEvent:
			// known misc types of event

		case *replication.GTIDEvent:
			if m.GTID {
				// TODO
			}

		default:
			log.Warn("[%s] unexpected event: %+v", m.masterAddr, e)
		}
	}

}

func (m *MySlave) emitFatalError(err error) {
	m.errors <- err
	if m.r != nil {
		m.r.Close()
	}
}
<|endoftext|>


<commit_before>package lnstate

import (
	"fmt"
	//"github.com/btcsuite/btcd/btcec"
	//"atomic"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"li.lan/labs/plasma/channeldb"
	"li.lan/labs/plasma/lnwire"
	"li.lan/labs/plasma/shachain"
	"sync"
)

//This is a state machine which allows for simultaneous high-volume
//bidirectional payments. The design goal is to for all steps to be
//non-blocking over-the-wire on signing/revoking with the counterparty.
//
//A *single* Lightning Network node on one reasonably high-speced computer
//should be able to conduct transactions at Visa-scale (one Lightning Network
//node can process thousands of transactions per second total), enabling
//incredible volume for the entire network. This state machine design should
//enable transaction volume for the entire network greater than current Visa
//card transaction load by many orders of magnitude.
//
//More info on the design is in lnwire/README.md

//NOTE: WORK IN PROGRESS (current working code in wallet is blocking -- it is
//implemented, but this version will be merged and replacing it soon)
//TODO: Will make channel-friendly & add in mutex stuff

//DUMMY FunCTIONS FOR UPDATING LATER
func disk() {
}
func net(msg lnwire.Message) {
	fmt.Println(msg)
}
func (l *LNChannel) sendErrorPkt() {
}

//Will copy in code from channel.go...

const (
	MAX_STAGED_HTLCS                 = 1000
	MAX_UNREVOKED_COMMITMENTS        = 16
	MAX_UPDATED_HTLCS_PER_COMMITMENT = 1000
)

//Currently, the mutex locks across the entire channel, it's possible to update
//it to be more granular and to have each PaymentDescriptor have its own locks
//(and lock both simultaneously for minimal parts). However, this is
//complicated and will be updated in the future. Right now, since there is a
//per-channel global-lock, this should work fine and be fairly performant.
//
//Before calling another function which does a lock, make sure to l.Unlock()
//first, e.g. if a packet received triggers a packet to be sent.

//PaymentDescriptor states
//PRESTAGE: We're not sure if the other guy wants to follow through
//STAGED: Both parties informally agree to add to their commitments
//SIGNING_AND_REVOKING: In the process of activating the HTLC
//
//Only one state is allowed at a time. timeout/settle can only begin on an
//ADD_COMPLETE state. If it times out before it hits completion, channel closes
//out uncooperatively.
//
//NOTE: Current design assumes if value%1000 == 200 as partially
//signed/revoked, this means that the code is reliant upon this as knowing when
//to force close out channels, etc.
const (
	//HTLC Add
	ADD_PRESTAGE             = 1000
	ADD_STAGED               = 1100
	ADD_SIGNING_AND_REVOKING = 1200
	ADD_COMPLETE             = 1300 //Most HTLCs should be this
	ADD_REJECTED             = 1999 //Staging request rejected

	//HTLC Timeout
	TIMEOUT_PRESTAGE             = 2000
	TIMEOUT_STAGED               = 2100
	TIMEOUT_SIGNING_AND_REVOKING = 2200
	TIMEOUT_COMPLETE             = 2300

	//HTLC Settle
	SETTLE_PRESTAGE             = 3000
	SETTLE_STAGED               = 3100
	SETTLE_SIGNING_AND_REVOKING = 3200
	SETTLE_COMPLETE             = 3300

	//TODO: Commitment states
)

//Example, Add Flow:
//1. Create the HTLC
//2. (you+they) Sign the commit
//3. (you+they) Send revocation when you get a new commit
//4. Update the state to account for revocation
//5. Both sides committed and revoked prior states, mark state as finished

type LNChannel struct {
	fundingTxIn *wire.TxIn
	channelDB   *channeldb.DB

	channelID uint64

	//The person who set up the channel is even, the person who responded
	//is odd. All HTLCKeys use even/odd numbering.
	//NOTE: should we change this to be a int64 and use positive/negative?
	isEven bool

	sync.RWMutex

	//Should update with new Commitments when fees are renegotiated
	feePerKb btcutil.Amount

	//HTLCs
	//Even/odd numbering in effect
	//Will just take the next number
	HTLCs map[lnwire.HTLCKey]*PaymentDescriptor
	//Last key in the map for incrementing
	//If the other party wants to use some stupid high number, then
	//they're basically wanting the channel to close out early...
	ourLastKey   lnwire.HTLCKey
	theirLastKey lnwire.HTLCKey

	//ourCommit/theirCommit is the most recent *signed* Commitment
	ourCommit                 *wire.MsgTx
	theirCommit               *wire.MsgTx
	ourCommitHeight           lnwire.CommitHeight
	ourUnrevokedCommitments   []lnwire.CommitHeight
	theirCommitHeight         lnwire.CommitHeight
	theirUnrevokedCommitments []lnwire.CommitHeight
	//UnrevoedStates indexed by CommitHeight, slice of HTLCKeys to update
	//as revoked/complete

	//NOTE: I think it's EASIER to reconstruct the Commitment than
	//to update because finding indicies and blah blah blah,
	//validation/verification screw that, the state is in the HTLCs
	//and the sig. We can optimize later.

	//ShaChain Height
	//Difference between ourCommitHeight and ourShaChain index is the
	//Commitments not yet revoked. If the difference is 1, then all prior
	//Commitments are revoked.
	ourShaChain   *shachain.HyperShaChain
	theirShaChain *shachain.HyperShaChain
}

type PaymentDescriptor struct {
	RHashes       []*[20]byte
	Timeout       uint32
	CreditsAmount lnwire.CreditsAmount
	Revocation    []*[20]byte
	Blob          []byte //next hop data
	PayToUs       bool

	State uint32 //Current state

	Index uint32   //Position in txout
	p2sh  [20]byte //cached p2sh script to use

	//Used during the SIGNING_AND_REVOKING process
	weSigned               bool
	theyRevoked            bool
	theySignedAndWeRevoked bool
	//This is the height which is revoked, anything before that
	//is assumed to still be valid.
	//This data point is useful for ...UnrevokedCommitments
	//if there's unrevoked commitments below these values,
	//then the state is not locked in
	//Used for add/settle/timeout
	ourRevokedHeight   lnwire.CommitHeight
	theirRevokedHeight lnwire.CommitHeight
}

//addHTLC will take in a PaymentDescriptor, adds to HTLCs and writes to disk
//Assumes the data has already been validated!
//NOTE: **MUST** HAVE THE MUTEX LOCKED ALREADY WHEN CALLED
func (l *LNChannel) addHTLC(h *PaymentDescriptor) (lnwire.HTLCKey, error) {
	//Sanity check
	if h.State != ADD_PRESTAGE {
		return 0, fmt.Errorf("addHTLC can only add PRESTAGE")
	}

	//Make sure we're not overfull
	//We should *never* hit this...
	//we subtract 3 in case we need to add one to correct for even/odd
	if l.ourLastKey >= ^lnwire.HTLCKey(0)-3 {
		return 0, fmt.Errorf("Channel full!!!")
	}

	//Assign a new value
	//We add 2 due to even/odd assignments
	l.ourLastKey += 2

	//Check whether the even/odd is invalid..
	//If it is, we iterate to the next one
	if l.ourLastKey%1 == 1 {
		if l.isEven {
			l.ourLastKey += 1
		}
	} else {
		if !l.isEven {
			l.ourLastKey += 1
		}
	}

	//Write the new HTLC
	l.HTLCs[l.ourLastKey] = h
	disk() //save l.ourLastKey and the htlcs

	return l.ourLastKey, nil
}

func (l *LNChannel) CreateHTLC(h *PaymentDescriptor) error {
	l.Lock()
	var err error
	//if h.State == ADD_PRESTAGE {
	//	//We already have it created, but let's re-send!
	//	//Send a payment request LNWire
	//}
	if h.State > ADD_PRESTAGE {
		l.Unlock()
		return fmt.Errorf("HTLC is already created!")
	}

	if !h.PayToUs { //We created the payment
		//Validate the data
		err = l.validateHTLC(h, false)
		if err != nil {
			return err
		}
		//Update state as pre-commit
		h.State = ADD_PRESTAGE
		l.addHTLC(h)
		//Send a ADDHTLC LNWire
		l.Unlock()
		net() //TODO
	} else {
		//Future version may be able to do this..
		l.Unlock()
		fmt.Errorf("Cannot pull money")
	}

	return nil
}

//Receives an HTLCAddRequest message
//Adds to the HTLC staging list
func (l *LNChannel) recvHTLCAddRequest(p *lnwire.HTLCAddRequest) error {
	l.Lock()
	var err error
	//Check if we already have a PaymentDescriptor
	if l.HTLCs[p.HTLCKey] != nil {
		//Don't do anything because we already have one
		l.Unlock()
		return fmt.Errorf("Counterparty attempted to re-send AddRequest")
	}
	//Make sure they aren't reusing
	if p.HTLCKey <= l.theirLastKey {
		//They're reusing.... uhoh
		l.Unlock()
		l.sendAddReject(p.HTLCKey) //This may be a dupe!
		return fmt.Errorf("Counterparty cannot re-use HTLCKeys")
	}

	//Update newest id
	l.theirLastKey = p.HTLCKey

	//Create a new HTLCKey entry
	htlc := new(PaymentDescriptor)
	//Populate the entries
	htlc.RHashes = p.RedemptionHashes
	htlc.Timeout = p.Expiry
	htlc.CreditsAmount = p.CreditsAmount
	htlc.Blob = p.Blob
	htlc.State = ADD_STAGED //mark as staged by both parties
	htlc.PayToUs = true     //assume this is paid to us, may change in the future

	//Validate the HTLC
	err = l.validateHTLC(htlc, true)
	if err != nil {
		//Update state just in case (not used but y'know..)
		htlc.State = ADD_REJECTED

		//currently not yet added to staging
		//so we don't need to worry about the above htlc
		//we also don't add to disk

		//However, we do need to send a AddReject packet
		l.Unlock()
		sendAddReject(p.HTLCKey)

		return err
	}

	//Validation passed, so we continue
	addHTLC(h)

	//Send add accept packet, and we're done
	l.Unlock()
	sendAddAccept(p.HTLCKey)
}

func (l *LNChannel) sendAddReject(htlckey lnwire.HTLCKey) error {
	l.Lock()
	defer l.Unlock()
	msg = new(lnwire.HTLCAddReject)
	msg.ChannelID = l.channelID
	msg.HTLCKey = h.HTLCKey
	net(msg)

	return nil
}

func (l *LNChannel) recvAddReject(htlckey lnwire.HTLCKey) error {
	l.Lock()
	defer l.Unlock()
	htlc = l.HTLCs[htlckey]
	if htlc == nil {
		return fmt.Errorf("Counterparty rejected non-existent HTLC")
	}
	if (*htlc).State != ADD_PRESTAGE {
		return fmt.Errorf("Counterparty atttempted to reject invalid state")
	}
	(*htlc).State = ADD_REJECTED
	disk()

	return nil
}

//Notifies the other party that it is now staged on our end
func (l *LNChannel) sendAddAccept(htlckey lnwire.HTLCKey) error {
	h = l.HTLCs[htlckey]
	msg = new(lnwire.HTLCAddAccept)
	msg.ChannelID = l.channelID
	msg.HTLCKey = h.HTLCKey

	disk()
	net(msg)

	return nil
}

//The other party has accepted the staging request, so we are staging now
func (l *LNChannel) recvAddAccept(p *HTLCAddAccept) error {
	htlc = l.HTLCs[p.HTLCKey]
	//Make sure it's in the list
	if htlc == nil {
		return fmt.Errorf("Counterparty accepted non-existent HTLC")
	}

	//Update pre-stage to staged
	//Everything else it won't do anything
	if (*htlc).State == ADD_PRESTAGE {
		//Update to staged
		(*htlc).State = ADD_STAGED
		disk()
	}
}

func (l *LNChannel) timeoutHTLC(htlcKey lnwire.HTLCKey) error {
}

func (l *LNChannel) settleHTLC(htlcKey lnwire.HTLCKey) error {
}

//receive AddAcceptHTLC: Find the HTLC and call createHTLC
func (l *LNChannel) addAccept(h *PaymentDescriptor) error {
	if h.State == ADD_PRESTAGE {
		//Mark stage as accepted
		h.State = ADD_SIGNING_AND_REVOKING
		//Write to disk
		disk()
	}
}

//Timeout, Settle

//Create a commitment
//	Update the markings on the HTLCs
func (l *LNChannel) createCommitment() error {
	//Take all staging marked as SIGNING_AND_REVOKING *and* we have not signed
	//	Mark each as weSigned
}

//They revoke prior
//After we send a Commitment, they should send a revocation
//	Update shaChain and HTLCs
func (l *LNChannel) receiveRevocation() error {
	//Revoke the prior Commitment
	//Update HTLCs with theyRevoked
	//Check each HTLC for being complete and mark as complete if so
}

//Receive a commitment & send out a revocation
//	Update the markings on the HTLCs
func (l *LNChannel) receiveCommitment() error {
	//Validate new Commitment (sigs, even/odd for commitment height, incremental commitment height, etc)
	//	Reconstruct Commitment
	//Update with new Commitment
	//Send revocation
	//Mark as theySignedAndWeRevoked
	//Check each HTLC for being complete and mark as complete if so
}

//Mark the HTLC as revoked if it is fully signed and revoked by both parties
func (l *LNChannel) addCompleteHTLC(h *PaymentDescriptor) error {
	//Check/validate values
	//Mark as ADD_COMPLETE
}

//Validate whether we want to add the HTLC
func (l *LNChannel) validateHTLC(h *PaymentDescriptor, toUs bool) error {
	//Make sure we have available spots; not too many outputs
	//Make sure there is available funds
	//Make sure there is sufficient reserve for fees
	//Make sure the money is going in the right direction
}
<commit_msg>minor edit<commit_after>package lnstate

import (
	"fmt"
	//"github.com/btcsuite/btcd/btcec"
	//"atomic"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"li.lan/labs/plasma/channeldb"
	"li.lan/labs/plasma/lnwire"
	"li.lan/labs/plasma/shachain"
	"sync"
)

//This is a state machine which allows for simultaneous high-volume
//bidirectional payments. The design goal is to for all steps to be
//non-blocking over-the-wire on signing/revoking with the counterparty.
//
//A *single* Lightning Network node on one reasonably high-speced computer
//should be able to conduct transactions at Visa-scale (one Lightning Network
//node can process thousands of transactions per second total), enabling
//incredible volume for the entire network. This state machine design should
//enable transaction volume for the entire network greater than current Visa
//card transaction load by many orders of magnitude.
//
//More info on the design is in lnwire/README.md

//NOTE: WORK IN PROGRESS (current working code in wallet is blocking -- it is
//implemented, but this version will be merged and replacing it soon)
//TODO: Will make channel-friendly & add in mutex stuff

//DUMMY FunCTIONS FOR UPDATING LATER
func disk() {
}
func net(msg lnwire.Message) {
	fmt.Println(msg)
}
func (l *LNChannel) sendErrorPkt() {
}

//Will copy in code from channel.go...

const (
	MAX_STAGED_HTLCS                 = 1000
	MAX_UNREVOKED_COMMITMENTS        = 16
	MAX_UPDATED_HTLCS_PER_COMMITMENT = 1000
)

//Currently, the mutex locks across the entire channel, it's possible to update
//it to be more granular and to have each PaymentDescriptor have its own locks
//(and lock both simultaneously for minimal parts). However, this is
//complicated and will be updated in the future. Right now, since there is a
//per-channel global-lock, this should work fine and be fairly performant.
//
//Before calling another function which does a lock, make sure to l.Unlock()
//first, e.g. if a packet received triggers a packet to be sent.

//PaymentDescriptor states
//PRESTAGE: We're not sure if the other guy wants to follow through
//STAGED: Both parties informally agree to add to their commitments
//SIGNING_AND_REVOKING: In the process of activating the HTLC
//
//Only one state is allowed at a time. timeout/settle can only begin on an
//ADD_COMPLETE state. If it times out before it hits completion, channel closes
//out uncooperatively.
//
//NOTE: Current design assumes if value%1000 == 200 as partially
//signed/revoked, this means that the code is reliant upon this as knowing when
//to force close out channels, etc.
const (
	//HTLC Add
	ADD_PRESTAGE             = 1000
	ADD_STAGED               = 1100
	ADD_SIGNING_AND_REVOKING = 1200
	ADD_COMPLETE             = 1300 //Most HTLCs should be this
	ADD_REJECTED             = 1999 //Staging request rejected

	//HTLC Timeout
	TIMEOUT_PRESTAGE             = 2000
	TIMEOUT_STAGED               = 2100
	TIMEOUT_SIGNING_AND_REVOKING = 2200
	TIMEOUT_COMPLETE             = 2300

	//HTLC Settle
	SETTLE_PRESTAGE             = 3000
	SETTLE_STAGED               = 3100
	SETTLE_SIGNING_AND_REVOKING = 3200
	SETTLE_COMPLETE             = 3300

	//TODO: Commitment states
)

//Example, Add Flow:
//1. Create the HTLC
//2. (you+they) Sign the commit
//3. (you+they) Send revocation when you get a new commit
//4. Update the state to account for revocation
//5. Both sides committed and revoked prior states, mark state as finished

type LNChannel struct {
	fundingTxIn *wire.TxIn
	channelDB   *channeldb.DB

	channelID uint64

	//The person who set up the channel is even, the person who responded
	//is odd. All HTLCKeys use even/odd numbering.
	//NOTE: should we change this to be a int64 and use positive/negative?
	isEven bool

	sync.RWMutex

	//Should update with new Commitments when fees are renegotiated
	feePerKb btcutil.Amount

	//HTLCs
	//Even/odd numbering in effect
	//Will just take the next number
	HTLCs map[lnwire.HTLCKey]*PaymentDescriptor
	//Last key in the map for incrementing
	//If the other party wants to use some stupid high number, then
	//they're basically wanting the channel to close out early...
	ourLastKey   lnwire.HTLCKey
	theirLastKey lnwire.HTLCKey

	//ourCommit/theirCommit is the most recent *signed* Commitment
	ourCommit                 *wire.MsgTx
	theirCommit               *wire.MsgTx
	ourCommitHeight           lnwire.CommitHeight
	ourUnrevokedCommitments   []lnwire.CommitHeight
	theirCommitHeight         lnwire.CommitHeight
	theirUnrevokedCommitments []lnwire.CommitHeight
	//UnrevoedStates indexed by CommitHeight, slice of HTLCKeys to update
	//as revoked/complete

	//NOTE: I think it's EASIER to reconstruct the Commitment than
	//to update because finding indicies and blah blah blah,
	//validation/verification screw that, the state is in the HTLCs
	//and the sig. We can optimize later.

	//ShaChain Height
	//Difference between ourCommitHeight and ourShaChain index is the
	//Commitments not yet revoked. If the difference is 1, then all prior
	//Commitments are revoked.
	ourShaChain   *shachain.HyperShaChain
	theirShaChain *shachain.HyperShaChain
}

type PaymentDescriptor struct {
	RHashes       []*[20]byte
	Timeout       uint32
	CreditsAmount lnwire.CreditsAmount
	Revocation    []*[20]byte
	Blob          []byte //next hop data
	PayToUs       bool

	State uint32 //Current state

	Index uint32   //Position in txout
	p2sh  [20]byte //cached p2sh script to use

	//Used during the SIGNING_AND_REVOKING process
	weSigned               bool
	theyRevoked            bool
	theySignedAndWeRevoked bool
	//This is the height which is revoked, anything before that
	//is assumed to still be valid.
	//This data point is useful for ...UnrevokedCommitments
	//if there's unrevoked commitments below these values,
	//then the state is not locked in
	//Used for add/settle/timeout
	ourRevokedHeight   lnwire.CommitHeight
	theirRevokedHeight lnwire.CommitHeight
}

//addHTLC will take in a PaymentDescriptor, adds to HTLCs and writes to disk
//Assumes the data has already been validated!
//NOTE: **MUST** HAVE THE MUTEX LOCKED ALREADY WHEN CALLED
func (l *LNChannel) addHTLC(h *PaymentDescriptor) (lnwire.HTLCKey, error) {
	//Sanity check
	if h.State != ADD_PRESTAGE {
		return 0, fmt.Errorf("addHTLC can only add PRESTAGE")
	}

	//Make sure we're not overfull
	//We should *never* hit this...
	//we subtract 3 in case we need to add one to correct for even/odd
	if l.ourLastKey >= ^lnwire.HTLCKey(0)-3 {
		return 0, fmt.Errorf("Channel full!!!")
	}

	//Assign a new value
	//We add 2 due to even/odd assignments
	l.ourLastKey += 2

	//Check whether the even/odd is invalid..
	//If it is, we iterate to the next one
	if l.ourLastKey%1 == 1 {
		if l.isEven {
			l.ourLastKey += 1
		}
	} else {
		if !l.isEven {
			l.ourLastKey += 1
		}
	}

	//Write the new HTLC
	l.HTLCs[l.ourLastKey] = h
	disk() //save l.ourLastKey and the htlcs

	return l.ourLastKey, nil
}

func (l *LNChannel) CreateHTLC(h *PaymentDescriptor) error {
	l.Lock()
	var err error
	//if h.State == ADD_PRESTAGE {
	//	//We already have it created, but let's re-send!
	//	//Send a payment request LNWire
	//}
	if h.State > ADD_PRESTAGE {
		l.Unlock()
		return fmt.Errorf("HTLC is already created!")
	}

	if !h.PayToUs { //We created the payment
		//Validate the data
		err = l.validateHTLC(h, false)
		if err != nil {
			l.Unlock()
			return err
		}
		//Update state as pre-commit
		h.State = ADD_PRESTAGE
		l.addHTLC(h)
		//Send a ADDHTLC LNWire
		l.Unlock()
		net() //TODO
	} else {
		//Future version may be able to do this..
		l.Unlock()
		return fmt.Errorf("Cannot pull money")
	}

	return nil
}

//Receives an HTLCAddRequest message
//Adds to the HTLC staging list
func (l *LNChannel) recvHTLCAddRequest(p *lnwire.HTLCAddRequest) error {
	l.Lock()
	var err error
	//Check if we already have a PaymentDescriptor
	if l.HTLCs[p.HTLCKey] != nil {
		//Don't do anything because we already have one
		l.Unlock()
		return fmt.Errorf("Counterparty attempted to re-send AddRequest")
	}
	//Make sure they aren't reusing
	if p.HTLCKey <= l.theirLastKey {
		//They're reusing.... uhoh
		l.Unlock()
		l.sendAddReject(p.HTLCKey) //This may be a dupe!
		return fmt.Errorf("Counterparty cannot re-use HTLCKeys")
	}

	//Update newest id
	l.theirLastKey = p.HTLCKey

	//Create a new HTLCKey entry
	htlc := new(PaymentDescriptor)
	//Populate the entries
	htlc.RHashes = p.RedemptionHashes
	htlc.Timeout = p.Expiry
	htlc.CreditsAmount = p.CreditsAmount
	htlc.Blob = p.Blob
	htlc.State = ADD_STAGED //mark as staged by both parties
	htlc.PayToUs = true     //assume this is paid to us, may change in the future

	//Validate the HTLC
	err = l.validateHTLC(htlc, true)
	if err != nil {
		//Update state just in case (not used but y'know..)
		htlc.State = ADD_REJECTED

		//currently not yet added to staging
		//so we don't need to worry about the above htlc
		//we also don't add to disk

		//However, we do need to send a AddReject packet
		l.Unlock()
		sendAddReject(p.HTLCKey)

		return err
	}

	//Validation passed, so we continue
	addHTLC(h)

	//Send add accept packet, and we're done
	l.Unlock()
	sendAddAccept(p.HTLCKey)
}

func (l *LNChannel) sendAddReject(htlckey lnwire.HTLCKey) error {
	l.Lock()
	defer l.Unlock()
	msg = new(lnwire.HTLCAddReject)
	msg.ChannelID = l.channelID
	msg.HTLCKey = h.HTLCKey
	net(msg)

	return nil
}

func (l *LNChannel) recvAddReject(htlckey lnwire.HTLCKey) error {
	l.Lock()
	defer l.Unlock()
	htlc = l.HTLCs[htlckey]
	if htlc == nil {
		return fmt.Errorf("Counterparty rejected non-existent HTLC")
	}
	if (*htlc).State != ADD_PRESTAGE {
		return fmt.Errorf("Counterparty atttempted to reject invalid state")
	}
	(*htlc).State = ADD_REJECTED
	disk()

	return nil
}

//Notifies the other party that it is now staged on our end
func (l *LNChannel) sendAddAccept(htlckey lnwire.HTLCKey) error {
	h = l.HTLCs[htlckey]
	msg = new(lnwire.HTLCAddAccept)
	msg.ChannelID = l.channelID
	msg.HTLCKey = h.HTLCKey

	disk()
	net(msg)

	return nil
}

//The other party has accepted the staging request, so we are staging now
func (l *LNChannel) recvAddAccept(p *HTLCAddAccept) error {
	htlc = l.HTLCs[p.HTLCKey]
	//Make sure it's in the list
	if htlc == nil {
		return fmt.Errorf("Counterparty accepted non-existent HTLC")
	}

	//Update pre-stage to staged
	//Everything else it won't do anything
	if (*htlc).State == ADD_PRESTAGE {
		//Update to staged
		(*htlc).State = ADD_STAGED
		disk()
	}
}

func (l *LNChannel) timeoutHTLC(htlcKey lnwire.HTLCKey) error {
}

func (l *LNChannel) settleHTLC(htlcKey lnwire.HTLCKey) error {
}

//receive AddAcceptHTLC: Find the HTLC and call createHTLC
func (l *LNChannel) addAccept(h *PaymentDescriptor) error {
	if h.State == ADD_PRESTAGE {
		//Mark stage as accepted
		h.State = ADD_SIGNING_AND_REVOKING
		//Write to disk
		disk()
	}
}

//Timeout, Settle

//Create a commitment
//	Update the markings on the HTLCs
func (l *LNChannel) createCommitment() error {
	//Take all staging marked as SIGNING_AND_REVOKING *and* we have not signed
	//	Mark each as weSigned
}

//They revoke prior
//After we send a Commitment, they should send a revocation
//	Update shaChain and HTLCs
func (l *LNChannel) receiveRevocation() error {
	//Revoke the prior Commitment
	//Update HTLCs with theyRevoked
	//Check each HTLC for being complete and mark as complete if so
}

//Receive a commitment & send out a revocation
//	Update the markings on the HTLCs
func (l *LNChannel) receiveCommitment() error {
	//Validate new Commitment (sigs, even/odd for commitment height, incremental commitment height, etc)
	//	Reconstruct Commitment
	//Update with new Commitment
	//Send revocation
	//Mark as theySignedAndWeRevoked
	//Check each HTLC for being complete and mark as complete if so
}

//Mark the HTLC as revoked if it is fully signed and revoked by both parties
func (l *LNChannel) addCompleteHTLC(h *PaymentDescriptor) error {
	//Check/validate values
	//Mark as ADD_COMPLETE
}

//Validate whether we want to add the HTLC
func (l *LNChannel) validateHTLC(h *PaymentDescriptor, toUs bool) error {
	//Make sure we have available spots; not too many outputs
	//Make sure there is available funds
	//Make sure there is sufficient reserve for fees
	//Make sure the money is going in the right direction
}
<|endoftext|>


<commit_before>package http

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"testing"

	"github.com/gansoi/gansoi/plugins"
)

func TestAgent(t *testing.T) {
	a := plugins.GetAgent("http")
	_ = a.(*HTTP)
}

func TestGetHostPort(t *testing.T) {
	cases := []struct {
		in   string
		host string
		port string
	}{
		{"http://gansoi.com/", "gansoi.com", "80"},
		{"http://gansoi.com:81/", "gansoi.com", "81"},
		{"https://gansoi.com/", "gansoi.com", "443"},
		{"https://gansoi.com:81/", "gansoi.com", "81"},
	}

	for _, c := range cases {
		URL, err := url.Parse(c.in)
		if err != nil {
			t.Fatalf("Test case '%s' cannot be parsed as an URL: %s", c.in, err.Error())
		}

		host, port := getHostPort(URL)
		if host != c.host {
			t.Fatalf("'%s' did not return expected hostname '%s', got '%s'", c.in, c.host, host)
		}

		if port != c.port {
			t.Fatalf("'%s' did not return expected port '%s', got '%s'", c.in, c.port, port)
		}
	}
}

func TestCheckFail(t *testing.T) {
	cases := []string{
		"",
		"http://",
		"example.com",
		"non://",
		"/",
		"https://127.0.0.1:0/",
		"http://127.0.0.1:0/",
		"http/////",
		"http://go-test-nonexisting/",
	}

	for _, u := range cases {
		a := &HTTP{
			URL: u,
		}

		result := plugins.NewAgentResult()
		err := a.Check(result)
		if err == nil {
			t.Fatalf("Failed to detect error for '%s'", u)
		}
	}
}

func okHandler(w http.ResponseWriter, r *http.Request) {
	if r.RequestURI == "/redirect" {
		http.Redirect(w, r, "/", 301)
		return
	}

	fmt.Fprintf(w, "ok\n")
}

func TestCheck(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(okHandler))
	a := &HTTP{
		URL: ts.URL + "/",
	}

	result := plugins.NewAgentResult()
	err := a.Check(result)
	if err != nil {
		t.Fatalf("Check failed: %s", err.Error())
	}
	ts.Close()

	ts = httptest.NewTLSServer(http.HandlerFunc(okHandler))
	ts.Config.ErrorLog = log.New(ioutil.Discard, "", 0)

	a.URL = ts.URL + "/"
	err = a.Check(result)
	if err == nil {
		t.Fatalf("Check did not catch unsigned cert")
	}

	a.Insecure = true
	a.IncludeBody = true
	err = a.Check(result)
	if err != nil {
		t.Fatalf("Check failed: %s", err.Error())
	}

	if strings.TrimSpace(result["Body"].(string)) != "ok" {
		t.Fatalf("Got wrong body, expected 'ok', got '%s'", result["Body"])
	}

	a.URL = ts.URL + "/redirect"
	a.FollowRedirect = true
	a.Insecure = true
	a.IncludeBody = true
	err = a.Check(result)
	if err != nil {
		t.Fatalf("Check failed: %s", err.Error())
	}

	ts.Close()
}

var _ plugins.Agent = (*HTTP)(nil)
<commit_msg>Don't assume 'http://' fails. It will try port 80 on localhost.<commit_after>package http

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"testing"

	"github.com/gansoi/gansoi/plugins"
)

func TestAgent(t *testing.T) {
	a := plugins.GetAgent("http")
	_ = a.(*HTTP)
}

func TestGetHostPort(t *testing.T) {
	cases := []struct {
		in   string
		host string
		port string
	}{
		{"http://gansoi.com/", "gansoi.com", "80"},
		{"http://gansoi.com:81/", "gansoi.com", "81"},
		{"https://gansoi.com/", "gansoi.com", "443"},
		{"https://gansoi.com:81/", "gansoi.com", "81"},
	}

	for _, c := range cases {
		URL, err := url.Parse(c.in)
		if err != nil {
			t.Fatalf("Test case '%s' cannot be parsed as an URL: %s", c.in, err.Error())
		}

		host, port := getHostPort(URL)
		if host != c.host {
			t.Fatalf("'%s' did not return expected hostname '%s', got '%s'", c.in, c.host, host)
		}

		if port != c.port {
			t.Fatalf("'%s' did not return expected port '%s', got '%s'", c.in, c.port, port)
		}
	}
}

func TestCheckFail(t *testing.T) {
	cases := []string{
		"",
		"http://:",
		"example.com",
		"non://",
		"/",
		"https://127.0.0.1:0/",
		"http://127.0.0.1:0/",
		"http/////",
		"http://go-test-nonexisting/",
	}

	for _, u := range cases {
		a := &HTTP{
			URL: u,
		}

		result := plugins.NewAgentResult()
		err := a.Check(result)
		if err == nil {
			t.Fatalf("Failed to detect error for '%s'", u)
		}
	}
}

func okHandler(w http.ResponseWriter, r *http.Request) {
	if r.RequestURI == "/redirect" {
		http.Redirect(w, r, "/", 301)
		return
	}

	fmt.Fprintf(w, "ok\n")
}

func TestCheck(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(okHandler))
	a := &HTTP{
		URL: ts.URL + "/",
	}

	result := plugins.NewAgentResult()
	err := a.Check(result)
	if err != nil {
		t.Fatalf("Check failed: %s", err.Error())
	}
	ts.Close()

	ts = httptest.NewTLSServer(http.HandlerFunc(okHandler))
	ts.Config.ErrorLog = log.New(ioutil.Discard, "", 0)

	a.URL = ts.URL + "/"
	err = a.Check(result)
	if err == nil {
		t.Fatalf("Check did not catch unsigned cert")
	}

	a.Insecure = true
	a.IncludeBody = true
	err = a.Check(result)
	if err != nil {
		t.Fatalf("Check failed: %s", err.Error())
	}

	if strings.TrimSpace(result["Body"].(string)) != "ok" {
		t.Fatalf("Got wrong body, expected 'ok', got '%s'", result["Body"])
	}

	a.URL = ts.URL + "/redirect"
	a.FollowRedirect = true
	a.Insecure = true
	a.IncludeBody = true
	err = a.Check(result)
	if err != nil {
		t.Fatalf("Check failed: %s", err.Error())
	}

	ts.Close()
}

var _ plugins.Agent = (*HTTP)(nil)
<|endoftext|>


<commit_before>/*
 Contains various general helper functions
*/

package server

import (
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	r "github.com/dancannon/gorethink"
	"io"
	"net/http"
	"os"
	"strconv"
)

// throw panics, if there is an error. Rob Pike must never know.
func throw(err error) {
	if err != nil {
		panic(err)
	}
}

// checkAuth checks if the suplied Ident is priveledged to perform an action
func checkAuth(action string, ident Ident) bool {
	if class, ok := config.Staff.Classes[ident.Auth]; ok {
		return class.Rights[action]
	}
	return false
}

// rGet is a shorthand for executing RethinkDB queries and panicing on error.
func rGet(query r.Term) *r.Cursor {
	cursor, err := query.Run(rSession)
	throw(err)
	return cursor
}

// rExec executes a RethinkDB query and panics on error. To be used, when the
// returned status is unneeded and we want the goroutine to crash on error.
func rExec(query r.Term) {
	throw(query.Exec(rSession))
}

// shorthand for constructing thread queries
func getThread(id uint64) r.Term {
	return r.Table("threads").Get(id)
}

// shorthand for constructing post queries
func getPost(id, op uint64) r.Term {
	return getThread(op).Field("posts").Field(idToString(id))
}

// Retrieve the current post counter number
func postCounter() uint64 {
	return getCounter(r.Table("main").Get("info").Field("postCtr"))
}

// Retrieve the history or "progress" counter of a board
func boardCounter(board string) uint64 {
	return getCounter(r.Table("main").Get("info").Field("postCtr"))
}

// Retrieve the history or "progress" counter of a thread
func threadCounter(id uint64) uint64 {
	return getCounter(getThread(id).Field("histCtr"))
}

// Helper function for retrieving an integer from the database
func getCounter(query r.Term) (counter uint64) {
	rGet(query).One(&counter)
	return counter
}

// Determine access rights of an IP
func lookUpIdent(ip string) Ident {
	ident := Ident{IP: ip}

	// TODO: BANS

	return ident
}

// Confirm client has rights to access board
func canAccessBoard(board string, ident Ident) bool {
	if board == config.Boards.Staff && !checkAuth("accessStaffBoard", ident) {
		return false
	}
	_, ok := config.Boards.Boards[board]
	if !ok && board == "all" {
		ok = true
	}
	return !ident.Banned && ok
}

// Confirm thread exists and client has rights to access it's board
func canAccessThread(id uint64, board string, ident Ident) bool {
	if !canAccessBoard(board, ident) {
		return false
	}
	var deleted bool
	rGet(getThread(id).
		Field("posts").
		Field(idToString(id)).
		Default(false),
	).
		One(&deleted)
	if deleted && !checkAuth("seeModeration", ident) {
		return false
	}
	return true
}

// Compute a truncated MD5 hash from a buffer
func hashBuffer(buf []byte) string {
	hasher := md5.New()
	hasher.Write(buf)
	return hex.EncodeToString(hasher.Sum(nil))[16:]
}

// Shorthand for marshaling JSON and handling the error
func marshalJSON(input interface{}) []byte {
	data, err := json.Marshal(input)
	throw(err)
	return data
}

// Shorthand for unmarshalling JSON
func unmarshalJSON(data []byte, store interface{}) {
	throw(json.Unmarshal(data, store))
}

// copyFile reads a file from disk and copies it into the writer
func copyFile(path string, writer io.Writer) {
	file, err := os.Open(path)
	throw(err)
	defer file.Close()
	_, err = io.Copy(writer, file)
	throw(err)
}

// Shorthand for converting a post ID to a string for JSON keys
func idToString(id uint64) string {
	return strconv.FormatUint(id, 10)
}

// chooseLang selects the language to use in responses to the client, by
// checking the language setting of the request's cookies and verifying it
// against the available selection on the server. Defaults to the server's
// default language.
func chooseLang(req *http.Request) string {
	cookie, err := req.Cookie("lang")
	if err == http.ErrNoCookie {
		return config.Lang.Default
	} else if err != nil {
		panic(err)
	}
	for _, lang := range config.Lang.Enabled {
		if cookie.Value == lang {
			return lang
		}
	}
	return config.Lang.Default
}
<commit_msg>server/util.go Check thread deletion on the thread hash<commit_after>/*
 Contains various general helper functions
*/

package server

import (
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	r "github.com/dancannon/gorethink"
	"io"
	"net/http"
	"os"
	"strconv"
)

// throw panics, if there is an error. Rob Pike must never know.
func throw(err error) {
	if err != nil {
		panic(err)
	}
}

// checkAuth checks if the suplied Ident is priveledged to perform an action
func checkAuth(action string, ident Ident) bool {
	if class, ok := config.Staff.Classes[ident.Auth]; ok {
		return class.Rights[action]
	}
	return false
}

// rGet is a shorthand for executing RethinkDB queries and panicing on error.
func rGet(query r.Term) *r.Cursor {
	cursor, err := query.Run(rSession)
	throw(err)
	return cursor
}

// rExec executes a RethinkDB query and panics on error. To be used, when the
// returned status is unneeded and we want the goroutine to crash on error.
func rExec(query r.Term) {
	throw(query.Exec(rSession))
}

// shorthand for constructing thread queries
func getThread(id uint64) r.Term {
	return r.Table("threads").Get(id)
}

// shorthand for constructing post queries
func getPost(id, op uint64) r.Term {
	return getThread(op).Field("posts").Field(idToString(id))
}

// Retrieve the current post counter number
func postCounter() uint64 {
	return getCounter(r.Table("main").Get("info").Field("postCtr"))
}

// Retrieve the history or "progress" counter of a board
func boardCounter(board string) uint64 {
	return getCounter(r.Table("main").Get("info").Field("postCtr"))
}

// Retrieve the history or "progress" counter of a thread
func threadCounter(id uint64) uint64 {
	return getCounter(getThread(id).Field("histCtr"))
}

// Helper function for retrieving an integer from the database
func getCounter(query r.Term) (counter uint64) {
	rGet(query).One(&counter)
	return counter
}

// Determine access rights of an IP
func lookUpIdent(ip string) Ident {
	ident := Ident{IP: ip}

	// TODO: BANS

	return ident
}

// Confirm client has rights to access board
func canAccessBoard(board string, ident Ident) bool {
	if board == config.Boards.Staff && !checkAuth("accessStaffBoard", ident) {
		return false
	}
	_, ok := config.Boards.Boards[board]
	if !ok && board == "all" {
		ok = true
	}
	return !ident.Banned && ok
}

// Confirm thread exists and client has rights to access it's board
func canAccessThread(id uint64, board string, ident Ident) bool {
	if !canAccessBoard(board, ident) {
		return false
	}
	var deleted bool
	rGet(getThread(id).Field("deleted").Default(false)).One(&deleted)
	if deleted && !checkAuth("seeModeration", ident) {
		return false
	}
	return true
}

// Compute a truncated MD5 hash from a buffer
func hashBuffer(buf []byte) string {
	hasher := md5.New()
	hasher.Write(buf)
	return hex.EncodeToString(hasher.Sum(nil))[16:]
}

// Shorthand for marshaling JSON and handling the error
func marshalJSON(input interface{}) []byte {
	data, err := json.Marshal(input)
	throw(err)
	return data
}

// Shorthand for unmarshalling JSON
func unmarshalJSON(data []byte, store interface{}) {
	throw(json.Unmarshal(data, store))
}

// copyFile reads a file from disk and copies it into the writer
func copyFile(path string, writer io.Writer) {
	file, err := os.Open(path)
	throw(err)
	defer file.Close()
	_, err = io.Copy(writer, file)
	throw(err)
}

// Shorthand for converting a post ID to a string for JSON keys
func idToString(id uint64) string {
	return strconv.FormatUint(id, 10)
}

// chooseLang selects the language to use in responses to the client, by
// checking the language setting of the request's cookies and verifying it
// against the available selection on the server. Defaults to the server's
// default language.
func chooseLang(req *http.Request) string {
	cookie, err := req.Cookie("lang")
	if err == http.ErrNoCookie {
		return config.Lang.Default
	} else if err != nil {
		panic(err)
	}
	for _, lang := range config.Lang.Enabled {
		if cookie.Value == lang {
			return lang
		}
	}
	return config.Lang.Default
}
<|endoftext|>


<commit_before>// Copyright 2012 Apcera Inc. All rights reserved.

package server

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"time"
)

// Varz will output server information on the monitoring port at /varz.
type Varz struct {
	Start       time.Time `json:"start"`
	Options     *Options  `json:"options"`
	Mem         int64     `json:"mem"`
	Cores       int       `json:"cores"`
	CPU         float64   `json:"cpu"`
	Connections int       `json:"connections"`
	InMsgs      int64     `json:"in_msgs"`
	OutMsgs     int64     `json:"out_msgs"`
	InBytes     int64     `json:"in_bytes"`
	OutBytes    int64     `json:"out_bytes"`
	Uptime      string    `json:"uptime"`
}

type usage struct {
	CPU   float32
	Cores int
	Mem   int64
}

// HandleVarz will process HTTP requests for server information.
func (s *Server) HandleVarz(w http.ResponseWriter, r *http.Request) {
	v := Varz{Start: s.start, Options: s.opts}
	v.Uptime = time.Since(s.start).String()

	updateUsage(&v)

	s.mu.Lock()
	v.Connections = len(s.clients)
	v.InMsgs = s.inMsgs
	v.InBytes = s.inBytes
	v.OutMsgs = s.outMsgs
	v.OutBytes = s.outBytes
	s.mu.Unlock()

	b, err := json.MarshalIndent(v, "", "  ")
	if err != nil {
		Log("Error marshalling response go /varz request: %v", err)
	}
	w.Write(b)
}

// FIXME(dlc): This is a big hack, make real..
func updateUsage(v *Varz) {
	v.Cores = runtime.NumCPU()
	pidStr := fmt.Sprintf("%d", os.Getpid())
	out, err := exec.Command("ps", "o", "pcpu=,rss=", "-p", pidStr).Output()
	if err != nil {
		// FIXME(dlc): Log?
		return
	}
	fmt.Sscanf(string(out), "%f %d", &v.CPU, &v.Mem)
	v.Mem *= 1024 // 1k blocks, want bytes.
}
<commit_msg>Add routes and remotes to varz<commit_after>// Copyright 2012 Apcera Inc. All rights reserved.

package server

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"time"
)

// Varz will output server information on the monitoring port at /varz.
type Varz struct {
	Start       time.Time `json:"start"`
	Options     *Options  `json:"options"`
	Mem         int64     `json:"mem"`
	Cores       int       `json:"cores"`
	CPU         float64   `json:"cpu"`
	Connections int       `json:"connections"`
	Routes      int       `json:"routes"`
	Remotes     int       `json:"remotes"`
	InMsgs      int64     `json:"in_msgs"`
	OutMsgs     int64     `json:"out_msgs"`
	InBytes     int64     `json:"in_bytes"`
	OutBytes    int64     `json:"out_bytes"`
	Uptime      string    `json:"uptime"`
}

type usage struct {
	CPU   float32
	Cores int
	Mem   int64
}

// HandleVarz will process HTTP requests for server information.
func (s *Server) HandleVarz(w http.ResponseWriter, r *http.Request) {
	v := Varz{Start: s.start, Options: s.opts}
	v.Uptime = time.Since(s.start).String()

	updateUsage(&v)

	s.mu.Lock()
	v.Connections = len(s.clients)
	v.Routes = len(s.routes)
	v.Remotes = len(s.remotes)
	v.InMsgs = s.inMsgs
	v.InBytes = s.inBytes
	v.OutMsgs = s.outMsgs
	v.OutBytes = s.outBytes
	s.mu.Unlock()

	b, err := json.MarshalIndent(v, "", "  ")
	if err != nil {
		Log("Error marshalling response go /varz request: %v", err)
	}
	w.Write(b)
}

// FIXME(dlc): This is a big hack, make real..
func updateUsage(v *Varz) {
	v.Cores = runtime.NumCPU()
	pidStr := fmt.Sprintf("%d", os.Getpid())
	out, err := exec.Command("ps", "o", "pcpu=,rss=", "-p", pidStr).Output()
	if err != nil {
		// FIXME(dlc): Log?
		return
	}
	fmt.Sscanf(string(out), "%f %d", &v.CPU, &v.Mem)
	v.Mem *= 1024 // 1k blocks, want bytes.
}
<|endoftext|>


<commit_before>package tcpez_test

import (
	proto "code.google.com/p/goprotobuf/proto"
	json "encoding/json"
	"fmt"
	"github.com/bmizerany/assert"
	"github.com/paperlesspost/tcpez"
	math "math"
	"testing"
)

type EchoHandler struct{}

func (h *EchoHandler) Respond(req []byte, span *tcpez.Span) (response []byte, err error) {
	return req, nil
}

func (h *EchoHandler) Record(span *tcpez.Span) {
}

// Reference proto, json, and math imports to suppress error if they are not otherwise used.
var _ = proto.Marshal
var _ = &json.SyntaxError{}
var _ = math.Inf

type Request struct {
	Command          *string `protobuf:"bytes,1,req,name=command" json:"command,omitempty"`
	Args             *string `protobuf:"bytes,2,req,name=args" json:"args,omitempty"`
	XXX_unrecognized []byte  `json:"-"`
}

func (m *Request) Reset()         { *m = Request{} }
func (m *Request) String() string { return proto.CompactTextString(m) }
func (*Request) ProtoMessage()    {}

func (m *Request) GetCommand() string {
	if m != nil && m.Command != nil {
		return *m.Command
	}
	return ""
}

func (m *Request) GetArgs() string {
	if m != nil && m.Args != nil {
		return *m.Args
	}
	return ""
}

type Response struct {
	Status           *string `protobuf:"bytes,1,req,name=status" json:"status,omitempty"`
	Message          *string `protobuf:"bytes,2,req,name=message" json:"message,omitempty"`
	XXX_unrecognized []byte  `json:"-"`
}

func (m *Response) Reset()         { *m = Response{} }
func (m *Response) String() string { return proto.CompactTextString(m) }
func (*Response) ProtoMessage()    {}

func (m *Response) GetStatus() string {
	if m != nil && m.Status != nil {
		return *m.Status
	}
	return ""
}

func (m *Response) GetMessage() string {
	if m != nil && m.Message != nil {
		return *m.Message
	}
	return ""
}

func init() {
}

func TestEchoServer(t *testing.T) {
	addr := "127.0.0.1:2000"
	l := tcpez.NewServer(addr, new(EchoHandler))
	assert.T(t, l != nil)
	go l.Start()
	defer l.Close()
	c := tcpez.NewClient([]string{addr}, 3, 3)
	assert.T(t, c != nil)
	var resp []byte
	var err error
	for i := 0; i < 1000; i++ {
		resp, err = c.SendRecv([]byte("PING"))
		assert.T(t, err == nil)
		assert.Equal(t, []byte("PING"), resp)
	}
}

func TestEchoServerPipelined(t *testing.T) {
	addr := "127.0.0.1:2000"
	l := tcpez.NewServer(addr, new(EchoHandler))
	assert.T(t, l != nil)
	go l.Start()
	defer l.Close()
	c := tcpez.NewClient([]string{addr}, 3, 3)
	assert.T(t, c != nil)
	pipe := c.Pipeline()
	for i := 0; i < 10; i++ {
		pipe.Send([]byte(fmt.Sprintf("PING%d", i)))
	}
	returned, _ := pipe.Flush()
	for i := 0; i < 10; i++ {
		assert.Equal(t, string(returned[i]), (fmt.Sprintf("PING%d", i)))
	}
}

func TestProtoServer(t *testing.T) {
	addr := "127.0.0.1:2000"
	protoFunc := tcpez.ProtoInitializerFunc(func() proto.Message {
		return new(Request)
	})
	handlerFunc := tcpez.ProtoHandlerFunc(func(req proto.Message, span *tcpez.Span) (res proto.Message) {
		r := req.(*Request)
		message := fmt.Sprintf("Got command: %s args: %s", r.GetCommand(), r.GetArgs())
		span.Increment("response")
		return &Response{
			Status:  proto.String("OK"),
			Message: proto.String(message),
		}
	})
	l := tcpez.NewProtoServer(addr, protoFunc, handlerFunc)
	assert.T(t, l != nil)
	assert.Equal(t, addr, l.Address)
	go l.Start()
	defer l.Close()
	c := tcpez.NewClient([]string{addr}, 3, 3)
	assert.T(t, c != nil)

	iter := 500

	for i := 0; i < iter; i++ {
		var req []byte
		var resp []byte
		var request *Request
		var response *Response
		var err error
		request = &Request{Command: proto.String("GET"), Args: proto.String("/")}
		req, err = proto.Marshal(request)
		assert.T(t, req != nil)
		assert.T(t, err == nil)
		resp, err = c.SendRecv(req)
		assert.T(t, err == nil)
		assert.T(t, resp != nil)
		response = new(Response)
		err = proto.Unmarshal(resp, response)
		assert.T(t, err == nil)
		assert.Equal(t, "OK", response.GetStatus())
	}
}
<commit_msg>Fix test scoping<commit_after>package tcpez

import (
	proto "code.google.com/p/goprotobuf/proto"
	json "encoding/json"
	"fmt"
	"github.com/bmizerany/assert"
	math "math"
	"testing"
)

type EchoHandler struct{}

func (h *EchoHandler) Respond(req []byte, span *Span) (response []byte, err error) {
	return req, nil
}

func (h *EchoHandler) Record(span *Span) {
}

// Reference proto, json, and math imports to suppress error if they are not otherwise used.
var _ = proto.Marshal
var _ = &json.SyntaxError{}
var _ = math.Inf

type Request struct {
	Command          *string `protobuf:"bytes,1,req,name=command" json:"command,omitempty"`
	Args             *string `protobuf:"bytes,2,req,name=args" json:"args,omitempty"`
	XXX_unrecognized []byte  `json:"-"`
}

func (m *Request) Reset()         { *m = Request{} }
func (m *Request) String() string { return proto.CompactTextString(m) }
func (*Request) ProtoMessage()    {}

func (m *Request) GetCommand() string {
	if m != nil && m.Command != nil {
		return *m.Command
	}
	return ""
}

func (m *Request) GetArgs() string {
	if m != nil && m.Args != nil {
		return *m.Args
	}
	return ""
}

type Response struct {
	Status           *string `protobuf:"bytes,1,req,name=status" json:"status,omitempty"`
	Message          *string `protobuf:"bytes,2,req,name=message" json:"message,omitempty"`
	XXX_unrecognized []byte  `json:"-"`
}

func (m *Response) Reset()         { *m = Response{} }
func (m *Response) String() string { return proto.CompactTextString(m) }
func (*Response) ProtoMessage()    {}

func (m *Response) GetStatus() string {
	if m != nil && m.Status != nil {
		return *m.Status
	}
	return ""
}

func (m *Response) GetMessage() string {
	if m != nil && m.Message != nil {
		return *m.Message
	}
	return ""
}

func init() {
}

func TestEchoServer(t *testing.T) {
	addr := "127.0.0.1:2000"
	l := NewServer(addr, new(EchoHandler))
	assert.T(t, l != nil)
	go l.Start()
	defer l.Close()
	c := NewClient([]string{addr}, 3, 3)
	assert.T(t, c != nil)
	var resp []byte
	var err error
	for i := 0; i < 1000; i++ {
		resp, err = c.SendRecv([]byte("PING"))
		assert.T(t, err == nil)
		assert.Equal(t, []byte("PING"), resp)
	}
}

func TestEchoServerPipelined(t *testing.T) {
	addr := "127.0.0.1:2000"
	l := NewServer(addr, new(EchoHandler))
	assert.T(t, l != nil)
	go l.Start()
	defer l.Close()
	c := NewClient([]string{addr}, 3, 3)
	assert.T(t, c != nil)
	pipe := c.Pipeline()
	for i := 0; i < 10; i++ {
		pipe.Send([]byte(fmt.Sprintf("PING%d", i)))
	}
	returned, _ := pipe.Flush()
	for i := 0; i < 10; i++ {
		assert.Equal(t, string(returned[i]), (fmt.Sprintf("PING%d", i)))
	}
}

func TestProtoServer(t *testing.T) {
	addr := "127.0.0.1:2000"
	protoFunc := ProtoInitializerFunc(func() proto.Message {
		return new(Request)
	})
	handlerFunc := ProtoHandlerFunc(func(req proto.Message, span *Span) (res proto.Message) {
		r := req.(*Request)
		message := fmt.Sprintf("Got command: %s args: %s", r.GetCommand(), r.GetArgs())
		span.Increment("response")
		return &Response{
			Status:  proto.String("OK"),
			Message: proto.String(message),
		}
	})
	l := NewProtoServer(addr, protoFunc, handlerFunc)
	assert.T(t, l != nil)
	assert.Equal(t, addr, l.Address)
	go l.Start()
	defer l.Close()
	c := NewClient([]string{addr}, 3, 3)
	assert.T(t, c != nil)

	iter := 500

	for i := 0; i < iter; i++ {
		var req []byte
		var resp []byte
		var request *Request
		var response *Response
		var err error
		request = &Request{Command: proto.String("GET"), Args: proto.String("/")}
		req, err = proto.Marshal(request)
		assert.T(t, req != nil)
		assert.T(t, err == nil)
		resp, err = c.SendRecv(req)
		assert.T(t, err == nil)
		assert.T(t, resp != nil)
		response = new(Response)
		err = proto.Unmarshal(resp, response)
		assert.T(t, err == nil)
		assert.Equal(t, "OK", response.GetStatus())
	}
}
<|endoftext|>


<commit_before>package httpserver

import (
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strconv"
	"testing"
)

func middlewareCreator(i int) func(next http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		fn := func(rw http.ResponseWriter, req *http.Request) {
			p := strconv.Itoa(i)
			io.WriteString(rw, p)
			next.ServeHTTP(rw, req)
			io.WriteString(rw, p)
		}

		return http.HandlerFunc(fn)
	}
}

func startListening(s *Server) net.Listener {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		log.Fatal(err)
	}

	go http.Serve(l, s)
	return l
}

func createAddr(l net.Listener, path string) string {
	return "http://" + l.Addr().String() + path
}

func TestGlobal(t *testing.T) {
	h := New(middlewareCreator(1))
	h.Get("/1", http.HandlerFunc(helloer))

	l := startListening(h)
	defer l.Close()

	const expected = "1Hello, World!1"
	expect(t, createAddr(l, "/1"), expected)

}

func TestHandler(t *testing.T) {
	h := New(middlewareCreator(1))
	h.Get("/1", NewHandler(http.HandlerFunc(helloer), middlewareCreator(2), middlewareCreator(3)))

	l := startListening(h)
	defer l.Close()

	const expected = "123Hello, World!321"
	expect(t, createAddr(l, "/1"), expected)

}

func TestClean(t *testing.T) {
	h := New()
	h.Get("/1", http.HandlerFunc(helloer))

	l := startListening(h)
	defer l.Close()

	const expected = "Hello, World!"
	expect(t, createAddr(l, "/1"), expected)
}

func expect(t *testing.T, url string, expected string) {
	resp, err := http.Get(url)
	if err != nil {
		t.Fatal(err.Error())
	}

	r, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err.Error())
	}

	if string(r) != expected {
		t.Fatalf("expected %s, bu got %s", expected, string(r))
	}
}

func helloer(rw http.ResponseWriter, req *http.Request) {
	io.WriteString(rw, "Hello, World!")
}
<commit_msg>Server: rename the function<commit_after>package httpserver

import (
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strconv"
	"testing"
)

func middlewareCreator(i int) func(next http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		fn := func(rw http.ResponseWriter, req *http.Request) {
			p := strconv.Itoa(i)
			io.WriteString(rw, p)
			next.ServeHTTP(rw, req)
			io.WriteString(rw, p)
		}

		return http.HandlerFunc(fn)
	}
}

func startListening(s *Server) net.Listener {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		log.Fatal(err)
	}

	go http.Serve(l, s)
	return l
}

func createAddr(l net.Listener, path string) string {
	return "http://" + l.Addr().String() + path
}

func TestGlobal(t *testing.T) {
	h := New(middlewareCreator(1))
	h.Get("/1", http.HandlerFunc(helloer))

	l := startListening(h)
	defer l.Close()

	const expected = "1Hello, World!1"
	process(t, createAddr(l, "/1"), expected)

}

func TestHandler(t *testing.T) {
	h := New(middlewareCreator(1))
	h.Get("/1", NewHandler(http.HandlerFunc(helloer), middlewareCreator(2), middlewareCreator(3)))

	l := startListening(h)
	defer l.Close()

	const expected = "123Hello, World!321"
	process(t, createAddr(l, "/1"), expected)

}

func TestClean(t *testing.T) {
	h := New()
	h.Get("/1", http.HandlerFunc(helloer))

	l := startListening(h)
	defer l.Close()

	const expected = "Hello, World!"
	process(t, createAddr(l, "/1"), expected)
}

func process(t *testing.T, url string, expected string) {
	resp, err := http.Get(url)
	if err != nil {
		t.Fatal(err.Error())
	}

	r, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err.Error())
	}

	if string(r) != expected {
		t.Fatalf("expected %s, bu got %s", expected, string(r))
	}
}

func helloer(rw http.ResponseWriter, req *http.Request) {
	io.WriteString(rw, "Hello, World!")
}
<|endoftext|>


<commit_before>package sftpd

import (
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/x509"
	"encoding/pem"
	"errors"
	"io"
	"net"
	"os"
	"strings"
	"testing"

	client "github.com/pkg/sftp"
	"github.com/taruti/sshutil"
	"golang.org/x/crypto/ssh"
)

const alnum = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"

var testUser = "test"
var testPass = func() []byte {
	bs := make([]byte, 16)
	_, e := io.ReadFull(rand.Reader, bs)
	if e != nil {
		panic(e)
	}
	for i, b := range bs {
		bs[i] = alnum[int(b)%len(alnum)]
	}
	return bs
}()

func createKeyPEM() ([]byte, error) {
	k, e := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
	if e != nil {
		return nil, e
	}
	b, e := x509.MarshalECPrivateKey(k)
	if e != nil {
		return nil, e
	}
	return pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: b}), nil

}

func failOnErr(t *testing.T, err error, reason string) {
	if err != nil {
		t.Fatalf("%s: %v", reason, err)
	}
}

func PrintDiscardRequests(in <-chan *ssh.Request) {
	for req := range in {
		debug("PRINTDISC", req.Type, *req)
		if req.WantReply {
			req.Reply(false, nil)
		}
	}
}

func TestServer(t *testing.T) {
	debugf("Listening on port 2022 user %s pass %s\n", testUser, testPass)

	config := &ssh.ServerConfig{
		PasswordCallback: sshutil.CreatePasswordCheck(testUser, testPass),
	}

	hkey, e := sshutil.KeyLoader{Flags: sshutil.Create}.Load()
	failOnErr(t, e, "Failed to parse host key")
	debugf("Public key: %s\n", sshutil.PublicKeyHash(hkey.PublicKey()))

	config.AddHostKey(hkey)

	listener, e := net.Listen("tcp", "127.0.0.1:2022")
	failOnErr(t, e, "Failed to listen")

	go ClientDo()

	//	for {
	nConn, e := listener.Accept()
	failOnErr(t, e, "Failed to accept")
	handleConn(nConn, config, t, EmptyFS{})
	//	}

	go ClientDo()

	//	for {
	nConn, e = listener.Accept()
	failOnErr(t, e, "Failed to accept")
	os.Mkdir("/tmp/test-sftpd", 0700)
	handleConn(nConn, config, t, rfs{})
	//	}
}

func handleConn(conn net.Conn, config *ssh.ServerConfig, t *testing.T, fs FileSystem) {
	sc, chans, reqs, e := ssh.NewServerConn(conn, config)
	failOnErr(t, e, "Failed to initiate new connection")

	debug("sc", sc)

	// The incoming Request channel must be serviced.
	go PrintDiscardRequests(reqs)

	// Service the incoming Channel channel.
	for newChannel := range chans {
		debug("NEWCHANNEL", newChannel, newChannel.ChannelType())
		if newChannel.ChannelType() != "session" {
			newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}
		channel, requests, err := newChannel.Accept()
		if err != nil {
			panic("could not accept channel.")
		}

		go func(in <-chan *ssh.Request) {
			for req := range in {
				debug("REQUEST:", *req)
				ok := false
				switch {
				case IsSftpRequest(req):
					ok = true
					go func() { debug(ServeChannel(channel, fs)) }()
				}
				req.Reply(ok, nil)
			}
		}(requests)

	}
}

func ClientDo() {
	e := clientDo()
	if e != nil {
		debug("CLIENT ERROR", e)
	}
}
func clientDo() error {
	var cc ssh.ClientConfig
	cc.User = string(testUser)
	cc.Auth = append(cc.Auth, ssh.Password(string(testPass)))
	conn, e := ssh.Dial("tcp4", "127.0.0.1:2022", &cc)
	if e != nil {
		return e
	}
	defer conn.Close()
	cl, e := client.NewClient(conn)
	if e != nil {
		return e
	}
	rs, e := cl.ReadDir("/")
	if e != nil {
		return e
	}
	debug(rs)
	return nil
}

func TestRandomInput(t *testing.T) {
	fs := EmptyFS{}
	rd := &fakeRandChannel{}
	for i := 0; i < 10000; i++ {
		rd.rem = 5
		ServeChannel(rd, fs)
	}
	for i := 0; i < 257; i++ {
		for j := 0; j < 100; j++ {
			rd.rem = i
			ServeChannel(rd, fs)
		}
	}
}

type fakeRandChannel struct{ rem int }

func (fr *fakeRandChannel) Read(data []byte) (int, error) {
	if len(data) > fr.rem {
		data = data[0:fr.rem]
	}
	n, e := rand.Read(data)
	fr.rem -= n
	if fr.rem <= 0 {
		fr.rem = 0
		e = io.EOF
	}
	if len(data) >= 4 && data[0]&1 == 0 {
		data[0], data[1], data[2] = 0, 0, 0
		for i := 5; len(data) >= 4+i; i += 4 {
			data[i], data[i+1], data[i+2] = 0, 0, 0
		}
	}
	return n, e
}
func (*fakeRandChannel) Write(bs []byte) (int, error) {
	tmp := make([]byte, 1)
	rand.Read(tmp)
	if tmp[0]&7 == 7 {
		return 0, errors.New("Random write error")
	}
	return len(bs), nil
}
func (*fakeRandChannel) Close() error      { return nil }
func (*fakeRandChannel) CloseWrite() error { return nil }
func (*fakeRandChannel) SendRequest(name string, wantReply bool, payload []byte) (bool, error) {
	return true, nil
}
func (fr *fakeRandChannel) Stderr() io.ReadWriter { return fr }

type rfile struct {
	EmptyFile
	f *os.File
}

func (rf rfile) Close() error                             { return rf.f.Close() }
func (rf rfile) ReadAt(bs []byte, pos int64) (int, error) { return rf.f.ReadAt(bs, pos) }

type rfs struct {
	EmptyFS
}

type rdir struct {
	d *os.File
}

func (d rdir) Readdir(count int) ([]NamedAttr, error) {
	fis, e := d.d.Readdir(count)
	if e != nil {
		return nil, e
	}
	nas := make([]NamedAttr, len(fis))
	for i, fi := range fis {
		nas[i].Name = fi.Name()
		nas[i].Flags = ATTR_SIZE | ATTR_MODE
		nas[i].Size = uint64(fi.Size())
		nas[i].Mode = modeMangle(fi)
	}
	return nas, nil
}
func (d rdir) Close() error {
	return d.d.Close()
}

func rfsMangle(path string) (string, error) {
	if strings.Contains(path, "..") {
		return "<invalid>", errors.New("Invalid path")
	}
	if len(path) > 0 && path[0] == '/' {
		path = path[1:]
	}
	path = "/tmp/test-sftpd/" + path
	debug("MANGLE -> " + path)
	return path, nil
}

func (fs rfs) OpenDir(path string) (Dir, error) {
	p, e := rfsMangle(path)
	if e != nil {
		return nil, e
	}
	f, e := os.Open(p)
	if e != nil {
		return nil, e
	}
	return rdir{f}, nil
}

func (fs rfs) OpenFile(path string, mode uint32, a *Attr) (File, error) {
	p, e := rfsMangle(path)
	if e != nil {
		return nil, e
	}
	f, e := os.Open(p)
	if e != nil {
		return nil, e
	}
	return rfile{f: f}, nil
}

func (fs rfs) Stat(name string, islstat bool) (*Attr, error) {
	p, e := rfsMangle(name)
	if e != nil {
		return nil, e
	}
	var fi os.FileInfo
	if islstat {
		fi, e = os.Lstat(p)
	} else {
		fi, e = os.Stat(p)
	}
	if e != nil {
		return nil, e
	}
	var a Attr
	a.Flags = ATTR_SIZE | ATTR_MODE
	a.Size = uint64(fi.Size())
	a.Mode = modeMangle(fi)

	return &a, nil
}

func modeMangle(fi os.FileInfo) uint32 {
	m := fi.Mode()
	r := uint32(m.Perm())
	switch {
	case m.IsDir():
		r |= 0040000
	case m.IsRegular():
		r |= 0100000
	}
	return r
}
<commit_msg>Hook debugging into testing.T.Log in server_test<commit_after>package sftpd

import (
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/x509"
	"encoding/pem"
	"errors"
	"io"
	"net"
	"os"
	"strings"
	"testing"

	client "github.com/pkg/sftp"
	"github.com/taruti/sshutil"
	"golang.org/x/crypto/ssh"
)

const alnum = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"

var testUser = "test"
var testPass = func() []byte {
	bs := make([]byte, 16)
	_, e := io.ReadFull(rand.Reader, bs)
	if e != nil {
		panic(e)
	}
	for i, b := range bs {
		bs[i] = alnum[int(b)%len(alnum)]
	}
	return bs
}()

func createKeyPEM() ([]byte, error) {
	k, e := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
	if e != nil {
		return nil, e
	}
	b, e := x509.MarshalECPrivateKey(k)
	if e != nil {
		return nil, e
	}
	return pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: b}), nil

}

func failOnErr(t *testing.T, err error, reason string) {
	if err != nil {
		t.Fatalf("%s: %v", reason, err)
	}
}

func PrintDiscardRequests(in <-chan *ssh.Request) {
	for req := range in {
		debug("PRINTDISC", req.Type, *req)
		if req.WantReply {
			req.Reply(false, nil)
		}
	}
}

func TestServer(t *testing.T) {
	debug = t.Log
	debugf = t.Logf
	debugf("Listening on port 2022 user %s pass %s\n", testUser, testPass)

	config := &ssh.ServerConfig{
		PasswordCallback: sshutil.CreatePasswordCheck(testUser, testPass),
	}

	hkey, e := sshutil.KeyLoader{Flags: sshutil.Create}.Load()
	failOnErr(t, e, "Failed to parse host key")
	debugf("Public key: %s\n", sshutil.PublicKeyHash(hkey.PublicKey()))

	config.AddHostKey(hkey)

	listener, e := net.Listen("tcp", "127.0.0.1:2022")
	failOnErr(t, e, "Failed to listen")

	go ClientDo()

	//	for {
	nConn, e := listener.Accept()
	failOnErr(t, e, "Failed to accept")
	handleConn(nConn, config, t, EmptyFS{})
	//	}

	go ClientDo()

	//	for {
	nConn, e = listener.Accept()
	failOnErr(t, e, "Failed to accept")
	os.Mkdir("/tmp/test-sftpd", 0700)
	handleConn(nConn, config, t, rfs{})
	//	}
}

func handleConn(conn net.Conn, config *ssh.ServerConfig, t *testing.T, fs FileSystem) {
	sc, chans, reqs, e := ssh.NewServerConn(conn, config)
	failOnErr(t, e, "Failed to initiate new connection")

	debug("sc", sc)

	// The incoming Request channel must be serviced.
	go PrintDiscardRequests(reqs)

	// Service the incoming Channel channel.
	for newChannel := range chans {
		debug("NEWCHANNEL", newChannel, newChannel.ChannelType())
		if newChannel.ChannelType() != "session" {
			newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}
		channel, requests, err := newChannel.Accept()
		if err != nil {
			panic("could not accept channel.")
		}

		go func(in <-chan *ssh.Request) {
			for req := range in {
				debug("REQUEST:", *req)
				ok := false
				switch {
				case IsSftpRequest(req):
					ok = true
					go func() { debug(ServeChannel(channel, fs)) }()
				}
				req.Reply(ok, nil)
			}
		}(requests)

	}
}

func ClientDo() {
	e := clientDo()
	if e != nil {
		debug("CLIENT ERROR", e)
	}
}
func clientDo() error {
	var cc ssh.ClientConfig
	cc.User = string(testUser)
	cc.Auth = append(cc.Auth, ssh.Password(string(testPass)))
	conn, e := ssh.Dial("tcp4", "127.0.0.1:2022", &cc)
	if e != nil {
		return e
	}
	defer conn.Close()
	cl, e := client.NewClient(conn)
	if e != nil {
		return e
	}
	rs, e := cl.ReadDir("/")
	if e != nil {
		return e
	}
	debug(rs)
	return nil
}

func TestRandomInput(t *testing.T) {
	fs := EmptyFS{}
	rd := &fakeRandChannel{}
	for i := 0; i < 10000; i++ {
		rd.rem = 5
		ServeChannel(rd, fs)
	}
	for i := 0; i < 257; i++ {
		for j := 0; j < 100; j++ {
			rd.rem = i
			ServeChannel(rd, fs)
		}
	}
}

type fakeRandChannel struct{ rem int }

func (fr *fakeRandChannel) Read(data []byte) (int, error) {
	if len(data) > fr.rem {
		data = data[0:fr.rem]
	}
	n, e := rand.Read(data)
	fr.rem -= n
	if fr.rem <= 0 {
		fr.rem = 0
		e = io.EOF
	}
	if len(data) >= 4 && data[0]&1 == 0 {
		data[0], data[1], data[2] = 0, 0, 0
		for i := 5; len(data) >= 4+i; i += 4 {
			data[i], data[i+1], data[i+2] = 0, 0, 0
		}
	}
	return n, e
}
func (*fakeRandChannel) Write(bs []byte) (int, error) {
	tmp := make([]byte, 1)
	rand.Read(tmp)
	if tmp[0]&7 == 7 {
		return 0, errors.New("Random write error")
	}
	return len(bs), nil
}
func (*fakeRandChannel) Close() error      { return nil }
func (*fakeRandChannel) CloseWrite() error { return nil }
func (*fakeRandChannel) SendRequest(name string, wantReply bool, payload []byte) (bool, error) {
	return true, nil
}
func (fr *fakeRandChannel) Stderr() io.ReadWriter { return fr }

type rfile struct {
	EmptyFile
	f *os.File
}

func (rf rfile) Close() error                             { return rf.f.Close() }
func (rf rfile) ReadAt(bs []byte, pos int64) (int, error) { return rf.f.ReadAt(bs, pos) }

type rfs struct {
	EmptyFS
}

type rdir struct {
	d *os.File
}

func (d rdir) Readdir(count int) ([]NamedAttr, error) {
	fis, e := d.d.Readdir(count)
	if e != nil {
		return nil, e
	}
	nas := make([]NamedAttr, len(fis))
	for i, fi := range fis {
		nas[i].Name = fi.Name()
		nas[i].Flags = ATTR_SIZE | ATTR_MODE
		nas[i].Size = uint64(fi.Size())
		nas[i].Mode = modeMangle(fi)
	}
	return nas, nil
}
func (d rdir) Close() error {
	return d.d.Close()
}

func rfsMangle(path string) (string, error) {
	if strings.Contains(path, "..") {
		return "<invalid>", errors.New("Invalid path")
	}
	if len(path) > 0 && path[0] == '/' {
		path = path[1:]
	}
	path = "/tmp/test-sftpd/" + path
	debug("MANGLE -> " + path)
	return path, nil
}

func (fs rfs) OpenDir(path string) (Dir, error) {
	p, e := rfsMangle(path)
	if e != nil {
		return nil, e
	}
	f, e := os.Open(p)
	if e != nil {
		return nil, e
	}
	return rdir{f}, nil
}

func (fs rfs) OpenFile(path string, mode uint32, a *Attr) (File, error) {
	p, e := rfsMangle(path)
	if e != nil {
		return nil, e
	}
	f, e := os.Open(p)
	if e != nil {
		return nil, e
	}
	return rfile{f: f}, nil
}

func (fs rfs) Stat(name string, islstat bool) (*Attr, error) {
	p, e := rfsMangle(name)
	if e != nil {
		return nil, e
	}
	var fi os.FileInfo
	if islstat {
		fi, e = os.Lstat(p)
	} else {
		fi, e = os.Stat(p)
	}
	if e != nil {
		return nil, e
	}
	var a Attr
	a.Flags = ATTR_SIZE | ATTR_MODE
	a.Size = uint64(fi.Size())
	a.Mode = modeMangle(fi)

	return &a, nil
}

func modeMangle(fi os.FileInfo) uint32 {
	m := fi.Mode()
	r := uint32(m.Perm())
	switch {
	case m.IsDir():
		r |= 0040000
	case m.IsRegular():
		r |= 0100000
	}
	return r
}
<|endoftext|>