<commit_before>// Copyright (c) 2016 Pulcy.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vultr

import (
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/JamesClonk/vultr/lib"
	"github.com/op/go-logging"

	"github.com/pulcy/quark/providers"
	"github.com/pulcy/quark/templates"
)

const (
	fileMode            = os.FileMode(0775)
	cloudConfigTemplate = "templates/cloud-config.tmpl"
)

// Create a machine instance
func (vp *vultrProvider) CreateInstance(log *logging.Logger, options providers.CreateInstanceOptions, dnsProvider providers.DnsProvider) (providers.ClusterInstance, error) {
	// Create server
	id, err := vp.createServer(options)
	if err != nil {
		return providers.ClusterInstance{}, maskAny(err)
	}

	// Wait for the server to be active
	server, err := vp.waitUntilServerActive(id)
	if err != nil {
		return providers.ClusterInstance{}, maskAny(err)
	}

	publicIpv4 := server.MainIP
	publicIpv6 := server.MainIPV6
	if err := providers.RegisterInstance(vp.Logger, dnsProvider, options, server.Name, publicIpv4, publicIpv6); err != nil {
		return providers.ClusterInstance{}, maskAny(err)
	}

	vp.Logger.Info("Server '%s' is ready", server.Name)

	return vp.clusterInstance(server), nil
}

// Create a single server
func (vp *vultrProvider) createServer(options providers.CreateInstanceOptions) (string, error) {
	// Find SSH key ID
	var sshid string
	if len(options.SSHKeyNames) > 0 {
		var err error
		sshid, err = vp.findSSHKeyID(options.SSHKeyNames[0])
		if err != nil {
			return "", maskAny(err)
		}
	}
	// Fetch SSH keys
	sshKeys, err := providers.FetchSSHKeys(options.SSHKeyGithubAccount)
	if err != nil {
		return "", maskAny(err)
	}

	// Create cloud-config
	// user-data
	ccOpts := options.NewCloudConfigOptions()
	ccOpts.PrivateIPv4 = "$private_ipv4"
	ccOpts.SshKeys = sshKeys
	userData, err := templates.Render(cloudConfigTemplate, ccOpts)
	if err != nil {
		return "", maskAny(err)
	}

	name := options.InstanceName
	opts := &lib.ServerOptions{
		IPV6:              true,
		PrivateNetworking: true,
		SSHKey:            sshid,
		UserData:          userData,
	}
	regionID, err := strconv.Atoi(options.RegionID)
	if err != nil {
		return "", maskAny(err)
	}
	planID, err := strconv.Atoi(options.TypeID)
	if err != nil {
		return "", maskAny(err)
	}
	osID, err := strconv.Atoi(options.ImageID)
	if err != nil {
		return "", maskAny(err)
	}
	server, err := vp.client.CreateServer(name, regionID, planID, osID, opts)
	if err != nil {
		vp.Logger.Debug("CreateServer failed: %#v", err)
		return "", maskAny(err)
	}
	vp.Logger.Info("Created server %s %s\n", server.ID, server.Name)

	return server.ID, nil
}

func (vp *vultrProvider) waitUntilServerActive(id string) (lib.Server, error) {
	for {
		server, err := vp.client.GetServer(id)
		if err != nil {
			return lib.Server{}, err
		}
		if server.Status == "active" {
			// Attempt an SSH connection
			instance := vp.clusterInstance(server)
			if _, err := instance.GetMachineID(vp.Logger); err == nil {
				// Success
				return server, nil
			}
		}
		// Wait a while
		time.Sleep(time.Second * 5)
	}
}

// Create an entire cluster
func (vp *vultrProvider) CreateCluster(log *logging.Logger, options providers.CreateClusterOptions, dnsProvider providers.DnsProvider) error {
	wg := sync.WaitGroup{}
	errors := make(chan error, options.InstanceCount)
	instanceDatas := make(chan instanceData, options.InstanceCount)
	for i := 1; i <= options.InstanceCount; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			instanceOptions, err := options.NewCreateInstanceOptions(true, i)
			if err != nil {
				errors <- maskAny(err)
				return
			}
			instance, err := vp.CreateInstance(log, instanceOptions, dnsProvider)
			if err != nil {
				errors <- maskAny(err)
			} else {
				instanceDatas <- instanceData{
					CreateInstanceOptions: instanceOptions,
					ClusterInstance:       instance,
					FleetMetadata:         instanceOptions.CreateFleetMetadata(true, i),
				}
			}
		}(i)
	}
	wg.Wait()
	close(errors)
	close(instanceDatas)
	err := <-errors
	if err != nil {
		return maskAny(err)
	}

	instances := []instanceData{}
	instanceList := providers.ClusterInstanceList{}
	for data := range instanceDatas {
		instances = append(instances, data)
		instanceList = append(instanceList, data.ClusterInstance)
	}

	clusterMembers, err := instanceList.AsClusterMemberList(log, nil)
	if err != nil {
		return maskAny(err)
	}

	if err := vp.setupInstances(log, instances, clusterMembers); err != nil {
		return maskAny(err)
	}

	return nil
}

type instanceData struct {
	CreateInstanceOptions providers.CreateInstanceOptions
	ClusterInstance       providers.ClusterInstance
	FleetMetadata         string
}

func (vp *vultrProvider) setupInstances(log *logging.Logger, instances []instanceData, clusterMembers providers.ClusterMemberList) error {
	wg := sync.WaitGroup{}
	errors := make(chan error, len(instances))
	for _, instance := range instances {
		wg.Add(1)
		go func(instance instanceData) {
			defer wg.Done()
			iso := providers.InitialSetupOptions{
				ClusterMembers: clusterMembers,
				FleetMetadata:  instance.FleetMetadata,
			}
			if err := instance.ClusterInstance.InitialSetup(log, instance.CreateInstanceOptions, iso); err != nil {
				errors <- maskAny(err)
				return
			}
		}(instance)
	}
	wg.Wait()
	close(errors)
	err := <-errors
	if err != nil {
		return maskAny(err)
	}

	return nil
}
<commit_msg>Added a bit of delay between the individual create calls<commit_after>// Copyright (c) 2016 Pulcy.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vultr

import (
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/JamesClonk/vultr/lib"
	"github.com/op/go-logging"

	"github.com/pulcy/quark/providers"
	"github.com/pulcy/quark/templates"
)

const (
	fileMode            = os.FileMode(0775)
	cloudConfigTemplate = "templates/cloud-config.tmpl"
)

// Create a machine instance
func (vp *vultrProvider) CreateInstance(log *logging.Logger, options providers.CreateInstanceOptions, dnsProvider providers.DnsProvider) (providers.ClusterInstance, error) {
	// Create server
	id, err := vp.createServer(options)
	if err != nil {
		return providers.ClusterInstance{}, maskAny(err)
	}

	// Wait for the server to be active
	server, err := vp.waitUntilServerActive(id)
	if err != nil {
		return providers.ClusterInstance{}, maskAny(err)
	}

	publicIpv4 := server.MainIP
	publicIpv6 := server.MainIPV6
	if err := providers.RegisterInstance(vp.Logger, dnsProvider, options, server.Name, publicIpv4, publicIpv6); err != nil {
		return providers.ClusterInstance{}, maskAny(err)
	}

	vp.Logger.Info("Server '%s' is ready", server.Name)

	return vp.clusterInstance(server), nil
}

// Create a single server
func (vp *vultrProvider) createServer(options providers.CreateInstanceOptions) (string, error) {
	// Find SSH key ID
	var sshid string
	if len(options.SSHKeyNames) > 0 {
		var err error
		sshid, err = vp.findSSHKeyID(options.SSHKeyNames[0])
		if err != nil {
			return "", maskAny(err)
		}
	}
	// Fetch SSH keys
	sshKeys, err := providers.FetchSSHKeys(options.SSHKeyGithubAccount)
	if err != nil {
		return "", maskAny(err)
	}

	// Create cloud-config
	// user-data
	ccOpts := options.NewCloudConfigOptions()
	ccOpts.PrivateIPv4 = "$private_ipv4"
	ccOpts.SshKeys = sshKeys
	userData, err := templates.Render(cloudConfigTemplate, ccOpts)
	if err != nil {
		return "", maskAny(err)
	}

	name := options.InstanceName
	opts := &lib.ServerOptions{
		IPV6:              true,
		PrivateNetworking: true,
		SSHKey:            sshid,
		UserData:          userData,
	}
	regionID, err := strconv.Atoi(options.RegionID)
	if err != nil {
		return "", maskAny(err)
	}
	planID, err := strconv.Atoi(options.TypeID)
	if err != nil {
		return "", maskAny(err)
	}
	osID, err := strconv.Atoi(options.ImageID)
	if err != nil {
		return "", maskAny(err)
	}
	server, err := vp.client.CreateServer(name, regionID, planID, osID, opts)
	if err != nil {
		vp.Logger.Debug("CreateServer failed: %#v", err)
		return "", maskAny(err)
	}
	vp.Logger.Info("Created server %s %s\n", server.ID, server.Name)

	return server.ID, nil
}

func (vp *vultrProvider) waitUntilServerActive(id string) (lib.Server, error) {
	for {
		server, err := vp.client.GetServer(id)
		if err != nil {
			return lib.Server{}, err
		}
		if server.Status == "active" {
			// Attempt an SSH connection
			instance := vp.clusterInstance(server)
			if _, err := instance.GetMachineID(vp.Logger); err == nil {
				// Success
				return server, nil
			}
		}
		// Wait a while
		time.Sleep(time.Second * 5)
	}
}

// Create an entire cluster
func (vp *vultrProvider) CreateCluster(log *logging.Logger, options providers.CreateClusterOptions, dnsProvider providers.DnsProvider) error {
	wg := sync.WaitGroup{}
	errors := make(chan error, options.InstanceCount)
	instanceDatas := make(chan instanceData, options.InstanceCount)
	for i := 1; i <= options.InstanceCount; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			time.Sleep(time.Duration((i - 1)) * time.Second * 10)
			instanceOptions, err := options.NewCreateInstanceOptions(true, i)
			if err != nil {
				errors <- maskAny(err)
				return
			}
			instance, err := vp.CreateInstance(log, instanceOptions, dnsProvider)
			if err != nil {
				errors <- maskAny(err)
			} else {
				instanceDatas <- instanceData{
					CreateInstanceOptions: instanceOptions,
					ClusterInstance:       instance,
					FleetMetadata:         instanceOptions.CreateFleetMetadata(true, i),
				}
			}
		}(i)
	}
	wg.Wait()
	close(errors)
	close(instanceDatas)
	err := <-errors
	if err != nil {
		return maskAny(err)
	}

	instances := []instanceData{}
	instanceList := providers.ClusterInstanceList{}
	for data := range instanceDatas {
		instances = append(instances, data)
		instanceList = append(instanceList, data.ClusterInstance)
	}

	clusterMembers, err := instanceList.AsClusterMemberList(log, nil)
	if err != nil {
		return maskAny(err)
	}

	if err := vp.setupInstances(log, instances, clusterMembers); err != nil {
		return maskAny(err)
	}

	return nil
}

type instanceData struct {
	CreateInstanceOptions providers.CreateInstanceOptions
	ClusterInstance       providers.ClusterInstance
	FleetMetadata         string
}

func (vp *vultrProvider) setupInstances(log *logging.Logger, instances []instanceData, clusterMembers providers.ClusterMemberList) error {
	wg := sync.WaitGroup{}
	errors := make(chan error, len(instances))
	for _, instance := range instances {
		wg.Add(1)
		go func(instance instanceData) {
			defer wg.Done()
			iso := providers.InitialSetupOptions{
				ClusterMembers: clusterMembers,
				FleetMetadata:  instance.FleetMetadata,
			}
			if err := instance.ClusterInstance.InitialSetup(log, instance.CreateInstanceOptions, iso); err != nil {
				errors <- maskAny(err)
				return
			}
		}(instance)
	}
	wg.Wait()
	close(errors)
	err := <-errors
	if err != nil {
		return maskAny(err)
	}

	return nil
}
<|endoftext|>


<commit_before>package prompt

import (
	// Stdlib
	"fmt"
	"reflect"
	"strconv"

	// Vendor
	"github.com/bgentry/speakeasy"
)

func Dialog(value interface{}, questionPrefix string) error {
	var (
		v = reflect.Indirect(reflect.ValueOf(value))
		t = v.Type()
	)
	if v.Kind() != reflect.Struct {
		panic(fmt.Errorf("not a struct: %v", v.Kind()))
	}

	fmt.Println("Just press Enter to use the default value (if available).")
	fmt.Println()

	numFields := t.NumField()
	for i := 0; i < numFields; i++ {
		fv := v.Field(i)
		ft := t.Field(i)

		// Skip unexported fields.
		if ft.PkgPath != "" {
			continue
		}

		var (
			questionSuffix = ft.Tag.Get("prompt")
			defaultValue   = ft.Tag.Get("default")
			secret         = ft.Tag.Get("secret") != ""
		)
		if questionSuffix == "" {
			continue
		}

		question := fmt.Sprintf("%v %v", questionPrefix, questionSuffix)
		var (
			answer string
			err    error
		)
		if secret {
			answer, err = speakeasy.Ask(question + ": ")
			if err == nil && answer == "" {
				err = ErrCanceled
			}
		} else {
			if defaultValue != "" {
				answer, err = PromptDefault(question, defaultValue)
			} else {
				answer, err = Prompt(question + ": ")
			}
		}
		if err != nil {
			if err == ErrCanceled {
				PanicCancel()
			}
			return err
		}

		switch fv.Kind() {
		case reflect.Int:
			i, err := strconv.Atoi(answer)
			if err != nil {
				return err
			}
			fv.SetInt(int64(i))

		case reflect.String:
			fv.SetString(answer)

		default:
			return fmt.Errorf("unsupported field type: %v", fv.Kind())
		}
	}
	return nil
}
<commit_msg>prompt: Extend Dialog to handle structs recursively<commit_after>package prompt

import (
	// Stdlib
	"fmt"
	"reflect"
	"strconv"

	// Vendor
	"github.com/bgentry/speakeasy"
)

func Dialog(value interface{}, questionPrefix string) error {
	fmt.Println("Just press Enter to use the default value (if available).")
	fmt.Println()
	return dialogStruct(value, questionPrefix)
}

func dialogStruct(value interface{}, questionPrefix string) error {
	var (
		v = reflect.Indirect(reflect.ValueOf(value))
		t = v.Type()
	)
	if kind := v.Kind(); kind != reflect.Struct {
		panic(fmt.Errorf("not a struct: %v", kind))
	}

	numFields := t.NumField()
	for i := 0; i < numFields; i++ {
		fv := v.Field(i)
		ft := t.Field(i)

		// Skip unexported fields.
		if ft.PkgPath != "" {
			continue
		}

		// Fill structs recursively.
		if kind := reflect.Indirect(fv).Kind(); kind == reflect.Struct {
			if err := dialogStruct(fv.Addr().Interface(), questionPrefix); err != nil {
				return err
			}
			continue
		}

		var (
			questionSuffix = ft.Tag.Get("prompt")
			defaultValue   = ft.Tag.Get("default")
			secret         = ft.Tag.Get("secret") != ""
		)
		if questionSuffix == "" {
			continue
		}

		question := fmt.Sprintf("%v %v", questionPrefix, questionSuffix)
		var (
			answer string
			err    error
		)
		if secret {
			answer, err = speakeasy.Ask(question + ": ")
			if err == nil && answer == "" {
				err = ErrCanceled
			}
		} else {
			if defaultValue != "" {
				answer, err = PromptDefault(question, defaultValue)
			} else {
				answer, err = Prompt(question + ": ")
			}
		}
		if err != nil {
			if err == ErrCanceled {
				PanicCancel()
			}
			return err
		}

		switch fv.Kind() {
		case reflect.Int:
			i, err := strconv.Atoi(answer)
			if err != nil {
				return err
			}
			fv.SetInt(int64(i))

		case reflect.String:
			fv.SetString(answer)

		default:
			return fmt.Errorf("unsupported field type: %v", fv.Kind())
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package eth

import (
	"fmt"
	"math/big"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/errs"
	"github.com/ethereum/go-ethereum/eth/downloader"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/p2p"
	"github.com/ethereum/go-ethereum/rlp"
)

const (
	ProtocolVersion    = 60
	NetworkId          = 0
	ProtocolLength     = uint64(8)
	ProtocolMaxMsgSize = 10 * 1024 * 1024
	maxHashes          = 512
	maxBlocks          = 128
)

// eth protocol message codes
const (
	StatusMsg = iota
	GetTxMsg  // unused
	TxMsg
	GetBlockHashesMsg
	BlockHashesMsg
	GetBlocksMsg
	BlocksMsg
	NewBlockMsg
)

const (
	ErrMsgTooLarge = iota
	ErrDecode
	ErrInvalidMsgCode
	ErrProtocolVersionMismatch
	ErrNetworkIdMismatch
	ErrGenesisBlockMismatch
	ErrNoStatusMsg
	ErrExtraStatusMsg
	ErrSuspendedPeer
)

var errorToString = map[int]string{
	ErrMsgTooLarge:             "Message too long",
	ErrDecode:                  "Invalid message",
	ErrInvalidMsgCode:          "Invalid message code",
	ErrProtocolVersionMismatch: "Protocol version mismatch",
	ErrNetworkIdMismatch:       "NetworkId mismatch",
	ErrGenesisBlockMismatch:    "Genesis block mismatch",
	ErrNoStatusMsg:             "No status message",
	ErrExtraStatusMsg:          "Extra status message",
	ErrSuspendedPeer:           "Suspended peer",
}

// ethProtocol represents the ethereum wire protocol
// instance is running on each peer
type ethProtocol struct {
	txPool          txPool
	chainManager    chainManager
	blockPool       blockPool
	downloader      *downloader.Downloader
	peer            *p2p.Peer
	id              string
	rw              p2p.MsgReadWriter
	errors          *errs.Errors
	protocolVersion int
	networkId       int
}

// backend is the interface the ethereum protocol backend should implement
// used as an argument to EthProtocol
type txPool interface {
	AddTransactions([]*types.Transaction)
	GetTransactions() types.Transactions
}

type chainManager interface {
	GetBlockHashesFromHash(hash common.Hash, amount uint64) (hashes []common.Hash)
	GetBlock(hash common.Hash) (block *types.Block)
	Status() (td *big.Int, currentBlock common.Hash, genesisBlock common.Hash)
}

type blockPool interface {
	AddBlockHashes(next func() (common.Hash, bool), peerId string)
	AddBlock(block *types.Block, peerId string)
	AddPeer(td *big.Int, currentBlock common.Hash, peerId string, requestHashes func(common.Hash) error, requestBlocks func([]common.Hash) error, peerError func(*errs.Error)) (best bool, suspended bool)
	RemovePeer(peerId string)
}

// message structs used for RLP serialization
type newBlockMsgData struct {
	Block *types.Block
	TD    *big.Int
}

type getBlockHashesMsgData struct {
	Hash   common.Hash
	Amount uint64
}

type statusMsgData struct {
	ProtocolVersion uint32
	NetworkId       uint32
	TD              *big.Int
	CurrentBlock    common.Hash
	GenesisBlock    common.Hash
}

// main entrypoint, wrappers starting a server running the eth protocol
// use this constructor to attach the protocol ("class") to server caps
// the Dev p2p layer then runs the protocol instance on each peer
func EthProtocol(protocolVersion, networkId int, txPool txPool, chainManager chainManager, blockPool blockPool, downloader *downloader.Downloader) p2p.Protocol {
	return p2p.Protocol{
		Name:    "eth",
		Version: uint(protocolVersion),
		Length:  ProtocolLength,
		Run: func(peer *p2p.Peer, rw p2p.MsgReadWriter) error {
			return runEthProtocol(protocolVersion, networkId, txPool, chainManager, blockPool, downloader, peer, rw)
		},
	}
}

// the main loop that handles incoming messages
// note RemovePeer in the post-disconnect hook
func runEthProtocol(protocolVersion, networkId int, txPool txPool, chainManager chainManager, blockPool blockPool, downloader *downloader.Downloader, peer *p2p.Peer, rw p2p.MsgReadWriter) (err error) {
	id := peer.ID()
	self := &ethProtocol{
		txPool:          txPool,
		chainManager:    chainManager,
		blockPool:       blockPool,
		downloader:      downloader,
		rw:              rw,
		peer:            peer,
		protocolVersion: protocolVersion,
		networkId:       networkId,
		errors: &errs.Errors{
			Package: "ETH",
			Errors:  errorToString,
		},
		id: fmt.Sprintf("%x", id[:8]),
	}

	// handshake.
	if err := self.handleStatus(); err != nil {
		return err
	}
	defer self.blockPool.RemovePeer(self.id)

	// propagate existing transactions. new transactions appearing
	// after this will be sent via broadcasts.
	if err := p2p.Send(rw, TxMsg, txPool.GetTransactions()); err != nil {
		return err
	}

	// main loop. handle incoming messages.
	for {
		if err := self.handle(); err != nil {
			return err
		}
	}
}

func (self *ethProtocol) handle() error {
	msg, err := self.rw.ReadMsg()
	if err != nil {
		return err
	}
	if msg.Size > ProtocolMaxMsgSize {
		return self.protoError(ErrMsgTooLarge, "%v > %v", msg.Size, ProtocolMaxMsgSize)
	}
	// make sure that the payload has been fully consumed
	defer msg.Discard()

	switch msg.Code {
	case GetTxMsg: // ignore
	case StatusMsg:
		return self.protoError(ErrExtraStatusMsg, "")

	case TxMsg:
		// TODO: rework using lazy RLP stream
		var txs []*types.Transaction
		if err := msg.Decode(&txs); err != nil {
			return self.protoError(ErrDecode, "msg %v: %v", msg, err)
		}
		for i, tx := range txs {
			if tx == nil {
				return self.protoError(ErrDecode, "transaction %d is nil", i)
			}
			jsonlogger.LogJson(&logger.EthTxReceived{
				TxHash:   tx.Hash().Hex(),
				RemoteId: self.peer.ID().String(),
			})
		}
		self.txPool.AddTransactions(txs)

	case GetBlockHashesMsg:
		var request getBlockHashesMsgData
		if err := msg.Decode(&request); err != nil {
			return self.protoError(ErrDecode, "->msg %v: %v", msg, err)
		}

		if request.Amount > maxHashes {
			request.Amount = maxHashes
		}
		hashes := self.chainManager.GetBlockHashesFromHash(request.Hash, request.Amount)
		return p2p.Send(self.rw, BlockHashesMsg, hashes)

	case BlockHashesMsg:
		msgStream := rlp.NewStream(msg.Payload)

		var hashes []common.Hash
		if err := msgStream.Decode(&hashes); err != nil {
			break
		}
		self.downloader.HashCh <- hashes

		/*
			if _, err := msgStream.List(); err != nil {
				return err
			}

			var i int
			iter := func() (hash common.Hash, err error) {
				err = msgStream.Decode(&hash)
				if err == rlp.EOL {
					return common.Hash{}, err
				} else if err != nil {
					return common.Hash{}, fmt.Errorf("Fetching hashes err (%d): %v", i, err)
				}

				i++
				return hash, nil
			}
			self.downloader.HashCh <- iter
			//self.blockPool.AddBlockHashes(iter, self.id)
		*/

	case GetBlocksMsg:
		msgStream := rlp.NewStream(msg.Payload)
		if _, err := msgStream.List(); err != nil {
			return err
		}

		var blocks []*types.Block
		var i int
		for {
			i++
			var hash common.Hash
			err := msgStream.Decode(&hash)
			if err == rlp.EOL {
				break
			} else if err != nil {
				return self.protoError(ErrDecode, "msg %v: %v", msg, err)
			}

			block := self.chainManager.GetBlock(hash)
			if block != nil {
				blocks = append(blocks, block)
			}
			if i == maxBlocks {
				break
			}
		}
		return p2p.Send(self.rw, BlocksMsg, blocks)

	case BlocksMsg:
		msgStream := rlp.NewStream(msg.Payload)

		var blocks []*types.Block
		if err := msgStream.Decode(&blocks); err != nil {
			glog.V(logger.Detail).Infoln("Decode error", err)
			fmt.Println("decode error", err)
			blocks = nil
		}
		self.downloader.DeliverChunk(self.id, blocks)
		/*
			msgStream := rlp.NewStream(msg.Payload)
			if _, err := msgStream.List(); err != nil {
				return err
			}
			for {
				var block types.Block
				if err := msgStream.Decode(&block); err != nil {
					if err == rlp.EOL {
						break
					} else {
						return self.protoError(ErrDecode, "msg %v: %v", msg, err)
					}
				}
				if err := block.ValidateFields(); err != nil {
					return self.protoError(ErrDecode, "block validation %v: %v", msg, err)
				}
				self.blockPool.AddBlock(&block, self.id)
			}
		*/

	case NewBlockMsg:
		var request newBlockMsgData
		if err := msg.Decode(&request); err != nil {
			return self.protoError(ErrDecode, "%v: %v", msg, err)
		}
		if err := request.Block.ValidateFields(); err != nil {
			return self.protoError(ErrDecode, "block validation %v: %v", msg, err)
		}
		hash := request.Block.Hash()
		_, chainHead, _ := self.chainManager.Status()

		jsonlogger.LogJson(&logger.EthChainReceivedNewBlock{
			BlockHash:     hash.Hex(),
			BlockNumber:   request.Block.Number(), // this surely must be zero
			ChainHeadHash: chainHead.Hex(),
			BlockPrevHash: request.Block.ParentHash().Hex(),
			RemoteId:      self.peer.ID().String(),
		})

		self.downloader.AddBlock(self.id, request.Block, request.TD)
		// to simplify backend interface adding a new block
		// uses AddPeer followed by AddBlock only if peer is the best peer
		// (or selected as new best peer)
		if _, suspended := self.blockPool.AddPeer(request.TD, hash, self.id, self.requestBlockHashes, self.requestBlocks, self.protoErrorDisconnect); !suspended {
			self.blockPool.AddBlock(request.Block, self.id)
		}

	default:
		return self.protoError(ErrInvalidMsgCode, "%v", msg.Code)
	}
	return nil
}

func (self *ethProtocol) handleStatus() error {
	if err := self.sendStatus(); err != nil {
		return err
	}

	// read and handle remote status
	msg, err := self.rw.ReadMsg()
	if err != nil {
		return err
	}
	if msg.Code != StatusMsg {
		return self.protoError(ErrNoStatusMsg, "first msg has code %x (!= %x)", msg.Code, StatusMsg)
	}
	if msg.Size > ProtocolMaxMsgSize {
		return self.protoError(ErrMsgTooLarge, "%v > %v", msg.Size, ProtocolMaxMsgSize)
	}

	var status statusMsgData
	if err := msg.Decode(&status); err != nil {
		return self.protoError(ErrDecode, "msg %v: %v", msg, err)
	}

	_, _, genesisBlock := self.chainManager.Status()

	if status.GenesisBlock != genesisBlock {
		return self.protoError(ErrGenesisBlockMismatch, "%x (!= %x)", status.GenesisBlock, genesisBlock)
	}

	if int(status.NetworkId) != self.networkId {
		return self.protoError(ErrNetworkIdMismatch, "%d (!= %d)", status.NetworkId, self.networkId)
	}

	if int(status.ProtocolVersion) != self.protocolVersion {
		return self.protoError(ErrProtocolVersionMismatch, "%d (!= %d)", status.ProtocolVersion, self.protocolVersion)
	}

	err = self.downloader.RegisterPeer(self.id, status.TD, status.CurrentBlock, self.requestBlockHashes, self.requestBlocks)
	if err != nil {
		return self.protoError(ErrSuspendedPeer, "something")
	}
	/*
		_, suspended := self.blockPool.AddPeer(status.TD, status.CurrentBlock, self.id, self.requestBlockHashes, self.requestBlocks, self.protoErrorDisconnect)
		if suspended {
			return self.protoError(ErrSuspendedPeer, "")
		}
	*/

	self.peer.Debugf("Peer is [eth] capable (%d/%d). TD=%v H=%x\n", status.ProtocolVersion, status.NetworkId, status.TD, status.CurrentBlock[:4])

	return nil
}

func (self *ethProtocol) requestBlockHashes(from common.Hash) error {
	self.peer.Debugf("fetching hashes (%d) %x...\n", maxHashes, from[0:4])
	return p2p.Send(self.rw, GetBlockHashesMsg, getBlockHashesMsgData{from, maxHashes})
}

func (self *ethProtocol) requestBlocks(hashes []common.Hash) error {
	self.peer.Debugf("fetching %v blocks", len(hashes))
	return p2p.Send(self.rw, GetBlocksMsg, hashes)
}

func (self *ethProtocol) protoError(code int, format string, params ...interface{}) (err *errs.Error) {
	err = self.errors.New(code, format, params...)
	//err.Log(self.peer.Logger)
	err.Log(glog.V(logger.Info))
	return
}

func (self *ethProtocol) sendStatus() error {
	td, currentBlock, genesisBlock := self.chainManager.Status()
	return p2p.Send(self.rw, StatusMsg, &statusMsgData{
		ProtocolVersion: uint32(self.protocolVersion),
		NetworkId:       uint32(self.networkId),
		TD:              td,
		CurrentBlock:    currentBlock,
		GenesisBlock:    genesisBlock,
	})
}

func (self *ethProtocol) protoErrorDisconnect(err *errs.Error) {
	err.Log(glog.V(logger.Info))
	if err.Fatal() {
		self.peer.Disconnect(p2p.DiscSubprotocolError)
	}

}
<commit_msg>eth: basic implementation of the downloader<commit_after>package eth

import (
	"fmt"
	"math/big"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/errs"
	"github.com/ethereum/go-ethereum/eth/downloader"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/p2p"
	"github.com/ethereum/go-ethereum/rlp"
)

const (
	ProtocolVersion    = 60
	NetworkId          = 0
	ProtocolLength     = uint64(8)
	ProtocolMaxMsgSize = 10 * 1024 * 1024
	maxHashes          = 512
	maxBlocks          = 128
)

// eth protocol message codes
const (
	StatusMsg = iota
	GetTxMsg  // unused
	TxMsg
	GetBlockHashesMsg
	BlockHashesMsg
	GetBlocksMsg
	BlocksMsg
	NewBlockMsg
)

const (
	ErrMsgTooLarge = iota
	ErrDecode
	ErrInvalidMsgCode
	ErrProtocolVersionMismatch
	ErrNetworkIdMismatch
	ErrGenesisBlockMismatch
	ErrNoStatusMsg
	ErrExtraStatusMsg
	ErrSuspendedPeer
)

var errorToString = map[int]string{
	ErrMsgTooLarge:             "Message too long",
	ErrDecode:                  "Invalid message",
	ErrInvalidMsgCode:          "Invalid message code",
	ErrProtocolVersionMismatch: "Protocol version mismatch",
	ErrNetworkIdMismatch:       "NetworkId mismatch",
	ErrGenesisBlockMismatch:    "Genesis block mismatch",
	ErrNoStatusMsg:             "No status message",
	ErrExtraStatusMsg:          "Extra status message",
	ErrSuspendedPeer:           "Suspended peer",
}

// ethProtocol represents the ethereum wire protocol
// instance is running on each peer
type ethProtocol struct {
	txPool          txPool
	chainManager    chainManager
	blockPool       blockPool
	downloader      *downloader.Downloader
	peer            *p2p.Peer
	id              string
	rw              p2p.MsgReadWriter
	errors          *errs.Errors
	protocolVersion int
	networkId       int
}

// backend is the interface the ethereum protocol backend should implement
// used as an argument to EthProtocol
type txPool interface {
	AddTransactions([]*types.Transaction)
	GetTransactions() types.Transactions
}

type chainManager interface {
	GetBlockHashesFromHash(hash common.Hash, amount uint64) (hashes []common.Hash)
	GetBlock(hash common.Hash) (block *types.Block)
	Status() (td *big.Int, currentBlock common.Hash, genesisBlock common.Hash)
}

type blockPool interface {
	AddBlockHashes(next func() (common.Hash, bool), peerId string)
	AddBlock(block *types.Block, peerId string)
	AddPeer(td *big.Int, currentBlock common.Hash, peerId string, requestHashes func(common.Hash) error, requestBlocks func([]common.Hash) error, peerError func(*errs.Error)) (best bool, suspended bool)
	RemovePeer(peerId string)
}

// message structs used for RLP serialization
type newBlockMsgData struct {
	Block *types.Block
	TD    *big.Int
}

type getBlockHashesMsgData struct {
	Hash   common.Hash
	Amount uint64
}

type statusMsgData struct {
	ProtocolVersion uint32
	NetworkId       uint32
	TD              *big.Int
	CurrentBlock    common.Hash
	GenesisBlock    common.Hash
}

// main entrypoint, wrappers starting a server running the eth protocol
// use this constructor to attach the protocol ("class") to server caps
// the Dev p2p layer then runs the protocol instance on each peer
func EthProtocol(protocolVersion, networkId int, txPool txPool, chainManager chainManager, blockPool blockPool, downloader *downloader.Downloader) p2p.Protocol {
	return p2p.Protocol{
		Name:    "eth",
		Version: uint(protocolVersion),
		Length:  ProtocolLength,
		Run: func(peer *p2p.Peer, rw p2p.MsgReadWriter) error {
			return runEthProtocol(protocolVersion, networkId, txPool, chainManager, blockPool, downloader, peer, rw)
		},
	}
}

// the main loop that handles incoming messages
// note RemovePeer in the post-disconnect hook
func runEthProtocol(protocolVersion, networkId int, txPool txPool, chainManager chainManager, blockPool blockPool, downloader *downloader.Downloader, peer *p2p.Peer, rw p2p.MsgReadWriter) (err error) {
	id := peer.ID()
	self := &ethProtocol{
		txPool:          txPool,
		chainManager:    chainManager,
		blockPool:       blockPool,
		downloader:      downloader,
		rw:              rw,
		peer:            peer,
		protocolVersion: protocolVersion,
		networkId:       networkId,
		errors: &errs.Errors{
			Package: "ETH",
			Errors:  errorToString,
		},
		id: fmt.Sprintf("%x", id[:8]),
	}

	// handshake.
	if err := self.handleStatus(); err != nil {
		return err
	}
	defer self.blockPool.RemovePeer(self.id)

	// propagate existing transactions. new transactions appearing
	// after this will be sent via broadcasts.
	if err := p2p.Send(rw, TxMsg, txPool.GetTransactions()); err != nil {
		return err
	}

	// main loop. handle incoming messages.
	for {
		if err := self.handle(); err != nil {
			return err
		}
	}
}

func (self *ethProtocol) handle() error {
	msg, err := self.rw.ReadMsg()
	if err != nil {
		return err
	}
	if msg.Size > ProtocolMaxMsgSize {
		return self.protoError(ErrMsgTooLarge, "%v > %v", msg.Size, ProtocolMaxMsgSize)
	}
	// make sure that the payload has been fully consumed
	defer msg.Discard()

	switch msg.Code {
	case GetTxMsg: // ignore
	case StatusMsg:
		return self.protoError(ErrExtraStatusMsg, "")

	case TxMsg:
		// TODO: rework using lazy RLP stream
		var txs []*types.Transaction
		if err := msg.Decode(&txs); err != nil {
			return self.protoError(ErrDecode, "msg %v: %v", msg, err)
		}
		for i, tx := range txs {
			if tx == nil {
				return self.protoError(ErrDecode, "transaction %d is nil", i)
			}
			jsonlogger.LogJson(&logger.EthTxReceived{
				TxHash:   tx.Hash().Hex(),
				RemoteId: self.peer.ID().String(),
			})
		}
		self.txPool.AddTransactions(txs)

	case GetBlockHashesMsg:
		var request getBlockHashesMsgData
		if err := msg.Decode(&request); err != nil {
			return self.protoError(ErrDecode, "->msg %v: %v", msg, err)
		}

		if request.Amount > maxHashes {
			request.Amount = maxHashes
		}
		hashes := self.chainManager.GetBlockHashesFromHash(request.Hash, request.Amount)
		return p2p.Send(self.rw, BlockHashesMsg, hashes)

	case BlockHashesMsg:
		msgStream := rlp.NewStream(msg.Payload)

		var hashes []common.Hash
		if err := msgStream.Decode(&hashes); err != nil {
			break
		}
		self.downloader.HashCh <- hashes

		/*
			if _, err := msgStream.List(); err != nil {
				return err
			}

			var i int
			iter := func() (hash common.Hash, err error) {
				err = msgStream.Decode(&hash)
				if err == rlp.EOL {
					return common.Hash{}, err
				} else if err != nil {
					return common.Hash{}, fmt.Errorf("Fetching hashes err (%d): %v", i, err)
				}

				i++
				return hash, nil
			}
			self.downloader.HashCh <- iter
			//self.blockPool.AddBlockHashes(iter, self.id)
		*/

	case GetBlocksMsg:
		msgStream := rlp.NewStream(msg.Payload)
		if _, err := msgStream.List(); err != nil {
			return err
		}

		var blocks []*types.Block
		var i int
		for {
			i++
			var hash common.Hash
			err := msgStream.Decode(&hash)
			if err == rlp.EOL {
				break
			} else if err != nil {
				return self.protoError(ErrDecode, "msg %v: %v", msg, err)
			}

			block := self.chainManager.GetBlock(hash)
			if block != nil {
				blocks = append(blocks, block)
			}
			if i == maxBlocks {
				break
			}
		}
		return p2p.Send(self.rw, BlocksMsg, blocks)

	case BlocksMsg:
		msgStream := rlp.NewStream(msg.Payload)

		var blocks []*types.Block
		if err := msgStream.Decode(&blocks); err != nil {
			glog.V(logger.Detail).Infoln("Decode error", err)
			fmt.Println("decode error", err)
			blocks = nil
		}
		self.downloader.DeliverChunk(self.id, blocks)
		/*
			msgStream := rlp.NewStream(msg.Payload)
			if _, err := msgStream.List(); err != nil {
				return err
			}
			for {
				var block types.Block
				if err := msgStream.Decode(&block); err != nil {
					if err == rlp.EOL {
						break
					} else {
						return self.protoError(ErrDecode, "msg %v: %v", msg, err)
					}
				}
				if err := block.ValidateFields(); err != nil {
					return self.protoError(ErrDecode, "block validation %v: %v", msg, err)
				}
				self.blockPool.AddBlock(&block, self.id)
			}
		*/

	case NewBlockMsg:
		var request newBlockMsgData
		if err := msg.Decode(&request); err != nil {
			return self.protoError(ErrDecode, "%v: %v", msg, err)
		}
		if err := request.Block.ValidateFields(); err != nil {
			return self.protoError(ErrDecode, "block validation %v: %v", msg, err)
		}
		hash := request.Block.Hash()
		_, chainHead, _ := self.chainManager.Status()

		jsonlogger.LogJson(&logger.EthChainReceivedNewBlock{
			BlockHash:     hash.Hex(),
			BlockNumber:   request.Block.Number(), // this surely must be zero
			ChainHeadHash: chainHead.Hex(),
			BlockPrevHash: request.Block.ParentHash().Hex(),
			RemoteId:      self.peer.ID().String(),
		})

		self.downloader.AddBlock(self.id, request.Block, request.TD)
		// to simplify backend interface adding a new block
		// uses AddPeer followed by AddBlock only if peer is the best peer
		// (or selected as new best peer)
		if _, suspended := self.blockPool.AddPeer(request.TD, hash, self.id, self.requestBlockHashes, self.requestBlocks, self.protoErrorDisconnect); !suspended {
			self.blockPool.AddBlock(request.Block, self.id)
		}

	default:
		return self.protoError(ErrInvalidMsgCode, "%v", msg.Code)
	}
	return nil
}

func (self *ethProtocol) handleStatus() error {
	if err := self.sendStatus(); err != nil {
		return err
	}

	// read and handle remote status
	msg, err := self.rw.ReadMsg()
	if err != nil {
		return err
	}
	if msg.Code != StatusMsg {
		return self.protoError(ErrNoStatusMsg, "first msg has code %x (!= %x)", msg.Code, StatusMsg)
	}
	if msg.Size > ProtocolMaxMsgSize {
		return self.protoError(ErrMsgTooLarge, "%v > %v", msg.Size, ProtocolMaxMsgSize)
	}

	var status statusMsgData
	if err := msg.Decode(&status); err != nil {
		return self.protoError(ErrDecode, "msg %v: %v", msg, err)
	}

	_, _, genesisBlock := self.chainManager.Status()

	if status.GenesisBlock != genesisBlock {
		return self.protoError(ErrGenesisBlockMismatch, "%x (!= %x)", status.GenesisBlock, genesisBlock)
	}

	if int(status.NetworkId) != self.networkId {
		return self.protoError(ErrNetworkIdMismatch, "%d (!= %d)", status.NetworkId, self.networkId)
	}

	if int(status.ProtocolVersion) != self.protocolVersion {
		return self.protoError(ErrProtocolVersionMismatch, "%d (!= %d)", status.ProtocolVersion, self.protocolVersion)
	}

	err = self.downloader.RegisterPeer(self.id, status.TD, status.CurrentBlock, self.requestBlockHashes, self.requestBlocks)
	if err != nil {
		return self.protoError(ErrSuspendedPeer, "something")
	}
	/*
		_, suspended := self.blockPool.AddPeer(status.TD, status.CurrentBlock, self.id, self.requestBlockHashes, self.requestBlocks, self.protoErrorDisconnect)
		if suspended {
			return self.protoError(ErrSuspendedPeer, "")
		}
	*/

	self.peer.Debugf("Peer is [eth] capable (%d/%d). TD=%v H=%x\n", status.ProtocolVersion, status.NetworkId, status.TD, status.CurrentBlock[:4])

	return nil
}

func (self *ethProtocol) requestBlockHashes(from common.Hash) error {
	self.peer.Debugf("fetching hashes (%d) %x...\n", maxHashes, from[0:4])
	return p2p.Send(self.rw, GetBlockHashesMsg, getBlockHashesMsgData{from, maxHashes})
}

func (self *ethProtocol) requestBlocks(hashes []common.Hash) error {
	self.peer.Debugf("fetching %v blocks", len(hashes))
	return p2p.Send(self.rw, GetBlocksMsg, hashes)
}

/*
func (self *ethProtocol) newRespBlockCh() BlockPack {
	self.blockRespCh = make(chan blockResp)
	return self.blockRespCh
}

func (self *ethProtocol) RequestBlocks(hashes *set.Set) <-chan []*types.Block {
	out := make(chan []*types.Block)
	go func() {
	done:
		for {
			select {
			case blockResp := <-self.newRespBlockCh():
				if len(blockResp.blocks) {
				}
			case <-time.After(5 * time.Second):
			}
		}

		close(out)
	}()
	return out
}
*/

func (self *ethProtocol) protoError(code int, format string, params ...interface{}) (err *errs.Error) {
	err = self.errors.New(code, format, params...)
	//err.Log(self.peer.Logger)
	err.Log(glog.V(logger.Info))
	return
}

func (self *ethProtocol) sendStatus() error {
	td, currentBlock, genesisBlock := self.chainManager.Status()
	return p2p.Send(self.rw, StatusMsg, &statusMsgData{
		ProtocolVersion: uint32(self.protocolVersion),
		NetworkId:       uint32(self.networkId),
		TD:              td,
		CurrentBlock:    currentBlock,
		GenesisBlock:    genesisBlock,
	})
}

func (self *ethProtocol) protoErrorDisconnect(err *errs.Error) {
	err.Log(glog.V(logger.Info))
	if err.Fatal() {
		self.peer.Disconnect(p2p.DiscSubprotocolError)
	}

}
<|endoftext|>


<commit_before>package eth

import (
	"container/list"
	"github.com/ethereum/ethchain-go"
	"github.com/ethereum/ethdb-go"
	"github.com/ethereum/ethutil-go"
	"github.com/ethereum/ethwire-go"
	"log"
	"net"
	"strconv"
	"sync"
	"sync/atomic"
	"time"
)

func eachPeer(peers *list.List, callback func(*Peer, *list.Element)) {
	// Loop thru the peers and close them (if we had them)
	for e := peers.Front(); e != nil; e = e.Next() {
		if peer, ok := e.Value.(*Peer); ok {
			callback(peer, e)
		}
	}
}

const (
	processReapingTimeout = 60 // TODO increase
)

type Ethereum struct {
	// Channel for shutting down the ethereum
	shutdownChan chan bool
	quit         chan bool
	// DB interface
	//db *ethdb.LDBDatabase
	db ethutil.Database
	// Block manager for processing new blocks and managing the block chain
	BlockManager *ethchain.BlockManager
	// The transaction pool. Transaction can be pushed on this pool
	// for later including in the blocks
	TxPool *ethchain.TxPool
	// Peers (NYI)
	peers *list.List
	// Nonce
	Nonce uint64

	Addr net.Addr

	peerMut sync.Mutex

	// Capabilities for outgoing peers
	serverCaps Caps

	nat NAT

	// Specifies the desired amount of maximum peers
	MaxPeers int
}

func New(caps Caps, usePnp bool) (*Ethereum, error) {
	//db, err := ethdb.NewLDBDatabase()
	db, err := ethdb.NewMemDatabase()
	if err != nil {
		return nil, err
	}

	var nat NAT
	if usePnp {
		nat, err = Discover()
		if err != nil {
			log.Println("UPnP failed", err)
		}
	}

	ethutil.Config.Db = db

	nonce, _ := ethutil.RandomUint64()
	ethereum := &Ethereum{
		shutdownChan: make(chan bool),
		quit:         make(chan bool),
		db:           db,
		peers:        list.New(),
		Nonce:        nonce,
		serverCaps:   caps,
		nat:          nat,
		MaxPeers:     5,
	}
	ethereum.TxPool = ethchain.NewTxPool()
	ethereum.TxPool.Speaker = ethereum
	ethereum.BlockManager = ethchain.NewBlockManager(ethereum)

	ethereum.TxPool.BlockManager = ethereum.BlockManager
	ethereum.BlockManager.TransactionPool = ethereum.TxPool

	return ethereum, nil
}

func (s *Ethereum) AddPeer(conn net.Conn) {
	peer := NewPeer(conn, s, true)

	if peer != nil && s.peers.Len() < s.MaxPeers {
		s.peers.PushBack(peer)
		peer.Start()
	}
}

func (s *Ethereum) ProcessPeerList(addrs []string) {
	for _, addr := range addrs {
		// TODO Probably requires some sanity checks
		s.ConnectToPeer(addr)
	}
}

func (s *Ethereum) ConnectToPeer(addr string) error {
	var alreadyConnected bool

	eachPeer(s.peers, func(p *Peer, v *list.Element) {
		if p.conn == nil {
			return
		}
		phost, _, _ := net.SplitHostPort(p.conn.RemoteAddr().String())
		ahost, _, _ := net.SplitHostPort(addr)

		if phost == ahost {
			alreadyConnected = true
			return
		}
	})

	if alreadyConnected {
		return nil
	}

	peer := NewOutboundPeer(addr, s, s.serverCaps)

	s.peers.PushBack(peer)

	return nil
}

func (s *Ethereum) OutboundPeers() []*Peer {
	// Create a new peer slice with at least the length of the total peers
	outboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if !p.inbound && p.conn != nil {
			outboundPeers[length] = p
			length++
		}
	})

	return outboundPeers[:length]
}

func (s *Ethereum) InboundPeers() []*Peer {
	// Create a new peer slice with at least the length of the total peers
	inboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if p.inbound {
			inboundPeers[length] = p
			length++
		}
	})

	return inboundPeers[:length]
}

func (s *Ethereum) InOutPeers() []*Peer {
	// Reap the dead peers first
	s.reapPeers()

	// Create a new peer slice with at least the length of the total peers
	inboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		// Only return peers with an actual ip
		if len(p.host) > 0 {
			inboundPeers[length] = p
			length++
		}
	})

	return inboundPeers[:length]
}

func (s *Ethereum) Broadcast(msgType ethwire.MsgType, data []interface{}) {
	msg := ethwire.NewMessage(msgType, data)
	s.BroadcastMsg(msg)
}

func (s *Ethereum) BroadcastMsg(msg *ethwire.Msg) {
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		p.QueueMessage(msg)
	})
}

func (s *Ethereum) Peers() *list.List {
	return s.peers
}

func (s *Ethereum) reapPeers() {
	s.peerMut.Lock()
	defer s.peerMut.Unlock()

	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if atomic.LoadInt32(&p.disconnect) == 1 || (p.inbound && (time.Now().Unix()-p.lastPong) > int64(5*time.Minute)) {
			s.peers.Remove(e)
		}
	})
}

func (s *Ethereum) ReapDeadPeerHandler() {
	reapTimer := time.NewTicker(processReapingTimeout * time.Second)

	for {
		select {
		case <-reapTimer.C:
			s.reapPeers()
		}
	}
}

// Start the ethereum
func (s *Ethereum) Start() {
	// Bind to addr and port
	ln, err := net.Listen("tcp", ":30303")
	if err != nil {
		log.Println("Connection listening disabled. Acting as client")
	} else {
		// Starting accepting connections
		log.Println("Ready and accepting connections")
		// Start the peer handler
		go s.peerHandler(ln)
	}

	if s.nat != nil {
		go s.upnpUpdateThread()
	}

	// Start the reaping processes
	go s.ReapDeadPeerHandler()

	// Start the tx pool
	s.TxPool.Start()
}

func (s *Ethereum) peerHandler(listener net.Listener) {
	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Println(err)

			continue
		}

		go s.AddPeer(conn)
	}
}

func (s *Ethereum) Stop() {
	// Close the database
	defer s.db.Close()

	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		p.Stop()
	})

	close(s.quit)

	s.TxPool.Stop()
	s.BlockManager.Stop()

	s.shutdownChan <- true
}

// This function will wait for a shutdown and resumes main thread execution
func (s *Ethereum) WaitForShutdown() {
	<-s.shutdownChan
}

func (s *Ethereum) upnpUpdateThread() {
	// Go off immediately to prevent code duplication, thereafter we renew
	// lease every 15 minutes.
	timer := time.NewTimer(0 * time.Second)
	lport, _ := strconv.ParseInt("30303", 10, 16)
	first := true
out:
	for {
		select {
		case <-timer.C:
			var err error
			_, err = s.nat.AddPortMapping("TCP", int(lport), int(lport), "eth listen port", 20*60)
			if err != nil {
				log.Println("can't add UPnP port mapping:", err)
				break out
			}
			if first && err == nil {
				_, err = s.nat.GetExternalAddress()
				if err != nil {
					log.Println("UPnP can't get external address:", err)
					continue out
				}
				first = false
			}
			timer.Reset(time.Minute * 15)
		case <-s.quit:
			break out
		}
	}

	timer.Stop()

	if err := s.nat.DeletePortMapping("TCP", int(lport), int(lport)); err != nil {
		log.Println("unable to remove UPnP port mapping:", err)
	} else {
		log.Println("succesfully disestablished UPnP port mapping")
	}
}
<commit_msg>Database<commit_after>package eth

import (
	"container/list"
	"github.com/ethereum/ethchain-go"
	"github.com/ethereum/ethdb-go"
	"github.com/ethereum/ethutil-go"
	"github.com/ethereum/ethwire-go"
	"log"
	"net"
	"strconv"
	"sync"
	"sync/atomic"
	"time"
)

func eachPeer(peers *list.List, callback func(*Peer, *list.Element)) {
	// Loop thru the peers and close them (if we had them)
	for e := peers.Front(); e != nil; e = e.Next() {
		if peer, ok := e.Value.(*Peer); ok {
			callback(peer, e)
		}
	}
}

const (
	processReapingTimeout = 60 // TODO increase
)

type Ethereum struct {
	// Channel for shutting down the ethereum
	shutdownChan chan bool
	quit         chan bool
	// DB interface
	//db *ethdb.LDBDatabase
	db ethutil.Database
	// Block manager for processing new blocks and managing the block chain
	BlockManager *ethchain.BlockManager
	// The transaction pool. Transaction can be pushed on this pool
	// for later including in the blocks
	TxPool *ethchain.TxPool
	// Peers (NYI)
	peers *list.List
	// Nonce
	Nonce uint64

	Addr net.Addr

	peerMut sync.Mutex

	// Capabilities for outgoing peers
	serverCaps Caps

	nat NAT

	// Specifies the desired amount of maximum peers
	MaxPeers int
}

func New(caps Caps, usePnp bool) (*Ethereum, error) {
	db, err := ethdb.NewLDBDatabase()
	//db, err := ethdb.NewMemDatabase()
	if err != nil {
		return nil, err
	}

	var nat NAT
	if usePnp {
		nat, err = Discover()
		if err != nil {
			log.Println("UPnP failed", err)
		}
	}

	ethutil.Config.Db = db

	nonce, _ := ethutil.RandomUint64()
	ethereum := &Ethereum{
		shutdownChan: make(chan bool),
		quit:         make(chan bool),
		db:           db,
		peers:        list.New(),
		Nonce:        nonce,
		serverCaps:   caps,
		nat:          nat,
		MaxPeers:     5,
	}
	ethereum.TxPool = ethchain.NewTxPool()
	ethereum.TxPool.Speaker = ethereum
	ethereum.BlockManager = ethchain.NewBlockManager(ethereum)

	ethereum.TxPool.BlockManager = ethereum.BlockManager
	ethereum.BlockManager.TransactionPool = ethereum.TxPool

	return ethereum, nil
}

func (s *Ethereum) AddPeer(conn net.Conn) {
	peer := NewPeer(conn, s, true)

	if peer != nil && s.peers.Len() < s.MaxPeers {
		s.peers.PushBack(peer)
		peer.Start()
	}
}

func (s *Ethereum) ProcessPeerList(addrs []string) {
	for _, addr := range addrs {
		// TODO Probably requires some sanity checks
		s.ConnectToPeer(addr)
	}
}

func (s *Ethereum) ConnectToPeer(addr string) error {
	var alreadyConnected bool

	eachPeer(s.peers, func(p *Peer, v *list.Element) {
		if p.conn == nil {
			return
		}
		phost, _, _ := net.SplitHostPort(p.conn.RemoteAddr().String())
		ahost, _, _ := net.SplitHostPort(addr)

		if phost == ahost {
			alreadyConnected = true
			return
		}
	})

	if alreadyConnected {
		return nil
	}

	peer := NewOutboundPeer(addr, s, s.serverCaps)

	s.peers.PushBack(peer)

	return nil
}

func (s *Ethereum) OutboundPeers() []*Peer {
	// Create a new peer slice with at least the length of the total peers
	outboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if !p.inbound && p.conn != nil {
			outboundPeers[length] = p
			length++
		}
	})

	return outboundPeers[:length]
}

func (s *Ethereum) InboundPeers() []*Peer {
	// Create a new peer slice with at least the length of the total peers
	inboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if p.inbound {
			inboundPeers[length] = p
			length++
		}
	})

	return inboundPeers[:length]
}

func (s *Ethereum) InOutPeers() []*Peer {
	// Reap the dead peers first
	s.reapPeers()

	// Create a new peer slice with at least the length of the total peers
	inboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		// Only return peers with an actual ip
		if len(p.host) > 0 {
			inboundPeers[length] = p
			length++
		}
	})

	return inboundPeers[:length]
}

func (s *Ethereum) Broadcast(msgType ethwire.MsgType, data []interface{}) {
	msg := ethwire.NewMessage(msgType, data)
	s.BroadcastMsg(msg)
}

func (s *Ethereum) BroadcastMsg(msg *ethwire.Msg) {
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		p.QueueMessage(msg)
	})
}

func (s *Ethereum) Peers() *list.List {
	return s.peers
}

func (s *Ethereum) reapPeers() {
	s.peerMut.Lock()
	defer s.peerMut.Unlock()

	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if atomic.LoadInt32(&p.disconnect) == 1 || (p.inbound && (time.Now().Unix()-p.lastPong) > int64(5*time.Minute)) {
			s.peers.Remove(e)
		}
	})
}

func (s *Ethereum) ReapDeadPeerHandler() {
	reapTimer := time.NewTicker(processReapingTimeout * time.Second)

	for {
		select {
		case <-reapTimer.C:
			s.reapPeers()
		}
	}
}

// Start the ethereum
func (s *Ethereum) Start() {
	// Bind to addr and port
	ln, err := net.Listen("tcp", ":30303")
	if err != nil {
		log.Println("Connection listening disabled. Acting as client")
	} else {
		// Starting accepting connections
		log.Println("Ready and accepting connections")
		// Start the peer handler
		go s.peerHandler(ln)
	}

	if s.nat != nil {
		go s.upnpUpdateThread()
	}

	// Start the reaping processes
	go s.ReapDeadPeerHandler()

	// Start the tx pool
	s.TxPool.Start()
}

func (s *Ethereum) peerHandler(listener net.Listener) {
	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Println(err)

			continue
		}

		go s.AddPeer(conn)
	}
}

func (s *Ethereum) Stop() {
	// Close the database
	defer s.db.Close()

	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		p.Stop()
	})

	close(s.quit)

	s.TxPool.Stop()
	s.BlockManager.Stop()

	s.shutdownChan <- true
}

// This function will wait for a shutdown and resumes main thread execution
func (s *Ethereum) WaitForShutdown() {
	<-s.shutdownChan
}

func (s *Ethereum) upnpUpdateThread() {
	// Go off immediately to prevent code duplication, thereafter we renew
	// lease every 15 minutes.
	timer := time.NewTimer(0 * time.Second)
	lport, _ := strconv.ParseInt("30303", 10, 16)
	first := true
out:
	for {
		select {
		case <-timer.C:
			var err error
			_, err = s.nat.AddPortMapping("TCP", int(lport), int(lport), "eth listen port", 20*60)
			if err != nil {
				log.Println("can't add UPnP port mapping:", err)
				break out
			}
			if first && err == nil {
				_, err = s.nat.GetExternalAddress()
				if err != nil {
					log.Println("UPnP can't get external address:", err)
					continue out
				}
				first = false
			}
			timer.Reset(time.Minute * 15)
		case <-s.quit:
			break out
		}
	}

	timer.Stop()

	if err := s.nat.DeletePortMapping("TCP", int(lport), int(lport)); err != nil {
		log.Println("unable to remove UPnP port mapping:", err)
	} else {
		log.Println("succesfully disestablished UPnP port mapping")
	}
}
<|endoftext|>


<commit_before>package thrift_rpc

import (
	"encoding/json"
	"fmt"

	"github.com/lightstep/lightstep-tracer-go/lightstep_thrift"
	"github.com/lightstep/lightstep-tracer-go/thrift_0_9_2/lib/go/thrift"
	"github.com/opentracing/opentracing-go/log"
)

const (
	deprecatedFieldKeyEvent   = "event"
	deprecatedFieldKeyPayload = "payload"
	ellipsis                  = "…"
)

// thrift_rpc.logFieldEncoder is an implementation of the log.Encoder interface
// that handles only the deprecated OpenTracing
// Span.LogEvent/LogEventWithPayload calls. (Since the thrift client is being
// phased out anyway)
type logFieldEncoder struct {
	logRecord *lightstep_thrift.LogRecord
	recorder  *Recorder
}

func (lfe *logFieldEncoder) EmitString(key, value string) {
	if key == deprecatedFieldKeyEvent {
		if len(value) > lfe.recorder.maxLogMessageLen {
			value = value[:(lfe.recorder.maxLogMessageLen-1)] + ellipsis
		}
		lfe.logRecord.StableName = thrift.StringPtr(value)
	}
}
func (lfe *logFieldEncoder) EmitObject(key string, value interface{}) {
	if key == deprecatedFieldKeyPayload {
		var thriftPayload string
		jsonString, err := json.Marshal(value)
		if err != nil {
			thriftPayload = fmt.Sprintf("Error encoding payload object: %v", err)
		} else {
			thriftPayload = string(jsonString)
		}
		if len(thriftPayload) > lfe.recorder.maxLogMessageLen {
			thriftPayload = thriftPayload[:(lfe.recorder.maxLogMessageLen-1)] + ellipsis
		}
		lfe.logRecord.PayloadJson = thrift.StringPtr(thriftPayload)
	}
}

// All other log.Encoder methods do nothing in the thrift_rpc implementation.
func (lfe *logFieldEncoder) EmitBool(key string, value bool)       {}
func (lfe *logFieldEncoder) EmitInt(key string, value int)         {}
func (lfe *logFieldEncoder) EmitInt32(key string, value int32)     {}
func (lfe *logFieldEncoder) EmitInt64(key string, value int64)     {}
func (lfe *logFieldEncoder) EmitUint32(key string, value uint32)   {}
func (lfe *logFieldEncoder) EmitUint64(key string, value uint64)   {}
func (lfe *logFieldEncoder) EmitFloat32(key string, value float32) {}
func (lfe *logFieldEncoder) EmitFloat64(key string, value float64) {}
func (lfe *logFieldEncoder) EmitLazyLogger(value log.LazyLogger)   {}
<commit_msg>Implements all log.Encoder using EmitString (#71)<commit_after>package thrift_rpc

import (
	"encoding/json"
	"fmt"

	"github.com/lightstep/lightstep-tracer-go/lightstep_thrift"
	"github.com/lightstep/lightstep-tracer-go/thrift_0_9_2/lib/go/thrift"
	"github.com/opentracing/opentracing-go/log"
)

const (
	deprecatedFieldKeyEvent   = "event"
	deprecatedFieldKeyPayload = "payload"
	ellipsis                  = "…"
)

// thrift_rpc.logFieldEncoder is an implementation of the log.Encoder interface
// that handles only the deprecated OpenTracing
// Span.LogEvent/LogEventWithPayload calls. (Since the thrift client is being
// phased out anyway)
type logFieldEncoder struct {
	logRecord *lightstep_thrift.LogRecord
	recorder  *Recorder
}

func (lfe *logFieldEncoder) EmitString(key, value string) {
	if key == deprecatedFieldKeyEvent {
		if len(value) > lfe.recorder.maxLogMessageLen {
			value = value[:(lfe.recorder.maxLogMessageLen-1)] + ellipsis
		}
		lfe.logRecord.StableName = thrift.StringPtr(value)
	}
}
func (lfe *logFieldEncoder) EmitObject(key string, value interface{}) {
	if key == deprecatedFieldKeyPayload {
		var thriftPayload string
		jsonString, err := json.Marshal(value)
		if err != nil {
			thriftPayload = fmt.Sprintf("Error encoding payload object: %v", err)
		} else {
			thriftPayload = string(jsonString)
		}
		if len(thriftPayload) > lfe.recorder.maxLogMessageLen {
			thriftPayload = thriftPayload[:(lfe.recorder.maxLogMessageLen-1)] + ellipsis
		}
		lfe.logRecord.PayloadJson = thrift.StringPtr(thriftPayload)
	}
}

func (lfe *logFieldEncoder) EmitBool(key string, value bool) {
	lfe.EmitString(key, fmt.Sprint(value))
}
func (lfe *logFieldEncoder) EmitInt(key string, value int) {
	lfe.EmitString(key, fmt.Sprint(value))
}
func (lfe *logFieldEncoder) EmitInt32(key string, value int32) {
	lfe.EmitString(key, fmt.Sprint(value))
}
func (lfe *logFieldEncoder) EmitInt64(key string, value int64) {
	lfe.EmitString(key, fmt.Sprint(value))
}
func (lfe *logFieldEncoder) EmitUint32(key string, value uint32) {
	lfe.EmitString(key, fmt.Sprint(value))
}
func (lfe *logFieldEncoder) EmitUint64(key string, value uint64) {
	lfe.EmitString(key, fmt.Sprint(value))
}
func (lfe *logFieldEncoder) EmitFloat32(key string, value float32) {
	lfe.EmitString(key, fmt.Sprint(value))
}
func (lfe *logFieldEncoder) EmitFloat64(key string, value float64) {
	lfe.EmitString(key, fmt.Sprint(value))
}
func (lfe *logFieldEncoder) EmitLazyLogger(value log.LazyLogger) {}
<|endoftext|>


<commit_before>package kubernetes

import (
	"github.com/hashicorp/terraform/helper/schema"
)

func labelSelectorFields() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"match_expressions": {
			Type:        schema.TypeList,
			Description: "A list of label selector requirements. The requirements are ANDed.",
			Optional:    true,
			ForceNew:    true,
			Elem: &schema.Resource{
				Schema: map[string]*schema.Schema{
					"key": {
						Type:        schema.TypeString,
						Description: "The label key that the selector applies to.",
						Optional:    true,
						ForceNew:    true,
					},
					"operator": {
						Type:        schema.TypeString,
						Description: "A key's relationship to a set of values. Valid operators ard `In`, `NotIn`, `Exists` and `DoesNotExist`.",
						Optional:    true,
						ForceNew:    true,
					},
					"values": {
						Type:        schema.TypeSet,
						Description: "An array of string values. If the operator is `In` or `NotIn`, the values array must be non-empty. If the operator is `Exists` or `DoesNotExist`, the values array must be empty. This array is replaced during a strategic merge patch.",
						Optional:    true,
						ForceNew:    true,
						Elem:        &schema.Schema{Type: schema.TypeString},
						Set:         schema.HashString,
					},
				},
			},
		},
		"match_labels": {
			Type:        schema.TypeMap,
			Description: "A map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of `match_expressions`, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.",
			Optional:    true,
			ForceNew:    true,
		},
	}
}
<commit_msg>fix: do not force to recreate deployment if matchExressions changed/added/removed<commit_after>package kubernetes

import (
	"github.com/hashicorp/terraform/helper/schema"
)

func labelSelectorFields() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"match_expressions": {
			Type:        schema.TypeList,
			Description: "A list of label selector requirements. The requirements are ANDed.",
			Optional:    true,
			Elem: &schema.Resource{
				Schema: map[string]*schema.Schema{
					"key": {
						Type:        schema.TypeString,
						Description: "The label key that the selector applies to.",
						Optional:    true,
					},
					"operator": {
						Type:        schema.TypeString,
						Description: "A key's relationship to a set of values. Valid operators ard `In`, `NotIn`, `Exists` and `DoesNotExist`.",
						Optional:    true,
					},
					"values": {
						Type:        schema.TypeSet,
						Description: "An array of string values. If the operator is `In` or `NotIn`, the values array must be non-empty. If the operator is `Exists` or `DoesNotExist`, the values array must be empty. This array is replaced during a strategic merge patch.",
						Optional:    true,
						Elem:        &schema.Schema{Type: schema.TypeString},
						Set:         schema.HashString,
					},
				},
			},
		},
		"match_labels": {
			Type:        schema.TypeMap,
			Description: "A map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of `match_expressions`, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.",
			Optional:    true,
			ForceNew:    true,
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"os"
)

func main() {
	os.Remove("./foo.db")

	db, err := sql.Open("sqlite3", "./foo.db")
	if err != nil {
		fmt.Println(err)
		return
	}

	sqls := []string{
		"create table foo (id integer not null primary key, name text)",
		"delete from foo",
	}
	for _, sql := range sqls {
		_, err = db.Exec(sql)
		if err != nil {
			fmt.Printf("%q: %s\n", err, sql)
			return
		}
	}

	tx, err := db.Begin()
	if err != nil {
		fmt.Println(err)
		return
	}
	stmt, err := tx.Prepare("insert into foo(id, name) values(?, ?)")
	if err != nil {
		fmt.Println(err)
		return
	}
	defer stmt.Close()

	for i := 0; i < 100; i++ {
		_, err = stmt.Exec(i, fmt.Sprintf("こんにちわ世界%03d", i))
		if err != nil {
			fmt.Println(err)
			return
		}
	}
	tx.Commit()

	rows, err := db.Query("select id, name from foo")
	if err != nil {
		fmt.Println(err)
		return
	}
	defer rows.Close()

	for rows.Next() {
		var id int
		var name string
		rows.Scan(&id, &name)
		println(id, name)
	}
}
<commit_msg>add example of Prepare()/QueryRow(). add db.Close(). close #7<commit_after>package main

import (
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"os"
)

func main() {
	os.Remove("./foo.db")

	db, err := sql.Open("sqlite3", "./foo.db")
	if err != nil {
		fmt.Println(err)
		return
	}
	defer db.Close()

	sqls := []string{
		"create table foo (id integer not null primary key, name text)",
		"delete from foo",
	}
	for _, sql := range sqls {
		_, err = db.Exec(sql)
		if err != nil {
			fmt.Printf("%q: %s\n", err, sql)
			return
		}
	}

	tx, err := db.Begin()
	if err != nil {
		fmt.Println(err)
		return
	}
	stmt, err := tx.Prepare("insert into foo(id, name) values(?, ?)")
	if err != nil {
		fmt.Println(err)
		return
	}
	defer stmt.Close()
	for i := 0; i < 100; i++ {
		_, err = stmt.Exec(i, fmt.Sprintf("こんにちわ世界%03d", i))
		if err != nil {
			fmt.Println(err)
			return
		}
	}
	tx.Commit()

	rows, err := db.Query("select id, name from foo")
	if err != nil {
		fmt.Println(err)
		return
	}
	defer rows.Close()
	for rows.Next() {
		var id int
		var name string
		rows.Scan(&id, &name)
		println(id, name)
	}
	rows.Close()

	stmt, err = db.Prepare("select name from foo where id = ?")
	if err != nil {
		fmt.Println(err)
		return
	}
	defer stmt.Close()
	var name string
	err = stmt.QueryRow("3").Scan(&name)
	if err != nil {
		fmt.Println(err)
		return
	}
	println(name)
}
<|endoftext|>


<commit_before>package ls

import "fmt"
import "io"
import "os"
import "path/filepath"
import "regexp"
import "sort"
import "strings"

import "../box"
import "../dos"
import "../exename"

const (
	O_STRIP_DIR = 1
	O_LONG      = 2
	O_INDICATOR = 4
	O_COLOR     = 8
	O_ALL       = 16
	O_TIME      = 32
	O_REVERSE   = 64
	O_RECURSIVE = 128
)

type fileInfoT struct {
	name        string
	os.FileInfo // anonymous
}

const (
	ANSI_EXEC     = "\x1B[1;35m"
	ANSI_DIR      = "\x1B[1;32m"
	ANSI_NORM     = "\x1B[1;37m"
	ANSI_READONLY = "\x1B[1;33m"
	ANSI_END      = "\x1B[39m"
)

func (this fileInfoT) Name() string { return this.name }

func newMyFileInfoT(name string, info os.FileInfo) *fileInfoT {
	return &fileInfoT{name, info}
}

func lsOneLong(status os.FileInfo, flag int, out io.Writer) {
	indicator := " "
	prefix := ""
	postfix := ""
	if status.IsDir() {
		io.WriteString(out, "d")
		indicator = "/"
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_DIR
			postfix = ANSI_END
		}
	} else {
		io.WriteString(out, "-")
	}
	mode := status.Mode()
	perm := mode.Perm()
	name := status.Name()
	if (perm & 4) > 0 {
		io.WriteString(out, "r")
	} else {
		io.WriteString(out, "-")
	}
	if (perm & 2) > 0 {
		io.WriteString(out, "w")
	} else {
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_READONLY
			postfix = ANSI_END
		}
		io.WriteString(out, "-")
	}
	if (perm & 1) > 0 {
		io.WriteString(out, "x")
	} else if exename.Suffixes[strings.ToLower(filepath.Ext(name))] {
		io.WriteString(out, "x")
		indicator = "*"
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_EXEC
			postfix = ANSI_END
		}
	} else {
		io.WriteString(out, "-")
	}
	if (flag & O_STRIP_DIR) > 0 {
		name = filepath.Base(name)
	}
	stamp := status.ModTime()
	fmt.Fprintf(out, " %8d %04d-%02d-%02d %02d:%02d %s%s%s",
		status.Size(),
		stamp.Year(),
		stamp.Month(),
		stamp.Day(),
		stamp.Hour(),
		stamp.Minute(),
		prefix,
		name,
		postfix)
	if (flag & O_INDICATOR) > 0 {
		io.WriteString(out, indicator)
	}
	io.WriteString(out, "\n")
}

func lsBox(nodes []os.FileInfo, flag int, out io.Writer) {
	nodes_ := make([]string, len(nodes))
	for key, val := range nodes {
		prefix := ""
		postfix := ""
		if val.IsDir() {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_DIR
				postfix = ANSI_END
			}
			if (flag & O_INDICATOR) != 0 {
				postfix += "/"
			}
		} else if (val.Mode().Perm() & 2) == 0 {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_READONLY
				postfix = ANSI_END
			}
		} else if exename.Suffixes[strings.ToLower(filepath.Ext(val.Name()))] {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_EXEC
				postfix = ANSI_END
			}
			if (flag & O_INDICATOR) != 0 {
				postfix += "*"
			}
		}
		nodes_[key] = prefix + val.Name() + postfix
	}
	box.Print(nodes_, 80, out)
}

func lsLong(nodes []os.FileInfo, flag int, out io.Writer) {
	for _, finfo := range nodes {
		lsOneLong(finfo, flag, out)
	}
}

type fileInfoCollection struct {
	flag  int
	nodes []os.FileInfo
}

func (this fileInfoCollection) Len() int {
	return len(this.nodes)
}
func (this fileInfoCollection) Less(i, j int) bool {
	var result bool
	if (this.flag & O_TIME) != 0 {
		result = this.nodes[i].ModTime().After(this.nodes[j].ModTime())
		if !result && !this.nodes[i].ModTime().Before(this.nodes[j].ModTime()) {
			result = (this.nodes[i].Name() < this.nodes[j].Name())
		}
	} else {
		result = (this.nodes[i].Name() < this.nodes[j].Name())
	}
	if (this.flag & O_REVERSE) != 0 {
		result = !result
	}
	return result
}
func (this fileInfoCollection) Swap(i, j int) {
	tmp := this.nodes[i]
	this.nodes[i] = this.nodes[j]
	this.nodes[j] = tmp
}

func lsFolder(folder string, flag int, out io.Writer) error {
	var folder_ string
	if rxDriveOnly.MatchString(folder) {
		folder_ = folder + "."
	} else {
		folder_ = folder
	}
	fd, err := os.Open(folder_)
	if err != nil {
		return err
	}
	var nodesArray fileInfoCollection
	nodesArray.nodes, err = fd.Readdir(-1)
	fd.Close()
	nodesArray.flag = flag
	if err != nil {
		return err
	}
	tmp := make([]os.FileInfo, 0)
	var folders []string = nil
	if (flag & O_RECURSIVE) != 0 {
		folders = make([]string, 0)
	}
	for _, f := range nodesArray.nodes {
		attr := dos.NewFileAttr(dos.Join(folder_, f.Name()))
		if (strings.HasPrefix(f.Name(), ".") || attr.IsHidden()) && (flag&O_ALL) == 0 {
			continue
		}
		if f.IsDir() && folders != nil {
			folders = append(folders, f.Name())
		} else {
			tmp = append(tmp, f)
		}
	}
	nodesArray.nodes = tmp
	sort.Sort(nodesArray)
	if (flag & O_LONG) > 0 {
		lsLong(nodesArray.nodes, O_STRIP_DIR|flag, out)
	} else {
		lsBox(nodesArray.nodes, O_STRIP_DIR|flag, out)
	}
	if folders != nil && len(folders) > 0 {
		for _, f1 := range folders {
			f1fullpath := dos.Join(folder, f1)
			fmt.Fprintf(out, "\n%s:\n", f1fullpath)
			lsFolder(f1fullpath, flag, out)
		}
	}
	return nil
}

var rxDriveOnly = regexp.MustCompile("^[a-zA-Z]:$")

func lsCore(paths []string, flag int, out io.Writer) error {
	if len(paths) <= 0 {
		return lsFolder(".", flag, out)
	}
	dirs := make([]string, 0)
	printCount := 0
	files := make([]os.FileInfo, 0)
	for _, name := range paths {
		var nameStat string
		if rxDriveOnly.MatchString(name) {
			nameStat = name + "."
		} else {
			nameStat = name
		}
		status, err := os.Stat(nameStat)
		if err != nil {
			continue
		} else if status.IsDir() {
			dirs = append(dirs, name)
		} else if (flag & O_LONG) != 0 {
			lsOneLong(newMyFileInfoT(name, status), flag, out)
			printCount += 1
		} else {
			files = append(files, newMyFileInfoT(name, status))
		}
	}
	if len(files) > 0 {
		lsBox(files, flag, out)
		printCount = len(files)
	}
	for _, name := range dirs {
		if len(paths) > 1 {
			if printCount > 0 {
				io.WriteString(out, "\n")
			}
			io.WriteString(out, name)
			io.WriteString(out, ":\n")
		}
		err := lsFolder(name, flag, out)
		if err != nil {
			return err
		}
		printCount++
	}
	return nil
}

var option = map[rune](func(*int) error){
	'l': func(flag *int) error {
		*flag |= O_LONG
		return nil
	},
	'F': func(flag *int) error {
		*flag |= O_INDICATOR
		return nil
	},
	'o': func(flag *int) error {
		*flag |= O_COLOR
		return nil
	},
	'a': func(flag *int) error {
		*flag |= O_ALL
		return nil
	},
	't': func(flag *int) error {
		*flag |= O_TIME
		return nil
	},
	'r': func(flag *int) error {
		*flag |= O_REVERSE
		return nil
	},
	'R': func(flag *int) error {
		*flag |= O_RECURSIVE
		return nil
	},
}

// 存在しないオプションに関するエラー
type OptionError struct {
	Option rune
}

func (this OptionError) Error() string {
	return fmt.Sprintf("-%c: No such option", this.Option)
}

// ls 機能のエントリ:引数をオプションとパスに分離する
func Main(args []string, out io.Writer) error {
	flag := 0
	paths := make([]string, 0)
	for _, arg := range args {
		if strings.HasPrefix(arg, "-") {
			for _, o := range arg[1:] {
				setter, ok := option[o]
				if !ok {
					var err OptionError
					err.Option = o
					return err
				}
				err := setter(&flag)
				if err != nil {
					return err
				}
			}
		} else {
			paths = append(paths, arg)
		}
	}
	return lsCore(paths, flag, out)
}

// vim:set fenc=utf8 ts=4 sw=4 noet:
<commit_msg>ls: ジャンクションの末尾の @ を付けられるようにした。<commit_after>package ls

import "fmt"
import "io"
import "os"
import "path/filepath"
import "regexp"
import "sort"
import "strings"

import "../box"
import "../dos"
import "../exename"

const (
	O_STRIP_DIR = 1
	O_LONG      = 2
	O_INDICATOR = 4
	O_COLOR     = 8
	O_ALL       = 16
	O_TIME      = 32
	O_REVERSE   = 64
	O_RECURSIVE = 128
)

type fileInfoT struct {
	name        string
	os.FileInfo // anonymous
}

const (
	ANSI_EXEC     = "\x1B[1;35m"
	ANSI_DIR      = "\x1B[1;32m"
	ANSI_NORM     = "\x1B[1;37m"
	ANSI_READONLY = "\x1B[1;33m"
	ANSI_END      = "\x1B[39m"
)

func (this fileInfoT) Name() string { return this.name }

func newMyFileInfoT(name string, info os.FileInfo) *fileInfoT {
	return &fileInfoT{name, info}
}

func lsOneLong(folder string, status os.FileInfo, flag int, out io.Writer) {
	indicator := " "
	prefix := ""
	postfix := ""
	if status.IsDir() {
		io.WriteString(out, "d")
		indicator = "/"
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_DIR
			postfix = ANSI_END
		}
	} else {
		io.WriteString(out, "-")
	}
	mode := status.Mode()
	perm := mode.Perm()
	name := status.Name()
	attr := dos.NewFileAttr(dos.Join(folder, status.Name()))
	if attr.IsReparse() {
		indicator = "@"
	}
	if (perm & 4) > 0 {
		io.WriteString(out, "r")
	} else {
		io.WriteString(out, "-")
	}
	if (perm & 2) > 0 {
		io.WriteString(out, "w")
	} else {
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_READONLY
			postfix = ANSI_END
		}
		io.WriteString(out, "-")
	}
	if (perm & 1) > 0 {
		io.WriteString(out, "x")
	} else if exename.Suffixes[strings.ToLower(filepath.Ext(name))] {
		io.WriteString(out, "x")
		indicator = "*"
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_EXEC
			postfix = ANSI_END
		}
	} else {
		io.WriteString(out, "-")
	}
	if (flag & O_STRIP_DIR) > 0 {
		name = filepath.Base(name)
	}
	stamp := status.ModTime()
	fmt.Fprintf(out, " %8d %04d-%02d-%02d %02d:%02d %s%s%s",
		status.Size(),
		stamp.Year(),
		stamp.Month(),
		stamp.Day(),
		stamp.Hour(),
		stamp.Minute(),
		prefix,
		name,
		postfix)
	if (flag & O_INDICATOR) > 0 {
		io.WriteString(out, indicator)
	}
	io.WriteString(out, "\n")
}

func lsBox(folder string, nodes []os.FileInfo, flag int, out io.Writer) {
	nodes_ := make([]string, len(nodes))
	for key, val := range nodes {
		prefix := ""
		postfix := ""
		attr := dos.NewFileAttr(dos.Join(folder, val.Name()))
		if attr.IsReparse() {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_DIR
				postfix = ANSI_END
			}
			if (flag & O_INDICATOR) != 0 {
				postfix += "@"
			}
		} else if val.IsDir() {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_DIR
				postfix = ANSI_END
			}
			if (flag & O_INDICATOR) != 0 {
				postfix += "/"
			}
		} else if (val.Mode().Perm() & 2) == 0 {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_READONLY
				postfix = ANSI_END
			}
		} else if exename.Suffixes[strings.ToLower(filepath.Ext(val.Name()))] {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_EXEC
				postfix = ANSI_END
			}
			if (flag & O_INDICATOR) != 0 {
				postfix += "*"
			}
		}
		nodes_[key] = prefix + val.Name() + postfix
	}
	box.Print(nodes_, 80, out)
}

func lsLong(folder string, nodes []os.FileInfo, flag int, out io.Writer) {
	for _, finfo := range nodes {
		lsOneLong(folder, finfo, flag, out)
	}
}

type fileInfoCollection struct {
	flag  int
	nodes []os.FileInfo
}

func (this fileInfoCollection) Len() int {
	return len(this.nodes)
}
func (this fileInfoCollection) Less(i, j int) bool {
	var result bool
	if (this.flag & O_TIME) != 0 {
		result = this.nodes[i].ModTime().After(this.nodes[j].ModTime())
		if !result && !this.nodes[i].ModTime().Before(this.nodes[j].ModTime()) {
			result = (this.nodes[i].Name() < this.nodes[j].Name())
		}
	} else {
		result = (this.nodes[i].Name() < this.nodes[j].Name())
	}
	if (this.flag & O_REVERSE) != 0 {
		result = !result
	}
	return result
}
func (this fileInfoCollection) Swap(i, j int) {
	tmp := this.nodes[i]
	this.nodes[i] = this.nodes[j]
	this.nodes[j] = tmp
}

func lsFolder(folder string, flag int, out io.Writer) error {
	var folder_ string
	if rxDriveOnly.MatchString(folder) {
		folder_ = folder + "."
	} else {
		folder_ = folder
	}
	fd, err := os.Open(folder_)
	if err != nil {
		return err
	}
	var nodesArray fileInfoCollection
	nodesArray.nodes, err = fd.Readdir(-1)
	fd.Close()
	nodesArray.flag = flag
	if err != nil {
		return err
	}
	tmp := make([]os.FileInfo, 0)
	var folders []string = nil
	if (flag & O_RECURSIVE) != 0 {
		folders = make([]string, 0)
	}
	for _, f := range nodesArray.nodes {
		attr := dos.NewFileAttr(dos.Join(folder_, f.Name()))
		if (strings.HasPrefix(f.Name(), ".") || attr.IsHidden()) && (flag&O_ALL) == 0 {
			continue
		}
		if f.IsDir() && folders != nil {
			folders = append(folders, f.Name())
		} else {
			tmp = append(tmp, f)
		}
	}
	nodesArray.nodes = tmp
	sort.Sort(nodesArray)
	if (flag & O_LONG) > 0 {
		lsLong(folder_, nodesArray.nodes, O_STRIP_DIR|flag, out)
	} else {
		lsBox(folder_, nodesArray.nodes, O_STRIP_DIR|flag, out)
	}
	if folders != nil && len(folders) > 0 {
		for _, f1 := range folders {
			f1fullpath := dos.Join(folder, f1)
			fmt.Fprintf(out, "\n%s:\n", f1fullpath)
			lsFolder(f1fullpath, flag, out)
		}
	}
	return nil
}

var rxDriveOnly = regexp.MustCompile("^[a-zA-Z]:$")

func lsCore(paths []string, flag int, out io.Writer) error {
	if len(paths) <= 0 {
		return lsFolder(".", flag, out)
	}
	dirs := make([]string, 0)
	printCount := 0
	files := make([]os.FileInfo, 0)
	for _, name := range paths {
		var nameStat string
		if rxDriveOnly.MatchString(name) {
			nameStat = name + "."
		} else {
			nameStat = name
		}
		status, err := os.Stat(nameStat)
		if err != nil {
			continue
		} else if status.IsDir() {
			dirs = append(dirs, name)
		} else if (flag & O_LONG) != 0 {
			lsOneLong(".", newMyFileInfoT(name, status), flag, out)
			printCount += 1
		} else {
			files = append(files, newMyFileInfoT(name, status))
		}
	}
	if len(files) > 0 {
		lsBox(".", files, flag, out)
		printCount = len(files)
	}
	for _, name := range dirs {
		if len(paths) > 1 {
			if printCount > 0 {
				io.WriteString(out, "\n")
			}
			io.WriteString(out, name)
			io.WriteString(out, ":\n")
		}
		err := lsFolder(name, flag, out)
		if err != nil {
			return err
		}
		printCount++
	}
	return nil
}

var option = map[rune](func(*int) error){
	'l': func(flag *int) error {
		*flag |= O_LONG
		return nil
	},
	'F': func(flag *int) error {
		*flag |= O_INDICATOR
		return nil
	},
	'o': func(flag *int) error {
		*flag |= O_COLOR
		return nil
	},
	'a': func(flag *int) error {
		*flag |= O_ALL
		return nil
	},
	't': func(flag *int) error {
		*flag |= O_TIME
		return nil
	},
	'r': func(flag *int) error {
		*flag |= O_REVERSE
		return nil
	},
	'R': func(flag *int) error {
		*flag |= O_RECURSIVE
		return nil
	},
}

// 存在しないオプションに関するエラー
type OptionError struct {
	Option rune
}

func (this OptionError) Error() string {
	return fmt.Sprintf("-%c: No such option", this.Option)
}

// ls 機能のエントリ:引数をオプションとパスに分離する
func Main(args []string, out io.Writer) error {
	flag := 0
	paths := make([]string, 0)
	for _, arg := range args {
		if strings.HasPrefix(arg, "-") {
			for _, o := range arg[1:] {
				setter, ok := option[o]
				if !ok {
					var err OptionError
					err.Option = o
					return err
				}
				err := setter(&flag)
				if err != nil {
					return err
				}
			}
		} else {
			paths = append(paths, arg)
		}
	}
	return lsCore(paths, flag, out)
}

// vim:set fenc=utf8 ts=4 sw=4 noet:
<|endoftext|>


<commit_before>// Copyright 2016 Openprovider Authors. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

package providers

import (
	"encoding/xml"
	"net/http"
	"time"

	"github.com/openprovider/rates"
	"golang.org/x/text/currency"
)

// ECB represents ECB provider
type ECB struct {
	currencies []currency.Unit
}

const (
	ratesLastURL    = "http://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml"
	rates90daysURL  = "http://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist-90d.xml"
	ratesHistoryURL = "http://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist.xml"
)

// ECBCurrencies are valid values for currency
var ECBCurrencies = []string{
	AUD, BGN, BRL, CAD, CHF, CNY, CZK, DKK, EUR, GBP, HKD,
	HRK, HUF, IDR, ILS, INR, JPY, KRW, MXN, MYR, NOK,
	NZD, PHP, PLN, RON, RUB, SEK, SGD, THB, TRY, USD, ZAR,
}

// NewECBProvider inits ECB provider record
func NewECBProvider(options rates.Options) *ECB {
	ecb := new(ECB)
	// init all units
	if len(options.Currencies) == 0 {
		options.Currencies = append(options.Currencies, ECBCurrencies...)
	}
	for _, unit := range options.Currencies {
		if c, err := currency.ParseISO(unit); err == nil {
			ecb.currencies = append(ecb.currencies, c)
		}
	}
	return ecb
}

// ECB XML envelope
type envelope struct {
	Data []struct {
		Date  string `xml:"time,attr"`
		Rates []struct {
			Currency string `xml:"currency,attr"`
			Rate     string `xml:"rate,attr"`
		} `xml:"Cube"`
	} `xml:"Cube>Cube"`
}

// FetchLast gets exchange rates for the last day
func (ecb *ECB) FetchLast() ([]rates.Rate, []error) {
	return ecb.fetch(ratesLastURL)
}

// Fetch90Days gets exchange rates for 90 days
func (ecb *ECB) Fetch90Days() ([]rates.Rate, []error) {
	return ecb.fetch(rates90daysURL)
}

// FetchHistory gets exchange rates for all existing days
func (ecb *ECB) FetchHistory() ([]rates.Rate, []error) {
	return ecb.fetch(ratesHistoryURL)
}

// FetchLast gets exchange rates for the last day
func (ecb *ECB) fetch(url string) (ecbRates []rates.Rate, errors []error) {
	currentTime := time.Now()
	timeString := currentTime.Format(stdDateTime)

	response, err := http.Get(url)
	if err != nil {
		errors = append(errors, err)
		return
	}
	defer response.Body.Close()

	var raw envelope

	if err := xml.NewDecoder(response.Body).Decode(&raw); err != nil {
		errors = append(errors, err)
		return
	}

	for _, day := range raw.Data {
		if currentTime.Format(stdDate) != day.Date {
			if t, err := time.Parse(stdDate, day.Date); err == nil {
				currentTime = t
				timeString = day.Date + " 00:00:00"
			} else {
				errors = append(errors, err)
			}
		}
		for _, unit := range ecb.currencies {
			if unit == currency.EUR {
				ecbRates = append(ecbRates, rates.Rate{
					Date:           currentTime,
					DateString:     timeString,
					Currency:       currency.EUR,
					CurrencyString: currency.EUR.String(),
					Value:          "1.0000",
				})
			}
			for _, item := range day.Rates {
				if item.Currency == unit.String() {
					ecbRates = append(ecbRates, rates.Rate{
						Date:           currentTime,
						DateString:     timeString,
						Currency:       unit,
						CurrencyString: unit.String(),
						Value:          item.Rate,
					})
				}
			}
		}
	}
	return
}
<commit_msg>pass options by reference<commit_after>// Copyright 2016 Openprovider Authors. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

package providers

import (
	"encoding/xml"
	"net/http"
	"time"

	"github.com/openprovider/rates"
	"golang.org/x/text/currency"
)

// ECB represents ECB provider
type ECB struct {
	currencies []currency.Unit
}

const (
	ratesLastURL    = "http://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml"
	rates90daysURL  = "http://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist-90d.xml"
	ratesHistoryURL = "http://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist.xml"
)

// ECBCurrencies are valid values for currency
var ECBCurrencies = []string{
	AUD, BGN, BRL, CAD, CHF, CNY, CZK, DKK, EUR, GBP, HKD,
	HRK, HUF, IDR, ILS, INR, JPY, KRW, MXN, MYR, NOK,
	NZD, PHP, PLN, RON, RUB, SEK, SGD, THB, TRY, USD, ZAR,
}

// NewECBProvider inits ECB provider record
func NewECBProvider(options *rates.Options) *ECB {
	ecb := new(ECB)
	// init all units
	if len(options.Currencies) == 0 {
		options.Currencies = append(options.Currencies, ECBCurrencies...)
	}
	for _, unit := range options.Currencies {
		if c, err := currency.ParseISO(unit); err == nil {
			ecb.currencies = append(ecb.currencies, c)
		}
	}
	return ecb
}

// ECB XML envelope
type envelope struct {
	Data []struct {
		Date  string `xml:"time,attr"`
		Rates []struct {
			Currency string `xml:"currency,attr"`
			Rate     string `xml:"rate,attr"`
		} `xml:"Cube"`
	} `xml:"Cube>Cube"`
}

// FetchLast gets exchange rates for the last day
func (ecb *ECB) FetchLast() ([]rates.Rate, []error) {
	return ecb.fetch(ratesLastURL)
}

// Fetch90Days gets exchange rates for 90 days
func (ecb *ECB) Fetch90Days() ([]rates.Rate, []error) {
	return ecb.fetch(rates90daysURL)
}

// FetchHistory gets exchange rates for all existing days
func (ecb *ECB) FetchHistory() ([]rates.Rate, []error) {
	return ecb.fetch(ratesHistoryURL)
}

// FetchLast gets exchange rates for the last day
func (ecb *ECB) fetch(url string) (ecbRates []rates.Rate, errors []error) {
	currentTime := time.Now()
	timeString := currentTime.Format(stdDateTime)

	response, err := http.Get(url)
	if err != nil {
		errors = append(errors, err)
		return
	}
	defer response.Body.Close()

	var raw envelope

	if err := xml.NewDecoder(response.Body).Decode(&raw); err != nil {
		errors = append(errors, err)
		return
	}

	for _, day := range raw.Data {
		if currentTime.Format(stdDate) != day.Date {
			if t, err := time.Parse(stdDate, day.Date); err == nil {
				currentTime = t
				timeString = day.Date + " 00:00:00"
			} else {
				errors = append(errors, err)
			}
		}
		for _, unit := range ecb.currencies {
			if unit == currency.EUR {
				ecbRates = append(ecbRates, rates.Rate{
					Date:           currentTime,
					DateString:     timeString,
					Currency:       currency.EUR,
					CurrencyString: currency.EUR.String(),
					Value:          "1.0000",
				})
			}
			for _, item := range day.Rates {
				if item.Currency == unit.String() {
					ecbRates = append(ecbRates, rates.Rate{
						Date:           currentTime,
						DateString:     timeString,
						Currency:       unit,
						CurrencyString: unit.String(),
						Value:          item.Rate,
					})
				}
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>package ls

import "fmt"
import "io"
import "os"
import "path"
import "sort"
import "strings"
import "time"

import "../box"

var exeSuffixes = map[string]bool{}

const (
	O_STRIP_DIR = 1
	O_LONG      = 2
	O_INDICATOR = 4
	O_COLOR     = 8
	O_ALL       = 16
	O_TIME      = 32
	O_REVERSE   = 64
)

type fileInfoT struct {
	name string
	info os.FileInfo
}

const (
	ANSI_EXEC     = "\x1B[1;35m"
	ANSI_DIR      = "\x1B[1;32m"
	ANSI_NORM     = "\x1B[1;37m"
	ANSI_READONLY = "\x1B[1;33m"
	ANSI_END      = "\x1B[39m"
)

func (this fileInfoT) Name() string       { return this.name }
func (this fileInfoT) Size() int64        { return this.info.Size() }
func (this fileInfoT) Mode() os.FileMode  { return this.info.Mode() }
func (this fileInfoT) ModTime() time.Time { return this.info.ModTime() }
func (this fileInfoT) IsDir() bool        { return this.info.IsDir() }
func (this fileInfoT) Sys() interface{}   { return this.info.Sys() }

func newMyFileInfoT(name string, info os.FileInfo) *fileInfoT {
	this := new(fileInfoT)
	this.name = name
	this.info = info
	return this
}

func lsOneLong(status os.FileInfo, flag int, out io.Writer) {
	indicator := " "
	prefix := ""
	postfix := ""
	if status.IsDir() {
		io.WriteString(out, "d")
		indicator = "/"
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_DIR
			postfix = ANSI_END
		}
	} else {
		io.WriteString(out, "-")
	}
	mode := status.Mode()
	perm := mode.Perm()
	name := status.Name()
	if (perm & 4) > 0 {
		io.WriteString(out, "r")
	} else {
		io.WriteString(out, "-")
	}
	if (perm & 2) > 0 {
		io.WriteString(out, "w")
	} else {
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_READONLY
			postfix = ANSI_END
		}
		io.WriteString(out, "-")
	}
	if (perm & 1) > 0 {
		io.WriteString(out, "x")
	} else if exeSuffixes[strings.ToLower(path.Ext(name))] {
		io.WriteString(out, "x")
		indicator = "*"
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_EXEC
			postfix = ANSI_END
		}
	} else {
		io.WriteString(out, "-")
	}
	if (flag & O_STRIP_DIR) > 0 {
		name = path.Base(name)
	}
	stamp := status.ModTime()
	fmt.Fprintf(out, " %8d %04d-%02d-%02d %02d:%02d %s%s%s",
		status.Size(),
		stamp.Year(),
		stamp.Month(),
		stamp.Day(),
		stamp.Hour(),
		stamp.Minute(),
		prefix,
		name,
		postfix)
	if (flag & O_INDICATOR) > 0 {
		io.WriteString(out, indicator)
	}
	io.WriteString(out, "\n")
}

func lsBox(nodes []os.FileInfo, flag int, out io.Writer) {
	nodes_ := make([]string, len(nodes))
	for key, val := range nodes {
		prefix := ""
		postfix := ""
		if val.IsDir() {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_DIR
				postfix = ANSI_END
			}
			if (flag & O_INDICATOR) != 0 {
				postfix += "/"
			}
		} else if (val.Mode().Perm() & 2) == 0 {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_READONLY
				postfix = ANSI_END
			}
		} else if exeSuffixes[strings.ToLower(path.Ext(val.Name()))] {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_EXEC
				postfix = ANSI_END
			}
			if (flag & O_INDICATOR) != 0 {
				postfix += "*"
			}
		}
		nodes_[key] = prefix + val.Name() + postfix
	}
	box.Print(nodes_, 80, out)
}

func lsLong(nodes []os.FileInfo, flag int, out io.Writer) {
	for _, finfo := range nodes {
		lsOneLong(finfo, flag, out)
	}
}

type fileInfoCollection struct {
	flag  int
	nodes []os.FileInfo
}

func (this fileInfoCollection) Len() int {
	return len(this.nodes)
}
func (this fileInfoCollection) Less(i, j int) bool {
	var result bool
	if (this.flag & O_TIME) != 0 {
		result = this.nodes[i].ModTime().After(this.nodes[j].ModTime())
		if !result && !this.nodes[i].ModTime().Before(this.nodes[j].ModTime()) {
			result = (this.nodes[i].Name() < this.nodes[j].Name())
		}
	} else {
		result = (this.nodes[i].Name() < this.nodes[j].Name())
	}
	if (this.flag & O_REVERSE) != 0 {
		result = !result
	}
	return result
}
func (this fileInfoCollection) Swap(i, j int) {
	tmp := this.nodes[i]
	this.nodes[i] = this.nodes[j]
	this.nodes[j] = tmp
}

func lsFolder(folder string, flag int, out io.Writer) error {
	fd, err := os.Open(folder)
	if err != nil {
		return err
	}
	defer fd.Close()
	var nodesArray fileInfoCollection
	nodesArray.nodes, err = fd.Readdir(-1)
	nodesArray.flag = flag
	if err != nil {
		return err
	}
	if (flag & O_ALL) == 0 {
		tmp := make([]os.FileInfo, 0)
		for _, f := range nodesArray.nodes {
			if !strings.HasPrefix(f.Name(), ".") {
				tmp = append(tmp, f)
			}
		}
		nodesArray.nodes = tmp
	}
	sort.Sort(nodesArray)
	if (flag & O_LONG) > 0 {
		lsLong(nodesArray.nodes, O_STRIP_DIR|flag, out)
	} else {
		lsBox(nodesArray.nodes, O_STRIP_DIR|flag, out)
	}
	return nil
}

func lsCore(paths []string, flag int, out io.Writer) error {
	if len(paths) <= 0 {
		return lsFolder(".", flag, out)
	}
	dirs := make([]string, 0)
	printCount := 0
	files := make([]os.FileInfo, 0)
	for _, name := range paths {
		status, err := os.Stat(name)
		if err != nil {
			return err
		}
		if status.IsDir() {
			dirs = append(dirs, name)
		} else if (flag & O_LONG) != 0 {
			lsOneLong(newMyFileInfoT(name, status), flag, out)
			printCount += 1
		} else {
			files = append(files, newMyFileInfoT(name, status))
		}
	}
	if len(files) > 0 {
		lsBox(files, flag, out)
		printCount = len(files)
	}
	for _, name := range dirs {
		if len(paths) > 1 {
			if printCount > 0 {
				io.WriteString(out, "\n")
			}
			io.WriteString(out, name)
			io.WriteString(out, ":\n")
		}
		err := lsFolder(name, flag, out)
		if err != nil {
			return err
		}
		printCount++
	}
	return nil
}

var option = map[rune](func(*int) error){
	'l': func(flag *int) error {
		*flag |= O_LONG
		return nil
	},
	'F': func(flag *int) error {
		*flag |= O_INDICATOR
		return nil
	},
	'o': func(flag *int) error {
		*flag |= O_COLOR
		return nil
	},
	'a': func(flag *int) error {
		*flag |= O_ALL
		return nil
	},
	't': func(flag *int) error {
		*flag |= O_TIME
		return nil
	},
	'r': func(flag *int) error {
		*flag |= O_REVERSE
		return nil
	},
}

// 存在しないオプションに関するエラー
type OptionError struct {
	Option rune
}

func (this OptionError) Error() string {
	return fmt.Sprintf("-%c: No such option", this.Option)
}

// ls 機能のエントリ:引数をオプションとパスに分離する
func Main(args []string, out io.Writer) error {
	if len(exeSuffixes) <= 0 {
		pathExt := os.Getenv("PATHEXT")
		for _, ext := range strings.Split(pathExt, ";") {
			exeSuffixes[strings.ToLower(ext)] = true
		}
	}

	flag := 0
	paths := make([]string, 0)
	for _, arg := range args {
		if strings.HasPrefix(arg, "-") {
			for _, o := range arg[1:] {
				setter, ok := option[o]
				if !ok {
					var err OptionError
					err.Option = o
					return err
				}
				err := setter(&flag)
				if err != nil {
					return err
				}
			}
		} else {
			paths = append(paths, arg)
		}
	}
	return lsCore(paths, flag, out)
}

// vim:set fenc=utf8 ts=4 sw=4 noet:
<commit_msg>ls -R 追加<commit_after>package ls

import "fmt"
import "io"
import "os"
import "path/filepath"
import "sort"
import "strings"
import "time"

import "../box"

var exeSuffixes = map[string]bool{}

const (
	O_STRIP_DIR = 1
	O_LONG      = 2
	O_INDICATOR = 4
	O_COLOR     = 8
	O_ALL       = 16
	O_TIME      = 32
	O_REVERSE   = 64
	O_RECURSIVE = 128
)

type fileInfoT struct {
	name string
	info os.FileInfo
}

const (
	ANSI_EXEC     = "\x1B[1;35m"
	ANSI_DIR      = "\x1B[1;32m"
	ANSI_NORM     = "\x1B[1;37m"
	ANSI_READONLY = "\x1B[1;33m"
	ANSI_END      = "\x1B[39m"
)

func (this fileInfoT) Name() string       { return this.name }
func (this fileInfoT) Size() int64        { return this.info.Size() }
func (this fileInfoT) Mode() os.FileMode  { return this.info.Mode() }
func (this fileInfoT) ModTime() time.Time { return this.info.ModTime() }
func (this fileInfoT) IsDir() bool        { return this.info.IsDir() }
func (this fileInfoT) Sys() interface{}   { return this.info.Sys() }

func newMyFileInfoT(name string, info os.FileInfo) *fileInfoT {
	this := new(fileInfoT)
	this.name = name
	this.info = info
	return this
}

func lsOneLong(status os.FileInfo, flag int, out io.Writer) {
	indicator := " "
	prefix := ""
	postfix := ""
	if status.IsDir() {
		io.WriteString(out, "d")
		indicator = "/"
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_DIR
			postfix = ANSI_END
		}
	} else {
		io.WriteString(out, "-")
	}
	mode := status.Mode()
	perm := mode.Perm()
	name := status.Name()
	if (perm & 4) > 0 {
		io.WriteString(out, "r")
	} else {
		io.WriteString(out, "-")
	}
	if (perm & 2) > 0 {
		io.WriteString(out, "w")
	} else {
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_READONLY
			postfix = ANSI_END
		}
		io.WriteString(out, "-")
	}
	if (perm & 1) > 0 {
		io.WriteString(out, "x")
	} else if exeSuffixes[strings.ToLower(filepath.Ext(name))] {
		io.WriteString(out, "x")
		indicator = "*"
		if (flag & O_COLOR) != 0 {
			prefix = ANSI_EXEC
			postfix = ANSI_END
		}
	} else {
		io.WriteString(out, "-")
	}
	if (flag & O_STRIP_DIR) > 0 {
		name = filepath.Base(name)
	}
	stamp := status.ModTime()
	fmt.Fprintf(out, " %8d %04d-%02d-%02d %02d:%02d %s%s%s",
		status.Size(),
		stamp.Year(),
		stamp.Month(),
		stamp.Day(),
		stamp.Hour(),
		stamp.Minute(),
		prefix,
		name,
		postfix)
	if (flag & O_INDICATOR) > 0 {
		io.WriteString(out, indicator)
	}
	io.WriteString(out, "\n")
}

func lsBox(nodes []os.FileInfo, flag int, out io.Writer) {
	nodes_ := make([]string, len(nodes))
	for key, val := range nodes {
		prefix := ""
		postfix := ""
		if val.IsDir() {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_DIR
				postfix = ANSI_END
			}
			if (flag & O_INDICATOR) != 0 {
				postfix += "/"
			}
		} else if (val.Mode().Perm() & 2) == 0 {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_READONLY
				postfix = ANSI_END
			}
		} else if exeSuffixes[strings.ToLower(filepath.Ext(val.Name()))] {
			if (flag & O_COLOR) != 0 {
				prefix = ANSI_EXEC
				postfix = ANSI_END
			}
			if (flag & O_INDICATOR) != 0 {
				postfix += "*"
			}
		}
		nodes_[key] = prefix + val.Name() + postfix
	}
	box.Print(nodes_, 80, out)
}

func lsLong(nodes []os.FileInfo, flag int, out io.Writer) {
	for _, finfo := range nodes {
		lsOneLong(finfo, flag, out)
	}
}

type fileInfoCollection struct {
	flag  int
	nodes []os.FileInfo
}

func (this fileInfoCollection) Len() int {
	return len(this.nodes)
}
func (this fileInfoCollection) Less(i, j int) bool {
	var result bool
	if (this.flag & O_TIME) != 0 {
		result = this.nodes[i].ModTime().After(this.nodes[j].ModTime())
		if !result && !this.nodes[i].ModTime().Before(this.nodes[j].ModTime()) {
			result = (this.nodes[i].Name() < this.nodes[j].Name())
		}
	} else {
		result = (this.nodes[i].Name() < this.nodes[j].Name())
	}
	if (this.flag & O_REVERSE) != 0 {
		result = !result
	}
	return result
}
func (this fileInfoCollection) Swap(i, j int) {
	tmp := this.nodes[i]
	this.nodes[i] = this.nodes[j]
	this.nodes[j] = tmp
}

func lsFolder(folder string, flag int, out io.Writer) error {
	fd, err := os.Open(folder)
	if err != nil {
		return err
	}
	var nodesArray fileInfoCollection
	nodesArray.nodes, err = fd.Readdir(-1)
	fd.Close()
	nodesArray.flag = flag
	if err != nil {
		return err
	}
	tmp := make([]os.FileInfo, 0)
	var folders []string = nil
	if (flag & O_RECURSIVE) != 0 {
		folders = make([]string, 0)
	}
	for _, f := range nodesArray.nodes {
		if strings.HasPrefix(f.Name(), ".") && (flag&O_ALL) == 0 {
			continue
		}
		if f.IsDir() && folders != nil {
			folders = append(folders, f.Name())
		} else {
			tmp = append(tmp, f)
		}
	}
	nodesArray.nodes = tmp
	sort.Sort(nodesArray)
	if (flag & O_LONG) > 0 {
		lsLong(nodesArray.nodes, O_STRIP_DIR|flag, out)
	} else {
		lsBox(nodesArray.nodes, O_STRIP_DIR|flag, out)
	}
	if folders != nil && len(folders) > 0 {
		for _, f1 := range folders {
			f1fullpath := filepath.Join(folder, f1)
			fmt.Fprintf(out, "\n%s:\n", f1fullpath)
			lsFolder(f1fullpath, flag, out)
		}
	}
	return nil
}

func lsCore(paths []string, flag int, out io.Writer) error {
	if len(paths) <= 0 {
		return lsFolder(".", flag, out)
	}
	dirs := make([]string, 0)
	printCount := 0
	files := make([]os.FileInfo, 0)
	for _, name := range paths {
		status, err := os.Stat(name)
		if err != nil {
			return err
		}
		if status.IsDir() {
			dirs = append(dirs, name)
		} else if (flag & O_LONG) != 0 {
			lsOneLong(newMyFileInfoT(name, status), flag, out)
			printCount += 1
		} else {
			files = append(files, newMyFileInfoT(name, status))
		}
	}
	if len(files) > 0 {
		lsBox(files, flag, out)
		printCount = len(files)
	}
	for _, name := range dirs {
		if len(paths) > 1 {
			if printCount > 0 {
				io.WriteString(out, "\n")
			}
			io.WriteString(out, name)
			io.WriteString(out, ":\n")
		}
		err := lsFolder(name, flag, out)
		if err != nil {
			return err
		}
		printCount++
	}
	return nil
}

var option = map[rune](func(*int) error){
	'l': func(flag *int) error {
		*flag |= O_LONG
		return nil
	},
	'F': func(flag *int) error {
		*flag |= O_INDICATOR
		return nil
	},
	'o': func(flag *int) error {
		*flag |= O_COLOR
		return nil
	},
	'a': func(flag *int) error {
		*flag |= O_ALL
		return nil
	},
	't': func(flag *int) error {
		*flag |= O_TIME
		return nil
	},
	'r': func(flag *int) error {
		*flag |= O_REVERSE
		return nil
	},
	'R': func(flag *int) error {
		*flag |= O_RECURSIVE
		return nil
	},
}

// 存在しないオプションに関するエラー
type OptionError struct {
	Option rune
}

func (this OptionError) Error() string {
	return fmt.Sprintf("-%c: No such option", this.Option)
}

// ls 機能のエントリ:引数をオプションとパスに分離する
func Main(args []string, out io.Writer) error {
	if len(exeSuffixes) <= 0 {
		pathExt := os.Getenv("PATHEXT")
		for _, ext := range strings.Split(pathExt, ";") {
			exeSuffixes[strings.ToLower(ext)] = true
		}
	}

	flag := 0
	paths := make([]string, 0)
	for _, arg := range args {
		if strings.HasPrefix(arg, "-") {
			for _, o := range arg[1:] {
				setter, ok := option[o]
				if !ok {
					var err OptionError
					err.Option = o
					return err
				}
				err := setter(&flag)
				if err != nil {
					return err
				}
			}
		} else {
			paths = append(paths, arg)
		}
	}
	return lsCore(paths, flag, out)
}

// vim:set fenc=utf8 ts=4 sw=4 noet:
<|endoftext|>