<commit_before>package cluster

import (
	"crypto/tls"
	"encoding/base64"
	"encoding/pem"
	"fmt"
	"net"
	"time"

	"github.com/pkg/errors"

	lxd "github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/cluster/request"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/state"
	storagePools "github.com/lxc/lxd/lxd/storage"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/version"
)

// ErrCertificateExists indicates that a certificate already exists.
var ErrCertificateExists error = fmt.Errorf("Certificate already in trust store")

// Connect is a convenience around lxd.ConnectLXD that configures the client
// with the correct parameters for node-to-node communication.
//
// If 'notify' switch is true, then the user agent will be set to the special
// to the UserAgentNotifier value, which can be used in some cases to distinguish
// between a regular client request and an internal cluster request.
func Connect(address string, cert *shared.CertInfo, notify bool) (lxd.InstanceServer, error) {
	// Wait for a connection to the events API first for non-notify connections.
	if !notify {
		connected := false
		for i := 0; i < 20; i++ {
			listenersLock.Lock()
			_, ok := listeners[address]
			listenersLock.Unlock()

			if ok {
				connected = true
				break
			}

			time.Sleep(500 * time.Millisecond)
		}

		if !connected {
			return nil, fmt.Errorf("Missing event connection with target cluster member")
		}
	}

	args := &lxd.ConnectionArgs{
		TLSServerCert: string(cert.PublicKey()),
		TLSClientCert: string(cert.PublicKey()),
		TLSClientKey:  string(cert.PrivateKey()),
		SkipGetServer: true,
		UserAgent:     version.UserAgent,
	}
	if notify {
		args.UserAgent = request.UserAgentNotifier
	}

	url := fmt.Sprintf("https://%s", address)
	return lxd.ConnectLXD(url, args)
}

// ConnectIfInstanceIsRemote figures out the address of the node which is
// running the container with the given name. If it's not the local node will
// connect to it and return the connected client, otherwise it will just return
// nil.
func ConnectIfInstanceIsRemote(cluster *db.Cluster, projectName string, name string, cert *shared.CertInfo, instanceType instancetype.Type) (lxd.InstanceServer, error) {
	var address string // Node address
	err := cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		address, err = tx.GetNodeAddressOfInstance(projectName, name, instanceType)
		return err
	})
	if err != nil {
		return nil, err
	}
	if address == "" {
		// The instance is running right on this node, no need to connect.
		return nil, nil
	}
	return Connect(address, cert, false)
}

// ConnectIfVolumeIsRemote figures out the address of the cluster member on which the volume with the given name is
// defined. If it's not the local cluster member it will connect to it and return the connected client, otherwise
// it just returns nil. If there is more than one cluster member with a matching volume name, an error is returned.
func ConnectIfVolumeIsRemote(s *state.State, poolName string, projectName string, volumeName string, volumeType int, cert *shared.CertInfo) (lxd.InstanceServer, error) {
	localNodeID := s.Cluster.GetNodeID()
	var err error
	var nodes []db.NodeInfo
	var poolID int64
	err = s.Cluster.Transaction(func(tx *db.ClusterTx) error {
		poolID, err = tx.GetStoragePoolID(poolName)
		if err != nil {
			return err
		}

		nodes, err = tx.GetStorageVolumeNodes(poolID, projectName, volumeName, volumeType)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil && err != db.ErrNoClusterMember {
		return nil, err
	}

	// If volume uses a remote storage driver and so has no explicit cluster member, then we need to check
	// whether it is exclusively attached to remote instance, and if so then we need to forward the request to
	// the node whereit is currently used. This avoids conflicting with another member when using it locally.
	if err == db.ErrNoClusterMember {
		// GetLocalStoragePoolVolume returns a volume with an empty Location field for remote drivers.
		_, vol, err := s.Cluster.GetLocalStoragePoolVolume(projectName, volumeName, volumeType, poolID)
		if err != nil {
			return nil, err
		}

		remoteInstance, err := storagePools.VolumeUsedByExclusiveRemoteInstancesWithProfiles(s, poolName, projectName, vol)
		if err != nil {
			return nil, errors.Wrapf(err, "Failed checking if volume %q is available", volumeName)
		}

		if remoteInstance != nil {
			var instNode db.NodeInfo
			err := s.Cluster.Transaction(func(tx *db.ClusterTx) error {
				instNode, err = tx.GetNodeByName(remoteInstance.Node)
				return err
			})
			if err != nil {
				return nil, errors.Wrapf(err, "Failed getting cluster member info for %q", remoteInstance.Node)
			}

			// Replace node list with instance's cluster member node (which might be local member).
			nodes = []db.NodeInfo{instNode}
		} else {
			// Volume isn't exclusively attached to an instance. Use local cluster member.
			return nil, nil
		}
	}

	nodeCount := len(nodes)
	if nodeCount > 1 {
		return nil, fmt.Errorf("More than one cluster member has a volume named %q. Please target a specific member", volumeName)
	} else if nodeCount < 1 {
		// Should never get here.
		return nil, fmt.Errorf("Volume %q has empty cluster member list", volumeName)
	}

	node := nodes[0]
	if node.ID == localNodeID {
		// Use local cluster member if volume belongs to this local node.
		return nil, nil
	}

	// Connect to remote cluster member.
	return Connect(node.Address, cert, false)
}

// SetupTrust is a convenience around InstanceServer.CreateCertificate that
// adds the given client certificate to the trusted pool of the cluster at the
// given address, using the given password.
func SetupTrust(cert, targetAddress, targetCert, targetPassword string) error {
	// Connect to the target cluster node.
	args := &lxd.ConnectionArgs{
		TLSServerCert: targetCert,
		UserAgent:     version.UserAgent,
	}

	target, err := lxd.ConnectLXD(fmt.Sprintf("https://%s", targetAddress), args)
	if err != nil {
		return errors.Wrap(err, "failed to connect to target cluster node")
	}

	block, _ := pem.Decode([]byte(cert))
	if block == nil {
		return fmt.Errorf("failed to decode certificate")
	}

	certificate := base64.StdEncoding.EncodeToString(block.Bytes)
	post := api.CertificatesPost{
		Password:    targetPassword,
		Certificate: certificate,
	}

	fingerprint, err := shared.CertFingerprintStr(cert)
	if err != nil {
		return errors.Wrap(err, "failed to calculate fingerprint")
	}

	post.Name = fmt.Sprintf("lxd.cluster.%s", fingerprint)
	post.Type = api.CertificateTypeClient

	err = target.CreateCertificate(post)
	if err != nil && err.Error() != "Certificate already in trust store" {
		return errors.Wrap(err, "Failed to add client cert to cluster")
	}

	return nil
}

// HasConnectivity probes the member with the given address for connectivity.
func HasConnectivity(cert *shared.CertInfo, address string) bool {
	config, err := tlsClientConfig(cert)
	if err != nil {
		return false
	}

	var conn net.Conn
	dialer := &net.Dialer{Timeout: time.Second}
	conn, err = tls.DialWithDialer(dialer, "tcp", address, config)
	if err == nil {
		conn.Close()
		return true
	}
	return false
}
<commit_msg>lxd/cluster/connect: Updates Connect to accept networkCert and serverCert args<commit_after>package cluster

import (
	"crypto/tls"
	"encoding/base64"
	"encoding/pem"
	"fmt"
	"net"
	"time"

	"github.com/pkg/errors"

	lxd "github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/cluster/request"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/state"
	storagePools "github.com/lxc/lxd/lxd/storage"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/version"
)

// ErrCertificateExists indicates that a certificate already exists.
var ErrCertificateExists error = fmt.Errorf("Certificate already in trust store")

// Connect is a convenience around lxd.ConnectLXD that configures the client
// with the correct parameters for node-to-node communication.
//
// If 'notify' switch is true, then the user agent will be set to the special
// to the UserAgentNotifier value, which can be used in some cases to distinguish
// between a regular client request and an internal cluster request.
func Connect(address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo, notify bool) (lxd.InstanceServer, error) {
	// Wait for a connection to the events API first for non-notify connections.
	if !notify {
		connected := false
		for i := 0; i < 20; i++ {
			listenersLock.Lock()
			_, ok := listeners[address]
			listenersLock.Unlock()

			if ok {
				connected = true
				break
			}

			time.Sleep(500 * time.Millisecond)
		}

		if !connected {
			return nil, fmt.Errorf("Missing event connection with target cluster member")
		}
	}

	args := &lxd.ConnectionArgs{
		TLSServerCert: string(networkCert.PublicKey()),
		TLSClientCert: string(serverCert.PublicKey()),
		TLSClientKey:  string(serverCert.PrivateKey()),
		SkipGetServer: true,
		UserAgent:     version.UserAgent,
	}

	if notify {
		args.UserAgent = request.UserAgentNotifier
	}

	url := fmt.Sprintf("https://%s", address)
	return lxd.ConnectLXD(url, args)
}

// ConnectIfInstanceIsRemote figures out the address of the node which is
// running the container with the given name. If it's not the local node will
// connect to it and return the connected client, otherwise it will just return
// nil.
func ConnectIfInstanceIsRemote(cluster *db.Cluster, projectName string, name string, networkCert *shared.CertInfo, serverCert *shared.CertInfo, instanceType instancetype.Type) (lxd.InstanceServer, error) {
	var address string // Node address
	err := cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		address, err = tx.GetNodeAddressOfInstance(projectName, name, instanceType)
		return err
	})
	if err != nil {
		return nil, err
	}
	if address == "" {
		// The instance is running right on this node, no need to connect.
		return nil, nil
	}
	return Connect(address, networkCert, serverCert, false)
}

// ConnectIfVolumeIsRemote figures out the address of the cluster member on which the volume with the given name is
// defined. If it's not the local cluster member it will connect to it and return the connected client, otherwise
// it just returns nil. If there is more than one cluster member with a matching volume name, an error is returned.
func ConnectIfVolumeIsRemote(s *state.State, poolName string, projectName string, volumeName string, volumeType int, networkCert *shared.CertInfo, serverCert *shared.CertInfo) (lxd.InstanceServer, error) {
	localNodeID := s.Cluster.GetNodeID()
	var err error
	var nodes []db.NodeInfo
	var poolID int64
	err = s.Cluster.Transaction(func(tx *db.ClusterTx) error {
		poolID, err = tx.GetStoragePoolID(poolName)
		if err != nil {
			return err
		}

		nodes, err = tx.GetStorageVolumeNodes(poolID, projectName, volumeName, volumeType)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil && err != db.ErrNoClusterMember {
		return nil, err
	}

	// If volume uses a remote storage driver and so has no explicit cluster member, then we need to check
	// whether it is exclusively attached to remote instance, and if so then we need to forward the request to
	// the node whereit is currently used. This avoids conflicting with another member when using it locally.
	if err == db.ErrNoClusterMember {
		// GetLocalStoragePoolVolume returns a volume with an empty Location field for remote drivers.
		_, vol, err := s.Cluster.GetLocalStoragePoolVolume(projectName, volumeName, volumeType, poolID)
		if err != nil {
			return nil, err
		}

		remoteInstance, err := storagePools.VolumeUsedByExclusiveRemoteInstancesWithProfiles(s, poolName, projectName, vol)
		if err != nil {
			return nil, errors.Wrapf(err, "Failed checking if volume %q is available", volumeName)
		}

		if remoteInstance != nil {
			var instNode db.NodeInfo
			err := s.Cluster.Transaction(func(tx *db.ClusterTx) error {
				instNode, err = tx.GetNodeByName(remoteInstance.Node)
				return err
			})
			if err != nil {
				return nil, errors.Wrapf(err, "Failed getting cluster member info for %q", remoteInstance.Node)
			}

			// Replace node list with instance's cluster member node (which might be local member).
			nodes = []db.NodeInfo{instNode}
		} else {
			// Volume isn't exclusively attached to an instance. Use local cluster member.
			return nil, nil
		}
	}

	nodeCount := len(nodes)
	if nodeCount > 1 {
		return nil, fmt.Errorf("More than one cluster member has a volume named %q. Please target a specific member", volumeName)
	} else if nodeCount < 1 {
		// Should never get here.
		return nil, fmt.Errorf("Volume %q has empty cluster member list", volumeName)
	}

	node := nodes[0]
	if node.ID == localNodeID {
		// Use local cluster member if volume belongs to this local node.
		return nil, nil
	}

	// Connect to remote cluster member.
	return Connect(node.Address, networkCert, serverCert, false)
}

// SetupTrust is a convenience around InstanceServer.CreateCertificate that
// adds the given client certificate to the trusted pool of the cluster at the
// given address, using the given password.
func SetupTrust(cert, targetAddress, targetCert, targetPassword string) error {
	// Connect to the target cluster node.
	args := &lxd.ConnectionArgs{
		TLSServerCert: targetCert,
		UserAgent:     version.UserAgent,
	}

	target, err := lxd.ConnectLXD(fmt.Sprintf("https://%s", targetAddress), args)
	if err != nil {
		return errors.Wrap(err, "failed to connect to target cluster node")
	}

	block, _ := pem.Decode([]byte(cert))
	if block == nil {
		return fmt.Errorf("failed to decode certificate")
	}

	certificate := base64.StdEncoding.EncodeToString(block.Bytes)
	post := api.CertificatesPost{
		Password:    targetPassword,
		Certificate: certificate,
	}

	fingerprint, err := shared.CertFingerprintStr(cert)
	if err != nil {
		return errors.Wrap(err, "failed to calculate fingerprint")
	}

	post.Name = fmt.Sprintf("lxd.cluster.%s", fingerprint)
	post.Type = api.CertificateTypeClient

	err = target.CreateCertificate(post)
	if err != nil && err.Error() != "Certificate already in trust store" {
		return errors.Wrap(err, "Failed to add client cert to cluster")
	}

	return nil
}

// HasConnectivity probes the member with the given address for connectivity.
func HasConnectivity(cert *shared.CertInfo, address string) bool {
	config, err := tlsClientConfig(cert)
	if err != nil {
		return false
	}

	var conn net.Conn
	dialer := &net.Dialer{Timeout: time.Second}
	conn, err = tls.DialWithDialer(dialer, "tcp", address, config)
	if err == nil {
		conn.Close()
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package mail

import (
	"bytes"
	"text/template"
	"github.com/WeisswurstSystems/WWM-BB/meeting"
)

type invitationData struct {
	Usermail       string
	MeetingCreator string
	MeetingID      string
	MeetingDate    string
	MeetingPlace   string
}

const invitationTopic = "Einladung zum Meeting am {{.MeetingPlace}}"

const invitationMessageTemplate = `
Hallo {{.Usermail}}!

{{.MeetingCreator}} hat dich zu einem Treffen eingeladen.

Wann? {{.MeetingDate}}
Wo? {{.MeetingDate}}

Unter dem folgenden Link kannst du beitreten und deine Bestellung aufgeben
https://weisswurstsystems.github.io/WWM-ITM/{{.MeetingID}}

Viel Spaß beim bestellen!
`

var invitationTemplate = template.New("invitation")

func init() {
	_, err := invitationTemplate.Parse(invitationMessageTemplate)
	if err != nil {
		panic(err)
	}
}

func NewInvitationMail(meeting meeting.Meeting, usermail string) (mail Mail) {
	data := invitationData{
		MeetingID: string(meeting.ID),
		Usermail: usermail,
		MeetingCreator:meeting.Creator,
		MeetingDate: meeting.Date.Format("2006-01-02 15:04:05"),
		MeetingPlace: meeting.Place,
	}

	var message bytes.Buffer
	err := invitationTemplate.Execute(&message, data)
	if err != nil {
		panic(err)
	}

	mail.Content, err = NewContent(invitationTopic, message.String())
	if err != nil {
		panic(err)
	}
	mail.Receivers = []string{usermail}
	return mail
}<commit_msg>invitation mail anpassungen<commit_after>package mail

import (
	"bytes"
	"text/template"
	"github.com/WeisswurstSystems/WWM-BB/meeting"
)

type invitationData struct {
	Usermail       string
	MeetingCreator string
	MeetingID      string
	MeetingDate    string
	MeetingPlace   string
}

const invitationTopic = "Du wurdest eingeladen!"

const invitationMessageTemplate = `
Hallo {{.Usermail}}!

{{.MeetingCreator}} hat dich zu einem Treffen eingeladen.

Wann? {{.MeetingDate}}

Wo? {{.MeetingPlace}}

Unter dem folgenden Link kannst du beitreten und deine Bestellung aufgeben:

https://weisswurstsystems.github.io/WWM-ITM/{{.MeetingID}}

Viel Spaß beim bestellen!
`

var invitationTemplate = template.New("invitation")

func init() {
	_, err := invitationTemplate.Parse(invitationMessageTemplate)
	if err != nil {
		panic(err)
	}
}

func NewInvitationMail(meeting meeting.Meeting, usermail string) (mail Mail) {
	data := invitationData{
		MeetingID: string(meeting.ID),
		Usermail: usermail,
		MeetingCreator:meeting.Creator,
		MeetingDate: meeting.Date.Format("2006-01-02 15:04:05"),
		MeetingPlace: meeting.Place,
	}

	var message bytes.Buffer
	err := invitationTemplate.Execute(&message, data)
	if err != nil {
		panic(err)
	}

	mail.Content, err = NewContent(invitationTopic, message.String())
	if err != nil {
		panic(err)
	}
	mail.Receivers = []string{usermail}
	return mail
}<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"github.com/v2ray/v2ray-core"
	_ "github.com/v2ray/v2ray-core/app/router/rules"
	"github.com/v2ray/v2ray-core/common/log"
	"github.com/v2ray/v2ray-core/shell/point"

	// The following are neccesary as they register handlers in their init functions.
	_ "github.com/v2ray/v2ray-core/proxy/blackhole"
	_ "github.com/v2ray/v2ray-core/proxy/dokodemo"
	_ "github.com/v2ray/v2ray-core/proxy/freedom"
	_ "github.com/v2ray/v2ray-core/proxy/http"
	_ "github.com/v2ray/v2ray-core/proxy/socks"
	_ "github.com/v2ray/v2ray-core/proxy/vmess/inbound"
	_ "github.com/v2ray/v2ray-core/proxy/vmess/outbound"
)

var (
	configFile string
	logLevel   = flag.String("loglevel", "warning", "Level of log info to be printed to console, available value: debug, info, warning, error")
	version    = flag.Bool("version", false, "Show current version of V2Ray.")
	test       = flag.Bool("test", false, "Test config file only, without launching V2Ray server.")
)

func init() {
	defaultConfigFile := ""
	workingDir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err == nil {
		defaultConfigFile = filepath.Join(workingDir, "config.json")
	}
	flag.StringVar(&configFile, "config", defaultConfigFile, "Config file for this Point server.")
}

func main() {
	flag.Parse()

	core.PrintVersion()

	if *version {
		return
	}

	switch *logLevel {
	case "debug":
		log.SetLogLevel(log.DebugLevel)
	case "info":
		log.SetLogLevel(log.InfoLevel)
	case "warning":
		log.SetLogLevel(log.WarningLevel)
	case "error":
		log.SetLogLevel(log.ErrorLevel)
	default:
		fmt.Println("Unknown log level: " + *logLevel)
		return
	}

	if len(configFile) == 0 {
		log.Error("Config file is not set.")
		return
	}
	config, err := point.LoadConfig(configFile)
	if err != nil {
		log.Error("Failed to read config file (", configFile, "): ", configFile, err)
		return
	}

	if config.LogConfig != nil && len(config.LogConfig.AccessLog) > 0 {
		log.InitAccessLogger(config.LogConfig.AccessLog)
	}

	vPoint, err := point.NewPoint(config)
	if err != nil {
		log.Error("Failed to create Point server: ", err)
		return
	}

	if *test {
		fmt.Println("Configuration OK.")
		return
	}

	err = vPoint.Start()
	if err != nil {
		log.Error("Error starting Point server: ", err)
		return
	}

	finish := make(chan bool)
	<-finish
}
<commit_msg>include shadowsocks as inbound in release<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"github.com/v2ray/v2ray-core"
	_ "github.com/v2ray/v2ray-core/app/router/rules"
	"github.com/v2ray/v2ray-core/common/log"
	"github.com/v2ray/v2ray-core/shell/point"

	// The following are neccesary as they register handlers in their init functions.
	_ "github.com/v2ray/v2ray-core/proxy/blackhole"
	_ "github.com/v2ray/v2ray-core/proxy/dokodemo"
	_ "github.com/v2ray/v2ray-core/proxy/freedom"
	_ "github.com/v2ray/v2ray-core/proxy/http"
	_ "github.com/v2ray/v2ray-core/proxy/shadowsocks"
	_ "github.com/v2ray/v2ray-core/proxy/socks"
	_ "github.com/v2ray/v2ray-core/proxy/vmess/inbound"
	_ "github.com/v2ray/v2ray-core/proxy/vmess/outbound"
)

var (
	configFile string
	logLevel   = flag.String("loglevel", "warning", "Level of log info to be printed to console, available value: debug, info, warning, error")
	version    = flag.Bool("version", false, "Show current version of V2Ray.")
	test       = flag.Bool("test", false, "Test config file only, without launching V2Ray server.")
)

func init() {
	defaultConfigFile := ""
	workingDir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err == nil {
		defaultConfigFile = filepath.Join(workingDir, "config.json")
	}
	flag.StringVar(&configFile, "config", defaultConfigFile, "Config file for this Point server.")
}

func main() {
	flag.Parse()

	core.PrintVersion()

	if *version {
		return
	}

	switch *logLevel {
	case "debug":
		log.SetLogLevel(log.DebugLevel)
	case "info":
		log.SetLogLevel(log.InfoLevel)
	case "warning":
		log.SetLogLevel(log.WarningLevel)
	case "error":
		log.SetLogLevel(log.ErrorLevel)
	default:
		fmt.Println("Unknown log level: " + *logLevel)
		return
	}

	if len(configFile) == 0 {
		log.Error("Config file is not set.")
		return
	}
	config, err := point.LoadConfig(configFile)
	if err != nil {
		log.Error("Failed to read config file (", configFile, "): ", configFile, err)
		return
	}

	if config.LogConfig != nil && len(config.LogConfig.AccessLog) > 0 {
		log.InitAccessLogger(config.LogConfig.AccessLog)
	}

	vPoint, err := point.NewPoint(config)
	if err != nil {
		log.Error("Failed to create Point server: ", err)
		return
	}

	if *test {
		fmt.Println("Configuration OK.")
		return
	}

	err = vPoint.Start()
	if err != nil {
		log.Error("Error starting Point server: ", err)
		return
	}

	finish := make(chan bool)
	<-finish
}
<|endoftext|>


<commit_before>package instructions

import (
    "jvmgo/rtda"
    rtclass "jvmgo/rtda/class"
)

// Create new object
type new_ struct {Index16Instruction}
func (self *new_) Execute(thread *rtda.Thread) {
    frame := thread.CurrentFrame()
    stack := frame.OperandStack()
    cp := frame.Method().Class().ConstantPool()
    cClass := cp.GetConstant(self.index).(*rtclass.ConstantClass)
    class := cClass.Class()

    if class.NotInitialized() {
        panic("todo new")
        // todo init class
    }

    ref := class.NewObj()
    stack.PushRef(ref)
}

// Create new array
type newarray struct {
    atype uint8
}
func (self *newarray) fetchOperands(bcr *BytecodeReader) {
    self.atype = bcr.readUint8()
}
func (self *newarray) Execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    count := stack.PopInt()
    ref := rtclass.NewPrimitiveArray(self.atype, count)
    stack.PushRef(ref)
}

// Create new array of reference
type anewarray struct {Index16Instruction}
func (self *anewarray) Execute(thread *rtda.Thread) {
    frame := thread.CurrentFrame()
    stack := frame.OperandStack()
    count := stack.PopInt()

    cp := frame.Method().Class().ConstantPool()
    cClass := cp.GetConstant(self.index).(*rtclass.ConstantClass)
    class := cClass.Class()

    if class.NotInitialized() {
        panic("todo anewarray")
        // todo init class
    }

    ref := rtclass.NewRefArray(count)
    stack.PushRef(ref)
}

// Create new multidimensional array
type multianewarray struct {
    index       uint16
    dimensions  uint8
}
func (self *multianewarray) fetchOperands(bcr *BytecodeReader) {
    self.index = bcr.readUint16()
    self.dimensions = bcr.readUint8()
}
func (self *multianewarray) Execute(thread *rtda.Thread) {
    // todo
    panic("todo multianewarray")
}
<commit_msg>finish anew array<commit_after>package instructions

import (
    "jvmgo/rtda"
    rtclass "jvmgo/rtda/class"
)

// Create new object
type new_ struct {Index16Instruction}
func (self *new_) Execute(thread *rtda.Thread) {
    frame := thread.CurrentFrame()
    stack := frame.OperandStack()
    cp := frame.Method().Class().ConstantPool()
    cClass := cp.GetConstant(self.index).(*rtclass.ConstantClass)
    class := cClass.Class()

    if class.NotInitialized() {
        panic("todo new")
        // todo init class
    }

    ref := class.NewObj()
    stack.PushRef(ref)
}

// Create new array
type newarray struct {
    atype uint8
}
func (self *newarray) fetchOperands(bcr *BytecodeReader) {
    self.atype = bcr.readUint8()
}
func (self *newarray) Execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    count := stack.PopInt()
    ref := rtclass.NewPrimitiveArray(self.atype, count)
    stack.PushRef(ref)
}

// Create new array of reference
type anewarray struct {Index16Instruction}
func (self *anewarray) Execute(thread *rtda.Thread) {
    frame := thread.CurrentFrame()
    cp := frame.Method().Class().ConstantPool()
    cClass := cp.GetConstant(self.index).(*rtclass.ConstantClass)
    class := cClass.Class()

    if class.NotInitialized() {
        frame.SetNextPC(thread.PC())
        initClass(class, thread)
    } else {
        stack := frame.OperandStack()
        count := stack.PopInt()
        ref := rtclass.NewRefArray(count)
        stack.PushRef(ref)
    }
}

// Create new multidimensional array
type multianewarray struct {
    index       uint16
    dimensions  uint8
}
func (self *multianewarray) fetchOperands(bcr *BytecodeReader) {
    self.index = bcr.readUint16()
    self.dimensions = bcr.readUint8()
}
func (self *multianewarray) Execute(thread *rtda.Thread) {
    // todo
    panic("todo multianewarray")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"

	"github.com/animenotifier/arn"
	"github.com/fatih/color"
)

type stats map[string]float64

func main() {
	color.Yellow("Generating statistics")

	userStats := getUserStats()
	animeStats := getAnimeStats()

	arn.PanicOnError(arn.DB.Set("Cache", "user statistics", &arn.StatisticsCategory{
		Name:      "Users",
		PieCharts: userStats,
	}))
	arn.PanicOnError(arn.DB.Set("Cache", "anime statistics", &arn.StatisticsCategory{
		Name:      "Anime",
		PieCharts: animeStats,
	}))

	color.Green("Finished.")
}

func getUserStats() []*arn.PieChart {
	println("Generating user statistics")

	analytics, err := arn.AllAnalytics()
	arn.PanicOnError(err)

	screenSize := stats{}
	pixelRatio := stats{}
	browser := stats{}
	country := stats{}
	gender := stats{}
	os := stats{}

	for _, info := range analytics {
		pixelRatio[fmt.Sprintf("%.1f", info.Screen.PixelRatio)]++

		size := arn.ToString(info.Screen.Width) + " x " + arn.ToString(info.Screen.Height)
		screenSize[size]++
	}

	for user := range arn.MustStreamUsers() {
		if user.Gender != "" {
			gender[user.Gender]++
		}

		if user.Browser.Name != "" {
			browser[user.Browser.Name]++
		}

		if user.Location.CountryName != "" {
			country[user.Location.CountryName]++
		}

		if user.OS.Name != "" {
			if strings.HasPrefix(user.OS.Name, "CrOS") {
				user.OS.Name = "Chrome OS"
			}

			os[user.OS.Name]++
		}
	}

	println("Finished user statistics")

	return []*arn.PieChart{
		arn.NewPieChart("OS", os),
		arn.NewPieChart("Screen size", screenSize),
		arn.NewPieChart("Browser", browser),
		arn.NewPieChart("Country", country),
		arn.NewPieChart("Gender", gender),
		arn.NewPieChart("Pixel ratio", pixelRatio),
	}
}

func getAnimeStats() []*arn.PieChart {
	println("Generating anime statistics")

	allAnime, err := arn.AllAnime()
	arn.PanicOnError(err)

	shoboi := stats{}
	anilist := stats{}
	mal := stats{}
	anidb := stats{}
	status := stats{}
	types := stats{}
	shoboiEdits := stats{}
	anilistEdits := stats{}
	malEdits := stats{}
	anidbEdits := stats{}
	rating := stats{}

	for _, anime := range allAnime {
		for _, external := range anime.Mappings {
			if external.Service == "shoboi/anime" {
				if external.CreatedBy == "" {
					shoboiEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					shoboiEdits[user.Nick]++
				}
			}

			if external.Service == "anilist/anime" {
				if external.CreatedBy == "" {
					anilistEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					anilistEdits[user.Nick]++
				}
			}

			if external.Service == "myanimelist/anime" {
				if external.CreatedBy == "" {
					malEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					malEdits[user.Nick]++
				}
			}

			if external.Service == "anidb/anime" {
				if external.CreatedBy == "" {
					anidbEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					anidbEdits[user.Nick]++
				}
			}
		}

		if anime.GetMapping("shoboi/anime") != "" {
			shoboi["Connected with Shoboi"]++
		} else {
			shoboi["Not connected with Shoboi"]++
		}

		if anime.GetMapping("anilist/anime") != "" {
			anilist["Connected with AniList"]++
		} else {
			anilist["Not connected with AniList"]++
		}

		if anime.GetMapping("myanimelist/anime") != "" {
			mal["Connected with MyAnimeList"]++
		} else {
			mal["Not connected with MyAnimeList"]++
		}

		if anime.GetMapping("anidb/anime") != "" {
			anidb["Connected with AniDB"]++
		} else {
			anidb["Not connected with AniDB"]++
		}

		rating[arn.ToString(int(anime.Rating.Overall+0.5))]++

		status[anime.Status]++
		types[anime.Type]++
	}

	println("Finished anime statistics")

	return []*arn.PieChart{
		arn.NewPieChart("Type", types),
		arn.NewPieChart("Status", status),
		arn.NewPieChart("Rating", rating),
		arn.NewPieChart("MyAnimeList", mal),
		arn.NewPieChart("AniList", anilist),
		arn.NewPieChart("AniDB", anidb),
		arn.NewPieChart("Shoboi", shoboi),
		// arn.NewPieChart("MyAnimeList Editors", malEdits),
		arn.NewPieChart("AniList Editors", anilistEdits),
		// arn.NewPieChart("AniDB Editors", anidbEdits),
		arn.NewPieChart("Shoboi Editors", shoboiEdits),
	}
}
<commit_msg>Added twist.moe statistics<commit_after>package main

import (
	"fmt"
	"strings"

	"github.com/animenotifier/arn"
	"github.com/fatih/color"
)

type stats map[string]float64

func main() {
	color.Yellow("Generating statistics")

	userStats := getUserStats()
	animeStats := getAnimeStats()

	arn.PanicOnError(arn.DB.Set("Cache", "user statistics", &arn.StatisticsCategory{
		Name:      "Users",
		PieCharts: userStats,
	}))
	arn.PanicOnError(arn.DB.Set("Cache", "anime statistics", &arn.StatisticsCategory{
		Name:      "Anime",
		PieCharts: animeStats,
	}))

	color.Green("Finished.")
}

func getUserStats() []*arn.PieChart {
	println("Generating user statistics")

	analytics, err := arn.AllAnalytics()
	arn.PanicOnError(err)

	screenSize := stats{}
	pixelRatio := stats{}
	browser := stats{}
	country := stats{}
	gender := stats{}
	os := stats{}

	for _, info := range analytics {
		pixelRatio[fmt.Sprintf("%.1f", info.Screen.PixelRatio)]++

		size := arn.ToString(info.Screen.Width) + " x " + arn.ToString(info.Screen.Height)
		screenSize[size]++
	}

	for user := range arn.MustStreamUsers() {
		if user.Gender != "" {
			gender[user.Gender]++
		}

		if user.Browser.Name != "" {
			browser[user.Browser.Name]++
		}

		if user.Location.CountryName != "" {
			country[user.Location.CountryName]++
		}

		if user.OS.Name != "" {
			if strings.HasPrefix(user.OS.Name, "CrOS") {
				user.OS.Name = "Chrome OS"
			}

			os[user.OS.Name]++
		}
	}

	println("Finished user statistics")

	return []*arn.PieChart{
		arn.NewPieChart("OS", os),
		arn.NewPieChart("Screen size", screenSize),
		arn.NewPieChart("Browser", browser),
		arn.NewPieChart("Country", country),
		arn.NewPieChart("Gender", gender),
		arn.NewPieChart("Pixel ratio", pixelRatio),
	}
}

func getAnimeStats() []*arn.PieChart {
	println("Generating anime statistics")

	allAnime, err := arn.AllAnime()
	arn.PanicOnError(err)

	shoboi := stats{}
	anilist := stats{}
	mal := stats{}
	anidb := stats{}
	status := stats{}
	types := stats{}
	shoboiEdits := stats{}
	anilistEdits := stats{}
	malEdits := stats{}
	anidbEdits := stats{}
	rating := stats{}
	twist := stats{}

	for _, anime := range allAnime {
		for _, external := range anime.Mappings {
			if external.Service == "shoboi/anime" {
				if external.CreatedBy == "" {
					shoboiEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					shoboiEdits[user.Nick]++
				}
			}

			if external.Service == "anilist/anime" {
				if external.CreatedBy == "" {
					anilistEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					anilistEdits[user.Nick]++
				}
			}

			if external.Service == "myanimelist/anime" {
				if external.CreatedBy == "" {
					malEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					malEdits[user.Nick]++
				}
			}

			if external.Service == "anidb/anime" {
				if external.CreatedBy == "" {
					anidbEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					anidbEdits[user.Nick]++
				}
			}
		}

		if anime.GetMapping("shoboi/anime") != "" {
			shoboi["Connected with Shoboi"]++
		} else {
			shoboi["Not connected with Shoboi"]++
		}

		if anime.GetMapping("anilist/anime") != "" {
			anilist["Connected with AniList"]++
		} else {
			anilist["Not connected with AniList"]++
		}

		if anime.GetMapping("myanimelist/anime") != "" {
			mal["Connected with MyAnimeList"]++
		} else {
			mal["Not connected with MyAnimeList"]++
		}

		if anime.GetMapping("anidb/anime") != "" {
			anidb["Connected with AniDB"]++
		} else {
			anidb["Not connected with AniDB"]++
		}

		rating[arn.ToString(int(anime.Rating.Overall+0.5))]++

		found := false
		for _, episode := range anime.Episodes().Items {
			if episode.Links != nil && episode.Links["twist.moe"] != "" {
				found = true
				break
			}
		}

		if found {
			twist["Connected with AnimeTwist"]++
		} else {
			twist["Not connected with AnimeTwist"]++
		}

		status[anime.Status]++
		types[anime.Type]++
	}

	println("Finished anime statistics")

	return []*arn.PieChart{
		arn.NewPieChart("Type", types),
		arn.NewPieChart("Status", status),
		arn.NewPieChart("Rating", rating),
		arn.NewPieChart("MyAnimeList", mal),
		arn.NewPieChart("AniList", anilist),
		arn.NewPieChart("AniDB", anidb),
		arn.NewPieChart("Shoboi", shoboi),
		arn.NewPieChart("AnimeTwist", twist),
		// arn.NewPieChart("MyAnimeList Editors", malEdits),
		arn.NewPieChart("AniList Editors", anilistEdits),
		// arn.NewPieChart("AniDB Editors", anidbEdits),
		arn.NewPieChart("Shoboi Editors", shoboiEdits),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"net"
	"time"
)

var dialTimeout, readTimeout time.Duration

// use a fast to fetch web site
const estimateSite = "www.baidu.com"

var estimateReq = []byte("GET / HTTP/1.1\r\n" +
	"Host: " + estimateSite + "\r\n" +
	"User-Agent: curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8r zlib/1.2.5\r\n" +
	"Accept: */*\r\n" +
	"Accept-Language: en-us,en;q=0.5\r\n" +
	"Accept-Encoding: gzip, deflate\r\n" +
	"Referer: http://www.baidu.com/\r\n" +
	"Connection: close\r\n\r\n")

// estimateTimeout tries to fetch a url and adjust timeout value according to
// how much time is spent on connect and fetch. This avoids incorrectly
// considering non-blocked sites as blocked when network connection is bad.
func estimateTimeout() {
	debug.Println("estimating timeout")
	buf := make([]byte, 4096)
	var est time.Duration

	start := time.Now()
	c, err := net.Dial("tcp", estimateSite+":80")
	if err != nil {
		errl.Printf("estimateTimeout: can't connect to %s: %v, network has problem?\n",
			estimateSite, err)
		goto onErr
	}
	defer c.Close()

	est = time.Now().Sub(start) * 5
	debug.Println("estimated dialTimeout:", est)
	if est > config.DialTimeout {
		dialTimeout = est
		info.Println("new dial timeout:", dialTimeout)
	} else if dialTimeout != config.DialTimeout {
		dialTimeout = config.DialTimeout
		info.Println("new dial timeout:", dialTimeout)
	}

	start = time.Now()
	// include time spent on sending request, reading all content to make it a
	// little longer
	c.Write(estimateReq)
	for err == nil {
		_, err = c.Read(buf)
	}
	if err != io.EOF {
		errl.Printf("estimateTimeout: error getting %s: %v, network has problem?\n",
			estimateSite, err)
	}
	est = time.Now().Sub(start) * 10
	debug.Println("estimated read timeout:", est)
	if est > time.Duration(config.ReadTimeout) {
		readTimeout = est
		info.Println("new read timeout:", readTimeout)
	} else if readTimeout != config.ReadTimeout {
		readTimeout = config.ReadTimeout
		info.Println("new read timeout:", readTimeout)
	}
	return
onErr:
	dialTimeout += 2
	readTimeout += 2
}

func runEstimateTimeout() {
	readTimeout = config.ReadTimeout
	dialTimeout = config.DialTimeout
	for {
		estimateTimeout()
		time.Sleep(time.Minute)
	}
}
<commit_msg>Set maximum timeout value to 15s.<commit_after>package main

import (
	"io"
	"net"
	"time"
)

var dialTimeout, readTimeout time.Duration
var maxTimeout = time.Duration(15)

// use a fast to fetch web site
const estimateSite = "www.baidu.com"

var estimateReq = []byte("GET / HTTP/1.1\r\n" +
	"Host: " + estimateSite + "\r\n" +
	"User-Agent: curl/7.24.0 (x86_64-apple-darwin12.0) libcurl/7.24.0 OpenSSL/0.9.8r zlib/1.2.5\r\n" +
	"Accept: */*\r\n" +
	"Accept-Language: en-us,en;q=0.5\r\n" +
	"Accept-Encoding: gzip, deflate\r\n" +
	"Referer: http://www.baidu.com/\r\n" +
	"Connection: close\r\n\r\n")

// estimateTimeout tries to fetch a url and adjust timeout value according to
// how much time is spent on connect and fetch. This avoids incorrectly
// considering non-blocked sites as blocked when network connection is bad.
func estimateTimeout() {
	debug.Println("estimating timeout")
	buf := make([]byte, 4096)
	var est time.Duration

	start := time.Now()
	c, err := net.Dial("tcp", estimateSite+":80")
	if err != nil {
		errl.Printf("estimateTimeout: can't connect to %s: %v, network has problem?\n",
			estimateSite, err)
		goto onErr
	}
	defer c.Close()

	est = time.Now().Sub(start) * 5
	debug.Println("estimated dialTimeout:", est)
	if est > maxTimeout {
		est = maxTimeout
	}
	if est > config.DialTimeout {
		dialTimeout = est
		info.Println("new dial timeout:", dialTimeout)
	} else if dialTimeout != config.DialTimeout {
		dialTimeout = config.DialTimeout
		info.Println("new dial timeout:", dialTimeout)
	}

	start = time.Now()
	// include time spent on sending request, reading all content to make it a
	// little longer
	c.Write(estimateReq)
	for err == nil {
		_, err = c.Read(buf)
	}
	if err != io.EOF {
		errl.Printf("estimateTimeout: error getting %s: %v, network has problem?\n",
			estimateSite, err)
	}
	est = time.Now().Sub(start) * 5
	if est > maxTimeout {
		est = maxTimeout
	}
	debug.Println("estimated read timeout:", est)
	if est > time.Duration(config.ReadTimeout) {
		readTimeout = est
		info.Println("new read timeout:", readTimeout)
	} else if readTimeout != config.ReadTimeout {
		readTimeout = config.ReadTimeout
		info.Println("new read timeout:", readTimeout)
	}
	return
onErr:
	dialTimeout += 2
	readTimeout += 2
}

func runEstimateTimeout() {
	readTimeout = config.ReadTimeout
	dialTimeout = config.DialTimeout
	for {
		estimateTimeout()
		time.Sleep(time.Minute)
	}
}
<|endoftext|>


<commit_before>/*
   Hockeypuck - OpenPGP key server
   Copyright (C) 2012  Casey Marshall

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, version 3.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

package sks

import (
	"bytes"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"gopkg.in/errgo.v1"
	"gopkg.in/tomb.v2"

	cf "hockeypuck/conflux"
	"hockeypuck/conflux/recon"
	"hockeypuck/conflux/recon/leveldb"
	"hockeypuck/hkp/storage"
	log "hockeypuck/logrus"
	"hockeypuck/openpgp"
)

const (
	RECON = "recon"
	maxKeyRecoveryAttempts = 10
	requestChunkSize = 100
)

type keyRecoveryCounter map[string]int

type Peer struct {
	peer     *recon.Peer
	storage  storage.Storage
	settings *recon.Settings
	ptree    recon.PrefixTree

	path  string
	stats *Stats

	t tomb.Tomb
}

func NewPrefixTree(path string, s *recon.Settings) (recon.PrefixTree, error) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		log.Debugf("creating prefix tree at: %q", path)
		err = os.MkdirAll(path, 0755)
		if err != nil {
			return nil, errgo.Mask(err)
		}
	}
	return leveldb.New(s.PTreeConfig, path)
}

func NewPeer(st storage.Storage, path string, s *recon.Settings) (*Peer, error) {
	if s == nil {
		s = recon.DefaultSettings()
	}

	ptree, err := NewPrefixTree(path, s)
	if err != nil {
		return nil, errgo.Mask(err)
	}
	err = ptree.Create()
	if err != nil {
		return nil, errgo.Mask(err)
	}

	peer := recon.NewPeer(s, ptree)
	sksPeer := &Peer{
		ptree:    ptree,
		storage:  st,
		settings: s,
		peer:     peer,
		path:     path,
	}
	sksPeer.readStats()
	st.Subscribe(sksPeer.updateDigests)
	return sksPeer, nil
}

func (p *Peer) log(label string) *log.Entry {
	return p.logFields(label, log.Fields{})
}

func (p *Peer) logAddr(label string, addr net.Addr) *log.Entry {
	return p.logFields(label, log.Fields{"remoteAddr": addr})
}

func (p *Peer) logFields(label string, fields log.Fields) *log.Entry {
	fields["label"] = fmt.Sprintf("%s %s", label, p.settings.ReconAddr)
	return log.WithFields(fields)
}

func StatsFilename(path string) string {
	dir, base := filepath.Dir(path), filepath.Base(path)
	return filepath.Join(dir, "."+base+".stats")
}

func (p *Peer) readStats() {
	fn := StatsFilename(p.path)
	stats := NewStats()
	err := stats.ReadFile(fn)
	if err != nil {
		p.log(RECON).Warningf("cannot open stats %q: %v", fn, err)
		stats = NewStats()
	}

	root, err := p.ptree.Root()
	if err != nil {
		p.log(RECON).Warningf("error accessing prefix tree root: %v", err)
	} else {
		stats.Total = root.Size()
	}

	p.stats = stats
}

func (p *Peer) writeStats() {
	fn := StatsFilename(p.path)
	err := p.stats.WriteFile(fn)
	if err != nil {
		p.log(RECON).Warningf("cannot write stats %q: %v", fn, err)
	}
}

func (p *Peer) pruneStats() error {
	timer := time.NewTimer(time.Hour)
	for {
		select {
		case <-p.t.Dying():
			return nil
		case <-timer.C:
			p.stats.prune()
			timer.Reset(time.Hour)
		}
	}
}

func (r *Peer) Stats() *Stats {
	return r.stats.clone()
}

func (r *Peer) Start() {
	r.t.Go(r.handleRecovery)
	r.t.Go(r.pruneStats)
	r.peer.Start()
}

func (r *Peer) Stop() {
	r.log(RECON).Info("recon processing: stopping")
	r.t.Kill(nil)
	err := r.t.Wait()
	if err != nil {
		r.log(RECON).Error(errgo.Details(err))
	}
	r.log(RECON).Info("recon processing: stopped")

	r.log(RECON).Info("recon peer: stopping")
	err = errgo.Mask(r.peer.Stop())
	if err != nil {
		r.log(RECON).Error(errgo.Details(err))
	}
	r.log(RECON).Info("recon peer: stopped")

	err = r.ptree.Close()
	if err != nil {
		r.log(RECON).Errorf("error closing prefix tree: %v", errgo.Details(err))
	}

	r.writeStats()
}

func DigestZp(digest string) (*cf.Zp, error) {
	buf, err := hex.DecodeString(digest)
	if err != nil {
		return nil, errgo.Mask(err)
	}
	buf = recon.PadSksElement(buf)
	return cf.Zb(cf.P_SKS, buf), nil
}

func (r *Peer) updateDigests(change storage.KeyChange) error {
	r.stats.Update(change)
	for _, digest := range change.InsertDigests() {
		digestZp, err := DigestZp(digest)
		if err != nil {
			return errgo.Notef(err, "bad digest %q", digest)
		}
		r.peer.Insert(digestZp)
	}
	for _, digest := range change.RemoveDigests() {
		digestZp, err := DigestZp(digest)
		if err != nil {
			return errgo.Notef(err, "bad digest %q", digest)
		}
		r.peer.Remove(digestZp)
	}
	return nil
}

func (r *Peer) handleRecovery() error {
	for {
		select {
		case <-r.t.Dying():
			return nil
		case rcvr := <-r.peer.RecoverChan:
			r.logAddr(RECON, rcvr.RemoteAddr).Infof("%d items to recover", len(rcvr.RemoteElements))
			if err := r.requestRecovered(rcvr); err != nil {
				r.logAddr(RECON, rcvr.RemoteAddr).Errorf("recovery completed with errors: %v", err)
			} else {
				r.logAddr(RECON, rcvr.RemoteAddr).Info("recovery complete")
			}
		}
	}
}

func (r *Peer) requestRecovered(rcvr *recon.Recover) error {
	items := rcvr.RemoteElements
	var resultErr error
	for len(items) > 0 {
		// Chunk requests to keep the hashquery message size and peer load reasonable.
		chunksize := requestChunkSize
		if chunksize > len(items) {
			chunksize = len(items)
		}
		chunk := items[:chunksize]
		items = items[chunksize:]

		err := r.requestChunk(rcvr, chunk)
		if err != nil {
			if resultErr == nil {
				resultErr = errgo.Mask(err)
			} else {
				resultErr = errgo.Notef(resultErr, "%s", errgo.Details(err))
			}
		}
	}
	return resultErr
}

func (r *Peer) requestChunk(rcvr *recon.Recover, chunk []*cf.Zp) error {
	var remoteAddr string
	remoteAddr, err := rcvr.HkpAddr()
	if err != nil {
		return errgo.Mask(err)
	}
	r.logAddr(RECON, rcvr.RemoteAddr).Debugf("requesting %d keys from %q via hashquery", len(chunk), remoteAddr)
	// Make an sks hashquery request
	hqBuf := bytes.NewBuffer(nil)
	err = recon.WriteInt(hqBuf, len(chunk))
	if err != nil {
		return errgo.Mask(err)
	}
	for _, z := range chunk {
		zb := z.Bytes()
		zb = recon.PadSksElement(zb)
		// Hashquery elements are 16 bytes (length_of(P_SKS)-1)
		zb = zb[:len(zb)-1]
		err = recon.WriteInt(hqBuf, len(zb))
		if err != nil {
			return errgo.Mask(err)
		}
		_, err = hqBuf.Write(zb)
		if err != nil {
			return errgo.Mask(err)
		}
	}

	url := fmt.Sprintf("http://%s/pks/hashquery", remoteAddr)
	resp, err := http.Post(url, "sks/hashquery", bytes.NewReader(hqBuf.Bytes()))
	if err != nil {
		return errgo.Mask(err)
	}

	// Store response in memory. Connection may timeout if we
	// read directly from it while loading.
	var body *bytes.Buffer
	bodyBuf, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return errgo.Mask(err)
	}
	body = bytes.NewBuffer(bodyBuf)
	resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return errgo.Newf("error response from %q: %v", remoteAddr, string(bodyBuf))
	}

	var nkeys, keyLen int
	nkeys, err = recon.ReadInt(body)
	if err != nil {
		return errgo.Mask(err)
	}
	r.logAddr(RECON, rcvr.RemoteAddr).Debugf("hashquery response from %q: %d keys found", remoteAddr, nkeys)
	for i := 0; i < nkeys; i++ {
		keyLen, err = recon.ReadInt(body)
		if err != nil {
			return errgo.Mask(err)
		}
		keyBuf := bytes.NewBuffer(nil)
		_, err = io.CopyN(keyBuf, body, int64(keyLen))
		if err != nil {
			return errgo.Mask(err)
		}
		r.logAddr(RECON, rcvr.RemoteAddr).Debugf("key# %d: %d bytes", i+1, keyLen)
		// Merge locally
		err = r.upsertKeys(rcvr, keyBuf.Bytes())
		if err != nil {
			r.logAddr(RECON, rcvr.RemoteAddr).Errorf("cannot upsert: %v", err)
		}
	}
	// Read last two bytes (CRLF, why?), or SKS will complain.
	body.Read(make([]byte, 2))
	return nil
}

func (r *Peer) upsertKeys(rcvr *recon.Recover, buf []byte) error {
	for readKey := range openpgp.ReadKeys(bytes.NewBuffer(buf)) {
		if readKey.Error != nil {
			return errgo.Mask(readKey.Error)
		}
		// TODO: collect duplicates to replicate SKS hashes?
		err := openpgp.DropDuplicates(readKey.PrimaryKey)
		if err != nil {
			return errgo.Mask(err)
		}
		keyChange, err := storage.UpsertKey(r.storage, readKey.PrimaryKey)
		if err != nil {
			return errgo.Mask(err)
		}
		r.logAddr(RECON, rcvr.RemoteAddr).Debug(keyChange)
	}
	return nil
}
<commit_msg>30 second timeout for hashquery POST requests<commit_after>/*
   Hockeypuck - OpenPGP key server
   Copyright (C) 2012  Casey Marshall

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, version 3.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

package sks

import (
	"bytes"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"gopkg.in/errgo.v1"
	"gopkg.in/tomb.v2"

	cf "hockeypuck/conflux"
	"hockeypuck/conflux/recon"
	"hockeypuck/conflux/recon/leveldb"
	"hockeypuck/hkp/storage"
	log "hockeypuck/logrus"
	"hockeypuck/openpgp"
)

const (
	RECON = "recon"
	maxKeyRecoveryAttempts = 10
	requestChunkSize = 100
)

const httpClientTimeout = 30

type keyRecoveryCounter map[string]int

type Peer struct {
	peer     *recon.Peer
	storage  storage.Storage
	settings *recon.Settings
	ptree    recon.PrefixTree
	http     *http.Client

	path  string
	stats *Stats

	t tomb.Tomb
}

func NewPrefixTree(path string, s *recon.Settings) (recon.PrefixTree, error) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		log.Debugf("creating prefix tree at: %q", path)
		err = os.MkdirAll(path, 0755)
		if err != nil {
			return nil, errgo.Mask(err)
		}
	}
	return leveldb.New(s.PTreeConfig, path)
}

func NewPeer(st storage.Storage, path string, s *recon.Settings) (*Peer, error) {
	if s == nil {
		s = recon.DefaultSettings()
	}

	ptree, err := NewPrefixTree(path, s)
	if err != nil {
		return nil, errgo.Mask(err)
	}
	err = ptree.Create()
	if err != nil {
		return nil, errgo.Mask(err)
	}

	peer := recon.NewPeer(s, ptree)
	sksPeer := &Peer{
		peer:     peer,
		storage:  st,
		settings: s,
		ptree:    ptree,
		http: &http.Client{
			Timeout: httpClientTimeout * time.Second,
		},
		path: path,
	}
	sksPeer.readStats()
	st.Subscribe(sksPeer.updateDigests)
	return sksPeer, nil
}

func (p *Peer) log(label string) *log.Entry {
	return p.logFields(label, log.Fields{})
}

func (p *Peer) logAddr(label string, addr net.Addr) *log.Entry {
	return p.logFields(label, log.Fields{"remoteAddr": addr})
}

func (p *Peer) logFields(label string, fields log.Fields) *log.Entry {
	fields["label"] = fmt.Sprintf("%s %s", label, p.settings.ReconAddr)
	return log.WithFields(fields)
}

func StatsFilename(path string) string {
	dir, base := filepath.Dir(path), filepath.Base(path)
	return filepath.Join(dir, "."+base+".stats")
}

func (p *Peer) readStats() {
	fn := StatsFilename(p.path)
	stats := NewStats()
	err := stats.ReadFile(fn)
	if err != nil {
		p.log(RECON).Warningf("cannot open stats %q: %v", fn, err)
		stats = NewStats()
	}

	root, err := p.ptree.Root()
	if err != nil {
		p.log(RECON).Warningf("error accessing prefix tree root: %v", err)
	} else {
		stats.Total = root.Size()
	}

	p.stats = stats
}

func (p *Peer) writeStats() {
	fn := StatsFilename(p.path)
	err := p.stats.WriteFile(fn)
	if err != nil {
		p.log(RECON).Warningf("cannot write stats %q: %v", fn, err)
	}
}

func (p *Peer) pruneStats() error {
	timer := time.NewTimer(time.Hour)
	for {
		select {
		case <-p.t.Dying():
			return nil
		case <-timer.C:
			p.stats.prune()
			timer.Reset(time.Hour)
		}
	}
}

func (r *Peer) Stats() *Stats {
	return r.stats.clone()
}

func (r *Peer) Start() {
	r.t.Go(r.handleRecovery)
	r.t.Go(r.pruneStats)
	r.peer.Start()
}

func (r *Peer) Stop() {
	r.log(RECON).Info("recon processing: stopping")
	r.t.Kill(nil)
	err := r.t.Wait()
	if err != nil {
		r.log(RECON).Error(errgo.Details(err))
	}
	r.log(RECON).Info("recon processing: stopped")

	r.log(RECON).Info("recon peer: stopping")
	err = errgo.Mask(r.peer.Stop())
	if err != nil {
		r.log(RECON).Error(errgo.Details(err))
	}
	r.log(RECON).Info("recon peer: stopped")

	err = r.ptree.Close()
	if err != nil {
		r.log(RECON).Errorf("error closing prefix tree: %v", errgo.Details(err))
	}

	r.writeStats()
}

func DigestZp(digest string) (*cf.Zp, error) {
	buf, err := hex.DecodeString(digest)
	if err != nil {
		return nil, errgo.Mask(err)
	}
	buf = recon.PadSksElement(buf)
	return cf.Zb(cf.P_SKS, buf), nil
}

func (r *Peer) updateDigests(change storage.KeyChange) error {
	r.stats.Update(change)
	for _, digest := range change.InsertDigests() {
		digestZp, err := DigestZp(digest)
		if err != nil {
			return errgo.Notef(err, "bad digest %q", digest)
		}
		r.peer.Insert(digestZp)
	}
	for _, digest := range change.RemoveDigests() {
		digestZp, err := DigestZp(digest)
		if err != nil {
			return errgo.Notef(err, "bad digest %q", digest)
		}
		r.peer.Remove(digestZp)
	}
	return nil
}

func (r *Peer) handleRecovery() error {
	for {
		select {
		case <-r.t.Dying():
			return nil
		case rcvr := <-r.peer.RecoverChan:
			r.logAddr(RECON, rcvr.RemoteAddr).Infof("%d items to recover", len(rcvr.RemoteElements))
			if err := r.requestRecovered(rcvr); err != nil {
				r.logAddr(RECON, rcvr.RemoteAddr).Errorf("recovery completed with errors: %v", err)
			} else {
				r.logAddr(RECON, rcvr.RemoteAddr).Info("recovery complete")
			}
		}
	}
}

func (r *Peer) requestRecovered(rcvr *recon.Recover) error {
	items := rcvr.RemoteElements
	var resultErr error
	for len(items) > 0 {
		// Chunk requests to keep the hashquery message size and peer load reasonable.
		chunksize := requestChunkSize
		if chunksize > len(items) {
			chunksize = len(items)
		}
		chunk := items[:chunksize]
		items = items[chunksize:]

		err := r.requestChunk(rcvr, chunk)
		if err != nil {
			if resultErr == nil {
				resultErr = errgo.Mask(err)
			} else {
				resultErr = errgo.Notef(resultErr, "%s", errgo.Details(err))
			}
		}
	}
	return resultErr
}

func (r *Peer) requestChunk(rcvr *recon.Recover, chunk []*cf.Zp) error {
	var remoteAddr string
	remoteAddr, err := rcvr.HkpAddr()
	if err != nil {
		return errgo.Mask(err)
	}
	r.logAddr(RECON, rcvr.RemoteAddr).Debugf("requesting %d keys from %q via hashquery", len(chunk), remoteAddr)
	// Make an sks hashquery request
	hqBuf := bytes.NewBuffer(nil)
	err = recon.WriteInt(hqBuf, len(chunk))
	if err != nil {
		return errgo.Mask(err)
	}
	for _, z := range chunk {
		zb := z.Bytes()
		zb = recon.PadSksElement(zb)
		// Hashquery elements are 16 bytes (length_of(P_SKS)-1)
		zb = zb[:len(zb)-1]
		err = recon.WriteInt(hqBuf, len(zb))
		if err != nil {
			return errgo.Mask(err)
		}
		_, err = hqBuf.Write(zb)
		if err != nil {
			return errgo.Mask(err)
		}
	}

	url := fmt.Sprintf("http://%s/pks/hashquery", remoteAddr)
	resp, err := r.http.Post(url, "sks/hashquery", bytes.NewReader(hqBuf.Bytes()))
	if err != nil {
		return errgo.Mask(err)
	}

	// Store response in memory. Connection may timeout if we
	// read directly from it while loading.
	var body *bytes.Buffer
	bodyBuf, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return errgo.Mask(err)
	}
	body = bytes.NewBuffer(bodyBuf)
	resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return errgo.Newf("error response from %q: %v", remoteAddr, string(bodyBuf))
	}

	var nkeys, keyLen int
	nkeys, err = recon.ReadInt(body)
	if err != nil {
		return errgo.Mask(err)
	}
	r.logAddr(RECON, rcvr.RemoteAddr).Debugf("hashquery response from %q: %d keys found", remoteAddr, nkeys)
	for i := 0; i < nkeys; i++ {
		keyLen, err = recon.ReadInt(body)
		if err != nil {
			return errgo.Mask(err)
		}
		keyBuf := bytes.NewBuffer(nil)
		_, err = io.CopyN(keyBuf, body, int64(keyLen))
		if err != nil {
			return errgo.Mask(err)
		}
		r.logAddr(RECON, rcvr.RemoteAddr).Debugf("key# %d: %d bytes", i+1, keyLen)
		// Merge locally
		err = r.upsertKeys(rcvr, keyBuf.Bytes())
		if err != nil {
			r.logAddr(RECON, rcvr.RemoteAddr).Errorf("cannot upsert: %v", err)
		}
	}
	// Read last two bytes (CRLF, why?), or SKS will complain.
	body.Read(make([]byte, 2))
	return nil
}

func (r *Peer) upsertKeys(rcvr *recon.Recover, buf []byte) error {
	for readKey := range openpgp.ReadKeys(bytes.NewBuffer(buf)) {
		if readKey.Error != nil {
			return errgo.Mask(readKey.Error)
		}
		// TODO: collect duplicates to replicate SKS hashes?
		err := openpgp.DropDuplicates(readKey.PrimaryKey)
		if err != nil {
			return errgo.Mask(err)
		}
		keyChange, err := storage.UpsertKey(r.storage, readKey.PrimaryKey)
		if err != nil {
			return errgo.Mask(err)
		}
		r.logAddr(RECON, rcvr.RemoteAddr).Debug(keyChange)
	}
	return nil
}
<|endoftext|>


<commit_before>// 23 january 2015
package iface

// Server describes a server.
type Server interface {
	// In returns a channel of messages to send to the server.
	// You may send to this channel.
	// In shall not change during the lifetime of the Server object.
	In() chan<- Message

	// Out returns a channel of messages to receive from the server.
	// You may receive from this channel.
	// Out shall not change during the lifetime of the Server object.
	// (TODO rename In and Out?)
	Out() <-chan Message

	// Connect establishes a connection to the server.
	// After Connect, In will start processing messages.
	// Out will also send a message when disconnected.
	// Connection errors will be sent across Out.
	// TODO what if allready connected?
	Connect()

	// Raw returns a Message that will result in the raw text being sent to the server when sent to In.
	Raw(message string) Message

	// Join returns a Message that, when sent to In, will result in that channel being joined.
	// key contains the channel password, if any.
	Join(channel string, key string) Message

	// Query returns a Message that, when sent to In, will result in the given nickname receiving the given line in a private message.
	// It is analogous to Join, and if the send succeeded, will return a QueryStarted message containing the Channel to use for future communications.
	Query(nick string, message string) Message

	// Quit returns a Message that, when sent to In, will result in the connection to the server being closed.
	// You can specify an optional reason.
	Quit(reason string) Message

	// Nick returns the current nickname.
	// It automatically responds to server-side nickname changes.
	Nick() string

	// SetNick returns a Message that, when sent to In, sets the current nickname.
	SetNick(nick string) Message
}

// Message represents messages.
type Message interface {
	Type() MessageType
	Server() Server
	Raw() []byte
	Time() time.Time

	// Err will be non-nil if Type is Error.
	// If Type is Disconnected, Err will be non-nil if an error led to the disconnect.
	Err() error

	// Channel returns the Channel that the Message belongs to, or nil if the message belongs to the server but not any channel.
	// It shall be nil for Error, Connected, and Disconnected messages.
	// If the message is of type Joined, use the returned value to initialize any of your own data structures for handling the Channel.
	// When sending a Message, this should return the destination Channel, or should be nil for the server itself.
	// Note that private messages count as channels; see Channel for details.
	Channel() Channel

	// Previous returns the old nickname if the message type is either YourNickChanged or OtherNickChanged.
	Previous() string

	// Nick returns the new nickname if the message type is either YourNickChanged or OtherNickChanged.
	// Nick returns the nickname of the person who joined or left if the message type is OtherJoined or OtherLeft.
	// Nick returns the nickname of the person in the query session if the message type is QueryStarted.
	Nick() string

	// The remainder of these only apply to Line messages.

	// Formatted returns a Formatted that represents the line with all protocol-specific formatting marks processed.
	Formatted() Formatted

	// LogLine returns a LogLine that represents the line as it should appear in chat logs.
	LogLine() []byte
}

// CurrentChannel represents the currently visible channel in the UI.
// This is used for Message.Channel() to indicate that the message should be displayed in the current channel.
// This should never be issued in a message to send.
// Calling methods on CurrentChannel will panic.
var CurrentChannel = currentChannel

// MessageType represents a message type.
type MessageType uint
const (
	Error MessageType = iota
	Line
	Connected
	Disconnected
	Joined
	YourNickChanged
	OtherNickChanged
	OtherJoined
	OtherLeft				// applies to both parts and disconnects (TODO)
	QueryStarted
)

// Channel represents a channel.
type Channel interface {
	Name() string
	Server() Server

	// Members returns a list of nicknames in the channel.
	// If the returned list has length zero, the Channel object represents a private message.
	// In that case, Name() returns the nickname of the person you are talking to.
	Members() []string

	// Say returns a Message that, when sent to Server().C(), says the line in the channel as a normal line.
	Say(what string) Message

	// Do returns a Message that, when sent to Server().In(), says the line in the channel as an action line.
	// For instance, IRC clients typically denote actions with "/me".
	Do(what string) Message

	// Leave returns a Message that, when sent to Server().In(), leaves the channel.
	// An optional reason may be specified.
	Leave(reason string) Message
}
<commit_msg>More TODOs.<commit_after>// 23 january 2015
package iface

// Server describes a server.
type Server interface {
	// In returns a channel of messages to send to the server.
	// You may send to this channel.
	// In shall not change during the lifetime of the Server object.
	In() chan<- Message

	// Out returns a channel of messages to receive from the server.
	// You may receive from this channel.
	// Out shall not change during the lifetime of the Server object.
	// (TODO rename In and Out?)
	// (TODO merge In and Out?)
	Out() <-chan Message

	// Connect establishes a connection to the server.
	// After Connect, In will start processing messages.
	// Out will also send a message when disconnected.
	// Connection errors will be sent across Out.
	// TODO what if allready connected?
	Connect()

	// Raw returns a Message that will result in the raw text being sent to the server when sent to In.
	Raw(message string) Message

	// Join returns a Message that, when sent to In, will result in that channel being joined.
	// key contains the channel password, if any.
	Join(channel string, key string) Message

	// Query returns a Message that, when sent to In, will result in the given nickname receiving the given line in a private message.
	// It is analogous to Join, and if the send succeeded, will return a QueryStarted message containing the Channel to use for future communications.
	Query(nick string, message string) Message

	// Quit returns a Message that, when sent to In, will result in the connection to the server being closed.
	// You can specify an optional reason.
	Quit(reason string) Message

	// Nick returns the current nickname.
	// It automatically responds to server-side nickname changes.
	Nick() string

	// SetNick returns a Message that, when sent to In, sets the current nickname.
	SetNick(nick string) Message
}

// Message represents messages.
type Message interface {
	Type() MessageType
	Server() Server
	Raw() []byte
	Time() time.Time

	// Err will be non-nil if Type is Error.
	// If Type is Disconnected, Err will be non-nil if an error led to the disconnect.
	Err() error

	// Channel returns the Channel that the Message belongs to, or nil if the message belongs to the server but not any channel.
	// It shall be nil for Error, Connected, and Disconnected messages.
	// If the message is of type Joined, use the returned value to initialize any of your own data structures for handling the Channel.
	// When sending a Message, this should return the destination Channel, or should be nil for the server itself.
	// Note that private messages count as channels; see Channel for details.
	Channel() Channel

	// Previous returns the old nickname if the message type is either YourNickChanged or OtherNickChanged.
	Previous() string

	// Nick returns the new nickname if the message type is either YourNickChanged or OtherNickChanged.
	// Nick returns the nickname of the person who joined or left if the message type is OtherJoined or OtherLeft.
	// Nick returns the nickname of the person in the query session if the message type is QueryStarted.
	Nick() string

	// The remainder of these only apply to Line messages.

	// Formatted returns a Formatted that represents the line with all protocol-specific formatting marks processed.
	Formatted() Formatted

	// LogLine returns a LogLine that represents the line as it should appear in chat logs.
	LogLine() []byte
}

// CurrentChannel represents the currently visible channel in the UI.
// This is used for Message.Channel() to indicate that the message should be displayed in the current channel.
// This should never be issued in a message to send.
// Calling methods on CurrentChannel will panic.
var CurrentChannel = currentChannel

// MessageType represents a message type.
type MessageType uint
const (
	Error MessageType = iota
	Line
	Connected
	Disconnected
	Joined
	YourNickChanged
	OtherNickChanged
	OtherJoined
	OtherLeft				// applies to both parts and disconnects (TODO)
	QueryStarted
)

// Channel represents a channel.
type Channel interface {
	Name() string
	Server() Server

	// Members returns a list of nicknames in the channel.
	// If the returned list has length zero, the Channel object represents a private message.
	// In that case, Name() returns the nickname of the person you are talking to.
	Members() []string

	// Say returns a Message that, when sent to Server().C(), says the line in the channel as a normal line.
	Say(what string) Message

	// Do returns a Message that, when sent to Server().In(), says the line in the channel as an action line.
	// For instance, IRC clients typically denote actions with "/me".
	Do(what string) Message

	// Leave returns a Message that, when sent to Server().In(), leaves the channel.
	// An optional reason may be specified.
	Leave(reason string) Message
}
<|endoftext|>


<commit_before>package geoip

import (
	"github.com/spf13/cobra"
	"github.com/stretchr/testify/assert"
	"log"
	"testing"
)

// todo: Write meaningful test
func TestGeoIP(t *testing.T) {
	assert.IsType(t, &cobra.Command{}, Command(log.Printf))
}
<commit_msg>Add some IP to test db<commit_after>package geoip

import (
	"github.com/spf13/cobra"
	"github.com/stretchr/testify/require"
	"log"
	"testing"
	"net"
	"github.com/stretchr/testify/assert"
)

// todo: Write meaningful test
func TestGeoIP(t *testing.T) {
	require.IsType(t, &cobra.Command{}, Command(log.Printf))
}

func TestIP(t *testing.T) {
	r, err := Reader()
	require.NoError(t, err)
	ip := net.ParseIP("2.60.177.126")
	c, _ := r.City(ip)
	assert.Equal(t, "Omsk", c.City.Names["en"])
	ip = net.ParseIP("195.19.132.64")
	c, _ = r.City(ip)
	assert.Equal(t, "Yekaterinburg", c.City.Names["en"])
	ip = net.ParseIP("77.45.128.13")
	c, _ = r.City(ip)
	assert.Equal(t, "Voronezh", c.City.Names["en"])
}<|endoftext|>


<commit_before>package audit

import (
	"github.com/stretchr/testify/assert"
	"gopkg.in/DATA-DOG/go-sqlmock.v1"
	"testing"

	"github.com/eirka/eirka-libs/db"
	e "github.com/eirka/eirka-libs/errors"
)

func TestAudit(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	rows := sqlmock.NewRows([]string{"role"}).AddRow(3)

	mock.ExpectPrepare(`INSERT INTO audit \(user_id,ib_id,audit_type,audit_ip,audit_time,audit_action,audit_info\)`).
		ExpectExec().
		WithArgs(1, 1, UserLog, "10.0.0.1", "NOW()", AuditEmailUpdate, "meta info")

	audit := Audit{
		User:   1,
		Ib:     1,
		Type:   UserLog,
		Ip:     "10.0.0.1",
		Action: AuditEmailUpdate,
		Info:   "meta info",
	}

	// submit audit
	err = audit.Submit()
	assert.NoError(t, err, "An error was not expected")

}
<commit_msg>add audit test<commit_after>package audit

import (
	"github.com/stretchr/testify/assert"
	"gopkg.in/DATA-DOG/go-sqlmock.v1"
	"testing"

	"github.com/eirka/eirka-libs/db"
)

func TestAudit(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	rows := sqlmock.NewRows([]string{"role"}).AddRow(3)

	mock.ExpectPrepare(`INSERT INTO audit \(user_id,ib_id,audit_type,audit_ip,audit_time,audit_action,audit_info\)`).
		ExpectExec().
		WithArgs(1, 1, UserLog, "10.0.0.1", "NOW()", AuditEmailUpdate, "meta info")

	audit := Audit{
		User:   1,
		Ib:     1,
		Type:   UserLog,
		Ip:     "10.0.0.1",
		Action: AuditEmailUpdate,
		Info:   "meta info",
	}

	// submit audit
	err = audit.Submit()
	assert.NoError(t, err, "An error was not expected")

}
<|endoftext|>


<commit_before>package controller_manager

import (
	"strings"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"

	exutil "github.com/openshift/origin/test/extended/util"
)

func waitForServiceAccountToken(client kubernetes.Interface, ns, name string, attempts int, interval time.Duration) (string, error) {
	for i := 0; i <= attempts; i++ {
		time.Sleep(interval)
		token, err := getServiceAccountToken(client, ns, name)
		if err != nil {
			return "", err
		}
		if len(token) > 0 {
			return token, nil
		}
	}
	return "", nil
}

func getServiceAccountToken(client kubernetes.Interface, ns, name string) (string, error) {
	secrets, err := client.CoreV1().Secrets(ns).List(metav1.ListOptions{})
	if err != nil {
		return "", err
	}
	for _, secret := range secrets.Items {
		if secret.Type == corev1.SecretTypeServiceAccountToken && secret.Annotations[corev1.ServiceAccountNameKey] == name {
			sa, err := client.CoreV1().ServiceAccounts(ns).Get(name, metav1.GetOptions{})
			if err != nil {
				return "", err
			}

			for _, ref := range sa.Secrets {
				if ref.Name == secret.Name {
					return string(secret.Data[corev1.ServiceAccountTokenKey]), nil
				}
			}

		}
	}

	return "", nil
}

var _ = g.Describe("[Feature:OpenShiftControllerManager]", func() {
	defer g.GinkgoRecover()
	oc := exutil.NewCLI("pull-secrets", exutil.KubeConfigPath())

	g.It("TestAutomaticCreationOfPullSecrets", func() {
		t := g.GinkgoT()

		clusterAdminKubeClient := oc.AdminKubeClient()
		saNamespace := oc.Namespace()
		saName := "default"

		// Get a service account token
		saToken, err := waitForServiceAccountToken(clusterAdminKubeClient, saNamespace, saName, 20, time.Second)
		if err != nil {
			t.Errorf("unexpected error: %v", err)
		}
		if len(saToken) == 0 {
			t.Errorf("token was not created")
		}

		// Get the matching dockercfg secret
		_, saPullSecret, err := waitForServiceAccountPullSecret(clusterAdminKubeClient, saNamespace, saName, 20, time.Second)
		if err != nil {
			t.Errorf("unexpected error: %v", err)
		}
		if len(saPullSecret) == 0 {
			t.Errorf("pull secret was not created")
		}

		imageConfig, err := oc.AdminConfigClient().ConfigV1().Images().Get("cluster", metav1.GetOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		if !strings.Contains(saPullSecret, imageConfig.Status.InternalRegistryHostname) {
			t.Errorf("missing %q in %v", imageConfig.Status.InternalRegistryHostname, saPullSecret)
		}

		if len(imageConfig.Spec.ExternalRegistryHostnames) > 0 {
			if !strings.Contains(saPullSecret, imageConfig.Spec.ExternalRegistryHostnames[0]) {
				t.Errorf("missing %q in %v", imageConfig.Spec.ExternalRegistryHostnames[0], saPullSecret)
			}
		}

	})
})

func waitForServiceAccountPullSecret(client kubernetes.Interface, ns, name string, attempts int, interval time.Duration) (string, string, error) {
	for i := 0; i <= attempts; i++ {
		time.Sleep(interval)
		secretName, dockerCfg, err := getServiceAccountPullSecret(client, ns, name)
		if err != nil {
			return "", "", err
		}
		if len(dockerCfg) > 2 {
			return secretName, dockerCfg, nil
		}
	}
	return "", "", nil
}

func getServiceAccountPullSecret(client kubernetes.Interface, ns, name string) (string, string, error) {
	secrets, err := client.CoreV1().Secrets(ns).List(metav1.ListOptions{})
	if err != nil {
		return "", "", err
	}
	for _, secret := range secrets.Items {
		if secret.Type == corev1.SecretTypeDockercfg && secret.Annotations[corev1.ServiceAccountNameKey] == name {
			return secret.Name, string(secret.Data[corev1.DockerConfigKey]), nil
		}
	}
	return "", "", nil
}

var _ = g.Describe("[Feature:OpenShiftControllerManager]", func() {
	defer g.GinkgoRecover()
	oc := exutil.NewCLI("pull-secrets", exutil.KubeConfigPath())

	// https://bugzilla.redhat.com/show_bug.cgi?id=1765294
	g.It("TestDockercfgTokenDeletedController [Disabled:Broken]", func() {
		t := g.GinkgoT()

		clusterAdminKubeClient := oc.AdminKubeClient()
		saNamespace := oc.Namespace()

		sa := &corev1.ServiceAccount{
			ObjectMeta: metav1.ObjectMeta{Name: "sa1", Namespace: saNamespace},
		}

		sa, err := clusterAdminKubeClient.CoreV1().ServiceAccounts(sa.Namespace).Create(sa)
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}

		secretsWatch, err := clusterAdminKubeClient.CoreV1().Secrets(sa.Namespace).Watch(metav1.ListOptions{ResourceVersion: sa.ResourceVersion})
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		defer secretsWatch.Stop()

		// Get the service account dockercfg secret's name
		dockercfgSecretName, _, err := waitForServiceAccountPullSecret(clusterAdminKubeClient, sa.Namespace, sa.Name, 20, time.Second)
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		if len(dockercfgSecretName) == 0 {
			t.Fatal("pull secret was not created")
		}

		// Get the matching secret's name
		dockercfgSecret, err := clusterAdminKubeClient.CoreV1().Secrets(sa.Namespace).Get(dockercfgSecretName, metav1.GetOptions{})
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		secretName := dockercfgSecret.Annotations["openshift.io/token-secret.name"]
		if len(secretName) == 0 {
			t.Fatal("secret was not created")
		}

		// Delete the service account's secret
		if err := clusterAdminKubeClient.CoreV1().Secrets(sa.Namespace).Delete(secretName, nil); err != nil {
			t.Fatalf("unexpected error: %v", err)
		}

		// Expect the matching dockercfg secret to also be deleted
		waitForSecretDelete(dockercfgSecretName, secretsWatch, t)
	})
})

func waitForSecretDelete(secretName string, w watch.Interface, t g.GinkgoTInterface) {
	for {
		select {
		case event := <-w.ResultChan():
			secret, ok := event.Object.(*corev1.Secret)
			if !ok {
				t.Fatalf("Got an event that was not a secret: %v %#v", event.Type, event.Object)
			}
			secret.Data = nil // reduce noise in log
			t.Logf("%s got %s %s %#v", time.Now().Format(time.RFC3339Nano), event.Type, secret.ResourceVersion, secret)
			if event.Type == watch.Deleted && secret.Name == secretName {
				return
			}

		case <-time.After(5 * time.Minute):
			t.Fatalf("timeout: %v", secretName)
		}
	}
}
<commit_msg>Changing from passive to active approach on secret deletion.<commit_after>package controller_manager

import (
	"strings"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"

	exutil "github.com/openshift/origin/test/extended/util"
)

func waitForServiceAccountToken(client kubernetes.Interface, ns, name string, attempts int, interval time.Duration) (string, error) {
	for i := 0; i <= attempts; i++ {
		time.Sleep(interval)
		token, err := getServiceAccountToken(client, ns, name)
		if err != nil {
			return "", err
		}
		if len(token) > 0 {
			return token, nil
		}
	}
	return "", nil
}

func getServiceAccountToken(client kubernetes.Interface, ns, name string) (string, error) {
	secrets, err := client.CoreV1().Secrets(ns).List(metav1.ListOptions{})
	if err != nil {
		return "", err
	}
	for _, secret := range secrets.Items {
		if secret.Type == corev1.SecretTypeServiceAccountToken && secret.Annotations[corev1.ServiceAccountNameKey] == name {
			sa, err := client.CoreV1().ServiceAccounts(ns).Get(name, metav1.GetOptions{})
			if err != nil {
				return "", err
			}

			for _, ref := range sa.Secrets {
				if ref.Name == secret.Name {
					return string(secret.Data[corev1.ServiceAccountTokenKey]), nil
				}
			}

		}
	}

	return "", nil
}

var _ = g.Describe("[Feature:OpenShiftControllerManager]", func() {
	defer g.GinkgoRecover()
	oc := exutil.NewCLI("pull-secrets", exutil.KubeConfigPath())

	g.It("TestAutomaticCreationOfPullSecrets", func() {
		t := g.GinkgoT()

		clusterAdminKubeClient := oc.AdminKubeClient()
		saNamespace := oc.Namespace()
		saName := "default"

		// Get a service account token
		saToken, err := waitForServiceAccountToken(clusterAdminKubeClient, saNamespace, saName, 20, time.Second)
		if err != nil {
			t.Errorf("unexpected error: %v", err)
		}
		if len(saToken) == 0 {
			t.Errorf("token was not created")
		}

		// Get the matching dockercfg secret
		_, saPullSecret, err := waitForServiceAccountPullSecret(clusterAdminKubeClient, saNamespace, saName, 20, time.Second)
		if err != nil {
			t.Errorf("unexpected error: %v", err)
		}
		if len(saPullSecret) == 0 {
			t.Errorf("pull secret was not created")
		}

		imageConfig, err := oc.AdminConfigClient().ConfigV1().Images().Get("cluster", metav1.GetOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		if !strings.Contains(saPullSecret, imageConfig.Status.InternalRegistryHostname) {
			t.Errorf("missing %q in %v", imageConfig.Status.InternalRegistryHostname, saPullSecret)
		}

		if len(imageConfig.Spec.ExternalRegistryHostnames) > 0 {
			if !strings.Contains(saPullSecret, imageConfig.Spec.ExternalRegistryHostnames[0]) {
				t.Errorf("missing %q in %v", imageConfig.Spec.ExternalRegistryHostnames[0], saPullSecret)
			}
		}

	})
})

func waitForServiceAccountPullSecret(client kubernetes.Interface, ns, name string, attempts int, interval time.Duration) (string, string, error) {
	for i := 0; i <= attempts; i++ {
		time.Sleep(interval)
		secretName, dockerCfg, err := getServiceAccountPullSecret(client, ns, name)
		if err != nil {
			return "", "", err
		}
		if len(dockerCfg) > 2 {
			return secretName, dockerCfg, nil
		}
	}
	return "", "", nil
}

func getServiceAccountPullSecret(client kubernetes.Interface, ns, name string) (string, string, error) {
	secrets, err := client.CoreV1().Secrets(ns).List(metav1.ListOptions{})
	if err != nil {
		return "", "", err
	}
	for _, secret := range secrets.Items {
		if secret.Type == corev1.SecretTypeDockercfg && secret.Annotations[corev1.ServiceAccountNameKey] == name {
			return secret.Name, string(secret.Data[corev1.DockerConfigKey]), nil
		}
	}
	return "", "", nil
}

var _ = g.Describe("[Feature:OpenShiftControllerManager]", func() {
	defer g.GinkgoRecover()
	oc := exutil.NewCLI("pull-secrets", exutil.KubeConfigPath())

	g.It("TestDockercfgTokenDeletedController", func() {
		t := g.GinkgoT()

		clusterAdminKubeClient := oc.AdminKubeClient()
		saNamespace := oc.Namespace()

		sa := &corev1.ServiceAccount{
			ObjectMeta: metav1.ObjectMeta{Name: "sa1", Namespace: saNamespace},
		}

		sa, err := clusterAdminKubeClient.CoreV1().ServiceAccounts(sa.Namespace).Create(sa)
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}

		// Get the service account dockercfg secret's name
		dockercfgSecretName, _, err := waitForServiceAccountPullSecret(clusterAdminKubeClient, sa.Namespace, sa.Name, 20, time.Second)
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		if len(dockercfgSecretName) == 0 {
			t.Fatal("pull secret was not created")
		}

		// Get the matching secret's name
		dockercfgSecret, err := clusterAdminKubeClient.CoreV1().Secrets(sa.Namespace).Get(dockercfgSecretName, metav1.GetOptions{})
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		secretName := dockercfgSecret.Annotations["openshift.io/token-secret.name"]
		if len(secretName) == 0 {
			t.Fatal("secret was not created")
		}

		// Delete the service account's secret
		if err := clusterAdminKubeClient.CoreV1().Secrets(sa.Namespace).Delete(secretName, nil); err != nil {
			t.Fatalf("unexpected error: %v", err)
		}

		// Expect the matching dockercfg secret to also be deleted
		if err := wait.Poll(5*time.Second, 5*time.Minute, func() (bool, error) {
			_, err := clusterAdminKubeClient.CoreV1().Secrets(sa.Namespace).Get(
				dockercfgSecretName,
				metav1.GetOptions{},
			)
			return errors.IsNotFound(err), nil
		}); err != nil {
			t.Fatalf("waiting for secret deletion: %v", err)
		}
	})
})
<|endoftext|>


<commit_before>// check_workers_health is an application that checks the health of all CT
// workers and reports results to the admins if any worker/device is down.
package main

import (
	"fmt"
	"strings"
	"time"

	"github.com/skia-dev/glog"
	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/go/common"
)

func main() {
	common.Init()
	defer glog.Flush()

	// Collect unhealthy machines
	offlineMachines := []string{}
	offlineDevices := []string{}
	missingDevices := []string{}
	deviceOfflineOutputs, err := util.SSH("adb devices", util.Slaves, 30*time.Minute)
	if err != nil {
		glog.Fatalf("Error while sshing into workers: %s", err)
		return
	}
	// Populate offlineMachines, offlineDevices and missingDevices.
	for hostname, out := range deviceOfflineOutputs {
		if out == "" {
			offlineMachines = append(offlineMachines, hostname)
			glog.Warningf("%s is offline", hostname)
		} else if strings.Contains(out, "offline") {
			// The adb output text contains offline devices.
			offlineDevices = append(offlineDevices, hostname)
			glog.Warningf("%s has an offline device", hostname)
		} else if strings.Count(out, "device") == 1 {
			// The adb output text only contains "List of devices attached"
			// without any devices listed below it.
			missingDevices = append(missingDevices, hostname)
			glog.Warningf("%s has missing devices", hostname)
		}
	}

	// Email admins if there are any unhealthy machines.
	if len(offlineMachines) != 0 || len(offlineDevices) != 0 || len(missingDevices) != 0 {
		emailSubject := "There are unhealthy Cluster telemetry machines"
		emailBody := "Please file a ticket to chrome-labs-tech-ticket@ for-

"
		if len(offlineMachines) != 0 {
			emailBody += fmt.Sprintf("The following machines are offline: %s
", strings.Join(offlineMachines, ","))
		}
		if len(offlineDevices) != 0 {
			emailBody += fmt.Sprintf("The following machines have offline devices: %s
", strings.Join(offlineDevices, ","))
		}
		if len(missingDevices) != 0 {
			emailBody += fmt.Sprintf("The following machines have missing devices: %s
", strings.Join(missingDevices, ","))
		}
		if err := util.SendEmail(util.CtAdmins, emailSubject, emailBody); err != nil {
			glog.Errorf("Error while sending email: %s", err)
			return
		}
	} else {
		glog.Info("All CT machines are healthy")
	}
}
<commit_msg>Add email address of ATL hardware group and link to machine mapping spreadsheet<commit_after>// check_workers_health is an application that checks the health of all CT
// workers and reports results to the admins if any worker/device is down.
package main

import (
	"fmt"
	"strings"
	"time"

	"github.com/skia-dev/glog"
	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/go/common"
)

func main() {
	common.Init()
	defer glog.Flush()

	// Collect unhealthy machines
	offlineMachines := []string{}
	offlineDevices := []string{}
	missingDevices := []string{}
	deviceOfflineOutputs, err := util.SSH("adb devices", util.Slaves, 30*time.Minute)
	if err != nil {
		glog.Fatalf("Error while sshing into workers: %s", err)
		return
	}
	// Populate offlineMachines, offlineDevices and missingDevices.
	for hostname, out := range deviceOfflineOutputs {
		if out == "" {
			offlineMachines = append(offlineMachines, hostname)
			glog.Warningf("%s is offline", hostname)
		} else if strings.Contains(out, "offline") {
			// The adb output text contains offline devices.
			offlineDevices = append(offlineDevices, hostname)
			glog.Warningf("%s has an offline device", hostname)
		} else if strings.Count(out, "device") == 1 {
			// The adb output text only contains "List of devices attached"
			// without any devices listed below it.
			missingDevices = append(missingDevices, hostname)
			glog.Warningf("%s has missing devices", hostname)
		}
	}

	// Email admins if there are any unhealthy machines.
	if len(offlineMachines) != 0 || len(offlineDevices) != 0 || len(missingDevices) != 0 {
		emailSubject := "There are unhealthy Cluster telemetry machines"
		emailBody := "Please file a ticket to hwops-interrupts-atl@ and chrome-labs-tech-ticket@ using https://docs.google.com/spreadsheets/d/1whlE4nDJB0XFBemJliupOORepdXf_vXyAfFgsprTAxY/edit#gid=0 for-

"
		if len(offlineMachines) != 0 {
			emailBody += fmt.Sprintf("The following machines are offline: %s
", strings.Join(offlineMachines, ","))
		}
		if len(offlineDevices) != 0 {
			emailBody += fmt.Sprintf("The following machines have offline devices: %s
", strings.Join(offlineDevices, ","))
		}
		if len(missingDevices) != 0 {
			emailBody += fmt.Sprintf("The following machines have missing devices: %s
", strings.Join(missingDevices, ","))
		}
		if err := util.SendEmail(util.CtAdmins, emailSubject, emailBody); err != nil {
			glog.Errorf("Error while sending email: %s", err)
			return
		}
	} else {
		glog.Info("All CT machines are healthy")
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Javier Arevalo <jare@iguanademos.com>

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Cross platform command execution and some file system operations

package main

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
)

// RunShell runs an interactive shell.
// Since the current program is still running, so may be
// its coroutines, which will trigger all sorts of weirdness
// For example, Ctrl-C may be caught by our Go runtime, killing us
// but leaving the spawned shell still running, with I/O shared with
// our parent (possibly another shell!). That's pretty ugly.
func RunShell(cwd string) error {
	attr := os.ProcAttr{
		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
		Dir:   cwd,
	}
	var args []string
	shell, ok := os.LookupEnv("COMSPEC")
	if !ok {
		args = []string{"-i"}
		shell, ok = os.LookupEnv("SHELL")
		if !ok {
			shell = "/bin/sh"
		}
	}
	process, err := os.StartProcess(shell, args, &attr)
	if err != nil {
		return err
	}
	state, err := process.Wait()
	if err != nil {
		return err
	}
	if state.Success() {
		return nil
	}
	return fmt.Errorf("<< Exited shell: %s", state.String())
}

// RunCommand executes the given command and arguments under the system'
// default shell. Really only tested under CMD and ksh
// If command fails, returns the system error and the output of the command
func RunCommand(command string, args ...string) error {
	var ret []string
	var finalargs []string
	var shell string
	if runtime.GOOS == "windows" {
		sh, ok := os.LookupEnv("COMSPEC")
		if !ok {
			shell = "C:\\Windows\\System32\\cmd.exe"
		}
		shell = sh
		args = append([]string{command}, args...)
		finalargs = []string{"/C", strings.Join(args, " ") + " 2>&1"}
	} else {
		sh, ok := os.LookupEnv("SHELL")
		if !ok {
			shell = "/bin/sh"
		}
		shell = sh
		for i, v := range args {
			args[i] = strconv.Quote(v)
		}
		args = append([]string{command}, args...)
		finalargs = []string{"-c", strings.Join(args, " ") + " 2>&1"}
	}
	Logf("Running command:\n>%s<\n>>%s<<\n", shell, strings.Join(finalargs, "<<\n>>"))
	cmd := exec.Command(shell, finalargs...)

	// See comment in sys_windows.go for this
	SetProcCmdline(cmd, strings.Join(finalargs, " "))

	outp, err := cmd.StdoutPipe()
	if err != nil {
		return fmt.Errorf("Failed to attach stdout: %v", err)
	}
	err = cmd.Start()
	if err != nil {
		Logf("ERROR: %s: %s\n", err, strings.Join(ret, "\n"))
		return fmt.Errorf("%s -> %s: %s", strings.Join(finalargs[1:], " "), err, strings.Join(ret, " \n"))
	}
	scanner := bufio.NewScanner(outp)
	for scanner.Scan() {
		line := scanner.Text()
		ret = append(ret, line)
	}
	err = cmd.Wait()
	Logf("Output: %s\n", strings.Join(ret, "\n"))
	if err != nil {
		Logf("ERROR: %s: %s\n", err, strings.Join(ret, "\n"))
		return fmt.Errorf("%s -> %s: %s", strings.Join(finalargs[1:], " "), err, strings.Join(ret, " \n"))
	}
	return nil
}

func quoteString(src string) string {
	if l := len(src); l > 2 && src[0] != '"' && src[l-1] != '"' {
		return "\"" + src + "\""
	}
	return src
}

// CommandCopy copies a given file or folder into the target folder
// Does not verify that the target folder exists nor if
// it is in fact a folder
// Fails if the target is the root folder
// If command fails, returns the system error and the output of the command
func CommandCopy(src string, dst string) error {
	dst = filepath.Clean(dst)
	if dst[len(dst)-1] == os.PathSeparator {
		return fmt.Errorf("Copy to root folder %s not allowed for safety", dst)
	}
	dst += string(os.PathSeparator)
	// Many safety checks to perform here...
	if runtime.GOOS == "windows" {
		// We end up using xcopy because copy will NOT handle hidden files ever
		stat, err := os.Stat(src)
		if err != nil {
			return err
		}
		if stat.IsDir() {
			fullDest := filepath.Join(dst, filepath.Base(src))
			return RunCommand("xcopy", "/Q", "/I", "/K", "/H", "/Y", "/R", "/S", "/E", quoteString(src), quoteString(fullDest))
		}
		return RunCommand("xcopy", "/Q", "/K", "/H", "/Y", "/R", quoteString(src), quoteString(dst))
	}
	return RunCommand("cp", "-R", quoteString(src), quoteString(dst))
}

// CommandMove moves a given file or folder into the target folder
// Does not verify that the target folder exists nor if
// it is in fact a folder
// Fails if the target is the root folder
// If command fails, returns the system error and the output of the command
func CommandMove(src string, dst string) error {
	dst = filepath.Clean(dst)
	if dst[len(dst)-1] == os.PathSeparator {
		return fmt.Errorf("Move to root folder %s not allowed for safety", dst)
	}
	dst += string(os.PathSeparator)
	dir := filepath.Dir(src)
	if dir[len(dir)-1] == os.PathSeparator {
		return fmt.Errorf("Moving %s from root folder not allowed for safety", dst)
	}
	// Many safety checks to perform here...
	if runtime.GOOS == "windows" {
		// hidden files will wreak havoc with move across devices
		err := RunCommand("move", "/Y", quoteString(src), quoteString(dst))
		if err != nil {
			// So if we get any errors we retry via copy & delete
			err = CommandCopy(src, dst)
			if err == nil {
				err = CommandDelete(src)
			}
		}
		return err
	}
	return RunCommand("mv", "-f", quoteString(src), quoteString(dst))
}

// CommandDelete deletes a given file or folder
// Fails if the target is the root folder
// If command fails, returns the system error and the output of the command
func CommandDelete(dst string) error {
	dst = filepath.Clean(dst)
	dir := filepath.Dir(dst)
	if dir[len(dir)-1] == os.PathSeparator {
		return fmt.Errorf("Deleting %s from root folder not allowed for safety", dst)
	}
	// Many safety checks to perform here...
	if runtime.GOOS == "windows" {
		// Deleting files in directories and deleting directories are two
		// separate things :(
		err := RunCommand("del", "/Q", "/A", "/F", quoteString(dst))
		if err == nil {
			// Must ignore the error because dst may have been fully deleted by prev command
			// UGH
			/*err = */
			RunCommand("rd", "/S", "/Q", quoteString(dst))
		}
		return err
	}
	return RunCommand("rm", "-rf", quoteString(dst))
}
<commit_msg>Remove wrong quotes on *nix commands<commit_after>// Copyright 2017 Javier Arevalo <jare@iguanademos.com>

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Cross platform command execution and some file system operations

package main

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
)

// RunShell runs an interactive shell.
// Since the current program is still running, so may be
// its coroutines, which will trigger all sorts of weirdness
// For example, Ctrl-C may be caught by our Go runtime, killing us
// but leaving the spawned shell still running, with I/O shared with
// our parent (possibly another shell!). That's pretty ugly.
func RunShell(cwd string) error {
	attr := os.ProcAttr{
		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
		Dir:   cwd,
	}
	var args []string
	shell, ok := os.LookupEnv("COMSPEC")
	if !ok {
		args = []string{"-i"}
		shell, ok = os.LookupEnv("SHELL")
		if !ok {
			shell = "/bin/sh"
		}
	}
	process, err := os.StartProcess(shell, args, &attr)
	if err != nil {
		return err
	}
	state, err := process.Wait()
	if err != nil {
		return err
	}
	if state.Success() {
		return nil
	}
	return fmt.Errorf("<< Exited shell: %s", state.String())
}

// RunCommand executes the given command and arguments under the system'
// default shell. Really only tested under CMD and ksh
// If command fails, returns the system error and the output of the command
func RunCommand(command string, args ...string) error {
	var ret []string
	var finalargs []string
	var shell string
	if runtime.GOOS == "windows" {
		sh, ok := os.LookupEnv("COMSPEC")
		if !ok {
			shell = "C:\\Windows\\System32\\cmd.exe"
		}
		shell = sh
		args = append([]string{command}, args...)
		finalargs = []string{"/C", strings.Join(args, " ") + " 2>&1"}
	} else {
		sh, ok := os.LookupEnv("SHELL")
		if !ok {
			shell = "/bin/sh"
		}
		shell = sh
		for i, v := range args {
			args[i] = strconv.Quote(v)
		}
		args = append([]string{command}, args...)
		finalargs = []string{"-c", strings.Join(args, " ") + " 2>&1"}
	}
	Logf("Running command:\n>%s<\n>>%s<<\n", shell, strings.Join(finalargs, "<<\n>>"))
	cmd := exec.Command(shell, finalargs...)

	// See comment in sys_windows.go for this
	SetProcCmdline(cmd, strings.Join(finalargs, " "))

	outp, err := cmd.StdoutPipe()
	if err != nil {
		return fmt.Errorf("Failed to attach stdout: %v", err)
	}
	err = cmd.Start()
	if err != nil {
		Logf("ERROR: %s: %s\n", err, strings.Join(ret, "\n"))
		return fmt.Errorf("%s -> %s: %s", strings.Join(finalargs[1:], " "), err, strings.Join(ret, " \n"))
	}
	scanner := bufio.NewScanner(outp)
	for scanner.Scan() {
		line := scanner.Text()
		ret = append(ret, line)
	}
	err = cmd.Wait()
	Logf("Output: %s\n", strings.Join(ret, "\n"))
	if err != nil {
		Logf("ERROR: %s: %s\n", err, strings.Join(ret, "\n"))
		return fmt.Errorf("%s -> %s: %s", strings.Join(finalargs[1:], " "), err, strings.Join(ret, " \n"))
	}
	return nil
}

func quoteString(src string) string {
	if l := len(src); l > 2 && src[0] != '"' && src[l-1] != '"' {
		return "\"" + src + "\""
	}
	return src
}

// CommandCopy copies a given file or folder into the target folder
// Does not verify that the target folder exists nor if
// it is in fact a folder
// Fails if the target is the root folder
// If command fails, returns the system error and the output of the command
func CommandCopy(src string, dst string) error {
	dst = filepath.Clean(dst)
	if dst[len(dst)-1] == os.PathSeparator {
		return fmt.Errorf("Copy to root folder %s not allowed for safety", dst)
	}
	dst += string(os.PathSeparator)
	// Many safety checks to perform here...
	if runtime.GOOS == "windows" {
		// We end up using xcopy because copy will NOT handle hidden files ever
		stat, err := os.Stat(src)
		if err != nil {
			return err
		}
		if stat.IsDir() {
			fullDest := filepath.Join(dst, filepath.Base(src))
			return RunCommand("xcopy", "/Q", "/I", "/K", "/H", "/Y", "/R", "/S", "/E", quoteString(src), quoteString(fullDest))
		}
		return RunCommand("xcopy", "/Q", "/K", "/H", "/Y", "/R", quoteString(src), quoteString(dst))
	}
	return RunCommand("cp", "-R", src, dst)
}

// CommandMove moves a given file or folder into the target folder
// Does not verify that the target folder exists nor if
// it is in fact a folder
// Fails if the target is the root folder
// If command fails, returns the system error and the output of the command
func CommandMove(src string, dst string) error {
	dst = filepath.Clean(dst)
	if dst[len(dst)-1] == os.PathSeparator {
		return fmt.Errorf("Move to root folder %s not allowed for safety", dst)
	}
	dst += string(os.PathSeparator)
	dir := filepath.Dir(src)
	if dir[len(dir)-1] == os.PathSeparator {
		return fmt.Errorf("Moving %s from root folder not allowed for safety", dst)
	}
	// Many safety checks to perform here...
	if runtime.GOOS == "windows" {
		// hidden files will wreak havoc with move across devices
		err := RunCommand("move", "/Y", quoteString(src), quoteString(dst))
		if err != nil {
			// So if we get any errors we retry via copy & delete
			err = CommandCopy(src, dst)
			if err == nil {
				err = CommandDelete(src)
			}
		}
		return err
	}
	return RunCommand("mv", "-f", src, dst)
}

// CommandDelete deletes a given file or folder
// Fails if the target is the root folder
// If command fails, returns the system error and the output of the command
func CommandDelete(dst string) error {
	dst = filepath.Clean(dst)
	dir := filepath.Dir(dst)
	if dir[len(dir)-1] == os.PathSeparator {
		return fmt.Errorf("Deleting %s from root folder not allowed for safety", dst)
	}
	// Many safety checks to perform here...
	if runtime.GOOS == "windows" {
		// Deleting files in directories and deleting directories are two
		// separate things :(
		err := RunCommand("del", "/Q", "/A", "/F", quoteString(dst))
		if err == nil {
			// Must ignore the error because dst may have been fully deleted by prev command
			// UGH
			/*err = */
			RunCommand("rd", "/S", "/Q", quoteString(dst))
		}
		return err
	}
	return RunCommand("rm", "-rf", dst)
}
<|endoftext|>