<commit_before>package provisioner

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/play-with-docker/play-with-docker/config"
	"github.com/play-with-docker/play-with-docker/docker"
	"github.com/play-with-docker/play-with-docker/pwd/types"
	"github.com/play-with-docker/play-with-docker/router"
	"github.com/play-with-docker/play-with-docker/storage"
)

var asgService *autoscaling.AutoScaling
var ec2Service *ec2.EC2

func init() {
	// Create a session to share configuration, and load external configuration.
	sess := session.Must(session.NewSession())
	//
	// // Create the service's client with the session.
	asgService = autoscaling.New(sess)
	ec2Service = ec2.New(sess)
}

type windows struct {
	factory docker.FactoryApi
	storage storage.StorageApi
}

type instanceInfo struct {
	publicIP  string
	privateIP string
	id        string
}

func NewWindowsASG(f docker.FactoryApi, st storage.StorageApi) *windows {
	return &windows{factory: f, storage: st}
}

func (d *windows) InstanceNew(session *types.Session, conf types.InstanceConfig) (*types.Instance, error) {

	conf.ImageName = config.GetSSHImage()

	winfo, err := d.getWindowsInstanceInfo(session.Id)

	if err != nil {
		return nil, err
	}

	if conf.Hostname == "" {
		var nodeName string
		for i := 1; ; i++ {
			nodeName = fmt.Sprintf("node%d", i)
			exists := checkHostnameExists(session, nodeName)
			if !exists {
				break
			}
		}
		conf.Hostname = nodeName
	}

	containerName := fmt.Sprintf("%s_%s", session.Id[:8], conf.Hostname)
	opts := docker.CreateContainerOpts{
		Image:           conf.ImageName,
		WindowsEndpoint: winfo.privateIP,
		SessionId:       session.Id,
		PwdIpAddress:    session.PwdIpAddress,
		ContainerName:   containerName,
		Hostname:        conf.Hostname,
		ServerCert:      conf.ServerCert,
		ServerKey:       conf.ServerKey,
		CACert:          conf.CACert,
		Privileged:      false,
		HostFQDN:        conf.Host,
	}

	dockerClient, err := d.factory.GetForSession(session.Id)
	if err != nil {
		d.releaseInstance(session.Id, winfo.id)
		return nil, err
	}
	_, err = dockerClient.CreateContainer(opts)
	if err != nil {
		d.releaseInstance(session.Id, winfo.id)
		return nil, err
	}

	instance := &types.Instance{}
	instance.Name = containerName
	instance.Image = opts.Image
	instance.IP = winfo.privateIP
	instance.SessionId = session.Id
	instance.WindowsId = winfo.id
	instance.Cert = conf.Cert
	instance.Key = conf.Key
	instance.Type = conf.Type
	instance.ServerCert = conf.ServerCert
	instance.ServerKey = conf.ServerKey
	instance.CACert = conf.CACert
	instance.Session = session
	instance.ProxyHost = router.EncodeHost(session.Id, instance.IP, router.HostOpts{})
	instance.SessionHost = session.Host
	// For now this condition holds through. In the future we might need a more complex logic.
	instance.IsDockerHost = opts.Privileged

	if cli, err := d.factory.GetForInstance(instance); err != nil {
		if derr := d.InstanceDelete(session, instance); derr != nil {
			log.Println("Error deleting instance: ", derr)
		}
		return nil, err
	} else {
		info, err := cli.GetDaemonInfo()
		if err != nil {
			if derr := d.InstanceDelete(session, instance); derr != nil {
				log.Println("Error deleting instance: ", derr)
			}
			return nil, err
		}
		instance.Hostname = info.Name
		instance.Name = fmt.Sprintf("%s_%s", session.Id[:8], info.Name)
		if err = dockerClient.ContainerRename(containerName, instance.Name); err != nil {
			// revert instance name to remove ssh container
			instance.Name = containerName
			if derr := d.InstanceDelete(session, instance); derr != nil {
				log.Println("Error deleting instance: ", derr)
			}
			return nil, err
		}
	}

	return instance, nil

}

func (d *windows) InstanceDelete(session *types.Session, instance *types.Instance) error {
	dockerClient, err := d.factory.GetForSession(session.Id)
	if err != nil {
		return err
	}
	err = dockerClient.DeleteContainer(instance.Name)
	if err != nil && !strings.Contains(err.Error(), "No such container") {
		return err
	}

	// TODO trigger deletion in AWS
	return d.releaseInstance(session.Id, instance.WindowsId)
}

func (d *windows) releaseInstance(sessionId, instanceId string) error {
	return d.storage.InstanceDeleteWindows(sessionId, instanceId)
}

func (d *windows) InstanceResizeTerminal(instance *types.Instance, rows, cols uint) error {
	dockerClient, err := d.factory.GetForSession(instance.SessionId)
	if err != nil {
		return err
	}
	return dockerClient.ContainerResize(instance.Name, rows, cols)
}

func (d *windows) InstanceGetTerminal(instance *types.Instance) (net.Conn, error) {
	dockerClient, err := d.factory.GetForSession(instance.SessionId)
	if err != nil {
		return nil, err
	}
	return dockerClient.CreateAttachConnection(instance.Name)
}

func (d *windows) InstanceUploadFromUrl(instance *types.Instance, fileName, dest, url string) error {
	return fmt.Errorf("Not implemented")
}

func (d *windows) InstanceUploadFromReader(instance *types.Instance, fileName, dest string, reader io.Reader) error {
	return fmt.Errorf("Not implemented")
}

func (d *windows) getWindowsInstanceInfo(sessionId string) (*instanceInfo, error) {

	input := &autoscaling.DescribeAutoScalingGroupsInput{
		AutoScalingGroupNames: []*string{aws.String("pwd-windows")},
	}
	out, err := asgService.DescribeAutoScalingGroups(input)

	if err != nil {
		return nil, err
	}

	// there should always be one asg
	instances := out.AutoScalingGroups[0].Instances
	availInstances := make([]string, len(instances))

	for i, inst := range instances {
		if *inst.LifecycleState == "InService" {
			availInstances[i] = *inst.InstanceId
		}
	}

	assignedInstances, err := d.storage.InstanceGetAllWindows()
	assignedInstancesIds := []string{}
	for _, ai := range assignedInstances {
		assignedInstancesIds = append(assignedInstancesIds, ai.ID)
	}

	if err != nil {
		return nil, err
	}

	avInstanceId := d.pickFreeInstance(sessionId, availInstances, assignedInstancesIds)

	if len(avInstanceId) == 0 {
		return nil, errors.New("No Windows instance available")
	}

	iout, err := ec2Service.DescribeInstances(&ec2.DescribeInstancesInput{
		InstanceIds: []*string{aws.String(avInstanceId)},
	})
	if err != nil {
		// TODO retry x times and free the instance that was picked?
		d.releaseInstance(sessionId, avInstanceId)
		return nil, err
	}

	instance := iout.Reservations[0].Instances[0]

	instanceInfo := &instanceInfo{
		publicIP:  *instance.PublicIpAddress,
		privateIP: *instance.PrivateIpAddress,
		id:        avInstanceId,
	}

	//TODO check for free instance, ASG capacity and return

	return instanceInfo, nil
}

// select free instance and lock it into db.
// additionally check if ASG needs to be resized
func (d *windows) pickFreeInstance(sessionId string, availInstances, assignedInstances []string) string {
	for _, av := range availInstances {
		found := false
		for _, as := range assignedInstances {
			if av == as {
				found = true
				break
			}
		}

		if !found {
			err := d.storage.InstanceCreateWindows(&types.WindowsInstance{SessionId: sessionId, ID: av})
			if err != nil {
				// TODO either storage error or instance is already assigned (race condition)
			}
			return av
		}
	}
	// all availalbe instances are assigned
	return ""
}
<commit_msg>Remove instances from ASG and terminate them when deleting<commit_after>package provisioner

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"sort"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/play-with-docker/play-with-docker/config"
	"github.com/play-with-docker/play-with-docker/docker"
	"github.com/play-with-docker/play-with-docker/pwd/types"
	"github.com/play-with-docker/play-with-docker/router"
	"github.com/play-with-docker/play-with-docker/storage"
)

var asgService *autoscaling.AutoScaling
var ec2Service *ec2.EC2

func init() {
	// Create a session to share configuration, and load external configuration.
	sess := session.Must(session.NewSession())
	//
	// // Create the service's client with the session.
	asgService = autoscaling.New(sess)
	ec2Service = ec2.New(sess)
}

type windows struct {
	factory docker.FactoryApi
	storage storage.StorageApi
}

type instanceInfo struct {
	publicIP  string
	privateIP string
	id        string
}

func NewWindowsASG(f docker.FactoryApi, st storage.StorageApi) *windows {
	return &windows{factory: f, storage: st}
}

func (d *windows) InstanceNew(session *types.Session, conf types.InstanceConfig) (*types.Instance, error) {

	conf.ImageName = config.GetSSHImage()

	winfo, err := d.getWindowsInstanceInfo(session.Id)

	if err != nil {
		return nil, err
	}

	if conf.Hostname == "" {
		var nodeName string
		for i := 1; ; i++ {
			nodeName = fmt.Sprintf("node%d", i)
			exists := checkHostnameExists(session, nodeName)
			if !exists {
				break
			}
		}
		conf.Hostname = nodeName
	}

	containerName := fmt.Sprintf("%s_%s", session.Id[:8], conf.Hostname)
	opts := docker.CreateContainerOpts{
		Image:           conf.ImageName,
		WindowsEndpoint: winfo.privateIP,
		SessionId:       session.Id,
		PwdIpAddress:    session.PwdIpAddress,
		ContainerName:   containerName,
		Hostname:        conf.Hostname,
		ServerCert:      conf.ServerCert,
		ServerKey:       conf.ServerKey,
		CACert:          conf.CACert,
		Privileged:      false,
		HostFQDN:        conf.Host,
	}

	dockerClient, err := d.factory.GetForSession(session.Id)
	if err != nil {
		d.releaseInstance(session.Id, winfo.id)
		return nil, err
	}
	_, err = dockerClient.CreateContainer(opts)
	if err != nil {
		d.releaseInstance(session.Id, winfo.id)
		return nil, err
	}

	instance := &types.Instance{}
	instance.Name = containerName
	instance.Image = opts.Image
	instance.IP = winfo.privateIP
	instance.SessionId = session.Id
	instance.WindowsId = winfo.id
	instance.Cert = conf.Cert
	instance.Key = conf.Key
	instance.Type = conf.Type
	instance.ServerCert = conf.ServerCert
	instance.ServerKey = conf.ServerKey
	instance.CACert = conf.CACert
	instance.Session = session
	instance.ProxyHost = router.EncodeHost(session.Id, instance.IP, router.HostOpts{})
	instance.SessionHost = session.Host
	// For now this condition holds through. In the future we might need a more complex logic.
	instance.IsDockerHost = opts.Privileged

	if cli, err := d.factory.GetForInstance(instance); err != nil {
		if derr := d.InstanceDelete(session, instance); derr != nil {
			log.Println("Error deleting instance: ", derr)
		}
		return nil, err
	} else {
		info, err := cli.GetDaemonInfo()
		if err != nil {
			if derr := d.InstanceDelete(session, instance); derr != nil {
				log.Println("Error deleting instance: ", derr)
			}
			return nil, err
		}
		instance.Hostname = info.Name
		instance.Name = fmt.Sprintf("%s_%s", session.Id[:8], info.Name)
		if err = dockerClient.ContainerRename(containerName, instance.Name); err != nil {
			// revert instance name to remove ssh container
			instance.Name = containerName
			if derr := d.InstanceDelete(session, instance); derr != nil {
				log.Println("Error deleting instance: ", derr)
			}
			return nil, err
		}
	}

	return instance, nil

}

func (d *windows) InstanceDelete(session *types.Session, instance *types.Instance) error {
	dockerClient, err := d.factory.GetForSession(session.Id)
	if err != nil {
		return err
	}

	_, err = asgService.DetachInstances(&autoscaling.DetachInstancesInput{
		AutoScalingGroupName:           aws.String("pwd-windows"),
		InstanceIds:                    []*string{aws.String(instance.WindowsId)},
		ShouldDecrementDesiredCapacity: aws.Bool(false),
	})

	if err != nil {
		return err
	}

	// return error and don't do anything else
	if _, err := ec2Service.TerminateInstances(&ec2.TerminateInstancesInput{InstanceIds: []*string{aws.String(instance.WindowsId)}}); err != nil {
		return err
	}

	err = dockerClient.DeleteContainer(instance.Name)
	if err != nil && !strings.Contains(err.Error(), "No such container") {
		return err
	}

	return d.releaseInstance(session.Id, instance.WindowsId)
}

func (d *windows) releaseInstance(sessionId, instanceId string) error {
	return d.storage.InstanceDeleteWindows(sessionId, instanceId)
}

func (d *windows) InstanceResizeTerminal(instance *types.Instance, rows, cols uint) error {
	dockerClient, err := d.factory.GetForSession(instance.SessionId)
	if err != nil {
		return err
	}
	return dockerClient.ContainerResize(instance.Name, rows, cols)
}

func (d *windows) InstanceGetTerminal(instance *types.Instance) (net.Conn, error) {
	dockerClient, err := d.factory.GetForSession(instance.SessionId)
	if err != nil {
		return nil, err
	}
	return dockerClient.CreateAttachConnection(instance.Name)
}

func (d *windows) InstanceUploadFromUrl(instance *types.Instance, fileName, dest, url string) error {
	return fmt.Errorf("Not implemented")
}

func (d *windows) InstanceUploadFromReader(instance *types.Instance, fileName, dest string, reader io.Reader) error {
	return fmt.Errorf("Not implemented")
}

func (d *windows) getWindowsInstanceInfo(sessionId string) (*instanceInfo, error) {

	input := &autoscaling.DescribeAutoScalingGroupsInput{
		AutoScalingGroupNames: []*string{aws.String("pwd-windows")},
	}
	out, err := asgService.DescribeAutoScalingGroups(input)

	if err != nil {
		return nil, err
	}

	// there should always be one asg
	instances := out.AutoScalingGroups[0].Instances
	availInstances := make([]string, len(instances))

	// reverse order so older instances are first served
	sort.Sort(sort.Reverse(sort.StringSlice(availInstances)))

	for i, inst := range instances {
		if *inst.LifecycleState == "InService" {
			availInstances[i] = *inst.InstanceId
		}
	}

	assignedInstances, err := d.storage.InstanceGetAllWindows()
	assignedInstancesIds := []string{}
	for _, ai := range assignedInstances {
		assignedInstancesIds = append(assignedInstancesIds, ai.ID)
	}

	if err != nil {
		return nil, err
	}

	avInstanceId := d.pickFreeInstance(sessionId, availInstances, assignedInstancesIds)

	if len(avInstanceId) == 0 {
		return nil, errors.New("No Windows instance available")
	}

	iout, err := ec2Service.DescribeInstances(&ec2.DescribeInstancesInput{
		InstanceIds: []*string{aws.String(avInstanceId)},
	})
	if err != nil {
		// TODO retry x times and free the instance that was picked?
		d.releaseInstance(sessionId, avInstanceId)
		return nil, err
	}

	instance := iout.Reservations[0].Instances[0]

	instanceInfo := &instanceInfo{
		publicIP:  *instance.PublicIpAddress,
		privateIP: *instance.PrivateIpAddress,
		id:        avInstanceId,
	}

	//TODO check for free instance, ASG capacity and return

	return instanceInfo, nil
}

// select free instance and lock it into db.
// additionally check if ASG needs to be resized
func (d *windows) pickFreeInstance(sessionId string, availInstances, assignedInstances []string) string {
	for _, av := range availInstances {
		found := false
		for _, as := range assignedInstances {
			if av == as {
				found = true
				break
			}
		}

		if !found {
			err := d.storage.InstanceCreateWindows(&types.WindowsInstance{SessionId: sessionId, ID: av})
			if err != nil {
				// TODO either storage error or instance is already assigned (race condition)
			}
			return av
		}
	}
	// all availalbe instances are assigned
	return ""
}
<|endoftext|>


<commit_before>package agent

import (
	"bufio"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/resourced/resourced/libstring"
)

func (a *Agent) setAccessTokens() error {
	a.AccessTokens = make([]string, 0)

	configDir := os.Getenv("RESOURCED_CONFIG_DIR")
	if configDir == "" {
		return nil
	}

	configDir = libstring.ExpandTildeAndEnv(configDir)
	accessTokensDir := path.Join(configDir, "access-tokens")

	tokenFiles, err := ioutil.ReadDir(accessTokensDir)
	if err != nil {
		return nil
	}

	for _, f := range tokenFiles {
		fullpath := path.Join(accessTokensDir, f.Name())

		file, err := os.Open(fullpath)
		if err != nil {
			continue
		}
		defer file.Close()

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			tokensPerLine := strings.Split(scanner.Text(), ",")

			// Ensure that we ignore comments "//" or "#"
			for _, tokenPerLine := range tokensPerLine {
				if !strings.HasPrefix(tokenPerLine, "#") && !strings.HasPrefix(tokenPerLine, "//") {
					a.AccessTokens = append(a.AccessTokens, strings.TrimSpace(tokenPerLine)
				}
			}
		}
	}

	return nil
}

// Check if a given access token is allowed.
func (a *Agent) IsAllowed(givenToken string) bool {
	// Allow all if there are no AccessTokens defined.
	if len(a.AccessTokens) == 0 {
		return true
	}

	for _, token := range a.AccessTokens {
		if token == givenToken {
			return true
		}
	}

	return false
}
<commit_msg>syntax error<commit_after>package agent

import (
	"bufio"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/resourced/resourced/libstring"
)

func (a *Agent) setAccessTokens() error {
	a.AccessTokens = make([]string, 0)

	configDir := os.Getenv("RESOURCED_CONFIG_DIR")
	if configDir == "" {
		return nil
	}

	configDir = libstring.ExpandTildeAndEnv(configDir)
	accessTokensDir := path.Join(configDir, "access-tokens")

	tokenFiles, err := ioutil.ReadDir(accessTokensDir)
	if err != nil {
		return nil
	}

	for _, f := range tokenFiles {
		fullpath := path.Join(accessTokensDir, f.Name())

		file, err := os.Open(fullpath)
		if err != nil {
			continue
		}
		defer file.Close()

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			tokensPerLine := strings.Split(scanner.Text(), ",")

			// Ensure that we ignore comments "//" or "#"
			for _, tokenPerLine := range tokensPerLine {
				if !strings.HasPrefix(tokenPerLine, "#") && !strings.HasPrefix(tokenPerLine, "//") {
					a.AccessTokens = append(a.AccessTokens, strings.TrimSpace(tokenPerLine))
				}
			}
		}
	}

	return nil
}

// Check if a given access token is allowed.
func (a *Agent) IsAllowed(givenToken string) bool {
	// Allow all if there are no AccessTokens defined.
	if len(a.AccessTokens) == 0 {
		return true
	}

	for _, token := range a.AccessTokens {
		if token == givenToken {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/samuel/go-zookeeper/zk"
	"github.com/wvanbergen/kazoo-go"
)

var (
	zkAddr      string
	topic       string
	partition   int
	offset      int64
	serializer  string
	contentType string
	callback    string
)

type MessageBody struct {
	Topic        string `json:"Topic"`
	PartitionKey string `json:"PartitionKey"`
	TimeStamp    int64  `json:"TimeStamp"`
	Data         string `json:"Data"`
	LogId        string `json:"LogId"`
	ContentType  string `json:"ContentType"`
}

func init() {
	flag.StringVar(&zkAddr, "zk", "127.0.0.1:2181", "zookeeper address which kafka belongs to")
	flag.StringVar(&topic, "topic", "", "topic name")
	flag.IntVar(&partition, "partition", -1, "partition id")
	flag.Int64Var(&offset, "offset", -1, "offset number")
	flag.StringVar(&serializer, "serializer", "raw", "message serializer name")
	flag.StringVar(&contentType, "type", "", "message content type")
	flag.StringVar(&callback, "callback", "http://url_to_replace", "callback url to build")
	zk.DefaultLogger = log.New(ioutil.Discard, "[Sarama] ", log.LstdFlags)
}

func escapeShellArg(arg string) string {
	return "'" + strings.Replace(arg, "'", "'\\''", -1) + "'"
}

func main() {
	flag.Parse()

	if topic == "" {
		panic("topic should not be empty")
	}

	if partition < 0 {
		panic("partition should be specified")
	}

	if offset < 0 {
		panic("offset should be specified")
	}

	kazooConf := kazoo.NewConfig()
	kazooInstance, err := kazoo.NewKazooFromConnectionString(zkAddr, kazooConf)
	if err != nil {
		panic(fmt.Sprintf("connect zookeeper failed: %v", err))
	}
	defer kazooInstance.Close()

	brokers, err := kazooInstance.BrokerList()
	if err != nil {
		panic(fmt.Sprintf("read broker list from zookeeper failed: %v", err))
	}

	saramaConfig := sarama.NewConfig()
	saramaConsumer, err := sarama.NewConsumer(brokers, saramaConfig)
	if err != nil {
		panic(fmt.Sprintf("connect to kafka failed: %v", err))
	}
	defer saramaConsumer.Close()

	partitionConsumer, err := saramaConsumer.ConsumePartition(topic, int32(partition), offset)
	if err != nil {
		panic(fmt.Sprintf("consume message failed, could not get partition consumer: %v", err))
	}
	defer partitionConsumer.Close()

	var message *sarama.ConsumerMessage

	select {
	case message = <-partitionConsumer.Messages():
	case <-time.After(10 * time.Second):
		panic("message not received after 10 seconds")
	}

	var messageBody MessageBody

	// got message
	switch serializer {
	case "raw":
		messageBody.Data = string(message.Value)
	case "json":
		fallthrough
	default:
		json.Unmarshal(message.Value, &messageBody)
	}

	if contentType != "" {
		messageBody.ContentType = contentType
	} else if messageBody.ContentType == "" {
		messageBody.ContentType = "application/x-www-form-urlencoded"
	}

	// build curl command
	var curlCommand []string

	curlCommand = append(curlCommand, "curl", escapeShellArg(callback))

	// append headers
	curlCommand = append(curlCommand, "-H", escapeShellArg(fmt.Sprintf("Content-Type: %s", messageBody.ContentType)))
	curlCommand = append(curlCommand, "-H", escapeShellArg("User-Agent: Taiji pusher consumer(go)/v2.0.0-simulator"))
	curlCommand = append(curlCommand, "-H", escapeShellArg("X-Retry-Times: 0"))
	curlCommand = append(curlCommand, "-H", escapeShellArg(fmt.Sprintf("X-Kmq-Topic: %s", message.Topic)))
	curlCommand = append(curlCommand, "-H", escapeShellArg(fmt.Sprintf("X-Kmq-Partition: %d", message.Partition)))
	curlCommand = append(curlCommand, "-H", escapeShellArg(fmt.Sprintf("X-Kmq-Partition-Key: %s", messageBody.PartitionKey)))
	curlCommand = append(curlCommand, "-H", escapeShellArg(fmt.Sprintf("X-Kmq-Offset: %d", message.Offset)))
	curlCommand = append(curlCommand, "-H", escapeShellArg(fmt.Sprintf("X-Kmq-Logid: %s", messageBody.LogId)))
	curlCommand = append(curlCommand, "-H", escapeShellArg(fmt.Sprintf("X-Kmq-Timestamp: %d", messageBody.TimeStamp)))
	curlCommand = append(curlCommand, "-H", escapeShellArg("Meilishuo uid:0;ip:0.0.0.0;v:0;master:0"))

	// append body
	curlCommand = append(curlCommand, "--data", escapeShellArg(messageBody.Data))

	fmt.Println(strings.Join(curlCommand, " "))
}
<commit_msg>Fix log prefix typo in tools/message_to_curl<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/samuel/go-zookeeper/zk"
	"github.com/wvanbergen/kazoo-go"
)

var (
	zkAddr      string
	topic       string
	partition   int
	offset      int64
	serializer  string
	contentType string
	callback    string
)

type MessageBody struct {
	Topic        string `json:"Topic"`
	PartitionKey string `json:"PartitionKey"`
	TimeStamp    int64  `json:"TimeStamp"`
	Data         string `json:"Data"`
	LogId        string `json:"LogId"`
	ContentType  string `json:"ContentType"`
}

func init() {
	flag.StringVar(&zkAddr, "zk", "127.0.0.1:2181", "zookeeper address which kafka belongs to")
	flag.StringVar(&topic, "topic", "", "topic name")
	flag.IntVar(&partition, "partition", -1, "partition id")
	flag.Int64Var(&offset, "offset", -1, "offset number")
	flag.StringVar(&serializer, "serializer", "raw", "message serializer name")
	flag.StringVar(&contentType, "type", "", "message content type")
	flag.StringVar(&callback, "callback", "http://url_to_replace", "callback url to build")
	zk.DefaultLogger = log.New(ioutil.Discard, "[Zookeeper] ", log.LstdFlags)
}

func escapeShellArg(arg string) string {
	return "'" + strings.Replace(arg, "'", "'\\''", -1) + "'"
}

func main() {
	flag.Parse()

	if topic == "" {
		panic("topic should not be empty")
	}

	if partition < 0 {
		panic("partition should be specified")
	}

	if offset < 0 {
		panic("offset should be specified")
	}

	kazooConf := kazoo.NewConfig()
	kazooInstance, err := kazoo.NewKazooFromConnectionString(zkAddr, kazooConf)
	if err != nil {
		panic(fmt.Sprintf("connect zookeeper failed: %v", err))
	}
	defer kazooInstance.Close()

	brokers, err := kazooInstance.BrokerList()
	if err != nil {
		panic(fmt.Sprintf("read broker list from zookeeper failed: %v", err))
	}

	saramaConfig := sarama.NewConfig()
	saramaConsumer, err := sarama.NewConsumer(brokers, saramaConfig)
	if err != nil {
		panic(fmt.Sprintf("connect to kafka failed: %v", err))
	}
	defer saramaConsumer.Close()

	partitionConsumer, err := saramaConsumer.ConsumePartition(topic, int32(partition), offset)
	if err != nil {
		panic(fmt.Sprintf("consume message failed, could not get partition consumer: %v", err))
	}
	defer partitionConsumer.Close()

	var message *sarama.ConsumerMessage

	select {
	case message = <-partitionConsumer.Messages():
	case <-time.After(10 * time.Second):
		panic("message not received after 10 seconds")
	}

	var messageBody MessageBody

	// got message
	switch serializer {
	case "raw":
		messageBody.Data = string(message.Value)
	case "json":
		fallthrough
	default:
		json.Unmarshal(message.Value, &messageBody)
	}

	if contentType != "" {
		messageBody.ContentType = contentType
	} else if messageBody.ContentType == "" {
		messageBody.ContentType = "application/x-www-form-urlencoded"
	}

	// build curl command
	var curlCommand []string

	curlCommand = append(curlCommand, "curl", escapeShellArg(callback))

	// append headers
	curlCommand = append(curlCommand, "-H", escapeShellArg(fmt.Sprintf("Content-Type: %s", messageBody.ContentType)))
	curlCommand = append(curlCommand, "-H", escapeShellArg("User-Agent: Taiji pusher consumer(go)/v2.0.0-simulator"))
	curlCommand = append(curlCommand, "-H", escapeShellArg("X-Retry-Times: 0"))
	curlCommand = append(curlCommand, "-H", escapeShellArg(fmt.Sprintf("X-Kmq-Topic: %s", message.Topic)))
	curlCommand = append(curlCommand, "-H", escapeShellArg(fmt.Sprintf("X-Kmq-Partition: %d", message.Partition)))
	curlCommand = append(curlCommand, "-H", escapeShellArg(fmt.Sprintf("X-Kmq-Partition-Key: %s", messageBody.PartitionKey)))
	curlCommand = append(curlCommand, "-H", escapeShellArg(fmt.Sprintf("X-Kmq-Offset: %d", message.Offset)))
	curlCommand = append(curlCommand, "-H", escapeShellArg(fmt.Sprintf("X-Kmq-Logid: %s", messageBody.LogId)))
	curlCommand = append(curlCommand, "-H", escapeShellArg(fmt.Sprintf("X-Kmq-Timestamp: %d", messageBody.TimeStamp)))
	curlCommand = append(curlCommand, "-H", escapeShellArg("Meilishuo uid:0;ip:0.0.0.0;v:0;master:0"))

	// append body
	curlCommand = append(curlCommand, "--data", escapeShellArg(messageBody.Data))

	fmt.Println(strings.Join(curlCommand, " "))
}
<|endoftext|>


<commit_before>package network

import (
	"net"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// Type represents an LXD network driver type.
type Type interface {
	FillConfig(config map[string]string) error
	Info() Info
	ValidateName(name string) error
	Type() string
	DBType() db.NetworkType
}

// Network represents an instantiated LXD network.
type Network interface {
	Type

	// Load.
	init(state *state.State, id int64, projectName string, name string, netType string, description string, config map[string]string, status string)

	// Config.
	Validate(config map[string]string) error
	ID() int64
	Name() string
	Description() string
	Status() string
	Config() map[string]string
	IsUsed() (bool, error)
	DHCPv4Subnet() *net.IPNet
	DHCPv6Subnet() *net.IPNet
	DHCPv4Ranges() []shared.IPRange
	DHCPv6Ranges() []shared.IPRange

	// Actions.
	Create(clientType cluster.ClientType) error
	Start() error
	Stop() error
	Rename(name string) error
	Update(newNetwork api.NetworkPut, targetNode string, clientType cluster.ClientType) error
	HandleHeartbeat(heartbeatData *cluster.APIHeartbeat) error
	Delete(clientType cluster.ClientType) error
}
<commit_msg>lxd/network/network/interface: Updates init to take api.Network and network nodes map<commit_after>package network

import (
	"net"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// Type represents an LXD network driver type.
type Type interface {
	FillConfig(config map[string]string) error
	Info() Info
	ValidateName(name string) error
	Type() string
	DBType() db.NetworkType
}

// Network represents an instantiated LXD network.
type Network interface {
	Type

	// Load.
	init(state *state.State, id int64, projectName string, netInfo *api.Network, netNodes map[int64]db.NetworkNode)

	// Config.
	Validate(config map[string]string) error
	ID() int64
	Name() string
	Description() string
	Status() string
	Config() map[string]string
	IsUsed() (bool, error)
	DHCPv4Subnet() *net.IPNet
	DHCPv6Subnet() *net.IPNet
	DHCPv4Ranges() []shared.IPRange
	DHCPv6Ranges() []shared.IPRange

	// Actions.
	Create(clientType cluster.ClientType) error
	Start() error
	Stop() error
	Rename(name string) error
	Update(newNetwork api.NetworkPut, targetNode string, clientType cluster.ClientType) error
	HandleHeartbeat(heartbeatData *cluster.APIHeartbeat) error
	Delete(clientType cluster.ClientType) error
}
<|endoftext|>


<commit_before>package configuration

import (
	"io/ioutil"
	"os"
	"path/filepath"
)

const (
	filePermissions = 0600
	dirPermissions  = 0700
)

type Persistor interface {
	Delete()
	Exists() bool
	Load(DataInterface) error
	Save(DataInterface) error
}

type DataInterface interface {
	JsonMarshalV3() ([]byte, error)
	JsonUnmarshalV3([]byte) error
}

type DiskPersistor struct {
	filePath string
}

func NewDiskPersistor(path string) (dp DiskPersistor) {
	return DiskPersistor{
		filePath: path,
	}
}

func (dp DiskPersistor) Exists() bool {
	_, err := os.Stat(dp.filePath)
	if err != nil && !os.IsExist(err) {
		return false
	}
	return true
}

func (dp DiskPersistor) Delete() {
	os.Remove(dp.filePath)
}

func (dp DiskPersistor) Load(data DataInterface) error {
	err := dp.read(data)
	if err != nil {
		err = dp.write(data)
	}
	return err
}

func (dp DiskPersistor) Save(data DataInterface) (err error) {
	return dp.write(data)
}

func (dp DiskPersistor) read(data DataInterface) error {
	err := os.MkdirAll(filepath.Dir(dp.filePath), dirPermissions)
	if err != nil {
		return err
	}

	jsonBytes, err := ioutil.ReadFile(dp.filePath)
	if err != nil {
		return err
	}

	err = data.JsonUnmarshalV3(jsonBytes)
	return err
}

func (dp DiskPersistor) write(data DataInterface) error {
	bytes, err := data.JsonMarshalV3()
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(dp.filePath, bytes, filePermissions)
	return err
}
<commit_msg>return correct error when unable to create config<commit_after>package configuration

import (
	"io/ioutil"
	"os"
	"path/filepath"
)

const (
	filePermissions = 0600
	dirPermissions  = 0700
)

type Persistor interface {
	Delete()
	Exists() bool
	Load(DataInterface) error
	Save(DataInterface) error
}

type DataInterface interface {
	JsonMarshalV3() ([]byte, error)
	JsonUnmarshalV3([]byte) error
}

type DiskPersistor struct {
	filePath string
}

func NewDiskPersistor(path string) (dp DiskPersistor) {
	return DiskPersistor{
		filePath: path,
	}
}

func (dp DiskPersistor) Exists() bool {
	_, err := os.Stat(dp.filePath)
	if err != nil && !os.IsExist(err) {
		return false
	}
	return true
}

func (dp DiskPersistor) Delete() {
	os.Remove(dp.filePath)
}

func (dp DiskPersistor) Load(data DataInterface) error {
	err := dp.read(data)
	if os.IsPermission(err) {
		return err
	}

	if err != nil {
		err = dp.write(data)
	}
	return err
}

func (dp DiskPersistor) Save(data DataInterface) (err error) {
	return dp.write(data)
}

func (dp DiskPersistor) read(data DataInterface) error {
	err := os.MkdirAll(filepath.Dir(dp.filePath), dirPermissions)
	if err != nil {
		return err
	}

	jsonBytes, err := ioutil.ReadFile(dp.filePath)
	if err != nil {
		return err
	}

	err = data.JsonUnmarshalV3(jsonBytes)
	return err
}

func (dp DiskPersistor) write(data DataInterface) error {
	bytes, err := data.JsonMarshalV3()
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(dp.filePath, bytes, filePermissions)
	return err
}
<|endoftext|>


<commit_before>// Package fastcgi has middleware that acts as a FastCGI client. Requests
// that get forwarded to FastCGI stop the middleware execution chain.
// The most common use for this package is to serve PHP websites via php-fpm.
package fastcgi

import (
	"errors"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/mholt/caddy/middleware"
)

// New generates a new FastCGI middleware.
func New(c middleware.Controller) (middleware.Middleware, error) {
	root, err := filepath.Abs(c.Root())
	if err != nil {
		return nil, err
	}

	rules, err := parse(c)
	if err != nil {
		return nil, err
	}

	return func(next middleware.Handler) middleware.Handler {
		return Handler{
			Next:            next,
			Rules:           rules,
			Root:            root,
			SoftwareName:    "Caddy", // TODO: Once generators are not in the same pkg as handler, obtain this from some global const
			SoftwareVersion: "",      // TODO: Get this from some global const too
			// TODO: Set ServerName and ServerPort to correct values... (as user defined in config)
		}
	}, nil
}

// Handler is a middleware type that can handle requests as a FastCGI client.
type Handler struct {
	Next  middleware.Handler
	Root  string // must be absolute path to site root
	Rules []Rule

	// These are sent to CGI scripts in env variables
	SoftwareName    string
	SoftwareVersion string
	ServerName      string
	ServerPort      string
}

// ServeHTTP satisfies the middleware.Handler interface.
func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
	for _, rule := range h.Rules {
		// In addition to matching the path, a request must meet some
		// other criteria before being proxied as FastCGI. For example,
		// we probably want to exclude static assets (CSS, JS, images...)
		// but we also want to be flexible for the script we proxy to.

		// These criteria work well in this order for PHP sites
		if middleware.Path(r.URL.Path).Matches(rule.Path) &&
			(r.URL.Path[len(r.URL.Path)-1] == '/' ||
				strings.HasSuffix(r.URL.Path, rule.Ext) ||
				!h.exists(r.URL.Path)) {

			// Create environment for CGI script
			env, err := h.buildEnv(r, rule)
			if err != nil {
				return http.StatusInternalServerError, err
			}

			// Connect to FastCGI gateway
			fcgi, err := Dial("tcp", rule.Address)
			if err != nil {
				return http.StatusBadGateway, err
			}

			// TODO: Allow more methods (requires refactoring fcgiclient first...)
			var resp *http.Response
			switch r.Method {
			case "GET":
				resp, err = fcgi.Get(env)
			case "POST":
				l, _ := strconv.Atoi(r.Header.Get("Content-Length"))
				resp, err = fcgi.Post(env, r.Header.Get("Content-Type"), r.Body, l)
			default:
				return http.StatusMethodNotAllowed, nil
			}
			defer resp.Body.Close()

			if err != nil && err != io.EOF {
				return http.StatusBadGateway, err
			}

			// Write the response header
			for key, vals := range resp.Header {
				for _, val := range vals {
					w.Header().Add(key, val)
				}
			}
			w.WriteHeader(resp.StatusCode)

			// Write the response body
			_, err = io.Copy(w, resp.Body)
			if err != nil {
				return http.StatusBadGateway, err
			}

			return resp.StatusCode, nil
		}
	}

	return h.Next.ServeHTTP(w, r)
}

func (h Handler) exists(path string) bool {
	if _, err := os.Stat(h.Root + path); err == nil {
		return true
	}
	return false
}

func (h Handler) buildEnv(r *http.Request, rule Rule) (map[string]string, error) {
	var env map[string]string

	// Get absolute path of requested resource
	absPath, err := filepath.Abs(h.Root + r.URL.Path)
	if err != nil {
		return env, err
	}

	// Separate remote IP and port; more lenient than net.SplitHostPort
	var ip, port string
	if idx := strings.Index(r.RemoteAddr, ":"); idx > -1 {
		ip = r.RemoteAddr[:idx]
		port = r.RemoteAddr[idx+1:]
	} else {
		ip = r.RemoteAddr
	}

	// Split path in preparation for env variables
	splitPos := strings.Index(r.URL.Path, rule.SplitPath)
	var docURI, scriptName, scriptFilename, pathInfo string
	if splitPos == -1 {
		// Request doesn't have the extension, so assume index file
		docURI = "/" + rule.IndexFile
		scriptName = "/" + rule.IndexFile
		scriptFilename = h.Root + "/" + rule.IndexFile
		pathInfo = r.URL.Path
	} else {
		// Request has the extension; path was split successfully
		docURI = r.URL.Path[:splitPos+len(rule.SplitPath)]
		pathInfo = r.URL.Path[splitPos+len(rule.SplitPath):]
		scriptName = r.URL.Path
		scriptFilename = absPath
	}

	// Some variables are unused but cleared explicitly to prevent
	// the parent environment from interfering.
	env = map[string]string{

		// Variables defined in CGI 1.1 spec
		"AUTH_TYPE":         "", // Not used
		"CONTENT_LENGTH":    r.Header.Get("Content-Length"),
		"CONTENT_TYPE":      r.Header.Get("Content-Type"),
		"GATEWAY_INTERFACE": "CGI/1.1",
		"PATH_INFO":         pathInfo,
		"PATH_TRANSLATED":   h.Root + "/" + pathInfo, // Source for path_translated: http://www.oreilly.com/openbook/cgi/ch02_04.html
		"QUERY_STRING":      r.URL.RawQuery,
		"REMOTE_ADDR":       ip,
		"REMOTE_HOST":       ip, // For speed, remote host lookups disabled
		"REMOTE_PORT":       port,
		"REMOTE_IDENT":      "", // Not used
		"REMOTE_USER":       "", // Not used
		"REQUEST_METHOD":    r.Method,
		"SERVER_NAME":       h.ServerName,
		"SERVER_PORT":       h.ServerPort,
		"SERVER_PROTOCOL":   r.Proto,
		"SERVER_SOFTWARE":   h.SoftwareName + "/" + h.SoftwareVersion,

		// Other variables
		"DOCUMENT_ROOT":   h.Root,
		"DOCUMENT_URI":    docURI,
		"HTTP_HOST":       r.Host, // added here, since not always part of headers
		"REQUEST_URI":     r.URL.RequestURI(),
		"SCRIPT_FILENAME": scriptFilename,
		"SCRIPT_NAME":     scriptName,
	}

	// Add all HTTP headers to env variables
	for field, val := range r.Header {
		header := strings.ToUpper(field)
		header = headerNameReplacer.Replace(header)
		env["HTTP_"+header] = strings.Join(val, ", ")
	}

	return env, nil
}

func parse(c middleware.Controller) ([]Rule, error) {
	var rules []Rule

	for c.Next() {
		var rule Rule

		args := c.RemainingArgs()

		switch len(args) {
		case 0:
			return rules, c.ArgErr()
		case 1:
			rule.Path = "/"
			rule.Address = args[0]
		case 2:
			rule.Path = args[0]
			rule.Address = args[1]
		case 3:
			rule.Path = args[0]
			rule.Address = args[1]
			err := preset(args[2], &rule)
			if err != nil {
				return rules, c.Err("Invalid fastcgi rule preset '" + args[2] + "'")
			}
		}

		for c.NextBlock() {
			switch c.Val() {
			case "ext":
				if !c.NextArg() {
					return rules, c.ArgErr()
				}
				rule.Ext = c.Val()
			case "split":
				if !c.NextArg() {
					return rules, c.ArgErr()
				}
				rule.SplitPath = c.Val()
			case "index":
				if !c.NextArg() {
					return rules, c.ArgErr()
				}
				rule.IndexFile = c.Val()
			}
		}

		rules = append(rules, rule)
	}

	return rules, nil
}

// preset configures rule according to name. It returns an error if
// name is not a recognized preset name.
func preset(name string, rule *Rule) error {
	switch name {
	case "php":
		rule.Ext = ".php"
		rule.SplitPath = ".php"
		rule.IndexFile = "index.php"
	default:
		return errors.New(name + " is not a valid preset name")
	}
	return nil
}

// Rule represents a FastCGI handling rule.
type Rule struct {
	// The base path to match. Required.
	Path string

	// The address of the FastCGI server. Required.
	Address string

	// Always process files with this extension with fastcgi.
	Ext string

	// The path in the URL will be split into two, with the first piece ending
	// with the value of SplitPath. The first piece will be assumed as the
	// actual resource (CGI script) name, and the second piece will be set to
	// PATH_INFO for the CGI script to use.
	SplitPath string

	// If the URL does not indicate a file, an index file with this name will be assumed.
	IndexFile string
}

var headerNameReplacer = strings.NewReplacer(" ", "_", "-", "_")
<commit_msg>adding support for fastcgi index files in subdirectories<commit_after>// Package fastcgi has middleware that acts as a FastCGI client. Requests
// that get forwarded to FastCGI stop the middleware execution chain.
// The most common use for this package is to serve PHP websites via php-fpm.
package fastcgi

import (
	"errors"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/mholt/caddy/middleware"
)

// New generates a new FastCGI middleware.
func New(c middleware.Controller) (middleware.Middleware, error) {
	root, err := filepath.Abs(c.Root())
	if err != nil {
		return nil, err
	}

	rules, err := parse(c)
	if err != nil {
		return nil, err
	}

	return func(next middleware.Handler) middleware.Handler {
		return Handler{
			Next:            next,
			Rules:           rules,
			Root:            root,
			SoftwareName:    "Caddy", // TODO: Once generators are not in the same pkg as handler, obtain this from some global const
			SoftwareVersion: "",      // TODO: Get this from some global const too
			// TODO: Set ServerName and ServerPort to correct values... (as user defined in config)
		}
	}, nil
}

// Handler is a middleware type that can handle requests as a FastCGI client.
type Handler struct {
	Next  middleware.Handler
	Root  string // must be absolute path to site root
	Rules []Rule

	// These are sent to CGI scripts in env variables
	SoftwareName    string
	SoftwareVersion string
	ServerName      string
	ServerPort      string
}

// ServeHTTP satisfies the middleware.Handler interface.
func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
	for _, rule := range h.Rules {
		// In addition to matching the path, a request must meet some
		// other criteria before being proxied as FastCGI. For example,
		// we probably want to exclude static assets (CSS, JS, images...)
		// but we also want to be flexible for the script we proxy to.

		path := r.URL.Path

		// These criteria work well in this order for PHP sites
		if middleware.Path(path).Matches(rule.Path) &&
			(path[len(path)-1] == '/' ||
				strings.HasSuffix(path, rule.Ext) ||
				!h.exists(path)) {

			if path[len(path)-1] == '/' && h.exists(path+rule.IndexFile) {
				// If index file in specified folder exists, send request to it
				path += rule.IndexFile
			}

			// Create environment for CGI script
			env, err := h.buildEnv(r, rule, path)
			if err != nil {
				return http.StatusInternalServerError, err
			}

			// Connect to FastCGI gateway
			fcgi, err := Dial("tcp", rule.Address)
			if err != nil {
				return http.StatusBadGateway, err
			}

			// TODO: Allow more methods (requires refactoring fcgiclient first...)
			var resp *http.Response
			switch r.Method {
			case "GET":
				resp, err = fcgi.Get(env)
			case "POST":
				l, _ := strconv.Atoi(r.Header.Get("Content-Length"))
				resp, err = fcgi.Post(env, r.Header.Get("Content-Type"), r.Body, l)
			default:
				return http.StatusMethodNotAllowed, nil
			}
			defer resp.Body.Close()

			if err != nil && err != io.EOF {
				return http.StatusBadGateway, err
			}

			// Write the response header
			for key, vals := range resp.Header {
				for _, val := range vals {
					w.Header().Add(key, val)
				}
			}
			w.WriteHeader(resp.StatusCode)

			// Write the response body
			_, err = io.Copy(w, resp.Body)
			if err != nil {
				return http.StatusBadGateway, err
			}

			return resp.StatusCode, nil
		}
	}

	return h.Next.ServeHTTP(w, r)
}

func (h Handler) exists(path string) bool {
	if _, err := os.Stat(h.Root + path); err == nil {
		return true
	}
	return false
}

func (h Handler) buildEnv(r *http.Request, rule Rule, path string) (map[string]string, error) {
	var env map[string]string

	// Get absolute path of requested resource
	absPath, err := filepath.Abs(h.Root + path)
	if err != nil {
		return env, err
	}

	// Separate remote IP and port; more lenient than net.SplitHostPort
	var ip, port string
	if idx := strings.Index(r.RemoteAddr, ":"); idx > -1 {
		ip = r.RemoteAddr[:idx]
		port = r.RemoteAddr[idx+1:]
	} else {
		ip = r.RemoteAddr
	}

	// Split path in preparation for env variables
	splitPos := strings.Index(path, rule.SplitPath)
	var docURI, scriptName, scriptFilename, pathInfo string
	if splitPos == -1 {
		// Request doesn't have the extension, so assume index file in root
		docURI = "/" + rule.IndexFile
		scriptName = "/" + rule.IndexFile
		scriptFilename = h.Root + "/" + rule.IndexFile
		pathInfo = path
	} else {
		// Request has the extension; path was split successfully
		docURI = path[:splitPos+len(rule.SplitPath)]
		pathInfo = path[splitPos+len(rule.SplitPath):]
		scriptName = path
		scriptFilename = absPath
	}

	// Some variables are unused but cleared explicitly to prevent
	// the parent environment from interfering.
	env = map[string]string{

		// Variables defined in CGI 1.1 spec
		"AUTH_TYPE":         "", // Not used
		"CONTENT_LENGTH":    r.Header.Get("Content-Length"),
		"CONTENT_TYPE":      r.Header.Get("Content-Type"),
		"GATEWAY_INTERFACE": "CGI/1.1",
		"PATH_INFO":         pathInfo,
		"PATH_TRANSLATED":   h.Root + "/" + pathInfo, // Source for path_translated: http://www.oreilly.com/openbook/cgi/ch02_04.html
		"QUERY_STRING":      r.URL.RawQuery,
		"REMOTE_ADDR":       ip,
		"REMOTE_HOST":       ip, // For speed, remote host lookups disabled
		"REMOTE_PORT":       port,
		"REMOTE_IDENT":      "", // Not used
		"REMOTE_USER":       "", // Not used
		"REQUEST_METHOD":    r.Method,
		"SERVER_NAME":       h.ServerName,
		"SERVER_PORT":       h.ServerPort,
		"SERVER_PROTOCOL":   r.Proto,
		"SERVER_SOFTWARE":   h.SoftwareName + "/" + h.SoftwareVersion,

		// Other variables
		"DOCUMENT_ROOT":   h.Root,
		"DOCUMENT_URI":    docURI,
		"HTTP_HOST":       r.Host, // added here, since not always part of headers
		"REQUEST_URI":     r.URL.RequestURI(),
		"SCRIPT_FILENAME": scriptFilename,
		"SCRIPT_NAME":     scriptName,
	}

	// Add all HTTP headers to env variables
	for field, val := range r.Header {
		header := strings.ToUpper(field)
		header = headerNameReplacer.Replace(header)
		env["HTTP_"+header] = strings.Join(val, ", ")
	}

	return env, nil
}

func parse(c middleware.Controller) ([]Rule, error) {
	var rules []Rule

	for c.Next() {
		var rule Rule

		args := c.RemainingArgs()

		switch len(args) {
		case 0:
			return rules, c.ArgErr()
		case 1:
			rule.Path = "/"
			rule.Address = args[0]
		case 2:
			rule.Path = args[0]
			rule.Address = args[1]
		case 3:
			rule.Path = args[0]
			rule.Address = args[1]
			err := preset(args[2], &rule)
			if err != nil {
				return rules, c.Err("Invalid fastcgi rule preset '" + args[2] + "'")
			}
		}

		for c.NextBlock() {
			switch c.Val() {
			case "ext":
				if !c.NextArg() {
					return rules, c.ArgErr()
				}
				rule.Ext = c.Val()
			case "split":
				if !c.NextArg() {
					return rules, c.ArgErr()
				}
				rule.SplitPath = c.Val()
			case "index":
				if !c.NextArg() {
					return rules, c.ArgErr()
				}
				rule.IndexFile = c.Val()
			}
		}

		rules = append(rules, rule)
	}

	return rules, nil
}

// preset configures rule according to name. It returns an error if
// name is not a recognized preset name.
func preset(name string, rule *Rule) error {
	switch name {
	case "php":
		rule.Ext = ".php"
		rule.SplitPath = ".php"
		rule.IndexFile = "index.php"
	default:
		return errors.New(name + " is not a valid preset name")
	}
	return nil
}

// Rule represents a FastCGI handling rule.
type Rule struct {
	// The base path to match. Required.
	Path string

	// The address of the FastCGI server. Required.
	Address string

	// Always process files with this extension with fastcgi.
	Ext string

	// The path in the URL will be split into two, with the first piece ending
	// with the value of SplitPath. The first piece will be assumed as the
	// actual resource (CGI script) name, and the second piece will be set to
	// PATH_INFO for the CGI script to use.
	SplitPath string

	// If the URL does not indicate a file, an index file with this name will be assumed.
	IndexFile string
}

var headerNameReplacer = strings.NewReplacer(" ", "_", "-", "_")
<|endoftext|>


<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package frontend

import (
	"encoding/hex"
	"fmt"
	"html/template"
	"net/http"

	"golang.org/x/net/context"

	"github.com/luci/luci-go/common/clock"
	"github.com/luci/luci-go/common/errors"
	"github.com/luci/luci-go/common/logging"
	"github.com/luci/luci-go/common/proto/google"
	"github.com/luci/luci-go/server/router"
	"github.com/luci/luci-go/server/templates"

	"github.com/luci/luci-go/milo/api/resp"
	"github.com/luci/luci-go/milo/buildsource"
	"github.com/luci/luci-go/milo/common"
	"github.com/luci/luci-go/milo/common/model"
	"github.com/luci/luci-go/milo/git"
)

// getConsoleDef finds the console definition as defined by any project.
// If the user is not a reader of the project, this will return a 404.
// TODO(hinoka): If the user is not a reader of any of of the builders returned,
// that builder will be removed from list of results.
func getConsoleDef(c context.Context, project, name string) (*common.Console, error) {
	cs, err := common.GetConsole(c, project, name)
	if err != nil {
		return nil, err
	}
	// TODO(hinoka): Remove builders that the user does not have access to.
	return cs, nil
}

func console(c context.Context, project, name string) (*resp.Console, error) {
	tStart := clock.Now(c)
	def, err := getConsoleDef(c, project, name)
	if err != nil {
		return nil, err
	}
	commitInfo, err := git.GetHistory(c, def.RepoURL, def.Ref, 25)
	if err != nil {
		return nil, err
	}
	tGitiles := clock.Now(c)
	logging.Debugf(c, "Loading commits took %s.", tGitiles.Sub(tStart))

	builderNames := make([]string, len(def.Builders))
	builders := make([]resp.BuilderRef, len(def.Builders))
	for i, b := range def.Builders {
		builderNames[i] = b
		builders[i].Name = b
		_, _, builders[i].ShortName, _ = buildsource.BuilderID(b).Split()
		// TODO(hinoka): Add Categories back in.
	}

	commitNames := make([]string, len(commitInfo.Commits))
	for i, commit := range commitInfo.Commits {
		commitNames[i] = hex.EncodeToString(commit.Hash)
	}
	rows, err := buildsource.GetConsoleRows(c, project, def, commitNames, builderNames)
	tConsole := clock.Now(c)
	logging.Debugf(c, "Loading the console took a total of %s.", tConsole.Sub(tGitiles))
	if err != nil {
		return nil, err
	}

	ccb := make([]resp.CommitBuild, len(commitInfo.Commits))
	for row, commit := range commitInfo.Commits {
		ccb[row].Build = make([]*model.BuildSummary, len(builders))
		ccb[row].Commit = resp.Commit{
			AuthorName:  commit.AuthorName,
			AuthorEmail: commit.AuthorEmail,
			CommitTime:  google.TimeFromProto(commit.CommitTime),
			Repo:        def.RepoURL,
			Branch:      def.Ref, // TODO(hinoka): Actually this doesn't match, change branch to ref.
			Description: commit.Msg,
			Revision:    resp.NewLink(commitNames[row], def.RepoURL+"/+/"+commitNames[row]),
		}

		for col, b := range builders {
			name := buildsource.BuilderID(b.Name)
			if summaries := rows[row].Builds[name]; len(summaries) > 0 {
				ccb[row].Build[col] = summaries[0]
			}
		}
	}

	return &resp.Console{
		Name:       def.ID,
		Commit:     ccb,
		BuilderRef: builders,
	}, nil
}

// consoleRenderer is a wrapper around Console to provide additional methods.
type consoleRenderer struct {
	*resp.Console
}

// Header generates the console header html.
func (c consoleRenderer) Header() template.HTML {
	// First, split things into nice rows and find the max depth.
	cat := make([][]string, len(c.BuilderRef))
	depth := 0
	for i, b := range c.BuilderRef {
		cat[i] = b.Category
		if len(cat[i]) > depth {
			depth = len(cat[i])
		}
	}

	result := ""
	for row := 0; row < depth; row++ {
		result += "
		"
		// "" is the first two nodes, " " is an empty node.
		current := ""
		colspan := 0
		for _, br := range cat {
			colspan++
			var s string
			if row >= len(br) {
				s = " "
			} else {
				s = br[row]
			}
			if s != current || current == " " {
				if current != "" || current == " " {
					result += fmt.Sprintf(`<th colspan="%d">%s</th>`, colspan, current)
					colspan = 0
				}
				current = s
			}
		}
		if colspan != 0 {
			result += fmt.Sprintf(`<th colspan="%d">%s</th>`, colspan, current)
		}
		result += "
"
	}

	// Last row: The actual builder shortnames.
	result += "
	"
	for _, br := range c.BuilderRef {
		result += fmt.Sprintf("%s	", br.ShortName)
	}
	result += "
"
	return template.HTML(result)
}

func (c consoleRenderer) BuilderLink(bs *model.BuildSummary) (*resp.Link, error) {
	_, _, builderName, err := buildsource.BuilderID(bs.BuilderID).Split()
	if err != nil {
		return nil, err
	}
	return resp.NewLink(builderName, "/"+bs.BuilderID), nil
}

// ConsoleHandler renders the console page.
func ConsoleHandler(c *router.Context) {
	project := c.Params.ByName("project")
	if project == "" {
		ErrorHandler(c, errors.New("Missing Project", common.CodeParameterError))
		return
	}
	name := c.Params.ByName("name")

	result, err := console(c.Context, project, name)
	if err != nil {
		ErrorHandler(c, err)
		return
	}

	templates.MustRender(c.Context, c.Writer, "pages/console.html", templates.Args{
		"Console": consoleRenderer{result},
	})
}

// ConsoleMainHandler is a redirect handler that redirects the user to the main
// console for a particular project.
func ConsoleMainHandler(ctx *router.Context) {
	w, r, p := ctx.Writer, ctx.Request, ctx.Params
	proj := p.ByName("project")
	http.Redirect(w, r, fmt.Sprintf("/console/%s/main", proj), http.StatusMovedPermanently)
	return
}
<commit_msg>Milo: Fix console table alignment<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package frontend

import (
	"encoding/hex"
	"fmt"
	"html/template"
	"net/http"

	"golang.org/x/net/context"

	"github.com/luci/luci-go/common/clock"
	"github.com/luci/luci-go/common/errors"
	"github.com/luci/luci-go/common/logging"
	"github.com/luci/luci-go/common/proto/google"
	"github.com/luci/luci-go/server/router"
	"github.com/luci/luci-go/server/templates"

	"github.com/luci/luci-go/milo/api/resp"
	"github.com/luci/luci-go/milo/buildsource"
	"github.com/luci/luci-go/milo/common"
	"github.com/luci/luci-go/milo/common/model"
	"github.com/luci/luci-go/milo/git"
)

// getConsoleDef finds the console definition as defined by any project.
// If the user is not a reader of the project, this will return a 404.
// TODO(hinoka): If the user is not a reader of any of of the builders returned,
// that builder will be removed from list of results.
func getConsoleDef(c context.Context, project, name string) (*common.Console, error) {
	cs, err := common.GetConsole(c, project, name)
	if err != nil {
		return nil, err
	}
	// TODO(hinoka): Remove builders that the user does not have access to.
	return cs, nil
}

func console(c context.Context, project, name string) (*resp.Console, error) {
	tStart := clock.Now(c)
	def, err := getConsoleDef(c, project, name)
	if err != nil {
		return nil, err
	}
	commitInfo, err := git.GetHistory(c, def.RepoURL, def.Ref, 25)
	if err != nil {
		return nil, err
	}
	tGitiles := clock.Now(c)
	logging.Debugf(c, "Loading commits took %s.", tGitiles.Sub(tStart))

	builderNames := make([]string, len(def.Builders))
	builders := make([]resp.BuilderRef, len(def.Builders))
	for i, b := range def.Builders {
		builderNames[i] = b
		builders[i].Name = b
		_, _, builders[i].ShortName, _ = buildsource.BuilderID(b).Split()
		// TODO(hinoka): Add Categories back in.
	}

	commitNames := make([]string, len(commitInfo.Commits))
	for i, commit := range commitInfo.Commits {
		commitNames[i] = hex.EncodeToString(commit.Hash)
	}
	rows, err := buildsource.GetConsoleRows(c, project, def, commitNames, builderNames)
	tConsole := clock.Now(c)
	logging.Debugf(c, "Loading the console took a total of %s.", tConsole.Sub(tGitiles))
	if err != nil {
		return nil, err
	}

	ccb := make([]resp.CommitBuild, len(commitInfo.Commits))
	for row, commit := range commitInfo.Commits {
		ccb[row].Build = make([]*model.BuildSummary, len(builders))
		ccb[row].Commit = resp.Commit{
			AuthorName:  commit.AuthorName,
			AuthorEmail: commit.AuthorEmail,
			CommitTime:  google.TimeFromProto(commit.CommitTime),
			Repo:        def.RepoURL,
			Branch:      def.Ref, // TODO(hinoka): Actually this doesn't match, change branch to ref.
			Description: commit.Msg,
			Revision:    resp.NewLink(commitNames[row], def.RepoURL+"/+/"+commitNames[row]),
		}

		for col, b := range builders {
			name := buildsource.BuilderID(b.Name)
			if summaries := rows[row].Builds[name]; len(summaries) > 0 {
				ccb[row].Build[col] = summaries[0]
			}
		}
	}

	return &resp.Console{
		Name:       def.ID,
		Commit:     ccb,
		BuilderRef: builders,
	}, nil
}

// consoleRenderer is a wrapper around Console to provide additional methods.
type consoleRenderer struct {
	*resp.Console
}

// Header generates the console header html.
func (c consoleRenderer) Header() template.HTML {
	// First, split things into nice rows and find the max depth.
	cat := make([][]string, len(c.BuilderRef))
	depth := 0
	for i, b := range c.BuilderRef {
		cat[i] = b.Category
		if len(cat[i]) > depth {
			depth = len(cat[i])
		}
	}

	result := ""
	for row := 0; row < depth; row++ {
		result += "
		"
		// "" is the first two nodes, " " is an empty node.
		current := ""
		colspan := 0
		for _, br := range cat {
			colspan++
			var s string
			if row >= len(br) {
				s = " "
			} else {
				s = br[row]
			}
			if s != current || current == " " {
				if current != "" || current == " " {
					result += fmt.Sprintf(`<th colspan="%d">%s</th>`, colspan, current)
					colspan = 0
				}
				current = s
			}
		}
		if colspan != 0 {
			result += fmt.Sprintf(`<th colspan="%d">%s</th>`, colspan, current)
		}
		result += "
"
	}

	// Last row: The actual builder shortnames.
	result += "
		"
	for _, br := range c.BuilderRef {
		result += fmt.Sprintf("%s	", br.ShortName)
	}
	result += "
"
	return template.HTML(result)
}

func (c consoleRenderer) BuilderLink(bs *model.BuildSummary) (*resp.Link, error) {
	_, _, builderName, err := buildsource.BuilderID(bs.BuilderID).Split()
	if err != nil {
		return nil, err
	}
	return resp.NewLink(builderName, "/"+bs.BuilderID), nil
}

// ConsoleHandler renders the console page.
func ConsoleHandler(c *router.Context) {
	project := c.Params.ByName("project")
	if project == "" {
		ErrorHandler(c, errors.New("Missing Project", common.CodeParameterError))
		return
	}
	name := c.Params.ByName("name")

	result, err := console(c.Context, project, name)
	if err != nil {
		ErrorHandler(c, err)
		return
	}

	templates.MustRender(c.Context, c.Writer, "pages/console.html", templates.Args{
		"Console": consoleRenderer{result},
	})
}

// ConsoleMainHandler is a redirect handler that redirects the user to the main
// console for a particular project.
func ConsoleMainHandler(ctx *router.Context) {
	w, r, p := ctx.Writer, ctx.Request, ctx.Params
	proj := p.ByName("project")
	http.Redirect(w, r, fmt.Sprintf("/console/%s/main", proj), http.StatusMovedPermanently)
	return
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*

Go Builder is a continuous build client for the Go project.
It integrates with the Go Dashboard AppEngine application.

Go Builder is intended to run continuously as a background process.

It periodically pulls updates from the Go Mercurial repository.

When a newer revision is found, Go Builder creates a clone of the repository,
runs all.bash, and reports build success or failure to the Go Dashboard.

For a successful build, Go Builder will also run benchmarks
(cd $GOROOT/src/pkg; make bench) and send the results to the Go Dashboard.

For a release revision (a change description that matches "release.YYYY-MM-DD"),
Go Builder will create a tar.gz archive of the GOROOT and deliver it to the
Go Google Code project's downloads section.

Usage:

  gobuilder goos-goarch...

  Several goos-goarch combinations can be provided, and the builder will
  build them in serial.

Optional flags:

  -dashboard="godashboard.appspot.com": Go Dashboard Host
    The location of the Go Dashboard application to which Go Builder will
    report its results.

  -bench: Run benchmarks

  -release: Build and deliver binary release archive

  -rev=N: Build revision N and exit

  -cmd="./all.bash": Build command (specify absolute or relative to go/src)

  -v: Verbose logging

  -external: External package builder mode (will not report Go build
     state to dashboard, issue releases, or run benchmarks)

The key file should be located at $HOME/.gobuilder or, for a builder-specific
key, $HOME/.gobuilder-$BUILDER (eg, $HOME/.gobuilder-linux-amd64).

The build key file is a text file of the format:

  godashboard-key
  googlecode-username
  googlecode-password

If the Google Code credentials are not provided the archival step
will be skipped.

*/
package documentation
<commit_msg>builder: fix documentation s/\.gobuilder/.gobuildkey/<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*

Go Builder is a continuous build client for the Go project.
It integrates with the Go Dashboard AppEngine application.

Go Builder is intended to run continuously as a background process.

It periodically pulls updates from the Go Mercurial repository.

When a newer revision is found, Go Builder creates a clone of the repository,
runs all.bash, and reports build success or failure to the Go Dashboard.

For a successful build, Go Builder will also run benchmarks
(cd $GOROOT/src/pkg; make bench) and send the results to the Go Dashboard.

For a release revision (a change description that matches "release.YYYY-MM-DD"),
Go Builder will create a tar.gz archive of the GOROOT and deliver it to the
Go Google Code project's downloads section.

Usage:

  gobuilder goos-goarch...

  Several goos-goarch combinations can be provided, and the builder will
  build them in serial.

Optional flags:

  -dashboard="godashboard.appspot.com": Go Dashboard Host
    The location of the Go Dashboard application to which Go Builder will
    report its results.

  -bench: Run benchmarks

  -release: Build and deliver binary release archive

  -rev=N: Build revision N and exit

  -cmd="./all.bash": Build command (specify absolute or relative to go/src)

  -v: Verbose logging

  -external: External package builder mode (will not report Go build
     state to dashboard, issue releases, or run benchmarks)

The key file should be located at $HOME/.gobuildkey or, for a builder-specific
key, $HOME/.gobuildkey-$BUILDER (eg, $HOME/.gobuildkey-linux-amd64).

The build key file is a text file of the format:

  godashboard-key
  googlecode-username
  googlecode-password

If the Google Code credentials are not provided the archival step
will be skipped.

*/
package documentation
<|endoftext|>


<commit_before>package models

import (
	"encoding/json"
	"github.com/APTrust/exchange/constants"
	"strings"
	"time"
)

/*
IntellectualObject is Fluctus' version of an IntellectualObject.
It belongs to an Institution and consists of one or more
GenericFiles and a number of events.
Institution is the owner of the intellectual object.
Title is the title.
Description is a free-text description of the object.
Identifier is the object's unique identifier. (Whose assigned
this id? APTrust or the owner?)
Access indicate who can access the object. Valid values are
consortial, institution and restricted.
*/
type IntellectualObject struct {
	// If Id is non-zero, this has been recorded in Pharos.
	Id                     int            `json:"id"`
	Identifier             string         `json:"identifier"`
	BagName                string         `json:"bag_name"`
	Institution            string         `json:"institution"`
	InstitutionId          int            `json:"institution_id"`
	Title                  string         `json:"title"`
	Description            string         `json:"description"`
	Access                 string         `json:"access"`
	AltIdentifier          string         `json:"alt_identifier"`
	GenericFiles           []*GenericFile `json:"generic_files"`
	PremisEvents           []*PremisEvent `json:"events"`
	CreatedAt              time.Time      `json:"created_at"`
	UpdatedAt              time.Time      `json:"updated_at"`

	// Exchange fields. These do not go to Pharos.
	IngestS3Bucket         string         `json:"ingest_s3_bucket"`
	IngestS3Key            string         `json:"ingest_s3_key"`
	IngestTarFilePath      string         `json:"ingest_tar_file_path"`
	IngestUntarredPath     string         `json:"ingest_untarred_path"`
	IngestRemoteMd5        string         `json:"ingest_remote_md5"`
	IngestLocalMd5         string         `json:"ingest_local_md5"`
	IngestMd5Verified      bool           `json:"ingest_md5_verified"`
	IngestMd5Verifiable    bool           `json:"ingest_md5_verifiable"`
	IngestManifests        []string       `json:"ingest_manifests"`
	IngestTagManifests     []string       `json:"ingest_tag_manifests"`
	IngestFilesIgnored     []string       `json:"ingest_files_ignored"`
	IngestTags             []*Tag         `json:"ingest_tags"`
	IngestMissingFiles     []*MissingFile `json:"ingest_missing_files"`
	IngestTopLevelDirNames []string     `json:"ingest_top_level_dir_names"`
	IngestErrorMessage     string         `json:"ingest_error_message"`

	genericFileMap         map[string]*GenericFile
	tagMap                 map[string][]*Tag
}

func NewIntellectualObject() (*IntellectualObject) {
	return &IntellectualObject{
		GenericFiles: make([]*GenericFile, 0),
		PremisEvents: make([]*PremisEvent, 0),
		IngestManifests: make([]string, 0),
		IngestTagManifests: make([]string, 0),
		IngestFilesIgnored: make([]string, 0),
		IngestTags: make([]*Tag, 0),
	}
}

// MissingFile defines a file that is not in the bag, despite the
// fact that its checksum was found in a manifest.
type MissingFile struct {
	Manifest    string
	LineNumber  int
	FilePath    string
	Digest      string
}

func NewMissingFile(manifest string, lineNumber int, filePath, digest string) (*MissingFile) {
	return &MissingFile{
		Manifest: manifest,
		LineNumber: lineNumber,
		FilePath: filePath,
		Digest: digest,
	}
}

// This Tag struct is essentially the same as the bagins
// TagField struct, but its properties are public and can
// be easily serialized to / deserialized from JSON.
type Tag struct {
	SourceFile string
	Label      string
	Value      string
}

func NewTag(sourceFile, label, value string) (*Tag) {
	return &Tag{
		SourceFile: sourceFile,
		Label: label,
		Value: value,
	}
}

// Returns the total number of bytes of all of the generic
// files in this object. The object's bag size will be slightly
// larger than this, because it will include a manifest, tag
// files and tar header.
func (obj *IntellectualObject) TotalFileSize() (int64) {
	total := int64(0)
	for _, genericFile := range obj.GenericFiles {
		total += genericFile.Size
	}
	return total
}

// AccessValid returns true or false to indicate whether the
// structure's Access property contains a valid value.
func (obj *IntellectualObject) AccessValid() bool {
	lcAccess := strings.ToLower(obj.Access)
	for _, value := range constants.AccessRights {
		if value == lcAccess {
			return true
		}
	}
	return false
}

// Returns the GenericFile record for the specified path, or nil.
// Param filePath should be the relative path of the file within
// the bag. E.g. "data/images/myphoto.jpg"
func (obj *IntellectualObject) FindGenericFile(filePath string) (*GenericFile) {
	if obj.genericFileMap == nil || len(obj.genericFileMap) != len(obj.GenericFiles) {
		obj.genericFileMap = make(map[string]*GenericFile, len(obj.GenericFiles))
		for i := range obj.GenericFiles {
			gf := obj.GenericFiles[i]
			obj.genericFileMap[gf.OriginalPath()] = gf
		}
	}
	return obj.genericFileMap[filePath]
}

// Returns the tag with the specified name, or nil. The bag spec at
// https://tools.ietf.org/html/draft-kunze-bagit-13#section-2.2.2
// says tags may be repeated, and their order must be preserved,
// so this returns a slice of tags if the tag is found. In most
// cases, the slice will contain one element.
func (obj *IntellectualObject) FindTag(tagName string) ([]*Tag) {
	if obj.tagMap == nil {
		obj.tagMap = make(map[string][]*Tag)
		for i := range obj.IngestTags {
			tag := obj.IngestTags[i]
			if obj.tagMap[tag.Label] == nil {
				obj.tagMap[tag.Label] = make([]*Tag, 0)
			}
			obj.tagMap[tag.Label] = append(obj.tagMap[tag.Label], tag)
		}
	}
	return obj.tagMap[tagName]
}

// Serialize the subset of IntellectualObject data that Pharos
// will accept. This is for post/put, where essential info, such
// as institution id and/or object id will be in the URL.
func (obj *IntellectualObject) SerializeForPharos() ([]byte, error) {
	return json.Marshal(map[string]interface{}{
		"identifier":     obj.Identifier,
		"bag_name":       obj.BagName,
		"institution":    obj.Institution,
		"title":          obj.Title,
		"description":    obj.Description,
		"alt_identifier": obj.AltIdentifier,
		"access":         obj.Access,
	})
}
<commit_msg>Clarified comment<commit_after>package models

import (
	"encoding/json"
	"github.com/APTrust/exchange/constants"
	"strings"
	"time"
)

/*
IntellectualObject is Fluctus' version of an IntellectualObject.
It belongs to an Institution and consists of one or more
GenericFiles and a number of events.
Institution is the owner of the intellectual object.
Title is the title.
Description is a free-text description of the object.
Identifier is the object's unique identifier. (Whose assigned
this id? APTrust or the owner?)
Access indicate who can access the object. Valid values are
consortial, institution and restricted.
*/
type IntellectualObject struct {
	// If Id is non-zero, this has been recorded in Pharos.
	Id                     int            `json:"id"`
	Identifier             string         `json:"identifier"`
	BagName                string         `json:"bag_name"`
	Institution            string         `json:"institution"`
	InstitutionId          int            `json:"institution_id"`
	Title                  string         `json:"title"`
	Description            string         `json:"description"`
	Access                 string         `json:"access"`
	AltIdentifier          string         `json:"alt_identifier"`
	GenericFiles           []*GenericFile `json:"generic_files"`
	PremisEvents           []*PremisEvent `json:"events"`
	CreatedAt              time.Time      `json:"created_at"`
	UpdatedAt              time.Time      `json:"updated_at"`

	// The following are fields populated and used by the Exchange
	// ingest code on ingest only. These fields are not stored in
	// Pharos, and will not be populated on any IntellectualObject
	// retrieved from Pharos.
	IngestS3Bucket         string         `json:"ingest_s3_bucket"`
	IngestS3Key            string         `json:"ingest_s3_key"`
	IngestTarFilePath      string         `json:"ingest_tar_file_path"`
	IngestUntarredPath     string         `json:"ingest_untarred_path"`
	IngestRemoteMd5        string         `json:"ingest_remote_md5"`
	IngestLocalMd5         string         `json:"ingest_local_md5"`
	IngestMd5Verified      bool           `json:"ingest_md5_verified"`
	IngestMd5Verifiable    bool           `json:"ingest_md5_verifiable"`
	IngestManifests        []string       `json:"ingest_manifests"`
	IngestTagManifests     []string       `json:"ingest_tag_manifests"`
	IngestFilesIgnored     []string       `json:"ingest_files_ignored"`
	IngestTags             []*Tag         `json:"ingest_tags"`
	IngestMissingFiles     []*MissingFile `json:"ingest_missing_files"`
	IngestTopLevelDirNames []string       `json:"ingest_top_level_dir_names"`
	IngestErrorMessage     string         `json:"ingest_error_message"`

	genericFileMap         map[string]*GenericFile
	tagMap                 map[string][]*Tag
}

func NewIntellectualObject() (*IntellectualObject) {
	return &IntellectualObject{
		GenericFiles: make([]*GenericFile, 0),
		PremisEvents: make([]*PremisEvent, 0),
		IngestManifests: make([]string, 0),
		IngestTagManifests: make([]string, 0),
		IngestFilesIgnored: make([]string, 0),
		IngestTags: make([]*Tag, 0),
	}
}

// MissingFile defines a file that is not in the bag, despite the
// fact that its checksum was found in a manifest.
type MissingFile struct {
	Manifest    string
	LineNumber  int
	FilePath    string
	Digest      string
}

func NewMissingFile(manifest string, lineNumber int, filePath, digest string) (*MissingFile) {
	return &MissingFile{
		Manifest: manifest,
		LineNumber: lineNumber,
		FilePath: filePath,
		Digest: digest,
	}
}

// This Tag struct is essentially the same as the bagins
// TagField struct, but its properties are public and can
// be easily serialized to / deserialized from JSON.
type Tag struct {
	SourceFile string
	Label      string
	Value      string
}

func NewTag(sourceFile, label, value string) (*Tag) {
	return &Tag{
		SourceFile: sourceFile,
		Label: label,
		Value: value,
	}
}

// Returns the total number of bytes of all of the generic
// files in this object. The object's bag size will be slightly
// larger than this, because it will include a manifest, tag
// files and tar header.
func (obj *IntellectualObject) TotalFileSize() (int64) {
	total := int64(0)
	for _, genericFile := range obj.GenericFiles {
		total += genericFile.Size
	}
	return total
}

// AccessValid returns true or false to indicate whether the
// structure's Access property contains a valid value.
func (obj *IntellectualObject) AccessValid() bool {
	lcAccess := strings.ToLower(obj.Access)
	for _, value := range constants.AccessRights {
		if value == lcAccess {
			return true
		}
	}
	return false
}

// Returns the GenericFile record for the specified path, or nil.
// Param filePath should be the relative path of the file within
// the bag. E.g. "data/images/myphoto.jpg"
func (obj *IntellectualObject) FindGenericFile(filePath string) (*GenericFile) {
	if obj.genericFileMap == nil || len(obj.genericFileMap) != len(obj.GenericFiles) {
		obj.genericFileMap = make(map[string]*GenericFile, len(obj.GenericFiles))
		for i := range obj.GenericFiles {
			gf := obj.GenericFiles[i]
			obj.genericFileMap[gf.OriginalPath()] = gf
		}
	}
	return obj.genericFileMap[filePath]
}

// Returns the tag with the specified name, or nil. The bag spec at
// https://tools.ietf.org/html/draft-kunze-bagit-13#section-2.2.2
// says tags may be repeated, and their order must be preserved,
// so this returns a slice of tags if the tag is found. In most
// cases, the slice will contain one element.
func (obj *IntellectualObject) FindTag(tagName string) ([]*Tag) {
	if obj.tagMap == nil {
		obj.tagMap = make(map[string][]*Tag)
		for i := range obj.IngestTags {
			tag := obj.IngestTags[i]
			if obj.tagMap[tag.Label] == nil {
				obj.tagMap[tag.Label] = make([]*Tag, 0)
			}
			obj.tagMap[tag.Label] = append(obj.tagMap[tag.Label], tag)
		}
	}
	return obj.tagMap[tagName]
}

// Serialize the subset of IntellectualObject data that Pharos
// will accept. This is for post/put, where essential info, such
// as institution id and/or object id will be in the URL.
func (obj *IntellectualObject) SerializeForPharos() ([]byte, error) {
	return json.Marshal(map[string]interface{}{
		"identifier":     obj.Identifier,
		"bag_name":       obj.BagName,
		"institution":    obj.Institution,
		"title":          obj.Title,
		"description":    obj.Description,
		"alt_identifier": obj.AltIdentifier,
		"access":         obj.Access,
	})
}
<|endoftext|>


<commit_before>package eventhandler

import (
	. "../definitions"
	"../driver"
	"../fsm"
	. "../network"
	"../queue"
	"fmt"
	"time"
)

func MessageTypeHandler(messageReciveChan chan Message, floorChan chan int, buttonChan chan Order) {

	for {
		msg := <-messageReciveChan
		switch msg.MessageType {

		case "Remove order up":
			fmt.Println("In MessageTypeHandler, Remove order up")
			if msg.SenderIP != GetLocalIP() {
				updateElevatorStatus(msg.SenderIP, msg.Elevator)
				queue.RemoveRemoteOrder(msg.Elevator.Floor, UP)
			}

		case "Remove order down":
			if msg.SenderIP != GetLocalIP() {
				updateElevatorStatus(msg.SenderIP, msg.Elevator)
				queue.RemoveRemoteOrder(msg.Elevator.Floor, DOWN)
			}

		case "Add order":
			fmt.Println("In MessageTypeHandler, Add order")
			if msg.SenderIP != GetLocalIP() {
				if msg.Order.Buttontype == UP {
					Elevators[msg.TargetIP].ExternalUp[msg.Order.Floor] = true
					driver.SetButtonLamp(msg.Order.Floor, UP, true)
				} else if msg.Order.Buttontype == DOWN {
					Elevators[msg.TargetIP].ExternalDown[msg.Order.Floor] = true
					driver.SetButtonLamp(msg.Order.Floor, DOWN, true)
				} else {
					Elevators[msg.TargetIP].InternalOrders[msg.Order.Floor] = true
				}
			}

			//Den Originale
			/*if (msg.SenderIP != msg.TargetIP) && (msg.SenderIP != GetLocalIP()) {
				queue.AddRemoteOrder(msg.TargetIP, msg.Elevator, msg.Order)
			}
			if msg.SenderIP != GetLocalIP() {
				updateElevatorStatus(msg.SenderIP, msg.Elevator)
			}
			if msg.SenderIP == GetLocalIP() {
				msg.Order.FromIP = GetLocalIP()
			}
			if msg.TargetIP == GetLocalIP() {
				buttonChan <- msg.Order
			}*/

			/*if msg.SenderIP != GetLocalIP() {
				fmt.Println("If this prints, its wrong.")
				if msg.SenderIP != msg.TargetIP {
					if msg.TargetIP != GetLocalIP() {
						queue.AddRemoteOrder(msg.TargetIP, msg.Elevator, msg.Order)
					} else {
						buttonChan <- msg.Order
					}
				} else {
					updateElevatorStatus(msg.SenderIP, msg.Elevator)
				}

			}
			*/
		}
	}
}

func updateElevatorStatus(MessageIP string, elevator Elevator) {
	Elevators[MessageIP].Active = elevator.Active
	Elevators[MessageIP].Floor = elevator.Floor
	Elevators[MessageIP].Direction = elevator.Direction
	Elevators[MessageIP].FsmState = elevator.FsmState

	for i := 0; i < driver.N_FLOORS; i++ {
		Elevators[MessageIP].InternalOrders[i] = elevator.InternalOrders[i]
		Elevators[MessageIP].ExternalUp[i] = elevator.ExternalUp[i]
		Elevators[MessageIP].ExternalDown[i] = elevator.ExternalDown[i]
	}
}

func HeartbeatEventHandler(newElevatorChan chan string, deadElevatorChan chan string) {
	for {
		select {
		case IP := <-newElevatorChan:
			fmt.Println("A new Elevator online:", IP)
			_, exist := Elevators[IP]
			if exist {
				Elevators[IP].Active = true
			} else {
				fmt.Println("Meeting new elevator")
				Elevators[IP] = &Elevator{true, -1, 0, IDLE, [driver.N_FLOORS]bool{false, false, false, false}, [driver.N_FLOORS]bool{false, false, false, false}, [driver.N_FLOORS]bool{false, false, false, false}}
			}
		case IP := <-deadElevatorChan:
			fmt.Println("We have lost an elevator:", IP)
			Elevators[IP].Active = false
		}
	}
}

func ButtonandFloorEventHandler(floorChan chan int, buttonChan chan Order) {

	go FloorEventCheck(floorChan)
	go ButtonEventCheck(buttonChan)
	PrevDirection := -1

	for {
		select {

		case floor := <-floorChan:
			dir := Elevators[GetLocalIP()].Direction
			if floor != -1 {
				Elevators[GetLocalIP()].Floor = floor
				if queue.ShouldStop(floor, dir) {
					queue.RemoveOrder(floor, PrevDirection)
					fsm.GoToDoorOpen()
				}
			} else {
				PrevDirection = dir
			}

		case order := <-buttonChan:
			queue.AddLocalOrder(order)
			if (Elevators[GetLocalIP()].Direction) != queue.QueueDirection(PrevDirection, Elevators[GetLocalIP()].Floor) {
				Elevators[GetLocalIP()].Direction = queue.QueueDirection(PrevDirection, Elevators[GetLocalIP()].Floor)
				if Elevators[GetLocalIP()].Direction != 0 {
					fsm.GoToElevating(Elevators[GetLocalIP()].Direction)
				}
			}
		default:
			switch Elevators[GetLocalIP()].FsmState {
			case IDLE:
				direction := queue.QueueDirection(PrevDirection, Elevators[GetLocalIP()].Floor)
				if direction == 0 && queue.EmptyQueue() {
					fsm.GoToIDLE()
				} else if direction == 0 && !queue.EmptyQueue() {
					fsm.GoToDoorOpen()
					queue.RemoveOrder(Elevators[GetLocalIP()].Floor, 0)
				} else {
					fsm.GoToElevating(direction)
				}

			}

		}

	}
}

func FloorEventCheck(event chan int) {
	prevFloor := 0
	for {
		newFloor := driver.GetFloorSignal()
		if newFloor != prevFloor {
			prevFloor = newFloor
			event <- newFloor
		}
		time.Sleep(200 * time.Millisecond)
	}
}

func ButtonEventCheck(buttonChan chan Order) {

	//We might not need this, we could change it to check the elevator[IP].queue instead.
	buttonPressed := make([][]bool, driver.N_FLOORS) //Makes row
	for i := range buttonPressed {
		buttonPressed[i] = make([]bool, driver.N_BUTTONS) //Makes column
	}

	for {
		var order Order
		for floor := 0; floor < driver.N_FLOORS; floor++ {
			for buttonType := 0; buttonType < driver.N_BUTTONS; buttonType++ {
				if driver.ElevGetButtonSignal(buttonType, floor) == 1 && !buttonPressed[floor][buttonType] {
					buttonPressed[floor][buttonType] = true
					order.Buttontype = buttonType
					order.Floor = floor
					order.FromIP = ""
					buttonChan <- order
				} else if buttonPressed[floor][buttonType] {
					buttonPressed[floor][buttonType] = false
				}
			}
		}
		time.Sleep(200 * time.Millisecond)
	}
}

/*func ElevateTowardNextDirection() {
	if Elevators[GetLocalIP()].Direction != queue.QueueDirection(PrevDirection, Elevators[GetLocalIP()].Floor) {
		Elevators[GetLocalIP()].Direction = queue.QueueDirection(PrevDirection, Elevators[GetLocalIP()].Floor)
		if Elevators[GetLocalIP()].Direction != 0 {
			fsm.GoToElevating(Elevators[GetLocalIP()].Direction)
		}
	}
}

/*else {
				msg.Order.FromIP = GetLocalIP()
				if msg.TargetIP == GetLocalIP() {
					buttonChan <- msg.Order
				}

*/
<commit_msg>Fixed remove order from other elevators<commit_after>package eventhandler

import (
	. "../definitions"
	"../driver"
	"../fsm"
	. "../network"
	"../queue"
	"fmt"
	"time"
)

func MessageTypeHandler(messageReciveChan chan Message, floorChan chan int, buttonChan chan Order) {

	for {
		msg := <-messageReciveChan
		switch msg.MessageType {

		case "Remove order up":
			fmt.Println("In MessageTypeHandler, Remove order up")
			if msg.SenderIP != GetLocalIP() {
				driver.SetButtonLamp(msg.Elevator.Floor, UP, false)
				Elevators[msg.SenderIP].ExternalUp[msg.Elevator.Floor] = false
				Elevators[msg.SenderIP].Floor = msg.Elevator.Floor
				Elevators[msg.SenderIP].Direction = 1

				//Originalen
				/*updateElevatorStatus(msg.SenderIP, msg.Elevator)
				queue.RemoveRemoteOrder(msg.Elevator.Floor, UP)*/
			}

		case "Remove order down":
			if msg.SenderIP != GetLocalIP() {
				driver.SetButtonLamp(msg.Elevator.Floor, DOWN, false)
				Elevators[msg.SenderIP].ExternalDown[msg.Elevator.Floor] = false
				Elevators[msg.SenderIP].Floor = msg.Elevator.Floor
				Elevators[msg.SenderIP].Direction = -1
				//Original
				/*updateElevatorStatus(msg.SenderIP, msg.Elevator)
				queue.RemoveRemoteOrder(msg.Elevator.Floor, DOWN)*/
			}

		case "Add order":
			fmt.Println("In MessageTypeHandler, Add order")
			if msg.SenderIP != GetLocalIP() {
				if msg.Order.Buttontype == UP {
					Elevators[msg.TargetIP].ExternalUp[msg.Order.Floor] = true
					driver.SetButtonLamp(msg.Order.Floor, UP, true)
				} else if msg.Order.Buttontype == DOWN {
					Elevators[msg.TargetIP].ExternalDown[msg.Order.Floor] = true
					driver.SetButtonLamp(msg.Order.Floor, DOWN, true)
				} else {
					Elevators[msg.TargetIP].InternalOrders[msg.Order.Floor] = true
				}
			}

			//Den Originale
			/*if (msg.SenderIP != msg.TargetIP) && (msg.SenderIP != GetLocalIP()) {
				queue.AddRemoteOrder(msg.TargetIP, msg.Elevator, msg.Order)
			}
			if msg.SenderIP != GetLocalIP() {
				updateElevatorStatus(msg.SenderIP, msg.Elevator)
			}
			if msg.SenderIP == GetLocalIP() {
				msg.Order.FromIP = GetLocalIP()
			}
			if msg.TargetIP == GetLocalIP() {
				buttonChan <- msg.Order
			}*/

			/*if msg.SenderIP != GetLocalIP() {
				fmt.Println("If this prints, its wrong.")
				if msg.SenderIP != msg.TargetIP {
					if msg.TargetIP != GetLocalIP() {
						queue.AddRemoteOrder(msg.TargetIP, msg.Elevator, msg.Order)
					} else {
						buttonChan <- msg.Order
					}
				} else {
					updateElevatorStatus(msg.SenderIP, msg.Elevator)
				}

			}
			*/
		}
	}
}

func updateElevatorStatus(MessageIP string, elevator Elevator) {
	Elevators[MessageIP].Active = elevator.Active
	Elevators[MessageIP].Floor = elevator.Floor
	Elevators[MessageIP].Direction = elevator.Direction
	Elevators[MessageIP].FsmState = elevator.FsmState

	for i := 0; i < driver.N_FLOORS; i++ {
		Elevators[MessageIP].InternalOrders[i] = elevator.InternalOrders[i]
		Elevators[MessageIP].ExternalUp[i] = elevator.ExternalUp[i]
		Elevators[MessageIP].ExternalDown[i] = elevator.ExternalDown[i]
	}
}

func HeartbeatEventHandler(newElevatorChan chan string, deadElevatorChan chan string) {
	for {
		select {
		case IP := <-newElevatorChan:
			fmt.Println("A new Elevator online:", IP)
			_, exist := Elevators[IP]
			if exist {
				Elevators[IP].Active = true
			} else {
				fmt.Println("Meeting new elevator")
				Elevators[IP] = &Elevator{true, -1, 0, IDLE, [driver.N_FLOORS]bool{false, false, false, false}, [driver.N_FLOORS]bool{false, false, false, false}, [driver.N_FLOORS]bool{false, false, false, false}}
			}
		case IP := <-deadElevatorChan:
			fmt.Println("We have lost an elevator:", IP)
			Elevators[IP].Active = false
		}
	}
}

func ButtonandFloorEventHandler(floorChan chan int, buttonChan chan Order) {

	go FloorEventCheck(floorChan)
	go ButtonEventCheck(buttonChan)
	PrevDirection := -1

	for {
		select {

		case floor := <-floorChan:
			dir := Elevators[GetLocalIP()].Direction
			if floor != -1 {
				Elevators[GetLocalIP()].Floor = floor
				if queue.ShouldStop(floor, dir) {
					queue.RemoveOrder(floor, PrevDirection)
					fsm.GoToDoorOpen()
				}
			} else {
				PrevDirection = dir
			}

		case order := <-buttonChan:
			queue.AddLocalOrder(order)
			if (Elevators[GetLocalIP()].Direction) != queue.QueueDirection(PrevDirection, Elevators[GetLocalIP()].Floor) {
				Elevators[GetLocalIP()].Direction = queue.QueueDirection(PrevDirection, Elevators[GetLocalIP()].Floor)
				if Elevators[GetLocalIP()].Direction != 0 {
					fsm.GoToElevating(Elevators[GetLocalIP()].Direction)
				}
			}
		default:
			switch Elevators[GetLocalIP()].FsmState {
			case IDLE:
				direction := queue.QueueDirection(PrevDirection, Elevators[GetLocalIP()].Floor)
				if direction == 0 && queue.EmptyQueue() {
					fsm.GoToIDLE()
				} else if direction == 0 && !queue.EmptyQueue() {
					fsm.GoToDoorOpen()
					queue.RemoveOrder(Elevators[GetLocalIP()].Floor, 0)
				} else {
					fsm.GoToElevating(direction)
				}

			}

		}

	}
}

func FloorEventCheck(event chan int) {
	prevFloor := 0
	for {
		newFloor := driver.GetFloorSignal()
		if newFloor != prevFloor {
			prevFloor = newFloor
			event <- newFloor
		}
		time.Sleep(200 * time.Millisecond)
	}
}

func ButtonEventCheck(buttonChan chan Order) {

	//We might not need this, we could change it to check the elevator[IP].queue instead.
	buttonPressed := make([][]bool, driver.N_FLOORS) //Makes row
	for i := range buttonPressed {
		buttonPressed[i] = make([]bool, driver.N_BUTTONS) //Makes column
	}

	for {
		var order Order
		for floor := 0; floor < driver.N_FLOORS; floor++ {
			for buttonType := 0; buttonType < driver.N_BUTTONS; buttonType++ {
				if driver.ElevGetButtonSignal(buttonType, floor) == 1 && !buttonPressed[floor][buttonType] {
					buttonPressed[floor][buttonType] = true
					order.Buttontype = buttonType
					order.Floor = floor
					order.FromIP = ""
					buttonChan <- order
				} else if buttonPressed[floor][buttonType] {
					buttonPressed[floor][buttonType] = false
				}
			}
		}
		time.Sleep(200 * time.Millisecond)
	}
}

/*func ElevateTowardNextDirection() {
	if Elevators[GetLocalIP()].Direction != queue.QueueDirection(PrevDirection, Elevators[GetLocalIP()].Floor) {
		Elevators[GetLocalIP()].Direction = queue.QueueDirection(PrevDirection, Elevators[GetLocalIP()].Floor)
		if Elevators[GetLocalIP()].Direction != 0 {
			fsm.GoToElevating(Elevators[GetLocalIP()].Direction)
		}
	}
}

/*else {
				msg.Order.FromIP = GetLocalIP()
				if msg.TargetIP == GetLocalIP() {
					buttonChan <- msg.Order
				}

*/
<|endoftext|>


<commit_before>package gateway

import (
	"net"
	"strconv"
	"testing"
	"time"

	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/inconshreveable/muxado"
)

func TestAddPeer(t *testing.T) {
	g := newTestingGateway("TestAddPeer", t)
	defer g.Close()
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	g.addPeer(&peer{addr: "foo", sess: muxado.Client(nil)})
	if len(g.peers) != 1 {
		t.Fatal("gateway did not add peer")
	}
}

func TestListen(t *testing.T) {
	g := newTestingGateway("TestListen", t)
	defer g.Close()

	// "compliant" connect
	conn, err := net.Dial("tcp", string(g.Address()))
	if err != nil {
		t.Fatal("dial failed:", err)
	}
	addr := modules.NetAddress(conn.LocalAddr().String())
	// send version
	if err := encoding.WriteObject(conn, version); err != nil {
		t.Fatal("couldn't write version")
	}
	// read ack
	var ack string
	if err := encoding.ReadObject(conn, &ack, maxAddrLength); err != nil {
		t.Fatal(err)
	} else if ack != "accept" {
		t.Fatal("gateway should have given ack")
	}

	// g should add foo
	var ok bool
	for !ok {
		id := g.mu.RLock()
		_, ok = g.peers[addr]
		g.mu.RUnlock(id)
	}

	conn.Close()

	// g should remove foo
	for ok {
		id := g.mu.RLock()
		_, ok = g.peers[addr]
		g.mu.RUnlock(id)
	}

	// "uncompliant" connect
	conn, err = net.Dial("tcp", string(g.Address()))
	if err != nil {
		t.Fatal("dial failed:", err)
	}
	if _, err := conn.Write([]byte("missing length prefix")); err != nil {
		t.Fatal("couldn't write malformed header")
	}
	// g should have closed the connection
	if n, err := conn.Write([]byte("closed")); err != nil && n > 0 {
		t.Error("write succeeded after closed connection")
	}
}

func TestConnect(t *testing.T) {
	// create bootstrap peer
	bootstrap := newTestingGateway("TestConnect1", t)
	defer bootstrap.Close()

	// give it a node
	bootstrap.addNode("foo")

	// create peer who will connect to bootstrap
	g := newTestingGateway("TestConnect2", t)
	defer g.Close()

	// first simulate a "bad" connect, where bootstrap won't share its nodes
	bootstrap.RegisterRPC("ShareNodes", func(modules.PeerConn) error {
		return nil
	})
	// connect
	err := g.Connect(bootstrap.Address())
	if err != nil {
		t.Fatal(err)
	}
	// g should not have foo
	if g.removeNode("foo") == nil {
		t.Fatal("bootstrapper should not have received foo:", g.nodes)
	}

	// split 'em up
	g.Disconnect(bootstrap.Address())
	bootstrap.Disconnect(g.Address())

	// now restore the correct ShareNodes RPC and try again
	bootstrap.RegisterRPC("ShareNodes", bootstrap.shareNodes)
	err = g.Connect(bootstrap.Address())
	if err != nil {
		t.Fatal(err)
	}
	// g should have foo
	if g.removeNode("foo") != nil {
		t.Fatal("bootstrapper should have received foo:", g.nodes)
	}
}

func TestDisconnect(t *testing.T) {
	g := newTestingGateway("TestDisconnect", t)
	defer g.Close()

	if err := g.Disconnect("bar"); err == nil {
		t.Fatal("disconnect removed unconnected peer")
	}

	// dummy listener to accept connection
	l, err := net.Listen("tcp", ":0")
	if err != nil {
		t.Fatal("couldn't start listener:", err)
	}
	go func() {
		conn, err := l.Accept()
		if err != nil {
			t.Fatal("accept failed:", err)
		}
		conn.Close()
	}()
	// skip standard connection protocol
	conn, err := net.Dial("tcp", string(g.Address()))
	if err != nil {
		t.Fatal("dial failed:", err)
	}
	id := g.mu.Lock()
	g.addPeer(&peer{addr: "foo", sess: muxado.Client(conn)})
	g.mu.Unlock(id)
	if err := g.Disconnect("foo"); err != nil {
		t.Fatal("disconnect failed:", err)
	}
}

func TestMakeOutboundConnections(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	g1 := newTestingGateway("TestMakeOutboundConnections1", t)
	defer g1.Close()

	// first add 8 dummy peers
	id := g1.mu.Lock()
	for i := 0; i < 8; i++ {
		peerAddr := modules.NetAddress("foo" + strconv.Itoa(i))
		g1.addPeer(&peer{addr: peerAddr, sess: muxado.Client(nil)})
	}
	g1.mu.Unlock(id)

	// makeOutboundConnections should now sleep for 5 seconds
	time.Sleep(1 * time.Second)
	// remove a peer while makeOutboundConnections is asleep, and add a new
	// connectable address to the node list
	g1.Disconnect("foo1")
	g2 := newTestingGateway("TestMakeOutboundConnections2", t)
	defer g2.Close()
	id = g1.mu.Lock()
	g1.addNode(g2.Address())
	g1.mu.Unlock(id)

	// when makeOutboundConnections wakes up, it should connect to g2.
	time.Sleep(5 * time.Second)

	id = g1.mu.RLock()
	defer g1.mu.RUnlock(id)
	if len(g1.peers) != 8 {
		t.Fatal("gateway did not reach 8 peers:", g1.peers)
	}
	if g1.peers[g2.Address()] == nil {
		t.Fatal("gateway did not connect to g2")
	}
}
<commit_msg>fix Testconnect race condition<commit_after>package gateway

import (
	"net"
	"strconv"
	"testing"
	"time"

	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/inconshreveable/muxado"
)

func TestAddPeer(t *testing.T) {
	g := newTestingGateway("TestAddPeer", t)
	defer g.Close()
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	g.addPeer(&peer{addr: "foo", sess: muxado.Client(nil)})
	if len(g.peers) != 1 {
		t.Fatal("gateway did not add peer")
	}
}

func TestListen(t *testing.T) {
	g := newTestingGateway("TestListen", t)
	defer g.Close()

	// "compliant" connect
	conn, err := net.Dial("tcp", string(g.Address()))
	if err != nil {
		t.Fatal("dial failed:", err)
	}
	addr := modules.NetAddress(conn.LocalAddr().String())
	// send version
	if err := encoding.WriteObject(conn, version); err != nil {
		t.Fatal("couldn't write version")
	}
	// read ack
	var ack string
	if err := encoding.ReadObject(conn, &ack, maxAddrLength); err != nil {
		t.Fatal(err)
	} else if ack != "accept" {
		t.Fatal("gateway should have given ack")
	}

	// g should add foo
	var ok bool
	for !ok {
		id := g.mu.RLock()
		_, ok = g.peers[addr]
		g.mu.RUnlock(id)
	}

	conn.Close()

	// g should remove foo
	for ok {
		id := g.mu.RLock()
		_, ok = g.peers[addr]
		g.mu.RUnlock(id)
	}

	// "uncompliant" connect
	conn, err = net.Dial("tcp", string(g.Address()))
	if err != nil {
		t.Fatal("dial failed:", err)
	}
	if _, err := conn.Write([]byte("missing length prefix")); err != nil {
		t.Fatal("couldn't write malformed header")
	}
	// g should have closed the connection
	if n, err := conn.Write([]byte("closed")); err != nil && n > 0 {
		t.Error("write succeeded after closed connection")
	}
}

func TestConnect(t *testing.T) {
	// create bootstrap peer
	bootstrap := newTestingGateway("TestConnect1", t)
	defer bootstrap.Close()

	// give it a node
	bootstrap.addNode("foo")

	// create peer who will connect to bootstrap
	g := newTestingGateway("TestConnect2", t)
	defer g.Close()

	// first simulate a "bad" connect, where bootstrap won't share its nodes
	bootstrap.RegisterRPC("ShareNodes", func(modules.PeerConn) error {
		return nil
	})
	// connect
	err := g.Connect(bootstrap.Address())
	if err != nil {
		t.Fatal(err)
	}
	// g should not have foo
	if g.removeNode("foo") == nil {
		t.Fatal("bootstrapper should not have received foo:", g.nodes)
	}

	// split 'em up
	g.Disconnect(bootstrap.Address())
	bootstrap.Disconnect(g.Address())

	// now restore the correct ShareNodes RPC and try again
	bootstrap.RegisterRPC("ShareNodes", bootstrap.shareNodes)
	err = g.Connect(bootstrap.Address())
	if err != nil {
		t.Fatal(err)
	}
	// g should have foo
	id := g.mu.RLock()
	defer g.mu.RUnlock(id)
	if _, ok := g.nodes["foo"]; !ok {
		t.Fatal("bootstrapper should have received foo:", g.nodes)
	}
}

func TestDisconnect(t *testing.T) {
	g := newTestingGateway("TestDisconnect", t)
	defer g.Close()

	if err := g.Disconnect("bar"); err == nil {
		t.Fatal("disconnect removed unconnected peer")
	}

	// dummy listener to accept connection
	l, err := net.Listen("tcp", ":0")
	if err != nil {
		t.Fatal("couldn't start listener:", err)
	}
	go func() {
		conn, err := l.Accept()
		if err != nil {
			t.Fatal("accept failed:", err)
		}
		conn.Close()
	}()
	// skip standard connection protocol
	conn, err := net.Dial("tcp", string(g.Address()))
	if err != nil {
		t.Fatal("dial failed:", err)
	}
	id := g.mu.Lock()
	g.addPeer(&peer{addr: "foo", sess: muxado.Client(conn)})
	g.mu.Unlock(id)
	if err := g.Disconnect("foo"); err != nil {
		t.Fatal("disconnect failed:", err)
	}
}

func TestMakeOutboundConnections(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	g1 := newTestingGateway("TestMakeOutboundConnections1", t)
	defer g1.Close()

	// first add 8 dummy peers
	id := g1.mu.Lock()
	for i := 0; i < 8; i++ {
		peerAddr := modules.NetAddress("foo" + strconv.Itoa(i))
		g1.addPeer(&peer{addr: peerAddr, sess: muxado.Client(nil)})
	}
	g1.mu.Unlock(id)

	// makeOutboundConnections should now sleep for 5 seconds
	time.Sleep(1 * time.Second)
	// remove a peer while makeOutboundConnections is asleep, and add a new
	// connectable address to the node list
	g1.Disconnect("foo1")
	g2 := newTestingGateway("TestMakeOutboundConnections2", t)
	defer g2.Close()
	id = g1.mu.Lock()
	g1.addNode(g2.Address())
	g1.mu.Unlock(id)

	// when makeOutboundConnections wakes up, it should connect to g2.
	time.Sleep(5 * time.Second)

	id = g1.mu.RLock()
	defer g1.mu.RUnlock(id)
	if len(g1.peers) != 8 {
		t.Fatal("gateway did not reach 8 peers:", g1.peers)
	}
	if g1.peers[g2.Address()] == nil {
		t.Fatal("gateway did not connect to g2")
	}
}
<|endoftext|>