<commit_before>package agent

import (
	"errors"
	"fmt"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/metrics"
	"github.com/buildkite/agent/retry"
	"github.com/buildkite/agent/signalwatcher"
	"github.com/buildkite/agent/system"
	"github.com/denisbrodbeck/machineid"
)

type AgentPool struct {
	Logger                *logger.Logger
	APIClient             *api.Client
	Token                 string
	ConfigFilePath        string
	Name                  string
	Priority              string
	Tags                  []string
	TagsFromEC2           bool
	TagsFromEC2Tags       bool
	TagsFromGCP           bool
	TagsFromHost          bool
	WaitForEC2TagsTimeout time.Duration
	Endpoint              string
	Debug                 bool
	DisableHTTP2          bool
	AgentConfiguration    *AgentConfiguration
	MetricsCollector      *metrics.Collector
	Spawn                 int

	interruptCount int
	signalLock     sync.Mutex
}

func (r *AgentPool) Start() error {
	// Show the welcome banner and config options used
	r.ShowBanner()

	// Create the agent registration API Client
	r.APIClient = APIClient{
		Endpoint:     r.Endpoint,
		Token:        r.Token,
		DisableHTTP2: r.DisableHTTP2,
		Logger:       r.Logger,
	}.Create()

	var wg sync.WaitGroup
	var errs = make(chan error, r.Spawn)

	for i := 0; i < r.Spawn; i++ {
		if r.Spawn == 1 {
			r.Logger.Info("Registering agent with Buildkite...")
		} else {
			r.Logger.Info("Registering agent %d of %d with Buildkite...", i+1, r.Spawn)
		}

		wg.Add(1)
		go func() {
			defer wg.Done()
			if err := r.startWorker(); err != nil {
				errs<-err
			}
		}()
	}

	go func() {
		wg.Wait()
		close(errs)
	}()

	return <-errs
}

func (r *AgentPool) startWorker() error {
	registered, err := r.RegisterAgent(r.CreateAgentTemplate())
	if err != nil {
		return err
	}

	r.Logger.Info("Successfully registered agent \"%s\" with tags [%s]", registered.Name,
		strings.Join(registered.Tags, ", "))

	r.Logger.Debug("Ping interval: %ds", registered.PingInterval)
	r.Logger.Debug("Job status interval: %ds", registered.JobStatusInterval)
	r.Logger.Debug("Heartbeat interval: %ds", registered.HearbeatInterval)

	// Now that we have a registered agent, we can connect it to the API,
	// and start running jobs.
	worker := AgentWorker{
		Logger:             r.Logger,
		Agent:              registered,
		AgentConfiguration: r.AgentConfiguration,
		Endpoint:           r.Endpoint,
		Debug:              r.Debug,
		DisableHTTP2:       r.DisableHTTP2,
		MetricsCollector:   r.MetricsCollector,
	}.Create()

	r.Logger.Info("Connecting to Buildkite...")
	if err := worker.Connect(); err != nil {
		return err
	}

	r.Logger.Info("Agent successfully connected")
	r.Logger.Info("You can press Ctrl-C to stop the agent")

	if r.AgentConfiguration.DisconnectAfterJob {
		r.Logger.Info("Waiting for job to be assigned...")
		r.Logger.Info("The agent will automatically disconnect after %d seconds if no job is assigned", r.AgentConfiguration.DisconnectAfterJobTimeout)
	} else {
		r.Logger.Info("Waiting for work...")
	}

	// Start a signalwatcher so we can monitor signals and handle shutdowns
	signalwatcher.Watch(func(sig signalwatcher.Signal) {
		r.signalLock.Lock()
		defer r.signalLock.Unlock()

		if sig == signalwatcher.QUIT {
			r.Logger.Debug("Received signal `%s`", sig.String())
			worker.Stop(false)
		} else if sig == signalwatcher.TERM || sig == signalwatcher.INT {
			r.Logger.Debug("Received signal `%s`", sig.String())
			if r.interruptCount == 0 {
				r.interruptCount++
				r.Logger.Info("Received CTRL-C, send again to forcefully kill the agent")
				worker.Stop(true)
			} else {
				r.Logger.Info("Forcefully stopping running jobs and stopping the agent")
				worker.Stop(false)
			}
		} else {
			r.Logger.Debug("Ignoring signal `%s`", sig.String())
		}
	})

	// Starts the agent worker.
	if err := worker.Start(); err != nil {
		return err
	}

	// Now that the agent has stopped, we can disconnect it
	r.Logger.Info("Disconnecting %s...", worker.Agent.Name)
	if err := worker.Disconnect(); err != nil {
		return err
	}

	return nil
}

// Takes the options passed to the CLI, and creates an api.Agent record that
// will be sent to the Buildkite Agent API for registration.
func (r *AgentPool) CreateAgentTemplate() *api.Agent {
	agent := &api.Agent{
		Name:              r.Name,
		Priority:          r.Priority,
		Tags:              r.Tags,
		ScriptEvalEnabled: r.AgentConfiguration.CommandEval,
		Version:           Version(),
		Build:             BuildVersion(),
		PID:               os.Getpid(),
		Arch:              runtime.GOARCH,
	}

	// get a unique identifier for the underlying host
	if machineID, err := machineid.ProtectedID("buildkite-agent"); err != nil {
		r.Logger.Warn("Failed to find unique machine-id: %v", err)
	} else {
		agent.MachineID = machineID
	}

	// Attempt to add the EC2 tags
	if r.TagsFromEC2 {
		r.Logger.Info("Fetching EC2 meta-data...")

		err := retry.Do(func(s *retry.Stats) error {
			tags, err := EC2MetaData{}.Get()
			if err != nil {
				r.Logger.Warn("%s (%s)", err, s)
			} else {
				r.Logger.Info("Successfully fetched EC2 meta-data")
				for tag, value := range tags {
					agent.Tags = append(agent.Tags, fmt.Sprintf("%s=%s", tag, value))
				}
				s.Break()
			}

			return err
		}, &retry.Config{Maximum: 5, Interval: 1 * time.Second, Jitter: true})

		// Don't blow up if we can't find them, just show a nasty error.
		if err != nil {
			r.Logger.Error(fmt.Sprintf("Failed to fetch EC2 meta-data: %s", err.Error()))
		}
	}

	// Attempt to add the EC2 tags
	if r.TagsFromEC2Tags {
		r.Logger.Info("Fetching EC2 tags...")
		err := retry.Do(func(s *retry.Stats) error {
			tags, err := EC2Tags{}.Get()
			// EC2 tags are apparently "eventually consistent" and sometimes take several seconds
			// to be applied to instances. This error will cause retries.
			if err == nil && len(tags) == 0 {
				err = errors.New("EC2 tags are empty")
			}
			if err != nil {
				r.Logger.Warn("%s (%s)", err, s)
			} else {
				r.Logger.Info("Successfully fetched EC2 tags")
				for tag, value := range tags {
					agent.Tags = append(agent.Tags, fmt.Sprintf("%s=%s", tag, value))
				}
				s.Break()
			}
			return err
		}, &retry.Config{Maximum: 5, Interval: r.WaitForEC2TagsTimeout / 5, Jitter: true})

		// Don't blow up if we can't find them, just show a nasty error.
		if err != nil {
			r.Logger.Error(fmt.Sprintf("Failed to find EC2 tags: %s", err.Error()))
		}
	}

	// Attempt to add the Google Cloud meta-data
	if r.TagsFromGCP {
		tags, err := GCPMetaData{}.Get()
		if err != nil {
			// Don't blow up if we can't find them, just show a nasty error.
			r.Logger.Error(fmt.Sprintf("Failed to fetch Google Cloud meta-data: %s", err.Error()))
		} else {
			for tag, value := range tags {
				agent.Tags = append(agent.Tags, fmt.Sprintf("%s=%s", tag, value))
			}
		}
	}

	var err error

	// Add the hostname
	agent.Hostname, err = os.Hostname()
	if err != nil {
		r.Logger.Warn("Failed to find hostname: %s", err)
	}

	// Add the OS dump
	agent.OS, err = system.VersionDump(r.Logger)
	if err != nil {
		r.Logger.Warn("Failed to find OS information: %s", err)
	}

	// Attempt to add the host tags
	if r.TagsFromHost {
		agent.Tags = append(agent.Tags,
			fmt.Sprintf("hostname=%s", agent.Hostname),
			fmt.Sprintf("os=%s", runtime.GOOS),
		)
		if agent.MachineID != "" {
			agent.Tags = append(agent.Tags, fmt.Sprintf("machine-id=%s", agent.MachineID))
		}
	}

	return agent
}

// Takes the agent template and returns a registered agent. The registered
// agent includes the Access Token used to communicate with the Buildkite Agent
// API
func (r *AgentPool) RegisterAgent(agent *api.Agent) (*api.Agent, error) {
	var registered *api.Agent
	var err error
	var resp *api.Response

	register := func(s *retry.Stats) error {
		registered, resp, err = r.APIClient.Agents.Register(agent)
		if err != nil {
			if resp != nil && resp.StatusCode == 401 {
				r.Logger.Warn("Buildkite rejected the registration (%s)", err)
				s.Break()
			} else {
				r.Logger.Warn("%s (%s)", err, s)
			}
		}

		return err
	}

	// Try to register, retrying every 10 seconds for a maximum of 30 attempts (5 minutes)
	err = retry.Do(register, &retry.Config{Maximum: 30, Interval: 10 * time.Second})

	return registered, err
}

// Shows the welcome banner and the configuration options used when starting
// this agent.
func (r *AgentPool) ShowBanner() {
	welcomeMessage :=
		"\n" +
			"%s  _           _ _     _ _    _ _                                _\n" +
			" | |         (_) |   | | |  (_) |                              | |\n" +
			" | |__  _   _ _| | __| | | ___| |_ ___    __ _  __ _  ___ _ __ | |_\n" +
			" | '_ \\| | | | | |/ _` | |/ / | __/ _ \\  / _` |/ _` |/ _ \\ '_ \\| __|\n" +
			" | |_) | |_| | | | (_| |   <| | ||  __/ | (_| | (_| |  __/ | | | |_\n" +
			" |_.__/ \\__,_|_|_|\\__,_|_|\\_\\_|\\__\\___|  \\__,_|\\__, |\\___|_| |_|\\__|\n" +
			"                                                __/ |\n" +
			" http://buildkite.com/agent                    |___/\n%s\n"

	if logger.ColorsEnabled() {
		fmt.Fprintf(os.Stderr, welcomeMessage, "\x1b[32m", "\x1b[0m")
	} else {
		fmt.Fprintf(os.Stderr, welcomeMessage, "", "")
	}

	r.Logger.Notice("Starting buildkite-agent v%s with PID: %s", Version(), fmt.Sprintf("%d", os.Getpid()))
	r.Logger.Notice("The agent source code can be found here: https://github.com/buildkite/agent")
	r.Logger.Notice("For questions and support, email us at: hello@buildkite.com")

	if r.ConfigFilePath != "" {
		r.Logger.Info("Configuration loaded from: %s", r.ConfigFilePath)
	}

	r.Logger.Debug("Bootstrap command: %s", r.AgentConfiguration.BootstrapScript)
	r.Logger.Debug("Build path: %s", r.AgentConfiguration.BuildPath)
	r.Logger.Debug("Hooks directory: %s", r.AgentConfiguration.HooksPath)
	r.Logger.Debug("Plugins directory: %s", r.AgentConfiguration.PluginsPath)

	if !r.AgentConfiguration.SSHKeyscan {
		r.Logger.Info("Automatic ssh-keyscan has been disabled")
	}

	if !r.AgentConfiguration.CommandEval {
		r.Logger.Info("Evaluating console commands has been disabled")
	}

	if !r.AgentConfiguration.PluginsEnabled {
		r.Logger.Info("Plugins have been disabled")
	}

	if !r.AgentConfiguration.RunInPty {
		r.Logger.Info("Running builds within a pseudoterminal (PTY) has been disabled")
	}

	if r.AgentConfiguration.DisconnectAfterJob {
		r.Logger.Info("Agent will disconnect after a job run has completed with a timeout of %d seconds", r.AgentConfiguration.DisconnectAfterJobTimeout)
	}
}
<commit_msg>Show a prefix per spawned agent<commit_after>package agent

import (
	"errors"
	"fmt"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/metrics"
	"github.com/buildkite/agent/retry"
	"github.com/buildkite/agent/signalwatcher"
	"github.com/buildkite/agent/system"
	"github.com/denisbrodbeck/machineid"
)

type AgentPool struct {
	Logger                *logger.Logger
	APIClient             *api.Client
	Token                 string
	ConfigFilePath        string
	Name                  string
	Priority              string
	Tags                  []string
	TagsFromEC2           bool
	TagsFromEC2Tags       bool
	TagsFromGCP           bool
	TagsFromHost          bool
	WaitForEC2TagsTimeout time.Duration
	Endpoint              string
	Debug                 bool
	DisableHTTP2          bool
	AgentConfiguration    *AgentConfiguration
	MetricsCollector      *metrics.Collector
	Spawn                 int

	interruptCount int
	signalLock     sync.Mutex
}

func (r *AgentPool) Start() error {
	// Show the welcome banner and config options used
	r.ShowBanner()

	// Create the agent registration API Client
	r.APIClient = APIClient{
		Endpoint:     r.Endpoint,
		Token:        r.Token,
		DisableHTTP2: r.DisableHTTP2,
		Logger:       r.Logger,
	}.Create()

	var wg sync.WaitGroup
	var errs = make(chan error, r.Spawn)

	for i := 0; i < r.Spawn; i++ {
		if r.Spawn == 1 {
			r.Logger.Info("Registering agent with Buildkite...")
		} else {
			r.Logger.Info("Registering agent %d of %d with Buildkite...", i+1, r.Spawn)
		}

		wg.Add(1)
		go func() {
			defer wg.Done()
			if err := r.startWorker(); err != nil {
				errs<-err
			}
		}()
	}

	go func() {
		wg.Wait()
		close(errs)
	}()

	r.Logger.Info("Started %d Agent(s)", r.Spawn)
	r.Logger.Info("You can press Ctrl-C to stop the agents")

	return <-errs
}

func (r *AgentPool) startWorker() error {
	registered, err := r.RegisterAgent(r.CreateAgentTemplate())
	if err != nil {
		return err
	}

	r.Logger.Info("Successfully registered agent \"%s\" with tags [%s]", registered.Name,
		strings.Join(registered.Tags, ", "))

	// Create a prefixed logger for some context in concurrent output
	l := r.Logger.WithPrefix(registered.Name)

	l.Debug("Ping interval: %ds", registered.PingInterval)
	l.Debug("Job status interval: %ds", registered.JobStatusInterval)
	l.Debug("Heartbeat interval: %ds", registered.HearbeatInterval)

	// Now that we have a registered agent, we can connect it to the API,
	// and start running jobs.
	worker := AgentWorker{
		Logger:         l,
		Agent:              registered,
		AgentConfiguration: r.AgentConfiguration,
		Endpoint:           r.Endpoint,
		Debug:              r.Debug,
		DisableHTTP2:       r.DisableHTTP2,
		MetricsCollector:   r.MetricsCollector,
	}.Create()

	l.Info("Connecting to Buildkite...")
	if err := worker.Connect(); err != nil {
		return err
	}

	if r.AgentConfiguration.DisconnectAfterJob {
		l.Info("Waiting for job to be assigned...")
		l.Info("The agent will automatically disconnect after %d seconds if no job is assigned", r.AgentConfiguration.DisconnectAfterJobTimeout)
	} else {
		l.Info("Waiting for work...")
	}

	// Start a signalwatcher so we can monitor signals and handle shutdowns
	signalwatcher.Watch(func(sig signalwatcher.Signal) {
		r.signalLock.Lock()
		defer r.signalLock.Unlock()

		if sig == signalwatcher.QUIT {
			l.Debug("Received signal `%s`", sig.String())
			worker.Stop(false)
		} else if sig == signalwatcher.TERM || sig == signalwatcher.INT {
			l.Debug("Received signal `%s`", sig.String())
			if r.interruptCount == 0 {
				r.interruptCount++
				l.Info("Received CTRL-C, send again to forcefully kill the agent")
				worker.Stop(true)
			} else {
				l.Info("Forcefully stopping running jobs and stopping the agent")
				worker.Stop(false)
			}
		} else {
			l.Debug("Ignoring signal `%s`", sig.String())
		}
	})

	// Starts the agent worker.
	if err := worker.Start(); err != nil {
		return err
	}

	// Now that the agent has stopped, we can disconnect it
	l.Info("Disconnecting %s...", worker.Agent.Name)
	if err := worker.Disconnect(); err != nil {
		return err
	}

	return nil
}

// Takes the options passed to the CLI, and creates an api.Agent record that
// will be sent to the Buildkite Agent API for registration.
func (r *AgentPool) CreateAgentTemplate() *api.Agent {
	agent := &api.Agent{
		Name:              r.Name,
		Priority:          r.Priority,
		Tags:              r.Tags,
		ScriptEvalEnabled: r.AgentConfiguration.CommandEval,
		Version:           Version(),
		Build:             BuildVersion(),
		PID:               os.Getpid(),
		Arch:              runtime.GOARCH,
	}

	// get a unique identifier for the underlying host
	if machineID, err := machineid.ProtectedID("buildkite-agent"); err != nil {
		r.Logger.Warn("Failed to find unique machine-id: %v", err)
	} else {
		agent.MachineID = machineID
	}

	// Attempt to add the EC2 tags
	if r.TagsFromEC2 {
		r.Logger.Info("Fetching EC2 meta-data...")

		err := retry.Do(func(s *retry.Stats) error {
			tags, err := EC2MetaData{}.Get()
			if err != nil {
				r.Logger.Warn("%s (%s)", err, s)
			} else {
				r.Logger.Info("Successfully fetched EC2 meta-data")
				for tag, value := range tags {
					agent.Tags = append(agent.Tags, fmt.Sprintf("%s=%s", tag, value))
				}
				s.Break()
			}

			return err
		}, &retry.Config{Maximum: 5, Interval: 1 * time.Second, Jitter: true})

		// Don't blow up if we can't find them, just show a nasty error.
		if err != nil {
			r.Logger.Error(fmt.Sprintf("Failed to fetch EC2 meta-data: %s", err.Error()))
		}
	}

	// Attempt to add the EC2 tags
	if r.TagsFromEC2Tags {
		r.Logger.Info("Fetching EC2 tags...")
		err := retry.Do(func(s *retry.Stats) error {
			tags, err := EC2Tags{}.Get()
			// EC2 tags are apparently "eventually consistent" and sometimes take several seconds
			// to be applied to instances. This error will cause retries.
			if err == nil && len(tags) == 0 {
				err = errors.New("EC2 tags are empty")
			}
			if err != nil {
				r.Logger.Warn("%s (%s)", err, s)
			} else {
				r.Logger.Info("Successfully fetched EC2 tags")
				for tag, value := range tags {
					agent.Tags = append(agent.Tags, fmt.Sprintf("%s=%s", tag, value))
				}
				s.Break()
			}
			return err
		}, &retry.Config{Maximum: 5, Interval: r.WaitForEC2TagsTimeout / 5, Jitter: true})

		// Don't blow up if we can't find them, just show a nasty error.
		if err != nil {
			r.Logger.Error(fmt.Sprintf("Failed to find EC2 tags: %s", err.Error()))
		}
	}

	// Attempt to add the Google Cloud meta-data
	if r.TagsFromGCP {
		tags, err := GCPMetaData{}.Get()
		if err != nil {
			// Don't blow up if we can't find them, just show a nasty error.
			r.Logger.Error(fmt.Sprintf("Failed to fetch Google Cloud meta-data: %s", err.Error()))
		} else {
			for tag, value := range tags {
				agent.Tags = append(agent.Tags, fmt.Sprintf("%s=%s", tag, value))
			}
		}
	}

	var err error

	// Add the hostname
	agent.Hostname, err = os.Hostname()
	if err != nil {
		r.Logger.Warn("Failed to find hostname: %s", err)
	}

	// Add the OS dump
	agent.OS, err = system.VersionDump(r.Logger)
	if err != nil {
		r.Logger.Warn("Failed to find OS information: %s", err)
	}

	// Attempt to add the host tags
	if r.TagsFromHost {
		agent.Tags = append(agent.Tags,
			fmt.Sprintf("hostname=%s", agent.Hostname),
			fmt.Sprintf("os=%s", runtime.GOOS),
		)
		if agent.MachineID != "" {
			agent.Tags = append(agent.Tags, fmt.Sprintf("machine-id=%s", agent.MachineID))
		}
	}

	return agent
}

// Takes the agent template and returns a registered agent. The registered
// agent includes the Access Token used to communicate with the Buildkite Agent
// API
func (r *AgentPool) RegisterAgent(agent *api.Agent) (*api.Agent, error) {
	var registered *api.Agent
	var err error
	var resp *api.Response

	register := func(s *retry.Stats) error {
		registered, resp, err = r.APIClient.Agents.Register(agent)
		if err != nil {
			if resp != nil && resp.StatusCode == 401 {
				r.Logger.Warn("Buildkite rejected the registration (%s)", err)
				s.Break()
			} else {
				r.Logger.Warn("%s (%s)", err, s)
			}
		}

		return err
	}

	// Try to register, retrying every 10 seconds for a maximum of 30 attempts (5 minutes)
	err = retry.Do(register, &retry.Config{Maximum: 30, Interval: 10 * time.Second})

	return registered, err
}

// Shows the welcome banner and the configuration options used when starting
// this agent.
func (r *AgentPool) ShowBanner() {
	welcomeMessage :=
		"\n" +
			"%s  _           _ _     _ _    _ _                                _\n" +
			" | |         (_) |   | | |  (_) |                              | |\n" +
			" | |__  _   _ _| | __| | | ___| |_ ___    __ _  __ _  ___ _ __ | |_\n" +
			" | '_ \\| | | | | |/ _` | |/ / | __/ _ \\  / _` |/ _` |/ _ \\ '_ \\| __|\n" +
			" | |_) | |_| | | | (_| |   <| | ||  __/ | (_| | (_| |  __/ | | | |_\n" +
			" |_.__/ \\__,_|_|_|\\__,_|_|\\_\\_|\\__\\___|  \\__,_|\\__, |\\___|_| |_|\\__|\n" +
			"                                                __/ |\n" +
			" http://buildkite.com/agent                    |___/\n%s\n"

	if logger.ColorsEnabled() {
		fmt.Fprintf(os.Stderr, welcomeMessage, "\x1b[32m", "\x1b[0m")
	} else {
		fmt.Fprintf(os.Stderr, welcomeMessage, "", "")
	}

	r.Logger.Notice("Starting buildkite-agent v%s with PID: %s", Version(), fmt.Sprintf("%d", os.Getpid()))
	r.Logger.Notice("The agent source code can be found here: https://github.com/buildkite/agent")
	r.Logger.Notice("For questions and support, email us at: hello@buildkite.com")

	if r.ConfigFilePath != "" {
		r.Logger.Info("Configuration loaded from: %s", r.ConfigFilePath)
	}

	r.Logger.Debug("Bootstrap command: %s", r.AgentConfiguration.BootstrapScript)
	r.Logger.Debug("Build path: %s", r.AgentConfiguration.BuildPath)
	r.Logger.Debug("Hooks directory: %s", r.AgentConfiguration.HooksPath)
	r.Logger.Debug("Plugins directory: %s", r.AgentConfiguration.PluginsPath)

	if !r.AgentConfiguration.SSHKeyscan {
		r.Logger.Info("Automatic ssh-keyscan has been disabled")
	}

	if !r.AgentConfiguration.CommandEval {
		r.Logger.Info("Evaluating console commands has been disabled")
	}

	if !r.AgentConfiguration.PluginsEnabled {
		r.Logger.Info("Plugins have been disabled")
	}

	if !r.AgentConfiguration.RunInPty {
		r.Logger.Info("Running builds within a pseudoterminal (PTY) has been disabled")
	}

	if r.AgentConfiguration.DisconnectAfterJob {
		r.Logger.Info("Agent will disconnect after a job run has completed with a timeout of %d seconds", r.AgentConfiguration.DisconnectAfterJobTimeout)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"log"
	"net/http"
	"strings"
	"time"

	"github.com/fsouza/go-dockerclient"
)

//Containers representing information about the running containers
type Containers struct {
	ID      string `json:"id"`
	Name    string `json:"name"`
	Command string `json:"command"`
	Port    string `json:"port"`
}

//Post represents a post to the server
type Post struct {
	AgentID             string       `json:"agentid"`
	ExpiredAfterSeconds int          `json:"expiredAfterSeconds"`
	Containers          []Containers `json:"containers"`
}

var (
	serverURL          string
	agentID            string
	refreshRate        int
	expireAfterSeconds int
)

func init() {
	flag.StringVar(&serverURL, "server", "http://localhost:8989", "The server uri where dockmaster is located.")
	flag.StringVar(&agentID, "agent", "localhost", "The name of an Agent. Example: TestQA1.")
	flag.IntVar(&refreshRate, "refresh", 60, "The rate at which this agent should check for changes in seconds.")
	flag.IntVar(&expireAfterSeconds, "expireAfterSeconds", 60, "The rate at which data sent by this agent should expire in seconds.")

	flag.Parse()
}

func main() {
	endpoint := "unix:///var/run/docker.sock"
	client, _ := docker.NewClient(endpoint)
	for {
		log.Println("Started listening... Refresh rate is :", refreshRate)
		post := Post{AgentID: agentID, ExpiredAfterSeconds: expireAfterSeconds}
		containers := []Containers{}
		runningContainers, err := client.ListContainers(docker.ListContainersOptions{All: false})
		if err != nil {
			panic("Failed to connect to Docker Client." + err.Error())
		}
		for _, v := range runningContainers {
			c := Containers{}
			c.ID = v.ID
			c.Name = strings.Join(v.Names, ",")
			for _, p := range v.Ports {
				c.Port += p.IP + ":" + p.Type
			}
			c.Command = v.Command
			containers = append(containers, c)
		}
		post.Containers = containers

		postString, err := json.Marshal(post)
		if err != nil {
			log.Println("Error occured while trying ot marshal POST:", err.Error())
		}
		req, err := http.NewRequest("POST", serverURL, bytes.NewBuffer(postString))
		if err != nil {
			log.Println("Failed to create post request... Trying again later.")
			time.Sleep(time.Second * time.Duration(refreshRate))
			continue
		}

		req.Header.Set("Content-Type", "application/json")

		client := &http.Client{}
		resp, err := client.Do(req)
		if err != nil {
			log.Println("Failed to receive from server... Trying again later.")
			time.Sleep(time.Second * time.Duration(refreshRate))
			continue
		}
		defer resp.Body.Close()
		//TODO: Verify the response
		time.Sleep(time.Second * time.Duration(refreshRate))
	}
}
<commit_msg>Fixed posting URL<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"log"
	"net/http"
	"strings"
	"time"

	"github.com/fsouza/go-dockerclient"
)

//Containers representing information about the running containers
type Containers struct {
	ID      string `json:"id"`
	Name    string `json:"name"`
	Command string `json:"command"`
	Port    string `json:"port"`
}

//Post represents a post to the server
type Post struct {
	AgentID             string       `json:"agentid"`
	ExpiredAfterSeconds int          `json:"expiredAfterSeconds"`
	Containers          []Containers `json:"containers"`
}

var (
	serverURL          string
	agentID            string
	refreshRate        int
	expireAfterSeconds int
)

func init() {
	flag.StringVar(&serverURL, "server", "http://localhost:8989", "The server uri where dockmaster is located.")
	flag.StringVar(&agentID, "agent", "localhost", "The name of an Agent. Example: TestQA1.")
	flag.IntVar(&refreshRate, "refresh", 60, "The rate at which this agent should check for changes in seconds.")
	flag.IntVar(&expireAfterSeconds, "expireAfterSeconds", 60, "The rate at which data sent by this agent should expire in seconds.")

	flag.Parse()
}

func main() {
	endpoint := "unix:///var/run/docker.sock"
	client, _ := docker.NewClient(endpoint)
	for {
		log.Println("Started listening... Refresh rate is :", refreshRate)
		post := Post{AgentID: agentID, ExpiredAfterSeconds: expireAfterSeconds}
		containers := []Containers{}
		runningContainers, err := client.ListContainers(docker.ListContainersOptions{All: false})
		if err != nil {
			panic("Failed to connect to Docker Client." + err.Error())
		}
		for _, v := range runningContainers {
			c := Containers{}
			c.ID = v.ID
			c.Name = strings.Join(v.Names, ",")
			for _, p := range v.Ports {
				c.Port += p.IP + ":" + p.Type
			}
			c.Command = v.Command
			containers = append(containers, c)
		}
		post.Containers = containers

		postString, err := json.Marshal(post)
		if err != nil {
			log.Println("Error occured while trying ot marshal POST:", err.Error())
		}
		req, err := http.NewRequest("POST", serverURL+"/api/1/add", bytes.NewBuffer(postString))
		if err != nil {
			log.Println("Failed to create post request... Trying again later.")
			time.Sleep(time.Second * time.Duration(refreshRate))
			continue
		}

		req.Header.Set("Content-Type", "application/json")

		client := &http.Client{}
		resp, err := client.Do(req)
		if err != nil {
			log.Println("Failed to receive from server... Trying again later.")
			time.Sleep(time.Second * time.Duration(refreshRate))
			continue
		}
		defer resp.Body.Close()
		//TODO: Verify the response
		time.Sleep(time.Second * time.Duration(refreshRate))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/daviddengcn/go-colortext"
	"github.com/korovkin/limiter"
	"github.com/korovkin/parallel"
)

type logger struct {
	ticket   int
	hostname string
	buf      *bytes.Buffer
}

var (
	loggerMutex     = new(sync.Mutex)
	loggerIndex     = int(0)
	loggerStartTime = time.Now()
	loggerHostname  = ""
)

var loggerColors = []ct.Color{
	ct.Green,
	ct.Cyan,
	ct.Magenta,
	ct.Yellow,
	ct.Blue,
	ct.Red,
}

func (l *logger) Write(p []byte) (int, error) {
	buf := bytes.NewBuffer(p)
	wrote := 0
	for {
		line, err := buf.ReadBytes('\n')
		if len(line) > 1 {
			now := time.Now().Format("15:04:05")
			s := string(line)
			ts := time.Since(loggerStartTime).String()

			loggerMutex.Lock()
			ct.ChangeColor(loggerColors[l.ticket%len(loggerColors)], false, ct.None, false)
			fmt.Printf("[%16s %s %s %d] ", ts, l.hostname, now, l.ticket)
			ct.ResetColor()

			if l.buf != nil {
				l.buf.Write([]byte(s))
			}

			fmt.Print(s)
			loggerMutex.Unlock()

			wrote += len(line)
		}
		if err != nil {
			break
		}
	}
	if len(p) > 0 && p[len(p)-1] != '\n' {
		fmt.Println()
	}

	return len(p), nil
}

func newLogger(ticket int, collectLines bool) *logger {
	l := &logger{ticket: ticket, buf: nil}
	if collectLines {
		l.buf = &bytes.Buffer{}
	}
	return l
}

func executeCommand(p *Parallel, ticket int, cmdLine string) (*parallel.Output, error) {
	T_START := time.Now()
	var err error
	output := &parallel.Output{}
	loggerOut := newLogger(ticket, true)
	loggerErr := newLogger(ticket, true)

	defer func() {
		fmt.Fprintf(
			loggerOut,
			"execute: done: dt: "+time.Since(T_START).String()+"\n",
		)
	}()

	// execute remotely:
	if len(p.Slaves) > 0 {
		slave := p.Slaves[ticket%len(p.Slaves)]

		var transport thrift.TTransport
		transport, err = thrift.NewTSocket(slave.Address)
		if err != nil {
			log.Fatalln("failed to dial slave:", err.Error())
		}

		transport, err = p.transportFactory.GetTransport(transport)
		if err != nil {
			log.Fatalln("failed to GetTransport:", err.Error())
		}

		err = transport.Open()
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		defer transport.Close()
		client := parallel.NewParallelClientFactory(transport, p.protocolFactory)

		cmd := parallel.Cmd{
			CmdLine: cmdLine,
			Ticket:  int64(ticket),
		}

		output, err = client.Execute(&cmd)
		if err != nil {
			log.Fatalln("failed to execute:", err.Error())
		}

		hostname := output.Tags["hostname"]
		loggerOut.hostname = hostname
		loggerErr.hostname = hostname

		fmt.Fprintf(loggerOut, "execute: remotely: host: %s stdout: [%s]\n", hostname, output.Stdout)
		fmt.Fprintf(loggerErr, "execute: remotely: host: %s stderr: [%s]\n", hostname, output.Stderr)

		return output, err
	}

	// execute locally:
	cs := []string{"/bin/sh", "-c", cmdLine}
	cmd := exec.Command(cs[0], cs[1:]...)
	cmd.Stdin = nil
	cmd.Stdout = loggerOut
	cmd.Stderr = loggerErr
	cmd.Env = append(
		os.Environ(),
		fmt.Sprintf("PARALLEL_TICKER=%d", ticket),
	)

	fmt.Fprintf(loggerOut, "run: '"+cmdLine+"'\n")

	err = cmd.Start()
	if err != nil {
		log.Fatalln("failed to start:", err)
		return output, err
	}

	if err == nil {
		err = cmd.Wait()
	}

	output.Tags = map[string]string{"hostname": loggerHostname}

	if loggerOut.buf != nil {
		output.Stdout = string(loggerOut.buf.Bytes())
	}

	if loggerErr.buf != nil {
		output.Stderr = string(loggerErr.buf.Bytes())
	}

	return output, err
}

type Slave struct {
	Address string `json:"address"`
}

type Parallel struct {
	jobs   int
	logger *logger
	worker *limiter.ConcurrencyLimiter

	// master / slave
	protocolFactory  thrift.TProtocolFactory
	transportFactory thrift.TTransportFactory

	// master:
	Slaves []*Slave

	// slave:
	handler         *ParallelSlaveHandler
	serverTransport thrift.TServerTransport
	address         string
}

func (p *Parallel) Close() {
	p.worker.Wait()
}

func mainMaster(p *Parallel) {
	var err error

	// connect to slaves:
	for _, slave := range p.Slaves {
		var transport thrift.TTransport
		transport, err = thrift.NewTSocket(slave.Address)
		if err != nil {
			log.Fatalln("failed to dial slave:", err.Error())
		}

		transport, err = p.transportFactory.GetTransport(transport)
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		err = transport.Open()
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		defer transport.Close()

		client := parallel.NewParallelClientFactory(transport, p.protocolFactory)
		ok, err := client.Ping()
		if err != nil {
			log.Fatalln("failed to ping client:", err.Error())
		}

		fmt.Fprintf(p.logger, fmt.Sprintf("adding slave: %s ok: %s", slave.Address, ok))

	}

	r := bufio.NewReaderSize(os.Stdin, 1*1024*1024)
	fmt.Fprintf(p.logger, "reading from stdin...\n")
	for {
		line, err := r.ReadString('\n')
		if err == io.EOF {
			break
		}
		line = strings.TrimSpace(line)

		p.worker.ExecuteWithTicket(func(ticket int) {
			executeCommand(p, ticket, line)
		})
	}
}

type ParallelSlaveHandler struct {
	p *Parallel
}

func NewParallelSlaveHandler() *ParallelSlaveHandler {
	return &ParallelSlaveHandler{}
}

func (p *ParallelSlaveHandler) Execute(command *parallel.Cmd) (output *parallel.Output, err error) {
	output = nil
	err = nil
	output, err = executeCommand(p.p, int(command.Ticket), command.CmdLine)

	// TODO:: recover, handle panics

	return output, err
}

func (p *ParallelSlaveHandler) Ping() (r string, err error) {
	log.Println("ParallelSlaveHandler: Ping")
	return "ping:ok", nil
}

func mainSlave(p *Parallel) {
	var err error

	p.serverTransport, err = thrift.NewTServerSocket(p.address)

	if err != nil {
		log.Fatalln("failed to start server:", err.Error())
		return
	}

	p.handler = NewParallelSlaveHandler()
	p.handler.p = p

	server := thrift.NewTSimpleServer4(
		parallel.NewParallelProcessor(p.handler),
		p.serverTransport,
		p.transportFactory,
		p.protocolFactory)

	err = server.Serve()
	if err != nil {
		log.Fatalln("failed to run slave:", err.Error())
	}
}

func main() {
	T_START := time.Now()
	logger := newLogger(0, false)
	defer func() {
		fmt.Fprintf(logger, "all done: dt: "+time.Since(T_START).String()+"\n")
	}()

	flag_jobs := flag.Int(
		"j",
		2,
		"num of concurrent jobs")

	flag_slave := flag.Bool(
		"slave",
		false,
		"run as slave")

	slaves := flag.String(
		"slaves",
		"",
		"CSV list of slave addresses")

	address := flag.String(
		"address",
		"localhost:9010",
		"slave address")

	loggerHostname, _ = os.Hostname()

	flag.Parse()
	fmt.Fprintf(logger, fmt.Sprintf("concurrency limit: %d", *flag_jobs))
	fmt.Fprintf(logger, fmt.Sprintf("slaves: %s", *slaves))

	p := Parallel{}
	p.jobs = *flag_jobs
	p.logger = logger
	p.worker = limiter.NewConcurrencyLimiter(p.jobs)
	p.address = *address
	p.Slaves = []*Slave{}
	for _, slaveAddr := range strings.Split(*slaves, ",") {
		if slaveAddr == "" {
			continue
		}
		slave := Slave{Address: slaveAddr}
		p.Slaves = append(p.Slaves, &slave)
	}

	defer p.Close()

	// thrift protocol
	p.protocolFactory = thrift.NewTBinaryProtocolFactoryDefault()

	// thrift transport
	p.transportFactory = thrift.NewTTransportFactory()
	p.transportFactory = thrift.NewTFramedTransportFactory(p.transportFactory)

	if *flag_slave == false {
		loggerHostname = p.address
		logger.hostname = loggerHostname

		fmt.Fprintf(logger, fmt.Sprintf("running as master\n"))
		mainMaster(&p)
	} else {
		loggerHostname = p.address
		logger.hostname = loggerHostname

		fmt.Fprintf(logger, fmt.Sprintf("running as slave on: %s\n", p.address))
		mainSlave(&p)
	}
}
<commit_msg>cleanup prints<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/daviddengcn/go-colortext"
	"github.com/korovkin/limiter"
	"github.com/korovkin/parallel"
)

type logger struct {
	ticket   int
	hostname string
	buf      *bytes.Buffer
}

var (
	loggerMutex     = new(sync.Mutex)
	loggerIndex     = int(0)
	loggerStartTime = time.Now()
	loggerHostname  = ""
)

var loggerColors = []ct.Color{
	ct.Green,
	ct.Cyan,
	ct.Magenta,
	ct.Yellow,
	ct.Blue,
	ct.Red,
}

func (l *logger) Write(p []byte) (int, error) {
	buf := bytes.NewBuffer(p)
	wrote := 0
	for {
		line, err := buf.ReadBytes('\n')
		if len(line) > 1 {
			now := time.Now().Format("15:04:05")
			s := string(line)
			ts := time.Since(loggerStartTime).String()

			loggerMutex.Lock()
			ct.ChangeColor(loggerColors[l.ticket%len(loggerColors)], false, ct.None, false)
			fmt.Printf("[%16s %s %s %d] ", ts, l.hostname, now, l.ticket)
			ct.ResetColor()

			if l.buf != nil {
				l.buf.Write([]byte(s))
			}

			fmt.Print(s)
			loggerMutex.Unlock()

			wrote += len(line)
		}
		if err != nil {
			break
		}
	}
	if len(p) > 0 && p[len(p)-1] != '\n' {
		fmt.Println()
	}

	return len(p), nil
}

func newLogger(ticket int, collectLines bool) *logger {
	l := &logger{ticket: ticket, buf: nil}
	if collectLines {
		l.buf = &bytes.Buffer{}
	}
	return l
}

func executeCommand(p *Parallel, ticket int, cmdLine string) (*parallel.Output, error) {
	T_START := time.Now()
	var err error
	output := &parallel.Output{}
	loggerOut := newLogger(ticket, true)
	loggerErr := newLogger(ticket, true)

	defer func() {
		fmt.Fprintf(
			loggerOut,
			"execute: done: dt: "+time.Since(T_START).String()+"\n",
		)
	}()

	// execute remotely:
	if len(p.Slaves) > 0 {
		slave := p.Slaves[ticket%len(p.Slaves)]

		var transport thrift.TTransport
		transport, err = thrift.NewTSocket(slave.Address)
		if err != nil {
			log.Fatalln("failed to dial slave:", err.Error())
		}

		transport, err = p.transportFactory.GetTransport(transport)
		if err != nil {
			log.Fatalln("failed to GetTransport:", err.Error())
		}

		err = transport.Open()
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		defer transport.Close()
		client := parallel.NewParallelClientFactory(transport, p.protocolFactory)

		cmd := parallel.Cmd{
			CmdLine: cmdLine,
			Ticket:  int64(ticket),
		}

		output, err = client.Execute(&cmd)
		if err != nil {
			log.Fatalln("failed to execute:", err.Error())
		}

		hostname := output.Tags["hostname"]
		loggerOut.hostname = hostname
		loggerErr.hostname = hostname

		fmt.Fprintf(loggerOut, "execute: remotely: host: %s stdout: [%s]\n", hostname, output.Stdout)
		fmt.Fprintf(loggerErr, "execute: remotely: host: %s stderr: [%s]\n", hostname, output.Stderr)

		return output, err
	}

	// execute locally:
	cs := []string{"/bin/sh", "-c", cmdLine}
	cmd := exec.Command(cs[0], cs[1:]...)
	cmd.Stdin = nil
	cmd.Stdout = loggerOut
	cmd.Stderr = loggerErr
	cmd.Env = append(
		os.Environ(),
		fmt.Sprintf("PARALLEL_TICKER=%d", ticket),
	)

	fmt.Fprintf(loggerOut, "run: '"+cmdLine+"'\n")

	err = cmd.Start()
	if err != nil {
		log.Fatalln("failed to start:", err)
		return output, err
	}

	if err == nil {
		err = cmd.Wait()
	}

	output.Tags = map[string]string{"hostname": loggerHostname}

	if loggerOut.buf != nil {
		output.Stdout = string(loggerOut.buf.Bytes())
	}

	if loggerErr.buf != nil {
		output.Stderr = string(loggerErr.buf.Bytes())
	}

	return output, err
}

type Slave struct {
	Address string `json:"address"`
}

type Parallel struct {
	jobs   int
	logger *logger
	worker *limiter.ConcurrencyLimiter

	// master / slave
	protocolFactory  thrift.TProtocolFactory
	transportFactory thrift.TTransportFactory

	// master:
	Slaves []*Slave

	// slave:
	handler         *ParallelSlaveHandler
	serverTransport thrift.TServerTransport
	address         string
}

func (p *Parallel) Close() {
	p.worker.Wait()
}

func mainMaster(p *Parallel) {
	var err error

	// connect to slaves:
	for _, slave := range p.Slaves {
		var transport thrift.TTransport
		transport, err = thrift.NewTSocket(slave.Address)
		if err != nil {
			log.Fatalln("failed to dial slave:", err.Error())
		}

		transport, err = p.transportFactory.GetTransport(transport)
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		err = transport.Open()
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		defer transport.Close()

		client := parallel.NewParallelClientFactory(transport, p.protocolFactory)
		ok, err := client.Ping()
		if err != nil {
			log.Fatalln("failed to ping client:", err.Error())
		}

		fmt.Fprintf(p.logger, fmt.Sprintf("adding slave: %s ok: %s", slave.Address, ok))

	}

	r := bufio.NewReaderSize(os.Stdin, 1*1024*1024)
	fmt.Fprintf(p.logger, "reading from stdin...\n")
	for {
		line, err := r.ReadString('\n')
		if err == io.EOF {
			break
		}
		line = strings.TrimSpace(line)

		p.worker.ExecuteWithTicket(func(ticket int) {
			executeCommand(p, ticket, line)
		})
	}
}

type ParallelSlaveHandler struct {
	p *Parallel
}

func NewParallelSlaveHandler() *ParallelSlaveHandler {
	return &ParallelSlaveHandler{}
}

func (p *ParallelSlaveHandler) Execute(command *parallel.Cmd) (output *parallel.Output, err error) {
	output = nil
	err = nil
	output, err = executeCommand(p.p, int(command.Ticket), command.CmdLine)

	// TODO:: recover, handle panics

	return output, err
}

func (p *ParallelSlaveHandler) Ping() (r string, err error) {
	return "ping:ok", nil
}

func mainSlave(p *Parallel) {
	var err error

	p.serverTransport, err = thrift.NewTServerSocket(p.address)

	if err != nil {
		log.Fatalln("failed to start server:", err.Error())
		return
	}

	p.handler = NewParallelSlaveHandler()
	p.handler.p = p

	server := thrift.NewTSimpleServer4(
		parallel.NewParallelProcessor(p.handler),
		p.serverTransport,
		p.transportFactory,
		p.protocolFactory)

	err = server.Serve()
	if err != nil {
		log.Fatalln("failed to run slave:", err.Error())
	}
}

func main() {
	T_START := time.Now()
	logger := newLogger(0, false)
	defer func() {
		fmt.Fprintf(logger, "all done: dt: "+time.Since(T_START).String()+"\n")
	}()

	flag_jobs := flag.Int(
		"j",
		2,
		"num of concurrent jobs")

	flag_slave := flag.Bool(
		"slave",
		false,
		"run as slave")

	slaves := flag.String(
		"slaves",
		"",
		"CSV list of slave addresses")

	address := flag.String(
		"address",
		"localhost:9010",
		"slave address")

	loggerHostname, _ = os.Hostname()

	flag.Parse()
	fmt.Fprintf(logger, fmt.Sprintf("concurrency limit: %d", *flag_jobs))
	fmt.Fprintf(logger, fmt.Sprintf("slaves: %s", *slaves))

	p := Parallel{}
	p.jobs = *flag_jobs
	p.logger = logger
	p.worker = limiter.NewConcurrencyLimiter(p.jobs)
	p.address = *address
	p.Slaves = []*Slave{}
	for _, slaveAddr := range strings.Split(*slaves, ",") {
		if slaveAddr == "" {
			continue
		}
		slave := Slave{Address: slaveAddr}
		p.Slaves = append(p.Slaves, &slave)
	}

	defer p.Close()

	// thrift protocol
	p.protocolFactory = thrift.NewTBinaryProtocolFactoryDefault()

	// thrift transport
	p.transportFactory = thrift.NewTTransportFactory()
	p.transportFactory = thrift.NewTFramedTransportFactory(p.transportFactory)

	if *flag_slave == false {
		loggerHostname = p.address
		logger.hostname = loggerHostname

		fmt.Fprintf(logger, fmt.Sprintf("running as master\n"))
		mainMaster(&p)
	} else {
		loggerHostname = p.address
		logger.hostname = loggerHostname

		fmt.Fprintf(logger, fmt.Sprintf("running as slave on: %s\n", p.address))
		mainSlave(&p)
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"strings"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
)

func resourceAwsCodePipelineWebhook() *schema.Resource {
	return &schema.Resource{
		Create:        resourceAwsCodePipelineWebhookCreate,
		Read:          resourceAwsCodePipelineWebhookRead,
		Update:        nil,
		Delete:        resourceAwsCodePipelineWebhookDelete,
		SchemaVersion: 1,

		Schema: map[string]*schema.Schema{
			"auth": {
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"type": {
							Type:     schema.TypeString,
							Required: true,
						},
						"secret_token": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"allowed_ip_range": {
							Type:     schema.TypeList,
							Default:  "0.0.0.0/0",
							Optional: true,
						},
					},
				},
			},
			"filter": {
				Type:     schema.TypeSet,
				Optional: true,
				ForceNew: true,
				Elem: &schema.Schema{
					Schema: map[string]*schema.Schema{
						"json_path": {
							Type:     schema.TypeString,
							Required: true,
						},

						"match_equals": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"target": {
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"action": {
							Type:     schema.TypeString,
							Required: true,
						},
						"pipeline": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
		},
	}
}

func resourceAwsCodePipelineWebhookCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn

	auth := d.Get("auth").(map[string]interface{})
	target := d.Get("target").(map[string]interface{})
	filters := d.Get("filter").(*schema.Set)

	var rules []aws.WebhookFilterRule

	for _, f := range filters.List() {
		r = f.(map[string]interface{})
		rules = append(rules, &aws.WebhookFilterRule{
			JsonPath:    aws.String(r["json_path"]),
			MatchEquals: aws.String(r["match_equals"]),
		})
	}

	request = &codepipeline.PutWebhookInput{
		Webhook: &codepipeline.WebhookDefinition{
			Authentication: &aws.String(auth["type"]),
			Filters:        &rules,
			Name:           &aws.String(d.Get("name").(string)),
			TargetAction:   &aws.String(target["action"]),
			TargetPipeline: &aws.String(target["pipeline"]),
		},
	}

	var authConfig WebhookAuthConfiguration
	switch auth["type"] {
	case "IP":
		authConfig.AllowedIPRange = auth["allowed_ip_range"]
		break
	case "GITHUB_HMAC":
		authConfig.SecretToken = auth["allowed_ip_range"]
		break
	case "UNAUTHENTICATED":
		break
	default:
		return fmt.Errorf("Invalid authentication type %s", auth["type"])
	}

	request.Webhook.AuthenticationConfiguration = &authConfig
	webhook, err := codepipeline.PutWebhook(request)
	if err != nil {
		return fmt.Errorf("Error creating webhook: %s", err)
	}

	d.SetId(webhook.Arn)

	return resourceAwsCodePipelineWebhookRead(d, meta)
}

func getAllCodePipelineWebhooks(conn *codepipeline.CodePipeline) ([]*codepipeline.ListWebhookItem, error) {
	var webhooks []*codepipeline.ListWebhookItem
	var nextToken string

	for {
		input := &codepipeline.ListWebhooksInput{
			MaxResults: aws.Int64(int64(60)),
		}
		if nextToken != "" {
			input.NextToken = aws.String(nextToken)
		}
		out, err := conn.ListWebhooks(input)
		if err != nil {
			return pools, err
		}
		pools = append(pools, out.Webhooks...)

		if out.NextToken == nil {
			break
		}
		nextToken = aws.StringValue(out.NextToken)
	}

	return webhooks, nil
}

func resourceAwsCodePipelineWebhookRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn
	arn := d.Id()
	webhooks, err := getAllCodePipelineWebhooks(conn)
	if err != nil {
		return fmt.Errorf("Error fetching webhooks: %s", err)
	}

	if len(webhooks) == 0 {
		return fmt.Errorf("No webhooks returned!")
	}

	found * codepipeline.ListWebhookItem
	for _, w := range webhooks {
		if w.Arn == arn {
			found = w
			break
		}
	}

	if found.Arn == "" {
		return fmt.Errorf("Webhook not found: %s", arn)
	}

	d.Set("name", found.Definition.Name)
}

func resourceAwsCodePipelineWebhookDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn
	name := d.Get("name").(string)

	resp, err := conn.DeleteWebhook(&DeleteWebhookInput{
		Name: &name,
	})

	if err != nil {
		return fmt.Errorf("Could not delete webhook: %s", err)
	}

	return nil
}
<commit_msg>Flesh out the read/set functions.<commit_after>package aws

import (
	"fmt"
	"strings"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
)

func resourceAwsCodePipelineWebhook() *schema.Resource {
	return &schema.Resource{
		Create:        resourceAwsCodePipelineWebhookCreate,
		Read:          resourceAwsCodePipelineWebhookRead,
		Update:        nil,
		Delete:        resourceAwsCodePipelineWebhookDelete,
		SchemaVersion: 1,

		Schema: map[string]*schema.Schema{
			"auth": {
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"type": {
							Type:     schema.TypeString,
							Required: true,
						},
						"secret_token": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"allowed_ip_range": {
							Type:     schema.TypeList,
							Default:  "0.0.0.0/0",
							Optional: true,
						},
					},
				},
			},
			"filter": {
				Type:     schema.TypeSet,
				Optional: true,
				ForceNew: true,
				Elem: &schema.Schema{
					Schema: map[string]*schema.Schema{
						"json_path": {
							Type:     schema.TypeString,
							Required: true,
						},

						"match_equals": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"target": {
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"action": {
							Type:     schema.TypeString,
							Required: true,
						},
						"pipeline": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
		},
	}
}

func resourceAwsCodePipelineWebhookCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn

	auth := d.Get("auth").(map[string]interface{})
	target := d.Get("target").(map[string]interface{})
	filters := d.Get("filter").(*schema.Set)

	var rules []aws.WebhookFilterRule

	for _, f := range filters.List() {
		r = f.(map[string]interface{})
		rules = append(rules, &aws.WebhookFilterRule{
			JsonPath:    aws.String(r["json_path"]),
			MatchEquals: aws.String(r["match_equals"]),
		})
	}

	request = &codepipeline.PutWebhookInput{
		Webhook: &codepipeline.WebhookDefinition{
			Authentication: &aws.String(auth["type"]),
			Filters:        &rules,
			Name:           &aws.String(d.Get("name").(string)),
			TargetAction:   &aws.String(target["action"]),
			TargetPipeline: &aws.String(target["pipeline"]),
		},
	}

	var authConfig WebhookAuthConfiguration
	switch auth["type"] {
	case "IP":
		authConfig.AllowedIPRange = auth["allowed_ip_range"]
		break
	case "GITHUB_HMAC":
		authConfig.SecretToken = auth["allowed_ip_range"]
		break
	case "UNAUTHENTICATED":
		break
	default:
		return fmt.Errorf("Invalid authentication type %s", auth["type"])
	}

	request.Webhook.AuthenticationConfiguration = &authConfig
	webhook, err := codepipeline.PutWebhook(request)
	if err != nil {
		return fmt.Errorf("Error creating webhook: %s", err)
	}

	d.SetId(webhook.Arn)

	return resourceAwsCodePipelineWebhookRead(d, meta)
}

func getAllCodePipelineWebhooks(conn *codepipeline.CodePipeline) ([]*codepipeline.ListWebhookItem, error) {
	var webhooks []*codepipeline.ListWebhookItem
	var nextToken string

	for {
		input := &codepipeline.ListWebhooksInput{
			MaxResults: aws.Int64(int64(60)),
		}
		if nextToken != "" {
			input.NextToken = aws.String(nextToken)
		}
		out, err := conn.ListWebhooks(input)
		if err != nil {
			return pools, err
		}
		pools = append(pools, out.Webhooks...)

		if out.NextToken == nil {
			break
		}
		nextToken = aws.StringValue(out.NextToken)
	}

	return webhooks, nil
}

func setFilters(webhook codepipeline.WebhookDefinition, d *schema.ResourceData) error {
	filters := []interface{}{}
	for _, filter := range webhook.Filters {
		f := map[string]interface{}{
			json_path:    filter.JsonPath,
			match_equals: filter.MatchEquals,
		}
		filters = append(filters, f)
	}

	if err := d.Set("auth", auth); err != nil {
		return err
	}

	return nil
}

func setAuthentication(webhook codepipeline.WebhookDefinition, d *schema.ResourceData) error {
	auth := map[string]interface{}{
		"type": webhook.Authentication,
	}

	if webhook.AuthenticationConfiguration.AllowedIPRange != "" {
		auth["allowed_ip_range"] = webhook.AuthenticationConfiguration.AllowedIPRange
	}

	if webhook.AuthenticationConfiguration.SecretToken != "" {
		auth["secret_token"] = webhook.AuthenticationConfiguration.SecretToken
	}

	if err := d.Set("auth", auth); err != nil {
		return err
	}

	return nil
}

func resourceAwsCodePipelineWebhookRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn
	arn := d.Id()
	webhooks, err := getAllCodePipelineWebhooks(conn)
	if err != nil {
		return fmt.Errorf("Error fetching webhooks: %s", err)
	}

	if len(webhooks) == 0 {
		return fmt.Errorf("No webhooks returned!")
	}

	found * codepipeline.ListWebhookItem
	for _, w := range webhooks {
		if w.Arn == arn {
			found = w
			break
		}
	}

	if found.Arn == "" {
		return fmt.Errorf("Webhook not found: %s", arn)
	}

	d.SetId(found.Arn)
	d.Set("name", found.Definition.Name)

	if err = setAuthentication(webhook, d); err != nil {
		return err
	}

	if err = setFilters(webhook, d); err != nil {
		return err
	}

	target := map[string]interface{}{
		action:   webhook.TargetAction,
		pipeline: webhook.TargetPipeline,
	}

	if err := d.Set("target", target); err != nil {
		return err
	}

	return nil
}

func resourceAwsCodePipelineWebhookDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn
	name := d.Get("name").(string)

	resp, err := conn.DeleteWebhook(&DeleteWebhookInput{
		Name: &name,
	})

	if err != nil {
		return fmt.Errorf("Could not delete webhook: %s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package lib

import (
	"crypto/tls"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/cheggaaa/pb"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/log"
)

type snap struct {
	Version string `json:"version"`
}

func getAvailable(name string) string {
	var update snap
	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	client := &http.Client{Transport: tr}
	if !config.Cdn.Allowinsecure {
		client = &http.Client{}
	}
	resp, err := client.Get("https://" + config.Cdn.Url + ":" + config.Cdn.Sslport + "/kurjun/rest/file/info?name=" + name)
	log.Check(log.FatalLevel, "GET: https://"+config.Cdn.Url+":"+config.Cdn.Sslport+"/kurjun/rest/file/info?name="+name, err)
	defer resp.Body.Close()
	js, err := ioutil.ReadAll(resp.Body)
	log.Check(log.FatalLevel, "Reading response", err)
	log.Check(log.FatalLevel, "Parsing file list", json.Unmarshal(js, &update))
	log.Debug("Available: " + update.Version)
	return update.Version
}

func getInstalled() string {
	f, err := ioutil.ReadFile(config.Agent.AppPrefix + "/meta/package.yaml")
	if !log.Check(log.DebugLevel, "Reading file package.yaml", err) {
		lines := strings.Split(string(f), "\n")
		for _, v := range lines {
			if strings.HasPrefix(v, "version: ") {
				if version := strings.Split(strings.TrimPrefix(v, "version: "), "-"); len(version) > 1 {
					log.Debug("Installed: " + version[1])
					return version[1]
				}
			}
		}
	}
	return "0"
}

func upgradeRh(packet string) {
	log.Info("Updating Resource host")
	file, err := os.Create("/tmp/" + packet)
	log.Check(log.FatalLevel, "Creating update file", err)
	defer file.Close()
	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	client := &http.Client{Transport: tr}
	if !config.Cdn.Allowinsecure {
		client = &http.Client{}
	}
	resp, err := client.Get("https://" + config.Cdn.Url + ":" + config.Cdn.Sslport + "/kurjun/rest/file/get?name=" + packet)
	log.Check(log.FatalLevel, "GET: https://"+config.Cdn.Url+":"+config.Cdn.Sslport+"/kurjun/rest/file/get?name="+packet, err)
	defer resp.Body.Close()
	log.Info("Downloading snap package")
	bar := pb.New(int(resp.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()
	rd := bar.NewProxyReader(resp.Body)

	_, err = io.Copy(file, rd)
	log.Check(log.FatalLevel, "Writing response to file", err)

	log.Check(log.FatalLevel, "Installing update /tmp/"+packet,
		exec.Command("snappy", "install", "--allow-unauthenticated", "/tmp/"+packet).Run())
	log.Check(log.FatalLevel, "Removing update file /tmp/"+packet, os.Remove("/tmp/"+packet))

}

func Update(name string, check bool) {
	if !lockSubutai(name + ".update") {
		log.Error("Another update process is already running")
	}
	defer unlockSubutai()
	switch name {
	case "rh":
		packet := "subutai_" + config.Template.Version + "_" + config.Template.Arch + ".snap"
		if len(config.Template.Branch) != 0 {
			packet = "subutai_" + config.Template.Version + "_" + config.Template.Arch + "-" + config.Template.Branch + ".snap"
		}

		installed, err := strconv.Atoi(getInstalled())
		log.Check(log.FatalLevel, "Converting installed package timestamp to int", err)
		available, err := strconv.Atoi(getAvailable(packet))
		log.Check(log.FatalLevel, "Converting available package timestamp to int", err)

		if installed >= available {
			log.Info("No update is available")
			os.Exit(1)
		} else if check {
			log.Info("Update is available")
			os.Exit(0)
		}

		upgradeRh(packet)

	default:
		if !container.IsContainer(name) {
			log.Error("no such instance \"" + name + "\"")
		}
		_, err := container.AttachExec(name, []string{"apt-get", "-qq", "update", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5"})
		log.Check(log.FatalLevel, "Updating apt index", err)
		output, err := container.AttachExec(name, []string{"apt-get", "-qq", "upgrade", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5", "-s"})
		log.Check(log.FatalLevel, "Checking for available update", err)
		if len(output) == 0 {
			log.Info("No update is available")
			os.Exit(1)
		} else if check {
			log.Info("Update is available")
			os.Exit(0)
		}
		_, err = container.AttachExec(name, []string{"apt-get", "-qq", "upgrade", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5"})
		log.Check(log.FatalLevel, "Updating container", err)
	}
}
<commit_msg>Update temporary switched to request by hash<commit_after>package lib

import (
	"crypto/tls"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/cheggaaa/pb"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/log"
)

type snap struct {
	Version string `json:"version"`
	Hash    string `json:"md5Sum"`
}

// func getAvailable(name string) string {
// 	var update snap
// 	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
// 	client := &http.Client{Transport: tr}
// 	if !config.Cdn.Allowinsecure {
// 		client = &http.Client{}
// 	}
// 	resp, err := client.Get("https://" + config.Cdn.Url + ":" + config.Cdn.Sslport + "/kurjun/rest/file/info?name=" + name)
// 	log.Check(log.FatalLevel, "GET: https://"+config.Cdn.Url+":"+config.Cdn.Sslport+"/kurjun/rest/file/info?name="+name, err)
// 	defer resp.Body.Close()
// 	js, err := ioutil.ReadAll(resp.Body)
// 	log.Check(log.FatalLevel, "Reading response", err)
// 	log.Check(log.FatalLevel, "Parsing file list", json.Unmarshal(js, &update))
// 	log.Debug("Available: " + update.Version)
// 	return update.Version
// }

//Temporary function until gorjun cache will be fixed
func ifUpdateable(installed int) string {
	var update snap
	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	client := &http.Client{Transport: tr}
	if !config.Cdn.Allowinsecure {
		client = &http.Client{}
	}
	packet := "subutai_" + config.Template.Version + "_" + config.Template.Arch + ".snap"
	if len(config.Template.Branch) != 0 {
		packet = "subutai_" + config.Template.Version + "_" + config.Template.Arch + "-" + config.Template.Branch + ".snap"
	}
	resp, err := client.Get("https://" + config.Cdn.Url + ":" + config.Cdn.Sslport + "/kurjun/rest/file/info?name=" + packet)
	log.Check(log.FatalLevel, "GET: https://"+config.Cdn.Url+":"+config.Cdn.Sslport+"/kurjun/rest/file/info?name="+packet, err)
	defer resp.Body.Close()
	js, err := ioutil.ReadAll(resp.Body)
	log.Check(log.FatalLevel, "Reading response", err)
	log.Check(log.FatalLevel, "Parsing file list", json.Unmarshal(js, &update))
	log.Debug("Available: " + update.Version)
	available, err := strconv.Atoi(update.Version)
	log.Check(log.ErrorLevel, "Converting available package timestamp to int", err)
	if installed >= available {
		return ""
	} else {
		return update.Hash
	}

}

func getInstalled() string {
	f, err := ioutil.ReadFile(config.Agent.AppPrefix + "/meta/package.yaml")
	if !log.Check(log.DebugLevel, "Reading file package.yaml", err) {
		lines := strings.Split(string(f), "\n")
		for _, v := range lines {
			if strings.HasPrefix(v, "version: ") {
				if version := strings.Split(strings.TrimPrefix(v, "version: "), "-"); len(version) > 1 {
					log.Debug("Installed: " + version[1])
					return version[1]
				}
			}
		}
	}
	return "0"
}

func upgradeRh(packet string) {
	log.Info("Updating Resource host")
	file, err := os.Create("/tmp/" + packet)
	log.Check(log.FatalLevel, "Creating update file", err)
	defer file.Close()
	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	client := &http.Client{Transport: tr}
	if !config.Cdn.Allowinsecure {
		client = &http.Client{}
	}
	resp, err := client.Get("https://" + config.Cdn.Url + ":" + config.Cdn.Sslport + "/kurjun/rest/file/get?id=" + packet)
	log.Check(log.FatalLevel, "GET: https://"+config.Cdn.Url+":"+config.Cdn.Sslport+"/kurjun/rest/file/get?id="+packet, err)
	defer resp.Body.Close()
	log.Info("Downloading snap package")
	bar := pb.New(int(resp.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()
	rd := bar.NewProxyReader(resp.Body)

	_, err = io.Copy(file, rd)
	log.Check(log.FatalLevel, "Writing response to file", err)

	log.Check(log.FatalLevel, "Installing update /tmp/"+packet,
		exec.Command("snappy", "install", "--allow-unauthenticated", "/tmp/"+packet).Run())
	log.Check(log.FatalLevel, "Removing update file /tmp/"+packet, os.Remove("/tmp/"+packet))

}

func Update(name string, check bool) {
	if !lockSubutai(name + ".update") {
		log.Error("Another update process is already running")
	}
	defer unlockSubutai()
	switch name {
	case "rh":
		// packet := "subutai_" + config.Template.Version + "_" + config.Template.Arch + ".snap"
		// if len(config.Template.Branch) != 0 {
		// 	packet = "subutai_" + config.Template.Version + "_" + config.Template.Arch + "-" + config.Template.Branch + ".snap"
		// }

		installed, err := strconv.Atoi(getInstalled())
		log.Check(log.FatalLevel, "Converting installed package timestamp to int", err)
		// available, err := strconv.Atoi(getAvailable(packet))
		// log.Check(log.FatalLevel, "Converting available package timestamp to int", err)

		//Temporary workaround until gorjun cache will be fixed
		newsnap := ifUpdateable(installed)
		if len(newsnap) == 0 {
			log.Info("No update is available")
			os.Exit(1)
		} else if check {
			log.Info("Update is available")
			os.Exit(0)
		}
		upgradeRh(newsnap)

		// if installed >= available {
		// 	log.Info("No update is available")
		// 	os.Exit(1)
		// } else if check {
		// 	log.Info("Update is available")
		// 	os.Exit(0)
		// }

		// upgradeRh(packet)

	default:
		if !container.IsContainer(name) {
			log.Error("no such instance \"" + name + "\"")
		}
		_, err := container.AttachExec(name, []string{"apt-get", "-qq", "update", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5"})
		log.Check(log.FatalLevel, "Updating apt index", err)
		output, err := container.AttachExec(name, []string{"apt-get", "-qq", "upgrade", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5", "-s"})
		log.Check(log.FatalLevel, "Checking for available update", err)
		if len(output) == 0 {
			log.Info("No update is available")
			os.Exit(1)
		} else if check {
			log.Info("Update is available")
			os.Exit(0)
		}
		_, err = container.AttachExec(name, []string{"apt-get", "-qq", "upgrade", "-y", "--force-yes", "-o", "Acquire::http::Timeout=5"})
		log.Check(log.FatalLevel, "Updating container", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/daviddengcn/go-colortext"
	"github.com/korovkin/limiter"
	"github.com/korovkin/parallel"
)

type logger struct {
	ticket int
}

var (
	loggerMutex     = new(sync.Mutex)
	loggerIndex     = int(0)
	loggerStartTime = time.Now()
)

var loggerColors = []ct.Color{
	ct.Green,
	ct.Cyan,
	ct.Magenta,
	ct.Yellow,
	ct.Blue,
	ct.Red,
}

func (l *logger) Write(p []byte) (int, error) {
	buf := bytes.NewBuffer(p)
	wrote := 0
	for {
		line, err := buf.ReadBytes('\n')
		if len(line) > 1 {
			now := time.Now().Format("15:04:05")
			s := string(line)
			ts := time.Since(loggerStartTime).String()

			loggerMutex.Lock()
			ct.ChangeColor(loggerColors[l.ticket%len(loggerColors)], false, ct.None, false)
			fmt.Printf("[%14s %s %d] ", ts, now, l.ticket)
			ct.ResetColor()
			fmt.Print(s)
			loggerMutex.Unlock()

			wrote += len(line)
		}
		if err != nil {
			break
		}
	}
	if len(p) > 0 && p[len(p)-1] != '\n' {
		fmt.Println()
	}

	return len(p), nil
}

func newLogger(ticket int) *logger {
	loggerMutex.Lock()
	defer loggerMutex.Unlock()
	l := &logger{ticket}
	return l
}

func executeCommand(p *Parallel, ticket int, cmdLine string) bool {
	T_START := time.Now()
	logger := newLogger(ticket)

	defer func() {
		fmt.Fprintf(logger, "done: dt: "+time.Since(T_START).String()+"\n")
	}()

	output, err := p.client.Execute(&parallel.Cmd{
		CmdLine: cmdLine,
		Ticket:  int64(ticket),
	})
	if err != nil {
		log.Fatalln("failed to execute:", err.Error())
	}
	fmt.Fprintf(logger, "execute: output: '"+output+"'\n")

	cs := []string{"/bin/sh", "-c", cmdLine}
	cmd := exec.Command(cs[0], cs[1:]...)
	cmd.Stdin = nil
	cmd.Stdout = logger
	cmd.Stderr = logger
	cmd.Env = append(
		os.Environ(),
		fmt.Sprintf("PARALLEL_TICKER=%d", ticket),
	)

	fmt.Fprintf(logger, "run: '"+cmdLine+"'\n")

	err = cmd.Start()
	if err != nil {
		log.Fatalln("failed to start:", err)
		return true
	}

	err = cmd.Wait()

	return true
}

type Parallel struct {
	jobs    int
	logger  *logger
	worker  *limiter.ConcurrencyLimiter
	address string

	// master / slave
	protocolFactory  thrift.TProtocolFactory
	transportFactory thrift.TTransportFactory

	// master:
	transport thrift.TTransport
	client    *parallel.ParallelClient

	// slave:
	handler *ParallelSlaveHandler
}

func (p *Parallel) Close() {
	if p.transport != nil {
		p.transport.Close()
	}
	p.worker.Wait()
}

func mainMaster(p *Parallel) {
	var err error
	p.transport, err = thrift.NewTSocket(p.address)

	if p.transport == nil {
		log.Fatalln("failed allocate transport")
	}

	if err != nil {
		log.Fatalln("failed to dial slave:", err.Error())
	}

	p.transport = p.transportFactory.GetTransport(p.transport)

	err = p.transport.Open()
	if err != nil {
		log.Fatalln("failed to open:", err.Error())
	}

	p.client = parallel.NewParallelClientFactory(p.transport, p.protocolFactory)
	p.client.Ping()

	r := bufio.NewReaderSize(os.Stdin, 1*1024*1024)
	fmt.Fprintf(p.logger, "reading from stdin...\n")
	for {
		line, err := r.ReadString('\n')
		if err == io.EOF {
			break
		}
		line = strings.TrimSpace(line)

		p.worker.ExecuteWithTicket(func(ticket int) {
			executeCommand(p, ticket, line)
		})
	}
}

type ParallelSlaveHandler struct {
}

func NewParallelSlaveHandler() *ParallelSlaveHandler {
	return &ParallelSlaveHandler{}
}

func (p *ParallelSlaveHandler) Execute(command *parallel.Cmd) (r string, err error) {
	log.Println("ParallelSlaveHandler: Execute: ", command.CmdLine)
	return "ok", nil
}

func (p *ParallelSlaveHandler) Ping() (r string, err error) {
	log.Println("ParallelSlaveHandler: Ping")
	return "ok", nil
}

func mainSlave(p *Parallel) {
	var err error

	var protocolFactory thrift.TProtocolFactory
	protocolFactory = thrift.NewTBinaryProtocolFactoryDefault()

	var transportFactory thrift.TTransportFactory
	transportFactory = thrift.NewTTransportFactory()
	transportFactory = thrift.NewTFramedTransportFactory(transportFactory)

	var transport thrift.TServerTransport
	transport, err = thrift.NewTServerSocket(p.address)

	if err != nil {
		log.Fatalln("failed to start server:", err.Error())
		return
	}

	p.handler = NewParallelSlaveHandler()

	server := thrift.NewTSimpleServer4(
		parallel.NewParallelProcessor(p.handler),
		transport,
		transportFactory,
		protocolFactory)

	err = server.Serve()
	if err != nil {
		log.Fatalln("failed to run slave:", err.Error())
	}
}

func main() {
	T_START := time.Now()
	logger := newLogger(0)
	defer func() {
		fmt.Fprintf(logger, "all done: dt: "+time.Since(T_START).String()+"\n")
	}()

	flag_jobs := flag.Int(
		"j",
		2,
		"num of concurrent jobs")
	flag_slave := flag.Bool(
		"slave",
		false,
		"run as slave")

	flag.Parse()
	fmt.Fprintf(logger, fmt.Sprintf("concurrency limit: %d", *flag_jobs))

	p := Parallel{}
	p.jobs = *flag_jobs
	p.logger = logger
	p.worker = limiter.NewConcurrencyLimiter(p.jobs)
	p.address = "localhost:9010"
	defer p.Close()

	// thrift protocol
	p.protocolFactory = thrift.NewTBinaryProtocolFactoryDefault()

	// thrift transport
	p.transportFactory = thrift.NewTTransportFactory()
	p.transportFactory = thrift.NewTFramedTransportFactory(p.transportFactory)

	if *flag_slave == false {
		fmt.Fprintf(logger, fmt.Sprintf("running as master\n"))
		mainMaster(&p)
	} else {
		fmt.Fprintf(logger, fmt.Sprintf("running as slave\n"))
		mainSlave(&p)
	}
}
<commit_msg>fix defer<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/daviddengcn/go-colortext"
	"github.com/korovkin/limiter"
	"github.com/korovkin/parallel"
)

type logger struct {
	ticket int
}

var (
	loggerMutex     = new(sync.Mutex)
	loggerIndex     = int(0)
	loggerStartTime = time.Now()
)

var loggerColors = []ct.Color{
	ct.Green,
	ct.Cyan,
	ct.Magenta,
	ct.Yellow,
	ct.Blue,
	ct.Red,
}

func (l *logger) Write(p []byte) (int, error) {
	buf := bytes.NewBuffer(p)
	wrote := 0
	for {
		line, err := buf.ReadBytes('\n')
		if len(line) > 1 {
			now := time.Now().Format("15:04:05")
			s := string(line)
			ts := time.Since(loggerStartTime).String()

			loggerMutex.Lock()
			ct.ChangeColor(loggerColors[l.ticket%len(loggerColors)], false, ct.None, false)
			fmt.Printf("[%14s %s %d] ", ts, now, l.ticket)
			ct.ResetColor()
			fmt.Print(s)
			loggerMutex.Unlock()

			wrote += len(line)
		}
		if err != nil {
			break
		}
	}
	if len(p) > 0 && p[len(p)-1] != '\n' {
		fmt.Println()
	}

	return len(p), nil
}

func newLogger(ticket int) *logger {
	loggerMutex.Lock()
	defer loggerMutex.Unlock()
	l := &logger{ticket}
	return l
}

func executeCommand(p *Parallel, ticket int, cmdLine string) bool {
	T_START := time.Now()
	logger := newLogger(ticket)

	defer func() {
		fmt.Fprintf(logger, "done: dt: "+time.Since(T_START).String()+"\n")
	}()

	output, err := p.client.Execute(&parallel.Cmd{
		CmdLine: cmdLine,
		Ticket:  int64(ticket),
	})
	if err != nil {
		log.Fatalln("failed to execute:", err.Error())
	}
	fmt.Fprintf(logger, "execute: output: '"+output+"'\n")

	cs := []string{"/bin/sh", "-c", cmdLine}
	cmd := exec.Command(cs[0], cs[1:]...)
	cmd.Stdin = nil
	cmd.Stdout = logger
	cmd.Stderr = logger
	cmd.Env = append(
		os.Environ(),
		fmt.Sprintf("PARALLEL_TICKER=%d", ticket),
	)

	fmt.Fprintf(logger, "run: '"+cmdLine+"'\n")

	err = cmd.Start()
	if err != nil {
		log.Fatalln("failed to start:", err)
		return true
	}

	err = cmd.Wait()

	return true
}

type Parallel struct {
	jobs    int
	logger  *logger
	worker  *limiter.ConcurrencyLimiter
	address string

	// master / slave
	protocolFactory  thrift.TProtocolFactory
	transportFactory thrift.TTransportFactory

	// master:
	transport thrift.TTransport
	client    *parallel.ParallelClient

	// slave:
	handler *ParallelSlaveHandler
}

func (p *Parallel) Close() {
	p.worker.Wait()
	if p.transport != nil {
		p.transport.Close()
	}
}

func mainMaster(p *Parallel) {
	var err error
	p.transport, err = thrift.NewTSocket(p.address)

	if p.transport == nil {
		log.Fatalln("failed allocate transport")
	}

	if err != nil {
		log.Fatalln("failed to dial slave:", err.Error())
	}

	p.transport = p.transportFactory.GetTransport(p.transport)

	err = p.transport.Open()
	if err != nil {
		log.Fatalln("failed to open:", err.Error())
	}

	p.client = parallel.NewParallelClientFactory(p.transport, p.protocolFactory)
	p.client.Ping()

	r := bufio.NewReaderSize(os.Stdin, 1*1024*1024)
	fmt.Fprintf(p.logger, "reading from stdin...\n")
	for {
		line, err := r.ReadString('\n')
		if err == io.EOF {
			break
		}
		line = strings.TrimSpace(line)

		p.worker.ExecuteWithTicket(func(ticket int) {
			executeCommand(p, ticket, line)
		})
	}
}

type ParallelSlaveHandler struct {
}

func NewParallelSlaveHandler() *ParallelSlaveHandler {
	return &ParallelSlaveHandler{}
}

func (p *ParallelSlaveHandler) Execute(command *parallel.Cmd) (r string, err error) {
	log.Println("ParallelSlaveHandler: Execute: ", command.CmdLine)
	return "ok", nil
}

func (p *ParallelSlaveHandler) Ping() (r string, err error) {
	log.Println("ParallelSlaveHandler: Ping")
	return "ok", nil
}

func mainSlave(p *Parallel) {
	var err error

	var protocolFactory thrift.TProtocolFactory
	protocolFactory = thrift.NewTBinaryProtocolFactoryDefault()

	var transportFactory thrift.TTransportFactory
	transportFactory = thrift.NewTTransportFactory()
	transportFactory = thrift.NewTFramedTransportFactory(transportFactory)

	var transport thrift.TServerTransport
	transport, err = thrift.NewTServerSocket(p.address)

	if err != nil {
		log.Fatalln("failed to start server:", err.Error())
		return
	}

	p.handler = NewParallelSlaveHandler()

	server := thrift.NewTSimpleServer4(
		parallel.NewParallelProcessor(p.handler),
		transport,
		transportFactory,
		protocolFactory)

	err = server.Serve()
	if err != nil {
		log.Fatalln("failed to run slave:", err.Error())
	}
}

func main() {
	T_START := time.Now()
	logger := newLogger(0)
	defer func() {
		fmt.Fprintf(logger, "all done: dt: "+time.Since(T_START).String()+"\n")
	}()

	flag_jobs := flag.Int(
		"j",
		2,
		"num of concurrent jobs")
	flag_slave := flag.Bool(
		"slave",
		false,
		"run as slave")

	flag.Parse()
	fmt.Fprintf(logger, fmt.Sprintf("concurrency limit: %d", *flag_jobs))

	p := Parallel{}
	p.jobs = *flag_jobs
	p.logger = logger
	p.worker = limiter.NewConcurrencyLimiter(p.jobs)
	p.address = "localhost:9010"
	defer p.Close()

	// thrift protocol
	p.protocolFactory = thrift.NewTBinaryProtocolFactoryDefault()

	// thrift transport
	p.transportFactory = thrift.NewTTransportFactory()
	p.transportFactory = thrift.NewTFramedTransportFactory(p.transportFactory)

	if *flag_slave == false {
		fmt.Fprintf(logger, fmt.Sprintf("running as master\n"))
		mainMaster(&p)
	} else {
		fmt.Fprintf(logger, fmt.Sprintf("running as slave\n"))
		mainSlave(&p)
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/codepipeline"
)

func resourceAwsCodePipelineWebhook() *schema.Resource {
	return &schema.Resource{
		Create:        resourceAwsCodePipelineWebhookCreate,
		Read:          resourceAwsCodePipelineWebhookRead,
		Update:        nil,
		Delete:        resourceAwsCodePipelineWebhookDelete,
		SchemaVersion: 1,

		Schema: map[string]*schema.Schema{
			"auth": {
				Type:     schema.TypeList,
				MaxItems: 1,
				MinItems: 1,
				Required: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"type": {
							Type:     schema.TypeString,
							Required: true,
							ValidateFunc: validation.StringInSlice([]string{
								codepipeline.WebhookAuthenticationTypeGithubHmac,
								codepipeline.WebhookAuthenticationTypeIp,
								codepipeline.WebhookAuthenticationTypeUnauthenticated,
							}, false),
						},
						"secret_token": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"allowed_ip_range": {
							Type:         schema.TypeString,
							Optional:     true,
							ValidateFunc: validation.CIDRNetwork(0, 32),
						},
					},
				},
			},
			"filter": {
				Type:     schema.TypeSet,
				Optional: true,
				ForceNew: true,
				MinItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"json_path": {
							Type:     schema.TypeString,
							Required: true,
						},

						"match_equals": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"url": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"target": {
				Type:     schema.TypeList,
				MaxItems: 1,
				MinItems: 1,
				Required: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"action": {
							Type:     schema.TypeString,
							Required: true,
						},
						"pipeline": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
		},
	}
}

func extractCodePipelineWebhookAttr(d *schema.ResourceData, attr string) (map[string]interface{}, error) {
	if v, ok := d.GetOk(attr); ok {
		l := v.([]interface{})
		if len(l) <= 0 {
			return nil, fmt.Errorf("Attribute %s is missing", attr)
		}

		data := l[0].(map[string]interface{})
		return data, nil
	}

	return nil, fmt.Errorf("Could not find attribute %s", attr)
}

func extractCodePipelineWebhookRules(filters *schema.Set) ([]*codepipeline.WebhookFilterRule, error) {
	var rules []*codepipeline.WebhookFilterRule

	for _, f := range filters.List() {
		r := f.(map[string]interface{})
		filter := codepipeline.WebhookFilterRule{
			JsonPath:    aws.String(r["json_path"].(string)),
			MatchEquals: aws.String(r["match_equals"].(string)),
		}

		rules = append(rules, &filter)
	}

	if len(rules) <= 0 {
		return nil, fmt.Errorf("One or more webhook filter rule is required (%d rules from %d filter blocks)", len(rules), len(filters.List()))
	}

	return rules, nil
}

func extractCodePipelineWebhookAuthConfig(auth map[string]interface{}) (*codepipeline.WebhookAuthConfiguration, error) {
	var authConfig codepipeline.WebhookAuthConfiguration
	switch auth["type"].(string) {
	case codepipeline.WebhookAuthenticationTypeIp:
		ipRange := auth["allowed_ip_range"].(string)
		if ipRange == "" {
			return nil, fmt.Errorf("An IP range must be set when using IP-based auth")
		}

		authConfig.AllowedIPRange = &ipRange

		break
	case codepipeline.WebhookAuthenticationTypeGithubHmac:
		secretToken := auth["secret_token"].(string)
		if secretToken == "" {
			return nil, fmt.Errorf("Secret token must be set when using GITHUB_HMAC")
		}

		authConfig.SecretToken = &secretToken
		break
	case codepipeline.WebhookAuthenticationTypeUnauthenticated:
		break
	default:
		return nil, fmt.Errorf("Invalid authentication type %s", auth["type"])
	}

	return &authConfig, nil
}

func resourceAwsCodePipelineWebhookCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn

	auth, err := extractCodePipelineWebhookAttr(d, "auth")
	if err != nil {
		return err
	}

	target, err := extractCodePipelineWebhookAttr(d, "target")
	if err != nil {
		return err
	}

	rules, err := extractCodePipelineWebhookRules(d.Get("filter").(*schema.Set))
	if err != nil {
		return err
	}

	authConfig, err := extractCodePipelineWebhookAuthConfig(auth)

	request := &codepipeline.PutWebhookInput{
		Webhook: &codepipeline.WebhookDefinition{
			Authentication:              aws.String(auth["type"].(string)),
			Filters:                     rules,
			Name:                        aws.String(d.Get("name").(string)),
			TargetAction:                aws.String(target["action"].(string)),
			TargetPipeline:              aws.String(target["pipeline"].(string)),
			AuthenticationConfiguration: authConfig,
		},
	}

	webhook, err := conn.PutWebhook(request)
	if err != nil {
		return fmt.Errorf("Error creating webhook: %s", err)
	}

	arn := *webhook.Webhook.Arn
	d.SetId(arn)

	url := *webhook.Webhook.Url
	d.Set("url", url)

	return resourceAwsCodePipelineWebhookRead(d, meta)
}

func getAllCodePipelineWebhooks(conn *codepipeline.CodePipeline) ([]*codepipeline.ListWebhookItem, error) {
	var webhooks []*codepipeline.ListWebhookItem
	var nextToken string

	for {
		input := &codepipeline.ListWebhooksInput{
			MaxResults: aws.Int64(int64(60)),
		}
		if nextToken != "" {
			input.NextToken = aws.String(nextToken)
		}

		out, err := conn.ListWebhooks(input)
		if err != nil {
			return webhooks, err
		}

		webhooks = append(webhooks, out.Webhooks...)

		if out.NextToken == nil {
			break
		}

		nextToken = aws.StringValue(out.NextToken)
	}

	return webhooks, nil
}

func setCodePipelineWebhookFilters(webhook codepipeline.WebhookDefinition, d *schema.ResourceData) error {
	filters := []interface{}{}
	for _, filter := range webhook.Filters {
		f := map[string]interface{}{
			"json_path":    *filter.JsonPath,
			"match_equals": *filter.MatchEquals,
		}
		filters = append(filters, f)
	}

	if err := d.Set("filter", filters); err != nil {
		return err
	}

	return nil
}

func setCodePipelineWebhookAuthentication(webhook codepipeline.WebhookDefinition, d *schema.ResourceData) error {
	var result []interface{}

	auth := map[string]interface{}{}

	authType := *webhook.Authentication
	auth["type"] = authType

	if webhook.AuthenticationConfiguration.AllowedIPRange != nil {
		ipRange := *webhook.AuthenticationConfiguration.AllowedIPRange
		auth["allowed_ip_range"] = ipRange
	}

	if webhook.AuthenticationConfiguration.SecretToken != nil {
		secretToken := *webhook.AuthenticationConfiguration.SecretToken
		auth["secret_token"] = secretToken
	}

	result = append(result, auth)
	if err := d.Set("auth", result); err != nil {
		return err
	}

	return nil
}

func setCodePipelineWebhookTarget(webhook codepipeline.WebhookDefinition, d *schema.ResourceData) error {
	var t []interface{}
	target := map[string]interface{}{
		"action":   *webhook.TargetAction,
		"pipeline": *webhook.TargetPipeline,
	}
	t = append(t, target)

	if err := d.Set("target", t); err != nil {
		return err
	}

	return nil
}

func resourceAwsCodePipelineWebhookRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn
	arn := d.Id()
	webhooks, err := getAllCodePipelineWebhooks(conn)
	if err != nil {
		return fmt.Errorf("Error fetching webhooks: %s", err)
	}

	if len(webhooks) == 0 {
		return fmt.Errorf("No webhooks returned!")
	}

	var found codepipeline.WebhookDefinition
	for _, w := range webhooks {
		a := *w.Arn
		if a == arn {
			found = *w.Definition
			break
		}
	}

	name := *found.Name
	if name == "" {
		return fmt.Errorf("Webhook not found: %s", arn)
	}

	d.Set("name", name)

	if err = setCodePipelineWebhookAuthentication(found, d); err != nil {
		return err
	}

	if err = setCodePipelineWebhookFilters(found, d); err != nil {
		return err
	}

	if err = setCodePipelineWebhookTarget(found, d); err != nil {
		return err
	}

	return nil
}

func resourceAwsCodePipelineWebhookDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn
	name := d.Get("name").(string)

	input := codepipeline.DeleteWebhookInput{
		Name: &name,
	}
	_, err := conn.DeleteWebhook(&input)

	if err != nil {
		return fmt.Errorf("Could not delete webhook: %s", err)
	}

	d.SetId("")

	return nil
}
<commit_msg>Make this required since AWS requires at least 1.<commit_after>package aws

import (
	"fmt"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/codepipeline"
)

func resourceAwsCodePipelineWebhook() *schema.Resource {
	return &schema.Resource{
		Create:        resourceAwsCodePipelineWebhookCreate,
		Read:          resourceAwsCodePipelineWebhookRead,
		Update:        nil,
		Delete:        resourceAwsCodePipelineWebhookDelete,
		SchemaVersion: 1,

		Schema: map[string]*schema.Schema{
			"auth": {
				Type:     schema.TypeList,
				MaxItems: 1,
				MinItems: 1,
				Required: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"type": {
							Type:     schema.TypeString,
							Required: true,
							ValidateFunc: validation.StringInSlice([]string{
								codepipeline.WebhookAuthenticationTypeGithubHmac,
								codepipeline.WebhookAuthenticationTypeIp,
								codepipeline.WebhookAuthenticationTypeUnauthenticated,
							}, false),
						},
						"secret_token": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"allowed_ip_range": {
							Type:         schema.TypeString,
							Optional:     true,
							ValidateFunc: validation.CIDRNetwork(0, 32),
						},
					},
				},
			},
			"filter": {
				Type:     schema.TypeSet,
				Required: true,
				ForceNew: true,
				MinItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"json_path": {
							Type:     schema.TypeString,
							Required: true,
						},

						"match_equals": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"url": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"target": {
				Type:     schema.TypeList,
				MaxItems: 1,
				MinItems: 1,
				Required: true,
				ForceNew: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"action": {
							Type:     schema.TypeString,
							Required: true,
						},
						"pipeline": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
		},
	}
}

func extractCodePipelineWebhookAttr(d *schema.ResourceData, attr string) (map[string]interface{}, error) {
	if v, ok := d.GetOk(attr); ok {
		l := v.([]interface{})
		if len(l) <= 0 {
			return nil, fmt.Errorf("Attribute %s is missing", attr)
		}

		data := l[0].(map[string]interface{})
		return data, nil
	}

	return nil, fmt.Errorf("Could not find attribute %s", attr)
}

func extractCodePipelineWebhookRules(filters *schema.Set) ([]*codepipeline.WebhookFilterRule, error) {
	var rules []*codepipeline.WebhookFilterRule

	for _, f := range filters.List() {
		r := f.(map[string]interface{})
		filter := codepipeline.WebhookFilterRule{
			JsonPath:    aws.String(r["json_path"].(string)),
			MatchEquals: aws.String(r["match_equals"].(string)),
		}

		rules = append(rules, &filter)
	}

	return rules, nil
}

func extractCodePipelineWebhookAuthConfig(auth map[string]interface{}) (*codepipeline.WebhookAuthConfiguration, error) {
	var authConfig codepipeline.WebhookAuthConfiguration
	switch auth["type"].(string) {
	case codepipeline.WebhookAuthenticationTypeIp:
		ipRange := auth["allowed_ip_range"].(string)
		if ipRange == "" {
			return nil, fmt.Errorf("An IP range must be set when using IP-based auth")
		}

		authConfig.AllowedIPRange = &ipRange

		break
	case codepipeline.WebhookAuthenticationTypeGithubHmac:
		secretToken := auth["secret_token"].(string)
		if secretToken == "" {
			return nil, fmt.Errorf("Secret token must be set when using GITHUB_HMAC")
		}

		authConfig.SecretToken = &secretToken
		break
	case codepipeline.WebhookAuthenticationTypeUnauthenticated:
		break
	default:
		return nil, fmt.Errorf("Invalid authentication type %s", auth["type"])
	}

	return &authConfig, nil
}

func resourceAwsCodePipelineWebhookCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn

	auth, err := extractCodePipelineWebhookAttr(d, "auth")
	if err != nil {
		return err
	}

	target, err := extractCodePipelineWebhookAttr(d, "target")
	if err != nil {
		return err
	}

	rules, err := extractCodePipelineWebhookRules(d.Get("filter").(*schema.Set))
	if err != nil {
		return err
	}

	authConfig, err := extractCodePipelineWebhookAuthConfig(auth)

	request := &codepipeline.PutWebhookInput{
		Webhook: &codepipeline.WebhookDefinition{
			Authentication:              aws.String(auth["type"].(string)),
			Filters:                     rules,
			Name:                        aws.String(d.Get("name").(string)),
			TargetAction:                aws.String(target["action"].(string)),
			TargetPipeline:              aws.String(target["pipeline"].(string)),
			AuthenticationConfiguration: authConfig,
		},
	}

	webhook, err := conn.PutWebhook(request)
	if err != nil {
		return fmt.Errorf("Error creating webhook: %s", err)
	}

	arn := *webhook.Webhook.Arn
	d.SetId(arn)

	url := *webhook.Webhook.Url
	d.Set("url", url)

	return resourceAwsCodePipelineWebhookRead(d, meta)
}

func getAllCodePipelineWebhooks(conn *codepipeline.CodePipeline) ([]*codepipeline.ListWebhookItem, error) {
	var webhooks []*codepipeline.ListWebhookItem
	var nextToken string

	for {
		input := &codepipeline.ListWebhooksInput{
			MaxResults: aws.Int64(int64(60)),
		}
		if nextToken != "" {
			input.NextToken = aws.String(nextToken)
		}

		out, err := conn.ListWebhooks(input)
		if err != nil {
			return webhooks, err
		}

		webhooks = append(webhooks, out.Webhooks...)

		if out.NextToken == nil {
			break
		}

		nextToken = aws.StringValue(out.NextToken)
	}

	return webhooks, nil
}

func setCodePipelineWebhookFilters(webhook codepipeline.WebhookDefinition, d *schema.ResourceData) error {
	filters := []interface{}{}
	for _, filter := range webhook.Filters {
		f := map[string]interface{}{
			"json_path":    *filter.JsonPath,
			"match_equals": *filter.MatchEquals,
		}
		filters = append(filters, f)
	}

	if err := d.Set("filter", filters); err != nil {
		return err
	}

	return nil
}

func setCodePipelineWebhookAuthentication(webhook codepipeline.WebhookDefinition, d *schema.ResourceData) error {
	var result []interface{}

	auth := map[string]interface{}{}

	authType := *webhook.Authentication
	auth["type"] = authType

	if webhook.AuthenticationConfiguration.AllowedIPRange != nil {
		ipRange := *webhook.AuthenticationConfiguration.AllowedIPRange
		auth["allowed_ip_range"] = ipRange
	}

	if webhook.AuthenticationConfiguration.SecretToken != nil {
		secretToken := *webhook.AuthenticationConfiguration.SecretToken
		auth["secret_token"] = secretToken
	}

	result = append(result, auth)
	if err := d.Set("auth", result); err != nil {
		return err
	}

	return nil
}

func setCodePipelineWebhookTarget(webhook codepipeline.WebhookDefinition, d *schema.ResourceData) error {
	var t []interface{}
	target := map[string]interface{}{
		"action":   *webhook.TargetAction,
		"pipeline": *webhook.TargetPipeline,
	}
	t = append(t, target)

	if err := d.Set("target", t); err != nil {
		return err
	}

	return nil
}

func resourceAwsCodePipelineWebhookRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn
	arn := d.Id()
	webhooks, err := getAllCodePipelineWebhooks(conn)
	if err != nil {
		return fmt.Errorf("Error fetching webhooks: %s", err)
	}

	if len(webhooks) == 0 {
		return fmt.Errorf("No webhooks returned!")
	}

	var found codepipeline.WebhookDefinition
	for _, w := range webhooks {
		a := *w.Arn
		if a == arn {
			found = *w.Definition
			break
		}
	}

	name := *found.Name
	if name == "" {
		return fmt.Errorf("Webhook not found: %s", arn)
	}

	d.Set("name", name)

	if err = setCodePipelineWebhookAuthentication(found, d); err != nil {
		return err
	}

	if err = setCodePipelineWebhookFilters(found, d); err != nil {
		return err
	}

	if err = setCodePipelineWebhookTarget(found, d); err != nil {
		return err
	}

	return nil
}

func resourceAwsCodePipelineWebhookDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).codepipelineconn
	name := d.Get("name").(string)

	input := codepipeline.DeleteWebhookInput{
		Name: &name,
	}
	_, err := conn.DeleteWebhook(&input)

	if err != nil {
		return fmt.Errorf("Could not delete webhook: %s", err)
	}

	d.SetId("")

	return nil
}
<|endoftext|>