<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"reflect"
	"time"

	docker_types "github.com/docker/docker/api/types"
	docker_types_container "github.com/docker/docker/api/types/container"
	docker_types_event "github.com/docker/docker/api/types/events"
	docker_types_network "github.com/docker/docker/api/types/network"
)

// DockerPorts describes docker ports for docker run
type DockerPorts struct {
	HostPort      int    `json:"host_port"`
	ContainerPort int    `json:"container_port"`
	Protocol      string `json:"protocol"`
}

// DockerCreateOpts describes docker run options
type DockerCreateOpts struct {
	Name       string         `json:"name"`
	Image      string         `json:"image"`
	Remove     bool           `json:"rm"`
	Ports      []*DockerPorts `json:"ports"`
	PublishAll bool           `json:"publish_all"`
	Command    string         `json:"command"`
	Env        []string       `json:"env"`
}

// DockerPingOpts describes the structure to ping docker containers in pikacloud API
type DockerPingOpts struct {
	Containers []string `json:"containers_id"`
}

// DockerPullOpts describes docker pull options
type DockerPullOpts struct {
	Image string `json:"image"`
}

// DockerUnpauseOpts describes docker unpause options
type DockerUnpauseOpts struct {
	ID string `json:"id"`
}

// DockerPauseOpts describes docker pause options
type DockerPauseOpts struct {
	ID string `json:"id"`
}

// DockerStopOpts describes docker stop options
type DockerStopOpts struct {
	ID string `json:"id"`
}

// DockerStartOpts describes docker start options
type DockerStartOpts struct {
	ID string `json:"id"`
}

// DockerRemoveOpts describes docker remove options
type DockerRemoveOpts struct {
	ID            string `json:"id"`
	Force         bool   `json:"force"`
	RemoveLinks   bool   `json:"remove_links"`
	RemoveVolumes bool   `json:"remove_volumes"`
}

// AgentDockerInfo describes docker info
type AgentDockerInfo struct {
	Info docker_types.Info `json:"info"`
}

// AgentContainer describes docker container
type AgentContainer struct {
	ID        string `json:"cid"`
	Container string `json:"container"`
	Config    string `json:"config"`
}

type syncDockerContainersOptions struct {
	ContainersID []string
}

func (agent *Agent) dockerPull(opts *DockerPullOpts) error {
	log.Printf("Pulling %s", opts.Image)
	ctx := context.Background()
	pullOpts := docker_types.ImagePullOptions{}
	out, err := agent.DockerClient.ImagePull(ctx, opts.Image, pullOpts)
	if err != nil {
		return err
	}
	defer out.Close()
	if _, err = io.Copy(ioutil.Discard, out); err != nil {
		return err
	}
	log.Printf("New image pulled %s", opts.Image)
	return nil
}

func (agent *Agent) dockerCreate(opts *DockerCreateOpts) (*docker_types_container.ContainerCreateCreatedBody, error) {
	ctx := context.Background()
	config := &docker_types_container.Config{
		Image: opts.Image,
		Env:   opts.Env,
	}
	hostConfig := &docker_types_container.HostConfig{}
	networkingConfig := &docker_types_network.NetworkingConfig{}

	container, err := agent.DockerClient.ContainerCreate(ctx, config, hostConfig, networkingConfig, opts.Name)
	if err != nil {
		return nil, err
	}
	log.Printf("New container created %s", container.ID)
	return &container, nil
}

func (agent *Agent) dockerStart(containerID string) error {
	ctx := context.Background()
	startOpts := docker_types.ContainerStartOptions{}
	if err := agent.DockerClient.ContainerStart(ctx, containerID, startOpts); err != nil {
		return err
	}
	log.Printf("New container started %s", containerID)
	return nil
}

func (agent *Agent) dockerUnpause(containerID string) error {
	ctx := context.Background()
	if err := agent.DockerClient.ContainerUnpause(ctx, containerID); err != nil {
		return err
	}
	log.Printf("Container %s unpaused", containerID)
	return nil
}

func (agent *Agent) dockerPause(containerID string) error {
	ctx := context.Background()
	if err := agent.DockerClient.ContainerPause(ctx, containerID); err != nil {
		return err
	}
	log.Printf("Container %s paused", containerID)
	return nil
}

func (agent *Agent) dockerStop(containerID string, timeout time.Duration) error {
	ctx := context.Background()
	if err := agent.DockerClient.ContainerStop(ctx, containerID, &timeout); err != nil {
		return err
	}
	log.Printf("Container %s stopped", containerID)
	return nil
}

func (agent *Agent) dockerRemove(containerID string, opts *DockerRemoveOpts) error {
	removeOpts := docker_types.ContainerRemoveOptions{
		Force:         opts.Force,
		RemoveVolumes: opts.RemoveVolumes,
		RemoveLinks:   opts.RemoveLinks,
	}
	ctx := context.Background()
	if err := agent.DockerClient.ContainerRemove(ctx, containerID, removeOpts); err != nil {
		return err
	}
	log.Printf("Container %s remove", containerID)
	return nil
}

func (agent *Agent) infiniteSyncDockerInfo() {
	dockerInfoState := docker_types.Info{}
	for {
		info, err := agent.DockerClient.Info(context.Background())
		if err != nil {
			time.Sleep(3 * time.Second)
			continue
		}
		// compare
		dockerInfoState.SystemTime = ""
		info.SystemTime = ""
		if !reflect.DeepEqual(dockerInfoState, info) {
			err := agent.syncDockerInfo(info)
			if err != nil {
				log.Printf("Cannot sync docker info: %+v", err)
			} else {
				dockerInfoState = info
				log.Println("Sync docker info OK")
			}
		}
		time.Sleep(3 * time.Second)
	}
}

func (agent *Agent) syncDockerInfo(info docker_types.Info) error {
	uri := fmt.Sprintf("run/agents/%s/docker/info/", agent.ID)
	pingInfo := AgentDockerInfo{
		Info: info,
	}
	status, err := agent.Client.Put(uri, pingInfo, nil)
	if err != nil {
		return err
	}
	if status != 200 {
		return fmt.Errorf("Failed to push docker info: %d", status)
	}
	return nil
}

func (agent *Agent) syncDockerContainers(opts syncDockerContainersOptions) error {
	containersListOpts := docker_types.ContainerListOptions{
		All: true,
	}
	var containersCreateList []AgentContainer
	uri := fmt.Sprintf("run/agents/%s/docker/containers/", agent.ID)
	containers, err := agent.DockerClient.ContainerList(context.Background(), containersListOpts)
	if err != nil {
		return err
	}
	for _, container := range containers {
		data, err := json.Marshal(container)
		if err != nil {
			log.Printf("Cannot decode %v", container)
			continue
		}
		if len(opts.ContainersID) > 0 {
			skip := true
			for _, c := range opts.ContainersID {
				if c == container.ID {
					skip = false
					break
				}
			}
			if skip {
				continue
			}
		}
		inspect, err := agent.DockerClient.ContainerInspect(context.Background(), container.ID)
		if err != nil {
			log.Printf("Cannot inspect container %v", err)
			continue
		}
		inspectConfig, err := json.Marshal(inspect)
		if err != nil {
			log.Printf("Cannot decode %v", inspect)
			continue
		}
		containersCreateList = append(containersCreateList,
			AgentContainer{
				ID:        container.ID,
				Container: string(data),
				Config:    string(inspectConfig),
			})
	}
	if len(containersCreateList) > 0 {
		status, err := agent.Client.Post(uri, containersCreateList, nil)
		if err != nil {
			return err
		}
		if status != 200 {
			return fmt.Errorf("Failed to push docker containers: %d", status)
		}
		log.Printf("Sync docker %d containers of %d OK", len(containersCreateList), len(containers))
	}

	return nil

}

func (agent *Agent) unsyncDockerContainer(containerID string) error {
	deleteContainerURI := fmt.Sprintf("run/agents/%s/docker/containers/%s/", agent.ID, containerID)
	_, err := agent.Client.Delete(deleteContainerURI, nil, nil)
	if err != nil {
		return err
	}
	log.Printf("Container %s garbage collected", containerID)
	return nil
}

func (agent *Agent) parseContainerEvent(msg docker_types_event.Message) error {
	containerID := msg.ID
	switch msg.Action {
	case "destroy":
		err := agent.unsyncDockerContainer(containerID)
		if err != nil {
			return err
		}
	default:

		opts := syncDockerContainersOptions{
			ContainersID: []string{containerID},
		}
		err := agent.syncDockerContainers(opts)
		if err != nil {
			return err
		}
	}
	return nil
}

func (agent *Agent) parseDockerEvent(msg docker_types_event.Message) error {
	switch msg.Type {
	case "container":
		err := agent.parseContainerEvent(msg)
		if err != nil {
			return err
		}
	}
	return nil
}

func (agent *Agent) listenDockerEvents() error {
	events, errs := agent.DockerClient.Events(context.Background(), docker_types.EventsOptions{})
	for {
		select {
		case dMsg := <-events:
			err := agent.parseDockerEvent(dMsg)
			if err != nil {
				log.Println(err)
			}
		case dErr := <-errs:
			if dErr != nil {
				fmt.Println(dErr)
			}
		}
	}
}

// Run docker container
func (agent *Agent) Run() {

}
<commit_msg>bind mounts and ports binding<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"reflect"
	"strconv"
	"time"

	docker_types "github.com/docker/docker/api/types"
	docker_types_container "github.com/docker/docker/api/types/container"
	docker_types_event "github.com/docker/docker/api/types/events"
	docker_types_network "github.com/docker/docker/api/types/network"
	docker_nat "github.com/docker/go-connections/nat"
)

// DockerPorts describes docker ports for docker run
type DockerPorts struct {
	HostIP        string `json:"host_ip"`
	HostPort      int    `json:"host_port"`
	ContainerPort int    `json:"container_port"`
	Protocol      string `json:"protocol"`
}

// DockerCreateOpts describes docker run options
type DockerCreateOpts struct {
	Name       string         `json:"name"`
	Image      string         `json:"image"`
	Remove     bool           `json:"rm"`
	Ports      []*DockerPorts `json:"ports"`
	PublishAll bool           `json:"publish_all"`
	Command    string         `json:"command"`
	Env        []string       `json:"env"`
	Binds      []string       `json:"binds"`
}

// DockerPingOpts describes the structure to ping docker containers in pikacloud API
type DockerPingOpts struct {
	Containers []string `json:"containers_id"`
}

// DockerPullOpts describes docker pull options
type DockerPullOpts struct {
	Image string `json:"image"`
}

// DockerUnpauseOpts describes docker unpause options
type DockerUnpauseOpts struct {
	ID string `json:"id"`
}

// DockerPauseOpts describes docker pause options
type DockerPauseOpts struct {
	ID string `json:"id"`
}

// DockerStopOpts describes docker stop options
type DockerStopOpts struct {
	ID string `json:"id"`
}

// DockerStartOpts describes docker start options
type DockerStartOpts struct {
	ID string `json:"id"`
}

// DockerRemoveOpts describes docker remove options
type DockerRemoveOpts struct {
	ID            string `json:"id"`
	Force         bool   `json:"force"`
	RemoveLinks   bool   `json:"remove_links"`
	RemoveVolumes bool   `json:"remove_volumes"`
}

// AgentDockerInfo describes docker info
type AgentDockerInfo struct {
	Info docker_types.Info `json:"info"`
}

// AgentContainer describes docker container
type AgentContainer struct {
	ID        string `json:"cid"`
	Container string `json:"container"`
	Config    string `json:"config"`
}

type syncDockerContainersOptions struct {
	ContainersID []string
}

func (agent *Agent) dockerPull(opts *DockerPullOpts) error {
	log.Printf("Pulling %s", opts.Image)
	ctx := context.Background()
	pullOpts := docker_types.ImagePullOptions{}
	out, err := agent.DockerClient.ImagePull(ctx, opts.Image, pullOpts)
	if err != nil {
		return err
	}
	defer out.Close()
	if _, err = io.Copy(ioutil.Discard, out); err != nil {
		return err
	}
	log.Printf("New image pulled %s", opts.Image)
	return nil
}

func (agent *Agent) dockerCreate(opts *DockerCreateOpts) (*docker_types_container.ContainerCreateCreatedBody, error) {
	ctx := context.Background()
	config := &docker_types_container.Config{
		Image: opts.Image,
		Env:   opts.Env,
	}
	natPortmap := docker_nat.PortMap{}
	for _, p := range opts.Ports {
		containerPortProto := docker_nat.Port(fmt.Sprintf("%d/%s", p.ContainerPort, p.Protocol))
		dockerHostConfig := docker_nat.PortBinding{
			HostIP:   p.HostIP,
			HostPort: strconv.Itoa(p.HostPort),
		}
		natPortmap[containerPortProto] = append(natPortmap[containerPortProto], dockerHostConfig)
	}
	hostConfig := &docker_types_container.HostConfig{
		Binds:           opts.Binds,
		PublishAllPorts: opts.PublishAll,
		PortBindings:    natPortmap,
	}
	networkingConfig := &docker_types_network.NetworkingConfig{}

	container, err := agent.DockerClient.ContainerCreate(ctx, config, hostConfig, networkingConfig, opts.Name)
	if err != nil {
		return nil, err
	}
	log.Printf("New container created %s", container.ID)
	return &container, nil
}

func (agent *Agent) dockerStart(containerID string) error {
	ctx := context.Background()
	startOpts := docker_types.ContainerStartOptions{}
	if err := agent.DockerClient.ContainerStart(ctx, containerID, startOpts); err != nil {
		return err
	}
	log.Printf("New container started %s", containerID)
	return nil
}

func (agent *Agent) dockerUnpause(containerID string) error {
	ctx := context.Background()
	if err := agent.DockerClient.ContainerUnpause(ctx, containerID); err != nil {
		return err
	}
	log.Printf("Container %s unpaused", containerID)
	return nil
}

func (agent *Agent) dockerPause(containerID string) error {
	ctx := context.Background()
	if err := agent.DockerClient.ContainerPause(ctx, containerID); err != nil {
		return err
	}
	log.Printf("Container %s paused", containerID)
	return nil
}

func (agent *Agent) dockerStop(containerID string, timeout time.Duration) error {
	ctx := context.Background()
	if err := agent.DockerClient.ContainerStop(ctx, containerID, &timeout); err != nil {
		return err
	}
	log.Printf("Container %s stopped", containerID)
	return nil
}

func (agent *Agent) dockerRemove(containerID string, opts *DockerRemoveOpts) error {
	removeOpts := docker_types.ContainerRemoveOptions{
		Force:         opts.Force,
		RemoveVolumes: opts.RemoveVolumes,
		RemoveLinks:   opts.RemoveLinks,
	}
	ctx := context.Background()
	if err := agent.DockerClient.ContainerRemove(ctx, containerID, removeOpts); err != nil {
		return err
	}
	log.Printf("Container %s remove", containerID)
	return nil
}

func (agent *Agent) infiniteSyncDockerInfo() {
	dockerInfoState := docker_types.Info{}
	for {
		info, err := agent.DockerClient.Info(context.Background())
		if err != nil {
			time.Sleep(3 * time.Second)
			continue
		}
		// compare
		dockerInfoState.SystemTime = ""
		info.SystemTime = ""
		if !reflect.DeepEqual(dockerInfoState, info) {
			err := agent.syncDockerInfo(info)
			if err != nil {
				log.Printf("Cannot sync docker info: %+v", err)
			} else {
				dockerInfoState = info
				log.Println("Sync docker info OK")
			}
		}
		time.Sleep(3 * time.Second)
	}
}

func (agent *Agent) syncDockerInfo(info docker_types.Info) error {
	uri := fmt.Sprintf("run/agents/%s/docker/info/", agent.ID)
	pingInfo := AgentDockerInfo{
		Info: info,
	}
	status, err := agent.Client.Put(uri, pingInfo, nil)
	if err != nil {
		return err
	}
	if status != 200 {
		return fmt.Errorf("Failed to push docker info: %d", status)
	}
	return nil
}

func (agent *Agent) syncDockerContainers(opts syncDockerContainersOptions) error {
	containersListOpts := docker_types.ContainerListOptions{
		All: true,
	}
	var containersCreateList []AgentContainer
	uri := fmt.Sprintf("run/agents/%s/docker/containers/", agent.ID)
	containers, err := agent.DockerClient.ContainerList(context.Background(), containersListOpts)
	if err != nil {
		return err
	}
	for _, container := range containers {
		data, err := json.Marshal(container)
		if err != nil {
			log.Printf("Cannot decode %v", container)
			continue
		}
		if len(opts.ContainersID) > 0 {
			skip := true
			for _, c := range opts.ContainersID {
				if c == container.ID {
					skip = false
					break
				}
			}
			if skip {
				continue
			}
		}
		inspect, err := agent.DockerClient.ContainerInspect(context.Background(), container.ID)
		if err != nil {
			log.Printf("Cannot inspect container %v", err)
			continue
		}
		inspectConfig, err := json.Marshal(inspect)
		if err != nil {
			log.Printf("Cannot decode %v", inspect)
			continue
		}
		containersCreateList = append(containersCreateList,
			AgentContainer{
				ID:        container.ID,
				Container: string(data),
				Config:    string(inspectConfig),
			})
	}
	if len(containersCreateList) > 0 {
		status, err := agent.Client.Post(uri, containersCreateList, nil)
		if err != nil {
			return err
		}
		if status != 200 {
			return fmt.Errorf("Failed to push docker containers: %d", status)
		}
		log.Printf("Sync docker %d containers of %d OK", len(containersCreateList), len(containers))
	}

	return nil

}

func (agent *Agent) unsyncDockerContainer(containerID string) error {
	deleteContainerURI := fmt.Sprintf("run/agents/%s/docker/containers/%s/", agent.ID, containerID)
	_, err := agent.Client.Delete(deleteContainerURI, nil, nil)
	if err != nil {
		return err
	}
	log.Printf("Container %s garbage collected", containerID)
	return nil
}

func (agent *Agent) parseContainerEvent(msg docker_types_event.Message) error {
	containerID := msg.ID
	switch msg.Action {
	case "destroy":
		err := agent.unsyncDockerContainer(containerID)
		if err != nil {
			return err
		}
	default:

		opts := syncDockerContainersOptions{
			ContainersID: []string{containerID},
		}
		err := agent.syncDockerContainers(opts)
		if err != nil {
			return err
		}
	}
	return nil
}

func (agent *Agent) parseDockerEvent(msg docker_types_event.Message) error {
	switch msg.Type {
	case "container":
		err := agent.parseContainerEvent(msg)
		if err != nil {
			return err
		}
	}
	return nil
}

func (agent *Agent) listenDockerEvents() error {
	events, errs := agent.DockerClient.Events(context.Background(), docker_types.EventsOptions{})
	for {
		select {
		case dMsg := <-events:
			err := agent.parseDockerEvent(dMsg)
			if err != nil {
				log.Println(err)
			}
		case dErr := <-errs:
			if dErr != nil {
				fmt.Println(dErr)
			}
		}
	}
}

// Run docker container
func (agent *Agent) Run() {

}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/client"
)

func isDockerImagePulled(p estafettePipeline) bool {

	fmt.Printf("[estafette] Checking if docker image '%v' exists...\n", p.ContainerImage)

	cli, err := client.NewEnvClient()
	if err != nil {
		return false
	}

	imageSummaries, err := cli.ImageList(context.Background(), types.ImageListOptions{})

	for _, summary := range imageSummaries {
		if contains(summary.RepoTags, p.ContainerImage) {
			return true
		}
	}

	return false
}

func runDockerPull(p estafettePipeline) (err error) {

	fmt.Printf("[estafette] Pulling docker image '%v'\n", p.ContainerImage)

	cli, err := client.NewEnvClient()
	if err != nil {
		return err
	}

	rc, err := cli.ImagePull(context.Background(), p.ContainerImage, types.ImagePullOptions{})
	defer rc.Close()
	if err != nil {
		return err
	}

	// wait for image pull to finish
	_, err = ioutil.ReadAll(rc)
	if err != nil {
		return err
	}

	return
}

func getDockerImageSize(p estafettePipeline) (totalSize int64, err error) {

	cli, err := client.NewEnvClient()
	if err != nil {
		return totalSize, err
	}

	items, err := cli.ImageHistory(context.Background(), p.ContainerImage)
	//summaries, err := cli.ImageList(context.Background(), type.ImageListOptions{})
	if err != nil {
		return totalSize, err
	}

	for _, item := range items {
		totalSize += item.Size
	}

	return totalSize, nil
}

func runDockerRun(dir string, envvars map[string]string, p estafettePipeline) (err error) {

	// run docker with image and commands from yaml
	ctx := context.Background()
	cli, err := client.NewEnvClient()
	if err != nil {
		return err
	}

	// define commands
	cmdSlice := make([]string, 0)
	cmdSlice = append(cmdSlice, "set -e;"+os.Expand(strings.Join(p.Commands, ";"), getEstafetteEnv))

	// define envvars
	envVars := make([]string, 0)
	if envvars != nil && len(envvars) > 0 {
		for k, v := range envvars {
			envVars = append(envVars, fmt.Sprintf("\"%v=%v\"", k, v))
		}
	}

	// define entrypoint
	entrypoint := make([]string, 0)
	entrypoint = append(entrypoint, p.Shell)
	entrypoint = append(entrypoint, "-c")

	// define binds
	binds := make([]string, 0)
	binds = append(binds, fmt.Sprintf("%v:%v", dir, os.Expand(p.WorkingDirectory, getEstafetteEnv)))
	binds = append(binds, "/var/run/docker.sock:/var/run/docker.sock")
	binds = append(binds, "/var/run/secrets/kubernetes.io/serviceaccount:/var/run/secrets/kubernetes.io/serviceaccount")

	// create container
	resp, err := cli.ContainerCreate(ctx, &container.Config{
		AttachStdout: true,
		AttachStderr: true,
		Env:          envVars,
		Cmd:          cmdSlice,
		Image:        p.ContainerImage,
		WorkingDir:   os.Expand(p.WorkingDirectory, getEstafetteEnv),
		Entrypoint:   entrypoint,
	}, &container.HostConfig{
		Binds:      binds,
		AutoRemove: true,
		Privileged: true,
	}, &network.NetworkingConfig{}, "")
	if err != nil {
		return err
	}

	// start container
	if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
		return err
	}

	// follow logs
	rc, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{
		ShowStdout: true,
		ShowStderr: true,
		Follow:     true,
	})
	defer rc.Close()
	if err != nil {
		return err
	}

	// stream logs to stdout with buffering
	in := bufio.NewScanner(rc)
	for in.Scan() {

		// strip first 8 bytes, they contain docker control characters (https://github.com/docker/docker/issues/7375)
		logLine := in.Text()
		output := os.Stdout
		if len(logLine) > 8 {

			headers := []byte(logLine[0:8])

			// first byte contains the streamType
			// -   0: stdin (will be written on stdout)
			// -   1: stdout
			// -   2: stderr
			streamType := headers[0]

			if streamType == 2 {
				output = os.Stderr
			}

			logLine = logLine[8:]
		}

		fmt.Fprintf(output, "[estafette] [%v] %v\n", p.Name, logLine)
	}
	if err := in.Err(); err != nil {
		fmt.Fprintf(os.Stderr, "[estafette] [%v] Error: %v\n", p.Name, err)
		return err
	}

	// wait for container to stop run
	exitCode, err := cli.ContainerWait(ctx, resp.ID)
	if err != nil {
		return err
	}

	if exitCode > 0 {
		return fmt.Errorf("Failed with exit code: %v", exitCode)
	}

	return
}
<commit_msg>don't wrap envvars in quotes<commit_after>package main

import (
	"bufio"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/client"
)

func isDockerImagePulled(p estafettePipeline) bool {

	fmt.Printf("[estafette] Checking if docker image '%v' exists...\n", p.ContainerImage)

	cli, err := client.NewEnvClient()
	if err != nil {
		return false
	}

	imageSummaries, err := cli.ImageList(context.Background(), types.ImageListOptions{})

	for _, summary := range imageSummaries {
		if contains(summary.RepoTags, p.ContainerImage) {
			return true
		}
	}

	return false
}

func runDockerPull(p estafettePipeline) (err error) {

	fmt.Printf("[estafette] Pulling docker image '%v'\n", p.ContainerImage)

	cli, err := client.NewEnvClient()
	if err != nil {
		return err
	}

	rc, err := cli.ImagePull(context.Background(), p.ContainerImage, types.ImagePullOptions{})
	defer rc.Close()
	if err != nil {
		return err
	}

	// wait for image pull to finish
	_, err = ioutil.ReadAll(rc)
	if err != nil {
		return err
	}

	return
}

func getDockerImageSize(p estafettePipeline) (totalSize int64, err error) {

	cli, err := client.NewEnvClient()
	if err != nil {
		return totalSize, err
	}

	items, err := cli.ImageHistory(context.Background(), p.ContainerImage)
	//summaries, err := cli.ImageList(context.Background(), type.ImageListOptions{})
	if err != nil {
		return totalSize, err
	}

	for _, item := range items {
		totalSize += item.Size
	}

	return totalSize, nil
}

func runDockerRun(dir string, envvars map[string]string, p estafettePipeline) (err error) {

	// run docker with image and commands from yaml
	ctx := context.Background()
	cli, err := client.NewEnvClient()
	if err != nil {
		return err
	}

	// define commands
	cmdSlice := make([]string, 0)
	cmdSlice = append(cmdSlice, "set -e;"+os.Expand(strings.Join(p.Commands, ";"), getEstafetteEnv))

	// define envvars
	envVars := make([]string, 0)
	if envvars != nil && len(envvars) > 0 {
		for k, v := range envvars {
			envVars = append(envVars, fmt.Sprintf("%v=%v", k, v))
		}
	}

	// define entrypoint
	entrypoint := make([]string, 0)
	entrypoint = append(entrypoint, p.Shell)
	entrypoint = append(entrypoint, "-c")

	// define binds
	binds := make([]string, 0)
	binds = append(binds, fmt.Sprintf("%v:%v", dir, os.Expand(p.WorkingDirectory, getEstafetteEnv)))
	binds = append(binds, "/var/run/docker.sock:/var/run/docker.sock")
	binds = append(binds, "/var/run/secrets/kubernetes.io/serviceaccount:/var/run/secrets/kubernetes.io/serviceaccount")

	// create container
	resp, err := cli.ContainerCreate(ctx, &container.Config{
		AttachStdout: true,
		AttachStderr: true,
		Env:          envVars,
		Cmd:          cmdSlice,
		Image:        p.ContainerImage,
		WorkingDir:   os.Expand(p.WorkingDirectory, getEstafetteEnv),
		Entrypoint:   entrypoint,
	}, &container.HostConfig{
		Binds:      binds,
		AutoRemove: true,
		Privileged: true,
	}, &network.NetworkingConfig{}, "")
	if err != nil {
		return err
	}

	// start container
	if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
		return err
	}

	// follow logs
	rc, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{
		ShowStdout: true,
		ShowStderr: true,
		Follow:     true,
	})
	defer rc.Close()
	if err != nil {
		return err
	}

	// stream logs to stdout with buffering
	in := bufio.NewScanner(rc)
	for in.Scan() {

		// strip first 8 bytes, they contain docker control characters (https://github.com/docker/docker/issues/7375)
		logLine := in.Text()
		output := os.Stdout
		if len(logLine) > 8 {

			headers := []byte(logLine[0:8])

			// first byte contains the streamType
			// -   0: stdin (will be written on stdout)
			// -   1: stdout
			// -   2: stderr
			streamType := headers[0]

			if streamType == 2 {
				output = os.Stderr
			}

			logLine = logLine[8:]
		}

		fmt.Fprintf(output, "[estafette] [%v] %v\n", p.Name, logLine)
	}
	if err := in.Err(); err != nil {
		fmt.Fprintf(os.Stderr, "[estafette] [%v] Error: %v\n", p.Name, err)
		return err
	}

	// wait for container to stop run
	exitCode, err := cli.ContainerWait(ctx, resp.ID)
	if err != nil {
		return err
	}

	if exitCode > 0 {
		return fmt.Errorf("Failed with exit code: %v", exitCode)
	}

	return
}
<|endoftext|>


<commit_before>package hoi

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
)

func TestMakePublic(t *testing.T) {
	publicDir := "public_test"
	os.MkdirAll(publicDir, 0755)
	defer os.RemoveAll(publicDir)

	hoi := Hoi{}
	linked := hoi.makePublic("hoi.go", publicDir)

	file, err := os.Lstat(filepath.Join(publicDir, linked))
	if err != nil {
		t.Errorf("It should be made symlink %s, %s", file, err)
	}
}

func TestClear(t *testing.T) {
	publicDir := "public_test"
	os.MkdirAll(publicDir, 0755)
	hoi := Hoi{publicDir: publicDir}
	hoi.makePublic("hoi.go", publicDir)
	hoi.Clear()
	contents, _ := ioutil.ReadDir(publicDir)
	if len(contents) > 0 {
		t.Errorf("It should be clear %s", publicDir)
	}
}

func TestRandomString(t *testing.T) {
	expect := 10
	random := randomString(expect)
	if len(random) != 10 {
		t.Errorf("It should equal %d", expect)
	}
}
<commit_msg>Removed test directory when finish test.<commit_after>package hoi

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
)

func TestMakePublic(t *testing.T) {
	publicDir := "public_test"
	os.MkdirAll(publicDir, 0755)
	defer os.RemoveAll(publicDir)

	hoi := Hoi{}
	linked := hoi.makePublic("hoi.go", publicDir)

	file, err := os.Lstat(filepath.Join(publicDir, linked))
	if err != nil {
		t.Errorf("It should be made symlink %s, %s", file, err)
	}
}

func TestClear(t *testing.T) {
	publicDir := "public_test"
	os.MkdirAll(publicDir, 0755)
	defer os.RemoveAll(publicDir)
	hoi := Hoi{publicDir: publicDir}
	hoi.makePublic("hoi.go", publicDir)
	hoi.Clear()
	contents, _ := ioutil.ReadDir(publicDir)
	if len(contents) > 0 {
		t.Errorf("It should be clear %s", publicDir)
	}
}

func TestRandomString(t *testing.T) {
	expect := 10
	random := randomString(expect)
	if len(random) != 10 {
		t.Errorf("It should equal %d", expect)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"net"
	"net/http"

	"github.com/flynn/flynn/host/types"
	"github.com/flynn/flynn/pkg/rpcplus"
	rpc "github.com/flynn/flynn/pkg/rpcplus/comborpc"
)

func serveHTTP(host *Host, attach *attachHandler, sh *shutdownHandler) error {
	if err := rpc.Register(host); err != nil {
		return err
	}
	rpc.Register(host)
	rpc.HandleHTTP()
	http.Handle("/attach", attach)

	l, err := net.Listen("tcp", ":1113")
	if err != nil {
		return err
	}
	sh.BeforeExit(func() { l.Close() })
	go http.Serve(l, nil)
	return nil
}

type Host struct {
	state   *State
	backend Backend
}

func (h *Host) ListJobs(arg struct{}, res *map[string]host.ActiveJob) error {
	*res = h.state.Get()
	return nil
}

func (h *Host) GetJob(id string, res *host.ActiveJob) error {
	job := h.state.GetJob(id)
	if job != nil {
		*res = *job
	}
	return nil
}

func (h *Host) StopJob(id string, res *struct{}) error {
	job := h.state.GetJob(id)
	if job == nil {
		return errors.New("host: unknown job")
	}
	if job.Status != host.StatusRunning {
		return errors.New("host: job is not running")
	}
	return h.backend.Stop(id)
}

func (h *Host) StreamEvents(id string, stream rpcplus.Stream) error {
	ch := h.state.AddListener(id)
	defer h.state.RemoveListener(id, ch)
	for {
		select {
		case event := <-ch:
			select {
			case stream.Send <- event:
			case <-stream.Error:
				return nil
			}
		case <-stream.Error:
			return nil
		}
	}
}
<commit_msg>host: Remove redundant rpc.Register<commit_after>package main

import (
	"errors"
	"net"
	"net/http"

	"github.com/flynn/flynn/host/types"
	"github.com/flynn/flynn/pkg/rpcplus"
	rpc "github.com/flynn/flynn/pkg/rpcplus/comborpc"
)

func serveHTTP(host *Host, attach *attachHandler, sh *shutdownHandler) error {
	if err := rpc.Register(host); err != nil {
		return err
	}
	rpc.HandleHTTP()
	http.Handle("/attach", attach)

	l, err := net.Listen("tcp", ":1113")
	if err != nil {
		return err
	}
	sh.BeforeExit(func() { l.Close() })
	go http.Serve(l, nil)
	return nil
}

type Host struct {
	state   *State
	backend Backend
}

func (h *Host) ListJobs(arg struct{}, res *map[string]host.ActiveJob) error {
	*res = h.state.Get()
	return nil
}

func (h *Host) GetJob(id string, res *host.ActiveJob) error {
	job := h.state.GetJob(id)
	if job != nil {
		*res = *job
	}
	return nil
}

func (h *Host) StopJob(id string, res *struct{}) error {
	job := h.state.GetJob(id)
	if job == nil {
		return errors.New("host: unknown job")
	}
	if job.Status != host.StatusRunning {
		return errors.New("host: job is not running")
	}
	return h.backend.Stop(id)
}

func (h *Host) StreamEvents(id string, stream rpcplus.Stream) error {
	ch := h.state.AddListener(id)
	defer h.state.RemoveListener(id, ch)
	for {
		select {
		case event := <-ch:
			select {
			case stream.Send <- event:
			case <-stream.Error:
				return nil
			}
		case <-stream.Error:
			return nil
		}
	}
}
<|endoftext|>


<commit_before>package stardust

import (
	"reflect"
	"testing"

	"github.com/juju/utils/set"
)

func TestJaccardSets(t *testing.T) {
	var jaccardSetsTests = []struct {
		a   set.Strings
		b   set.Strings
		out float64
	}{
		{set.NewStrings("a"), set.NewStrings("a"), 1.0},
		{set.NewStrings("a"), set.NewStrings("a", "b"), 0.5},
		{set.NewStrings("a"), set.NewStrings("a", "b", "c"), 1.0 / 3},
		{set.NewStrings("a", "b"), set.NewStrings("a", "b", "c"), 2.0 / 3},
	}

	for _, tt := range jaccardSetsTests {
		out := JaccardSets(tt.a, tt.b)
		if out != tt.out {
			t.Errorf("Jaccard(%v, %v) => %f, want: %f", tt.a, tt.b, out, tt.out)
		}
	}
}

func TestNgram(t *testing.T) {
	var ngramTests = []struct {
		s   string
		n   int
		out set.Strings
	}{
		{"abc", 2, set.NewStrings("ab", "bc")},
		{"abc", 6, set.NewStrings()},
		{"abc", 3, set.NewStrings("abc")},
		{"abc", 1, set.NewStrings("a", "b", "c")},
		{"abc", 0, set.NewStrings()},
		{"abc", -10, set.NewStrings()},
		{"abc def", 3, set.NewStrings("abc", "bc ", "c d", " de", "def")},
	}
	for _, tt := range ngramTests {
		out := Ngrams(tt.s, tt.n)
		if !reflect.DeepEqual(out, tt.out) {
			t.Errorf("Ngrams(%s, %d) => %v, want: %v", tt.s, tt.n, out, tt.out)
		}
	}
}

func TestNgramSimilarity(t *testing.T) {
	var ngramSimilarityTests = []struct {
		a   string
		b   string
		out float64
	}{
		{"Hello World", "Hello Earth", 0.285714},
		{"Hello World", "Hello Wookie", 0.461538},
		{"Hello World", "Hello", 0.333333},
		{"The quick brown fox", "The qiuck brown fox", 0.619048},
		{"The quick brown fox", "The quick brown fox", 1.000000},
	}
	for _, tt := range ngramSimilarityTests {
		out := NgramSimilarity(tt.a, tt.b)
		if out != tt.out {
			t.Errorf("NgramSimilarity(%s, %s) => %f, want: %f", tt.a, tt.b, out, tt.out)
		}
	}
}

func TestNgramSimilaritySize(t *testing.T) {
	var ngramSimilaritySizeTests = []struct {
		a    string
		b    string
		size int
		out  float64
	}{
		{"Hello World", "Hello Earth", 2, 0.333333},
		{"Hello World", "Hello Wookie", 7, 0.222222},
		{"Hello World", "Hello", 1, 0.500000},
		{"The quick brown fox", "The qiuck brown fox", 3, 0.619048},
		{"The quick brown fox", "The qiuck brown fox", 2, 0.714286},
		{"The quick brown fox", "The qiuck brown fox", 1, 1.000000},
	}
	for _, tt := range ngramSimilaritySizeTests {
		out := NgramSimilaritySize(tt.a, tt.b, tt.size)
		if out != tt.out {
			t.Errorf("NgramSimilaritySize(%s, %s, %d) => %f, want: %f", tt.a, tt.b, tt.size, out, tt.out)
		}
	}
}
<commit_msg>added AlmostEqualRelative test helper<commit_after>package stardust

import (
	"math"
	"reflect"
	"testing"

	"github.com/juju/utils/set"
)

// AlmostEqualRelative is a float comparison helper
// Via http://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/
func AlmostEqualRelative(a, b, maxRelDiff float64) bool {
	diff := math.Abs(a - b)
	A := math.Abs(a)
	B := math.Abs(b)
	largest := A
	if B > A {
		largest = B
	}
	if diff <= largest*maxRelDiff {
		return true
	}
	return false
}

func TestJaccardSets(t *testing.T) {
	var jaccardSetsTests = []struct {
		a   set.Strings
		b   set.Strings
		out float64
	}{
		{set.NewStrings("a"), set.NewStrings("a"), 1.0},
		{set.NewStrings("a"), set.NewStrings("a", "b"), 0.5},
		{set.NewStrings("a"), set.NewStrings("a", "b", "c"), 1.0 / 3},
		{set.NewStrings("a", "b"), set.NewStrings("a", "b", "c"), 2.0 / 3},
	}

	for _, tt := range jaccardSetsTests {
		out := JaccardSets(tt.a, tt.b)
		if out != tt.out {
			t.Errorf("Jaccard(%v, %v) => %f, want: %f", tt.a, tt.b, out, tt.out)
		}
	}
}

func TestNgram(t *testing.T) {
	var ngramTests = []struct {
		s   string
		n   int
		out set.Strings
	}{
		{"abc", 2, set.NewStrings("ab", "bc")},
		{"abc", 6, set.NewStrings()},
		{"abc", 3, set.NewStrings("abc")},
		{"abc", 1, set.NewStrings("a", "b", "c")},
		{"abc", 0, set.NewStrings()},
		{"abc", -10, set.NewStrings()},
		{"abc def", 3, set.NewStrings("abc", "bc ", "c d", " de", "def")},
	}
	for _, tt := range ngramTests {
		out := Ngrams(tt.s, tt.n)
		if !reflect.DeepEqual(out, tt.out) {
			t.Errorf("Ngrams(%s, %d) => %v, want: %v", tt.s, tt.n, out, tt.out)
		}
	}
}

func TestNgramSimilarity(t *testing.T) {
	var ngramSimilarityTests = []struct {
		a   string
		b   string
		out float64
	}{
		{"Hello World", "Hello Earth", 0.285714},
		{"Hello World", "Hello Wookie", 0.461538},
		{"Hello World", "Hello", 0.333333},
		{"The quick brown fox", "The qiuck brown fox", 0.619048},
		{"The quick brown fox", "The quick brown fox", 1.000000},
	}
	for _, tt := range ngramSimilarityTests {
		out := NgramSimilarity(tt.a, tt.b)
		if !AlmostEqualRelative(out, tt.out, 1e-5) {
			t.Errorf("NgramSimilarity(%s, %s) => %f, want: %f", tt.a, tt.b, out, tt.out)
		}
	}
}

func TestNgramSimilaritySize(t *testing.T) {
	var ngramSimilaritySizeTests = []struct {
		a    string
		b    string
		size int
		out  float64
	}{
		{"Hello World", "Hello Earth", 2, 0.333333},
		{"Hello World", "Hello Wookie", 7, 0.222222},
		{"Hello World", "Hello", 1, 0.500000},
		{"The quick brown fox", "The qiuck brown fox", 3, 0.619048},
		{"The quick brown fox", "The qiuck brown fox", 2, 0.714286},
		{"The quick brown fox", "The qiuck brown fox", 1, 1.000000},
	}
	for _, tt := range ngramSimilaritySizeTests {
		out := NgramSimilaritySize(tt.a, tt.b, tt.size)
		if !AlmostEqualRelative(out, tt.out, 1e-5) {
			t.Errorf("NgramSimilaritySize(%s, %s, %d) => %f, want: %f", tt.a, tt.b, tt.size, out, tt.out)
		}
	}
}
<|endoftext|>


<commit_before>package hoverfly

import (
	"encoding/json"
	"fmt"
	"net/url"
	"os"
	"path"
	"regexp"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
)

// Import is a function that based on input decides whether it is a local resource or whether
// it should fetch it from remote server. It then imports given payload into the database
// or returns an error
func (d *DBClient) Import(uri string) error {

	// assuming file URI is URL:
	if isURL(uri) {
		log.WithFields(log.Fields{
			"isURL":      isURL(uri),
			"importFrom": uri,
		}).Info("URL")
		return d.ImportFromURL(uri)
	}
	// assuming file URI is disk location
	ext := path.Ext(uri)
	if ext != ".json" {
		return fmt.Errorf("Failed to import payloads, only JSON files are acceppted. Given file: %s", uri)
	}
	// checking whether it exists
	exists, err := exists(uri)
	if err != nil {
		return fmt.Errorf("Failed to import payloads from %s. Got error: %s", uri, err.Error())
	}
	if exists {
		// file is JSON and it exist
		return d.ImportFromDisk(uri)
	}
	return fmt.Errorf("Failed to import payloads, given file '%s' does not exist", uri)
}

// URL is regexp to match http urls
const URL string = `^((ftp|https?):\/\/)(\S+(:\S*)?@)?((([1-9]\d?|1\d\d|2[01]\d|22[0-3])(\.(1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.([0-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(([a-zA-Z0-9]+([-\.][a-zA-Z0-9]+)*)|((www\.)?))?(([a-z\x{00a1}-\x{ffff}0-9]+-?-?)*[a-z\x{00a1}-\x{ffff}0-9]+)(?:\.([a-z\x{00a1}-\x{ffff}]{2,}))?))(:(\d{1,5}))?((\/|\?|#)[^\s]*)?$`

var rxURL = regexp.MustCompile(URL)

func isURL(str string) bool {
	if str == "" || len(str) >= 2083 || len(str) <= 3 || strings.HasPrefix(str, ".") {
		return false
	}
	u, err := url.Parse(str)
	if err != nil {
		return false
	}
	if strings.HasPrefix(u.Host, ".") {
		return false
	}
	if u.Host == "" && (u.Path != "" && !strings.Contains(u.Path, ".")) {
		return false
	}

	return rxURL.MatchString(str)

}

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return true, err
}

// ImportFromDisk - takes one string value and tries to open a file, then parse it into recordedRequests structure
// (which is default format in which Hoverfly exports captured requests) and imports those requests into the database
func (d *DBClient) ImportFromDisk(path string) error {
	payloadsFile, err := os.Open(path)
	if err != nil {
		return fmt.Errorf("Got error while opening payloads file, error %s", err.Error())
	}

	var requests recordedRequests

	jsonParser := json.NewDecoder(payloadsFile)
	if err = jsonParser.Decode(&requests); err != nil {
		return fmt.Errorf("Got error while parsing payloads file, error %s", err.Error())
	}

	return d.ImportPayloads(requests.Data)
}

// ImportFromURL - takes one string value and tries connect to a remote server, then parse response body into
// recordedRequests structure (which is default format in which Hoverfly exports captured requests) and
// imports those requests into the database
func (d *DBClient) ImportFromURL(url string) error {

	resp, err := d.HTTP.Get(url)
	if err != nil {
		return fmt.Errorf("Failed to fetch given URL, error %s", err.Error())
	}

	var requests recordedRequests

	jsonParser := json.NewDecoder(resp.Body)
	if err = jsonParser.Decode(&requests); err != nil {
		return fmt.Errorf("Got error while parsing payloads, error %s", err.Error())
	}

	return d.ImportPayloads(requests.Data)
}

func isJSON(s string) bool {
	var js map[string]interface{}
	return json.Unmarshal([]byte(s), &js) == nil

}

// ImportPayloads - a function to save given payloads into the database.
func (d *DBClient) ImportPayloads(payloads []Payload) error {
	if len(payloads) > 0 {
		success := 0
		failed := 0
		for _, pl := range payloads {
			pl.Request.Headers = make(map[string][]string)
			// sniffing content types
			if isJSON(pl.Request.Body) {
				pl.Request.Headers["Content-Type"] = []string{"application/json"}
			} else {
				ct := http.DetectContentType([]byte(pl.Request.Body))
				pl.Request.Headers["Content-Type"] = []string{ct}
			}

			// recalculating request hash and storing it in database
			r := RequestContainer{Details: pl.Request}
			key := r.Hash()

			// regenerating key
			pl.ID = key

			bts, err := pl.Encode()
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
				}).Error("Failed to encode payload")
				failed++
			} else {
				// hook
				var en Entry
				en.ActionType = ActionTypeRequestCaptured
				en.Message = "imported"
				en.Time = time.Now()
				en.Data = bts

				if err := d.Hooks.Fire(ActionTypeRequestCaptured, &en); err != nil {
					log.WithFields(log.Fields{
						"error":      err.Error(),
						"message":    en.Message,
						"actionType": ActionTypeRequestCaptured,
					}).Error("failed to fire hook")
				}

				d.Cache.Set([]byte(key), bts)
				if err == nil {
					success++
				} else {
					failed++
				}
			}
		}
		log.WithFields(log.Fields{
			"total":      len(payloads),
			"successful": success,
			"failed":     failed,
		}).Info("payloads imported")
		return nil
	}
	return fmt.Errorf("Bad request. Nothing to import!")
}
<commit_msg>imports<commit_after>package hoverfly

import (
	"encoding/json"
	"fmt"
	"net/url"
	"os"
	"path"
	"regexp"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"net/http"
)

// Import is a function that based on input decides whether it is a local resource or whether
// it should fetch it from remote server. It then imports given payload into the database
// or returns an error
func (d *DBClient) Import(uri string) error {

	// assuming file URI is URL:
	if isURL(uri) {
		log.WithFields(log.Fields{
			"isURL":      isURL(uri),
			"importFrom": uri,
		}).Info("URL")
		return d.ImportFromURL(uri)
	}
	// assuming file URI is disk location
	ext := path.Ext(uri)
	if ext != ".json" {
		return fmt.Errorf("Failed to import payloads, only JSON files are acceppted. Given file: %s", uri)
	}
	// checking whether it exists
	exists, err := exists(uri)
	if err != nil {
		return fmt.Errorf("Failed to import payloads from %s. Got error: %s", uri, err.Error())
	}
	if exists {
		// file is JSON and it exist
		return d.ImportFromDisk(uri)
	}
	return fmt.Errorf("Failed to import payloads, given file '%s' does not exist", uri)
}

// URL is regexp to match http urls
const URL string = `^((ftp|https?):\/\/)(\S+(:\S*)?@)?((([1-9]\d?|1\d\d|2[01]\d|22[0-3])(\.(1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.([0-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(([a-zA-Z0-9]+([-\.][a-zA-Z0-9]+)*)|((www\.)?))?(([a-z\x{00a1}-\x{ffff}0-9]+-?-?)*[a-z\x{00a1}-\x{ffff}0-9]+)(?:\.([a-z\x{00a1}-\x{ffff}]{2,}))?))(:(\d{1,5}))?((\/|\?|#)[^\s]*)?$`

var rxURL = regexp.MustCompile(URL)

func isURL(str string) bool {
	if str == "" || len(str) >= 2083 || len(str) <= 3 || strings.HasPrefix(str, ".") {
		return false
	}
	u, err := url.Parse(str)
	if err != nil {
		return false
	}
	if strings.HasPrefix(u.Host, ".") {
		return false
	}
	if u.Host == "" && (u.Path != "" && !strings.Contains(u.Path, ".")) {
		return false
	}

	return rxURL.MatchString(str)

}

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return true, err
}

// ImportFromDisk - takes one string value and tries to open a file, then parse it into recordedRequests structure
// (which is default format in which Hoverfly exports captured requests) and imports those requests into the database
func (d *DBClient) ImportFromDisk(path string) error {
	payloadsFile, err := os.Open(path)
	if err != nil {
		return fmt.Errorf("Got error while opening payloads file, error %s", err.Error())
	}

	var requests recordedRequests

	jsonParser := json.NewDecoder(payloadsFile)
	if err = jsonParser.Decode(&requests); err != nil {
		return fmt.Errorf("Got error while parsing payloads file, error %s", err.Error())
	}

	return d.ImportPayloads(requests.Data)
}

// ImportFromURL - takes one string value and tries connect to a remote server, then parse response body into
// recordedRequests structure (which is default format in which Hoverfly exports captured requests) and
// imports those requests into the database
func (d *DBClient) ImportFromURL(url string) error {

	resp, err := d.HTTP.Get(url)
	if err != nil {
		return fmt.Errorf("Failed to fetch given URL, error %s", err.Error())
	}

	var requests recordedRequests

	jsonParser := json.NewDecoder(resp.Body)
	if err = jsonParser.Decode(&requests); err != nil {
		return fmt.Errorf("Got error while parsing payloads, error %s", err.Error())
	}

	return d.ImportPayloads(requests.Data)
}

func isJSON(s string) bool {
	var js map[string]interface{}
	return json.Unmarshal([]byte(s), &js) == nil

}

// ImportPayloads - a function to save given payloads into the database.
func (d *DBClient) ImportPayloads(payloads []Payload) error {
	if len(payloads) > 0 {
		success := 0
		failed := 0
		for _, pl := range payloads {
			pl.Request.Headers = make(map[string][]string)
			// sniffing content types
			if isJSON(pl.Request.Body) {
				pl.Request.Headers["Content-Type"] = []string{"application/json"}
			} else {
				ct := http.DetectContentType([]byte(pl.Request.Body))
				pl.Request.Headers["Content-Type"] = []string{ct}
			}

			// recalculating request hash and storing it in database
			r := RequestContainer{Details: pl.Request}
			key := r.Hash()

			// regenerating key
			pl.ID = key

			bts, err := pl.Encode()
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
				}).Error("Failed to encode payload")
				failed++
			} else {
				// hook
				var en Entry
				en.ActionType = ActionTypeRequestCaptured
				en.Message = "imported"
				en.Time = time.Now()
				en.Data = bts

				if err := d.Hooks.Fire(ActionTypeRequestCaptured, &en); err != nil {
					log.WithFields(log.Fields{
						"error":      err.Error(),
						"message":    en.Message,
						"actionType": ActionTypeRequestCaptured,
					}).Error("failed to fire hook")
				}

				d.Cache.Set([]byte(key), bts)
				if err == nil {
					success++
				} else {
					failed++
				}
			}
		}
		log.WithFields(log.Fields{
			"total":      len(payloads),
			"successful": success,
			"failed":     failed,
		}).Info("payloads imported")
		return nil
	}
	return fmt.Errorf("Bad request. Nothing to import!")
}
<|endoftext|>


<commit_before>package frank

import (
	"code.google.com/p/go.net/html"
	"code.google.com/p/go.net/html/atom"
	_ "crypto/sha512"
	"errors"
	frankconf "github.com/breunigs/frank/config"
	irc "github.com/fluffle/goirc/client"
	"io"
	"log"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// how many URLs can the cache store
const cacheSize = 500

// how many hours an entry should be considered valid
const cacheValidHours = 24

// how many kilo bytes should be considered when looking for the title
// tag.
const httpReadKByte = 100

// don’t repost the same title within this period
const noRepostWithinSeconds = 30

// matches all whitespace and zero bytes. Additionally, all Unicode
// characters of class Cf (format chars, e.g. right-to-left) and Cc
// (control chars) are matched.
var whitespaceRegex = regexp.MustCompile(`[\s\0\p{Cf}\p{Cc}]+`)

var ignoreDomainsRegex = regexp.MustCompile(`^http://p\.nnev\.de`)

var twitterDomainRegex = regexp.MustCompile(`(?i)^https?://(?:[a-z0-9]\.)?twitter.com`)
var twitterPicsRegex = regexp.MustCompile(`(?i)(?:\b|^)pic\.twitter\.com/[a-z0-9]+(?:\b|$)`)

var noSpoilerRegex = regexp.MustCompile(`(?i)(don't|no|kein|nicht) spoiler`)

// blacklist pointless titles /////////////////////////////////////////
var pointlessTitles = []string{"", "imgur: the simple image sharer"}

func isIn(needle string, haystack []string) bool {
	for _, s := range haystack {
		if s == needle {
			return true
		}
	}
	return false
}

func UriFind(conn *irc.Conn, line *irc.Line) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("MEGA-WTF:pkg: %v", r)
		}
	}()

	msg := line.Args[1]

	if noSpoilerRegex.MatchString(msg) {
		log.Printf("not spoilering this line: %s", msg)
		return
	}

	urls := extract(msg)

	for _, url := range urls {
		if url == "" {
			continue
		}

		if cp := cacheGetByUrl(url); cp != nil {
			log.Printf("using cache for URL: %s", cp.url)
			ago := cacheGetTimeAgo(cp)
			postTitle(conn, line, cp.title, "cached "+ago+" ago")
			// Hack: add title to the cache again so we can correctly check
			// for reposts, even if the original link has been cached quite
			// some time ago. Since the repost check searches by title, but
			// here we search by URL wie get the correct time when it was
			// cached while still preventing people from using frank to
			// multiply their spamming.
			cacheAdd("", cp.title)
			continue
		}

		go func(url string) {
			if ignoreDomainsRegex.MatchString(url) {
				log.Printf("ignoring this URL: %s", url)
				return
			}

			log.Printf("testing URL: %s", url)
			title, _, err := TitleGet(url)
			if err != nil {
				//postTitle(conn, line, err.Error(), "Error")
			} else if !isIn(title, pointlessTitles) {
				postTitle(conn, line, title, "")
				cacheAdd(url, title)
			}
		}(url)
	}
}

// regexing ////////////////////////////////////////////////////////////

func extract(msg string) []string {
	results := make([]string, 0)
	for idx := strings.Index(msg, "http"); idx > -1; idx = strings.Index(msg, "http") {
		url := msg[idx:]
		if !strings.HasPrefix(url, "http://") &&
			!strings.HasPrefix(url, "https://") {
			msg = msg[idx+len("http"):]
			continue
		}

		// End on commas, but only if they are followed by a space.
		// spiegel.de URLs have commas in them, that would be a
		// false positive otherwise.
		if end := strings.Index(url, ", "); end > -1 {
			url = url[:end]
		}

		// use special handling if the URL contains closing parens
		closingParen := strings.Index(url, ")")
		if closingParen > -1 {
			absPos := idx + closingParen + 1
			if len(msg) > absPos && msg[absPos] == ')' {
				// if an URL ends with double closing parens, assume that the
				// former one belongs to the URL
				url = url[:closingParen+1]
			} else if idx > 0 && msg[idx-1] == '(' {
				// if it ends on a single closing parens (follow by other chars)
				// only remove that closing parens if the URL is directly
				// preceded by one
				url = url[:closingParen]
			}
		}

		// Whitespace always ends a URL.
		if end := strings.IndexAny(url, " \t"); end > -1 {
			url = url[:end]
		}

		results = append(results, url)
		msg = msg[idx+len(url):]
	}
	return results
}

// http/html stuff /////////////////////////////////////////////////////

func TitleGet(url string) (string, string, error) {
	c := HttpClientWithTimeout()

	r, err := c.Get(url)
	if err != nil {
		log.Printf("WTF: could not resolve %s: %s\n", url, err)
		return "", url, err
	}
	defer r.Body.Close()

	lastUrl := r.Request.URL.String()

	// TODO: r.Body → utf8?
	title, tweet := titleParseHtml(io.LimitReader(r.Body, 1024*httpReadKByte))

	if r.StatusCode != 200 {
		return "", lastUrl, errors.New("[" + strconv.Itoa(r.StatusCode) + "] " + title)
	}

	if tweet != "" && twitterDomainRegex.MatchString(lastUrl) {
		title = tweet
	}

	log.Printf("Title for URL %s: %s\n", url, title)

	return title, lastUrl, nil
}

// parses the incoming HTML fragment and tries to extract text from
// suitable tags. Currently this is the page’s title tag and tweets
// when the HTML-code is similar enough to twitter.com. Returns
// title and tweet.
func titleParseHtml(r io.Reader) (string, string) {
	doc, err := html.Parse(r)
	if err != nil {
		log.Printf("WTF: html parser blew up: %s\n", err)
		return "", ""
	}

	title := ""
	tweetText := ""
	tweetUserName := ""
	tweetUserScreenName := ""
	tweetPicUrl := ""

	var f func(*html.Node)
	f = func(n *html.Node) {
		if title == "" && n.Type == html.ElementNode && n.DataAtom == atom.Title {
			title = extractText(n)
			return
		}

		if hasClass(n, "permalink-tweet") {
			tweetUserName = getAttr(n, "data-name")
			tweetUserScreenName = getAttr(n, "data-screen-name")
			// find next child “tweet-text”
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				if hasClass(c, "tweet-text") {
					tweetText = extractText(c)
					break
				}
			}
		}

		isMedia := hasClass(n, "media") || hasClass(n, "media-thumbnail")
		if tweetPicUrl == "" && isMedia && !hasClass(n, "profile-picture") {
			attrVal := getAttr(n, "data-url")
			if attrVal != "" {
				tweetPicUrl = attrVal
				return
			}
		}

		// recurse down
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}

	}
	f(doc)

	// cleanup
	tweet := ""
	tweetUser := ""
	if tweetText != "" {
		tweetText = twitterPicsRegex.ReplaceAllString(tweetText, "")
		tweetUser = tweetUserName + " (@" + tweetUserScreenName + "): "
		tweet = tweetUser + tweetText + " " + tweetPicUrl
		tweet = clean(tweet)
	}

	return strings.TrimSpace(title), strings.TrimSpace(tweet)
}

func extractText(n *html.Node) string {
	text := ""
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if c.Type == html.TextNode {
			text += c.Data
		} else {
			text += extractText(c)
		}
	}
	return clean(text)
}

func hasClass(n *html.Node, class string) bool {
	if n.Type != html.ElementNode {
		return false
	}

	class = " " + strings.TrimSpace(class) + " "
	attr := strings.Replace(getAttr(n, "class"), "\n", " ", -1)
	if strings.Contains(" "+attr+" ", class) {
		return true
	}
	return false
}

func getAttr(n *html.Node, findAttr string) string {
	for _, attr := range n.Attr {
		if attr.Key == findAttr {
			return attr.Val
		}
	}
	return ""
}

// Cache ///////////////////////////////////////////////////////////////
type Cache struct {
	url   string
	title string
	date  time.Time
}

var cache = [cacheSize]Cache{}
var cacheIndex = 0

func cacheAdd(url string, title string) {
	if len(cache) == cacheIndex {
		cacheIndex = 0
	}
	cache[cacheIndex] = Cache{url, title, time.Now()}
	cacheIndex += 1
}

func cacheGetByUrl(url string) *Cache {
	for _, cc := range cache {
		if cc.url == url && time.Since(cc.date).Hours() <= cacheValidHours {
			return &cc
		}
	}
	return nil
}

func cacheGetTimeAgo(cc *Cache) string {
	ago := time.Since(cc.date).Minutes()
	if ago < 60 {
		return strconv.Itoa(int(ago)) + "m"
	} else {
		hours := strconv.Itoa(int(ago/60.0 + 0.5))
		return hours + "h"
	}
}

func cacheGetSecondsToLastPost(title string) int {
	var secondsAgo = int(^uint(0) >> 1)
	for _, cc := range cache {
		var a = int(time.Since(cc.date).Seconds())
		if cc.title == title && a < secondsAgo {
			secondsAgo = a
		}
	}
	return secondsAgo
}

// util ////////////////////////////////////////////////////////////////

func postTitle(conn *irc.Conn, line *irc.Line, title string, prefix string) {
	tgt := line.Args[0]

	secondsAgo := cacheGetSecondsToLastPost(title)
	if secondsAgo <= noRepostWithinSeconds {
		log.Printf("Skipping, because posted %d seconds ago (“%s”)", secondsAgo, title)
		return
	}

	if frankconf.Debug {
		log.Printf("Title was last posted: %#v (“%s”)", secondsAgo, title)
	}

	log.Printf("nick=%s, target=%s, title=%s", line.Nick, tgt, title)
	// if target is our current nick, it was a private message.
	// Answer the users in this case.
	if tgt == conn.Me().Nick {
		tgt = line.Nick
	}
	if prefix == "" {
		prefix = "Link Info"
	} else {
		prefix = clean(prefix)
	}
	title = clean(title)
	// the IRC spec states that notice should be used instead of msg
	// and that bots should not react to notice at all. However, no
	// real world bot adheres to this. Furthermore, people who can’t
	// configure their client to not highlight them on notices will
	// complain.
	conn.Privmsg(tgt, "["+prefix+"] "+title)
}

func clean(text string) string {
	text = whitespaceRegex.ReplaceAllString(text, " ")
	return strings.TrimSpace(text)
}
<commit_msg>added some titles to title-blacklist<commit_after>package frank

import (
	"code.google.com/p/go.net/html"
	"code.google.com/p/go.net/html/atom"
	_ "crypto/sha512"
	"errors"
	frankconf "github.com/breunigs/frank/config"
	irc "github.com/fluffle/goirc/client"
	"io"
	"log"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// how many URLs can the cache store
const cacheSize = 500

// how many hours an entry should be considered valid
const cacheValidHours = 24

// how many kilo bytes should be considered when looking for the title
// tag.
const httpReadKByte = 100

// don’t repost the same title within this period
const noRepostWithinSeconds = 30

// matches all whitespace and zero bytes. Additionally, all Unicode
// characters of class Cf (format chars, e.g. right-to-left) and Cc
// (control chars) are matched.
var whitespaceRegex = regexp.MustCompile(`[\s\0\p{Cf}\p{Cc}]+`)

var ignoreDomainsRegex = regexp.MustCompile(`^http://p\.nnev\.de`)

var twitterDomainRegex = regexp.MustCompile(`(?i)^https?://(?:[a-z0-9]\.)?twitter.com`)
var twitterPicsRegex = regexp.MustCompile(`(?i)(?:\b|^)pic\.twitter\.com/[a-z0-9]+(?:\b|$)`)

var noSpoilerRegex = regexp.MustCompile(`(?i)(don't|no|kein|nicht) spoiler`)

// blacklist pointless titles /////////////////////////////////////////
var pointlessTitles = []string{"",
	"imgur: the simple image sharer",
	"Fefes Blog",
	"Gmane Loom",
	"i3 - A better tiling and dynamic window manager",
	"i3 - improved tiling wm",
	"IT-News, c't, iX, Technology Review, Telepolis | heise online",
	"debian Pastezone",
	"Index of /docs/",
	"NoName e.V. pastebin",
	"Nopaste - powered by project-mindstorm IT Services",
	"Diff NoName e.V. pastebin",
	"",
	"",
	"",
	"",
	"Google"
}

func isIn(needle string, haystack []string) bool {
	for _, s := range haystack {
		if s == needle {
			return true
		}
	}
	return false
}

func UriFind(conn *irc.Conn, line *irc.Line) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("MEGA-WTF:pkg: %v", r)
		}
	}()

	msg := line.Args[1]

	if noSpoilerRegex.MatchString(msg) {
		log.Printf("not spoilering this line: %s", msg)
		return
	}

	urls := extract(msg)

	for _, url := range urls {
		if url == "" {
			continue
		}

		if cp := cacheGetByUrl(url); cp != nil {
			log.Printf("using cache for URL: %s", cp.url)
			ago := cacheGetTimeAgo(cp)
			postTitle(conn, line, cp.title, "cached "+ago+" ago")
			// Hack: add title to the cache again so we can correctly check
			// for reposts, even if the original link has been cached quite
			// some time ago. Since the repost check searches by title, but
			// here we search by URL wie get the correct time when it was
			// cached while still preventing people from using frank to
			// multiply their spamming.
			cacheAdd("", cp.title)
			continue
		}

		go func(url string) {
			if ignoreDomainsRegex.MatchString(url) {
				log.Printf("ignoring this URL: %s", url)
				return
			}

			log.Printf("testing URL: %s", url)
			title, _, err := TitleGet(url)
			if err != nil {
				//postTitle(conn, line, err.Error(), "Error")
			} else if !isIn(title, pointlessTitles) {
				postTitle(conn, line, title, "")
				cacheAdd(url, title)
			}
		}(url)
	}
}

// regexing ////////////////////////////////////////////////////////////

func extract(msg string) []string {
	results := make([]string, 0)
	for idx := strings.Index(msg, "http"); idx > -1; idx = strings.Index(msg, "http") {
		url := msg[idx:]
		if !strings.HasPrefix(url, "http://") &&
			!strings.HasPrefix(url, "https://") {
			msg = msg[idx+len("http"):]
			continue
		}

		// End on commas, but only if they are followed by a space.
		// spiegel.de URLs have commas in them, that would be a
		// false positive otherwise.
		if end := strings.Index(url, ", "); end > -1 {
			url = url[:end]
		}

		// use special handling if the URL contains closing parens
		closingParen := strings.Index(url, ")")
		if closingParen > -1 {
			absPos := idx + closingParen + 1
			if len(msg) > absPos && msg[absPos] == ')' {
				// if an URL ends with double closing parens, assume that the
				// former one belongs to the URL
				url = url[:closingParen+1]
			} else if idx > 0 && msg[idx-1] == '(' {
				// if it ends on a single closing parens (follow by other chars)
				// only remove that closing parens if the URL is directly
				// preceded by one
				url = url[:closingParen]
			}
		}

		// Whitespace always ends a URL.
		if end := strings.IndexAny(url, " \t"); end > -1 {
			url = url[:end]
		}

		results = append(results, url)
		msg = msg[idx+len(url):]
	}
	return results
}

// http/html stuff /////////////////////////////////////////////////////

func TitleGet(url string) (string, string, error) {
	c := HttpClientWithTimeout()

	r, err := c.Get(url)
	if err != nil {
		log.Printf("WTF: could not resolve %s: %s\n", url, err)
		return "", url, err
	}
	defer r.Body.Close()

	lastUrl := r.Request.URL.String()

	// TODO: r.Body → utf8?
	title, tweet := titleParseHtml(io.LimitReader(r.Body, 1024*httpReadKByte))

	if r.StatusCode != 200 {
		return "", lastUrl, errors.New("[" + strconv.Itoa(r.StatusCode) + "] " + title)
	}

	if tweet != "" && twitterDomainRegex.MatchString(lastUrl) {
		title = tweet
	}

	log.Printf("Title for URL %s: %s\n", url, title)

	return title, lastUrl, nil
}

// parses the incoming HTML fragment and tries to extract text from
// suitable tags. Currently this is the page’s title tag and tweets
// when the HTML-code is similar enough to twitter.com. Returns
// title and tweet.
func titleParseHtml(r io.Reader) (string, string) {
	doc, err := html.Parse(r)
	if err != nil {
		log.Printf("WTF: html parser blew up: %s\n", err)
		return "", ""
	}

	title := ""
	tweetText := ""
	tweetUserName := ""
	tweetUserScreenName := ""
	tweetPicUrl := ""

	var f func(*html.Node)
	f = func(n *html.Node) {
		if title == "" && n.Type == html.ElementNode && n.DataAtom == atom.Title {
			title = extractText(n)
			return
		}

		if hasClass(n, "permalink-tweet") {
			tweetUserName = getAttr(n, "data-name")
			tweetUserScreenName = getAttr(n, "data-screen-name")
			// find next child “tweet-text”
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				if hasClass(c, "tweet-text") {
					tweetText = extractText(c)
					break
				}
			}
		}

		isMedia := hasClass(n, "media") || hasClass(n, "media-thumbnail")
		if tweetPicUrl == "" && isMedia && !hasClass(n, "profile-picture") {
			attrVal := getAttr(n, "data-url")
			if attrVal != "" {
				tweetPicUrl = attrVal
				return
			}
		}

		// recurse down
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}

	}
	f(doc)

	// cleanup
	tweet := ""
	tweetUser := ""
	if tweetText != "" {
		tweetText = twitterPicsRegex.ReplaceAllString(tweetText, "")
		tweetUser = tweetUserName + " (@" + tweetUserScreenName + "): "
		tweet = tweetUser + tweetText + " " + tweetPicUrl
		tweet = clean(tweet)
	}

	return strings.TrimSpace(title), strings.TrimSpace(tweet)
}

func extractText(n *html.Node) string {
	text := ""
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if c.Type == html.TextNode {
			text += c.Data
		} else {
			text += extractText(c)
		}
	}
	return clean(text)
}

func hasClass(n *html.Node, class string) bool {
	if n.Type != html.ElementNode {
		return false
	}

	class = " " + strings.TrimSpace(class) + " "
	attr := strings.Replace(getAttr(n, "class"), "\n", " ", -1)
	if strings.Contains(" "+attr+" ", class) {
		return true
	}
	return false
}

func getAttr(n *html.Node, findAttr string) string {
	for _, attr := range n.Attr {
		if attr.Key == findAttr {
			return attr.Val
		}
	}
	return ""
}

// Cache ///////////////////////////////////////////////////////////////
type Cache struct {
	url   string
	title string
	date  time.Time
}

var cache = [cacheSize]Cache{}
var cacheIndex = 0

func cacheAdd(url string, title string) {
	if len(cache) == cacheIndex {
		cacheIndex = 0
	}
	cache[cacheIndex] = Cache{url, title, time.Now()}
	cacheIndex += 1
}

func cacheGetByUrl(url string) *Cache {
	for _, cc := range cache {
		if cc.url == url && time.Since(cc.date).Hours() <= cacheValidHours {
			return &cc
		}
	}
	return nil
}

func cacheGetTimeAgo(cc *Cache) string {
	ago := time.Since(cc.date).Minutes()
	if ago < 60 {
		return strconv.Itoa(int(ago)) + "m"
	} else {
		hours := strconv.Itoa(int(ago/60.0 + 0.5))
		return hours + "h"
	}
}

func cacheGetSecondsToLastPost(title string) int {
	var secondsAgo = int(^uint(0) >> 1)
	for _, cc := range cache {
		var a = int(time.Since(cc.date).Seconds())
		if cc.title == title && a < secondsAgo {
			secondsAgo = a
		}
	}
	return secondsAgo
}

// util ////////////////////////////////////////////////////////////////

func postTitle(conn *irc.Conn, line *irc.Line, title string, prefix string) {
	tgt := line.Args[0]

	secondsAgo := cacheGetSecondsToLastPost(title)
	if secondsAgo <= noRepostWithinSeconds {
		log.Printf("Skipping, because posted %d seconds ago (“%s”)", secondsAgo, title)
		return
	}

	if frankconf.Debug {
		log.Printf("Title was last posted: %#v (“%s”)", secondsAgo, title)
	}

	log.Printf("nick=%s, target=%s, title=%s", line.Nick, tgt, title)
	// if target is our current nick, it was a private message.
	// Answer the users in this case.
	if tgt == conn.Me().Nick {
		tgt = line.Nick
	}
	if prefix == "" {
		prefix = "Link Info"
	} else {
		prefix = clean(prefix)
	}
	title = clean(title)
	// the IRC spec states that notice should be used instead of msg
	// and that bots should not react to notice at all. However, no
	// real world bot adheres to this. Furthermore, people who can’t
	// configure their client to not highlight them on notices will
	// complain.
	conn.Privmsg(tgt, "["+prefix+"] "+title)
}

func clean(text string) string {
	text = whitespaceRegex.ReplaceAllString(text, " ")
	return strings.TrimSpace(text)
}
<|endoftext|>


<commit_before>// The idx package provides a metadata index for metrics
// it's based on the index from https://github.com/dgryski/carbonmem/
// it seems promising and performant for our current needs, but also experimental
// We don't necessarily want to start maintaining our own little search engine.
// Future needs (scale, features) may deprecate this package in favor of something else
// like ES, bleve, etc.  But for now it seems to work well, so let's see where it takes us

// callers are responsible for thread safety, period pruning if desired
package idx

import (
	"path/filepath"
	"sort"
	"strings"
	//	"time"

	"github.com/armon/go-radix"
	"github.com/dgryski/go-trigram"
)

type MatchType uint8

const (
	MatchLiteral MatchType = iota
	MatchPrefix
	MatchTrigram
)

type MetricID uint32 // same thing as trigram.DocID. just syntactic sugar

type Glob struct {
	Id     MetricID
	Metric string
	IsLeaf bool
}

const OTTBase = uint32(2147483648)             // 1 << 31
const PublicOrgTrigram = trigram.T(2147483648) // 1 << 31. eg base + 0

// orgToTrigram takes in an orgid between -1 (special) and 214748345 and converts it to a special trigram
func orgToTrigram(org int) trigram.T {
	if org < -1 {
		panic("orgid must be >= -1")
	}
	// org+1 cannot be 2147483647 because we would end up at MaxUint32 == 4294967295 == 0xffffffff which is reserved
	if org > 2147483645 {
		panic("org too high")
	}
	ret := OTTBase + uint32(org) + 1
	return trigram.T(ret)
}

type globByName []Glob

func (g globByName) Len() int           { return len(g) }
func (g globByName) Swap(i, j int)      { g[i], g[j] = g[j], g[i] }
func (g globByName) Less(i, j int) bool { return g[i].Metric < g[j].Metric }

type Idx struct {
	// all metrics
	keys  []map[string]MetricID // slice is per org
	revs  []string              // MetricID to string key. doesn't have to be isolated per org
	count int                   // global across all orgs

	// currently 'active'
	active map[MetricID]int // doesn't have to be isolated per org

	prefix []*radix.Tree // per org. each org is stored at pos orgid+1 because we have a special org -1

	// stores all orgs in one index, but with special org-id trigrams
	Pathidx trigram.Index
}

func New() *Idx {
	return &Idx{
		keys: make([]map[string]MetricID, 0),

		active: make(map[MetricID]int),
		prefix: make([]*radix.Tree, 0),

		Pathidx: trigram.NewIndex(nil),
	}

}

func (l *Idx) Len() int {
	return len(l.keys)
}

// Get retrieves the metricId and whether it was found
// it first checks the given org, but also falls back to org -1
func (l *Idx) Get(org int, key string) (MetricID, bool) {
	pos := org + 1 // org -1 is pos 0, etc

	for len(l.keys) < pos+1 {
		l.keys = append(l.keys, make(map[string]MetricID))
	}

	id, ok := l.keys[pos][key]
	if !ok {
		id, ok = l.keys[0][key]
	}

	return id, ok
}

// GetsOrAdd retrieves the key if it exists for the org (but doesn't do additional check for org -1)
// and adds it for the org if non-existant
func (l *Idx) GetOrAdd(org int, key string) MetricID {
	pos := org + 1 // org -1 is pos 0, etc

	for len(l.keys) < pos+1 {
		l.keys = append(l.keys, make(map[string]MetricID))
	}

	id, ok := l.keys[pos][key]

	if ok {
		return id
	}

	id = MetricID(l.count)
	l.count++
	l.revs = append(l.revs, key)

	l.keys[pos][key] = id

	// like l.Pathidx.Insert(key, trigram.DocID(id)) but with added org trigram
	ts := trigram.ExtractAll(key, nil)
	ts = append(ts, orgToTrigram(org))
	l.Pathidx.InsertTrigrams(ts, trigram.DocID(id))

	return id
}

func (l *Idx) Key(id MetricID) string {
	return l.revs[id]
}

func (l *Idx) AddRef(org int, id MetricID) {
	v, ok := l.active[id]
	if !ok {
		pos := org + 1
		for len(l.prefix) < pos+1 {
			l.prefix = append(l.prefix, radix.New())
		}
		l.prefix[pos].Insert(l.revs[id], id)
	}

	l.active[id] = v + 1
}

// only call this after having made sure the org entries exists! e.g. after Get/GetOrAdd and AddRef
func (l *Idx) DelRef(org int, id MetricID) {
	l.active[id]--
	if l.active[id] == 0 {
		delete(l.active, id)
		delete(l.keys[org], l.revs[id])
		l.prefix[org+1].Delete(l.revs[id])
		if l.Pathidx != nil {
			l.Pathidx.Delete(l.revs[id], trigram.DocID(id))
		}
		l.revs[id] = ""
	}
}

func (l *Idx) Active(id MetricID) bool {
	return l.active[id] != 0
}

func (l *Idx) Prefix(org int, query string, fn radix.WalkFn) {
	pos := org + 1
	for len(l.prefix) < pos+1 {
		l.prefix = append(l.prefix, radix.New())
	}
	l.prefix[pos].WalkPrefix(query, fn)
}

// for a "filesystem glob" query (e.g. supports * and [] but not {})
// looks in the trigrams index
func (l *Idx) QueryPath(org int, query string) []Glob {

	if l.Pathidx == nil {
		return nil
	}

	ts := extractTrigrams(query)
	ids := l.Pathidx.QueryTrigrams(ts)
	ids = l.Pathidx.FilterOr(ids, [][]trigram.T{{PublicOrgTrigram}, {orgToTrigram(org)}})

	seen := make(map[string]bool)
	out := make([]Glob, 0)

	for _, id := range ids {

		p := l.revs[MetricID(id)]

		dir := filepath.Dir(p)

		if seen[dir] {
			continue
		}

		if matched, err := filepath.Match(query, dir); err == nil && matched {
			seen[dir] = true
			out = append(out, Glob{0, dir, false}) // for directories we leave id=0
			continue
		}

		if matched, err := filepath.Match(query, p); err == nil && matched {
			seen[p] = true
			out = append(out, Glob{MetricID(id), p, true})
		}
	}

	return out
}

func (l *Idx) List(org int) []MetricID {
	var response []MetricID
	fn := func(k string, v interface{}) bool {
		response = append(response, v.(MetricID))
		return false
	}
	if org == -1 {
		for i := 0; i < len(l.prefix); i++ {
			l.prefix[i].Walk(fn)
		}
	} else {
		l.Prefix(-1, "", fn)
		l.Prefix(org, "", fn)
	}
	return response
}
func (i *Idx) QueryRadix(org int, query string) []Glob {
	var response []Glob
	l := len(query)
	seen := make(map[string]bool)
	fn := func(k string, v interface{}) bool {
		// figure out if we're a leaf or not
		dot := strings.IndexByte(k[l:], '.')
		var leaf bool
		m := k
		if dot == -1 {
			leaf = true
		} else {
			m = k[:dot+l]
		}
		if !seen[m] {
			seen[m] = true
			response = append(response, Glob{Id: v.(MetricID), Metric: m, IsLeaf: leaf})
		}
		// false == "don't terminate iteration"
		return false
	}
	i.Prefix(-1, query, fn)
	i.Prefix(org, query, fn)
	return response
}

// TODO(dgryski): this needs most of the logic in grobian/carbsonerver:findHandler()
// Dieter: this doesn't support {, }, [, ]

func (i *Idx) Match(org int, query string) (MatchType, []Glob) {

	// no wildcard == exact match only
	var star int
	if star = strings.Index(query, "*"); star == -1 {
		var id MetricID
		var ok bool
		if id, ok = i.Get(org, query); !ok {
			return MatchLiteral, nil
		}
		return MatchLiteral, []Glob{{Id: id, Metric: query, IsLeaf: true}}
	}

	var response []Glob
	var matchType MatchType

	if star == len(query)-1 {
		// only one trailing star
		query = query[:len(query)-1]
		response = i.QueryRadix(org, query)
		matchType = MatchPrefix
	} else {
		// at least one interior star
		response = i.QueryPath(org, query)
		matchType = MatchTrigram
	}

	sort.Sort(globByName(response))

	return matchType, response
}

func extractTrigrams(query string) []trigram.T {

	if len(query) < 3 {
		return nil
	}

	var start int
	var i int

	var trigrams []trigram.T

	for i < len(query) {
		if query[i] == '[' || query[i] == '*' || query[i] == '?' {
			trigrams = trigram.Extract(query[start:i], trigrams)

			if query[i] == '[' {
				for i < len(query) && query[i] != ']' {
					i++
				}
			}

			start = i + 1
		}
		i++
	}

	trigrams = trigram.Extract(query[start:i], trigrams)

	return trigrams
}

// Prune is a custom pruning function for the trigram index
// it removes all trigrams that are present in more than the specified percentage of the documents.
// However, it makes sure that neither tAllDocIDs,
// nor any of the org specific trigrams used for access control are ever pruned.
func (i *Idx) Prune(pct float64) {
	// 0xFFFFFFFF is just tAllDocIDs in the trigram package but it's not yet exported
	// https://github.com/dgryski/go-trigram/pull/18
	maxDocs := int(pct * float64(len(i.Pathidx[0xFFFFFFFF])))
	for k, v := range i.Pathidx {
		// all values >= 1<<31 are special trigrams that should be kept
		if uint32(k) < uint32(2147483648) && len(v) > maxDocs {
			i.Pathidx[k] = nil
		}
	}
}
<commit_msg>can now use exported TAllDocIDs<commit_after>// The idx package provides a metadata index for metrics
// it's based on the index from https://github.com/dgryski/carbonmem/
// it seems promising and performant for our current needs, but also experimental
// We don't necessarily want to start maintaining our own little search engine.
// Future needs (scale, features) may deprecate this package in favor of something else
// like ES, bleve, etc.  But for now it seems to work well, so let's see where it takes us

// callers are responsible for thread safety, period pruning if desired
package idx

import (
	"path/filepath"
	"sort"
	"strings"
	//	"time"

	"github.com/armon/go-radix"
	"github.com/dgryski/go-trigram"
)

type MatchType uint8

const (
	MatchLiteral MatchType = iota
	MatchPrefix
	MatchTrigram
)

type MetricID uint32 // same thing as trigram.DocID. just syntactic sugar

type Glob struct {
	Id     MetricID
	Metric string
	IsLeaf bool
}

const OTTBase = uint32(2147483648)             // 1 << 31
const PublicOrgTrigram = trigram.T(2147483648) // 1 << 31. eg base + 0

// orgToTrigram takes in an orgid between -1 (special) and 214748345 and converts it to a special trigram
func orgToTrigram(org int) trigram.T {
	if org < -1 {
		panic("orgid must be >= -1")
	}
	// org+1 cannot be 2147483647 because we would end up at MaxUint32 == 4294967295 == 0xffffffff which is reserved
	if org > 2147483645 {
		panic("org too high")
	}
	ret := OTTBase + uint32(org) + 1
	return trigram.T(ret)
}

type globByName []Glob

func (g globByName) Len() int           { return len(g) }
func (g globByName) Swap(i, j int)      { g[i], g[j] = g[j], g[i] }
func (g globByName) Less(i, j int) bool { return g[i].Metric < g[j].Metric }

type Idx struct {
	// all metrics
	keys  []map[string]MetricID // slice is per org
	revs  []string              // MetricID to string key. doesn't have to be isolated per org
	count int                   // global across all orgs

	// currently 'active'
	active map[MetricID]int // doesn't have to be isolated per org

	prefix []*radix.Tree // per org. each org is stored at pos orgid+1 because we have a special org -1

	// stores all orgs in one index, but with special org-id trigrams
	Pathidx trigram.Index
}

func New() *Idx {
	return &Idx{
		keys: make([]map[string]MetricID, 0),

		active: make(map[MetricID]int),
		prefix: make([]*radix.Tree, 0),

		Pathidx: trigram.NewIndex(nil),
	}

}

func (l *Idx) Len() int {
	return len(l.keys)
}

// Get retrieves the metricId and whether it was found
// it first checks the given org, but also falls back to org -1
func (l *Idx) Get(org int, key string) (MetricID, bool) {
	pos := org + 1 // org -1 is pos 0, etc

	for len(l.keys) < pos+1 {
		l.keys = append(l.keys, make(map[string]MetricID))
	}

	id, ok := l.keys[pos][key]
	if !ok {
		id, ok = l.keys[0][key]
	}

	return id, ok
}

// GetsOrAdd retrieves the key if it exists for the org (but doesn't do additional check for org -1)
// and adds it for the org if non-existant
func (l *Idx) GetOrAdd(org int, key string) MetricID {
	pos := org + 1 // org -1 is pos 0, etc

	for len(l.keys) < pos+1 {
		l.keys = append(l.keys, make(map[string]MetricID))
	}

	id, ok := l.keys[pos][key]

	if ok {
		return id
	}

	id = MetricID(l.count)
	l.count++
	l.revs = append(l.revs, key)

	l.keys[pos][key] = id

	// like l.Pathidx.Insert(key, trigram.DocID(id)) but with added org trigram
	ts := trigram.ExtractAll(key, nil)
	ts = append(ts, orgToTrigram(org))
	l.Pathidx.InsertTrigrams(ts, trigram.DocID(id))

	return id
}

func (l *Idx) Key(id MetricID) string {
	return l.revs[id]
}

func (l *Idx) AddRef(org int, id MetricID) {
	v, ok := l.active[id]
	if !ok {
		pos := org + 1
		for len(l.prefix) < pos+1 {
			l.prefix = append(l.prefix, radix.New())
		}
		l.prefix[pos].Insert(l.revs[id], id)
	}

	l.active[id] = v + 1
}

// only call this after having made sure the org entries exists! e.g. after Get/GetOrAdd and AddRef
func (l *Idx) DelRef(org int, id MetricID) {
	l.active[id]--
	if l.active[id] == 0 {
		delete(l.active, id)
		delete(l.keys[org], l.revs[id])
		l.prefix[org+1].Delete(l.revs[id])
		if l.Pathidx != nil {
			l.Pathidx.Delete(l.revs[id], trigram.DocID(id))
		}
		l.revs[id] = ""
	}
}

func (l *Idx) Active(id MetricID) bool {
	return l.active[id] != 0
}

func (l *Idx) Prefix(org int, query string, fn radix.WalkFn) {
	pos := org + 1
	for len(l.prefix) < pos+1 {
		l.prefix = append(l.prefix, radix.New())
	}
	l.prefix[pos].WalkPrefix(query, fn)
}

// for a "filesystem glob" query (e.g. supports * and [] but not {})
// looks in the trigrams index
func (l *Idx) QueryPath(org int, query string) []Glob {

	if l.Pathidx == nil {
		return nil
	}

	ts := extractTrigrams(query)
	ids := l.Pathidx.QueryTrigrams(ts)
	ids = l.Pathidx.FilterOr(ids, [][]trigram.T{{PublicOrgTrigram}, {orgToTrigram(org)}})

	seen := make(map[string]bool)
	out := make([]Glob, 0)

	for _, id := range ids {

		p := l.revs[MetricID(id)]

		dir := filepath.Dir(p)

		if seen[dir] {
			continue
		}

		if matched, err := filepath.Match(query, dir); err == nil && matched {
			seen[dir] = true
			out = append(out, Glob{0, dir, false}) // for directories we leave id=0
			continue
		}

		if matched, err := filepath.Match(query, p); err == nil && matched {
			seen[p] = true
			out = append(out, Glob{MetricID(id), p, true})
		}
	}

	return out
}

func (l *Idx) List(org int) []MetricID {
	var response []MetricID
	fn := func(k string, v interface{}) bool {
		response = append(response, v.(MetricID))
		return false
	}
	if org == -1 {
		for i := 0; i < len(l.prefix); i++ {
			l.prefix[i].Walk(fn)
		}
	} else {
		l.Prefix(-1, "", fn)
		l.Prefix(org, "", fn)
	}
	return response
}
func (i *Idx) QueryRadix(org int, query string) []Glob {
	var response []Glob
	l := len(query)
	seen := make(map[string]bool)
	fn := func(k string, v interface{}) bool {
		// figure out if we're a leaf or not
		dot := strings.IndexByte(k[l:], '.')
		var leaf bool
		m := k
		if dot == -1 {
			leaf = true
		} else {
			m = k[:dot+l]
		}
		if !seen[m] {
			seen[m] = true
			response = append(response, Glob{Id: v.(MetricID), Metric: m, IsLeaf: leaf})
		}
		// false == "don't terminate iteration"
		return false
	}
	i.Prefix(-1, query, fn)
	i.Prefix(org, query, fn)
	return response
}

// TODO(dgryski): this needs most of the logic in grobian/carbsonerver:findHandler()
// Dieter: this doesn't support {, }, [, ]

func (i *Idx) Match(org int, query string) (MatchType, []Glob) {

	// no wildcard == exact match only
	var star int
	if star = strings.Index(query, "*"); star == -1 {
		var id MetricID
		var ok bool
		if id, ok = i.Get(org, query); !ok {
			return MatchLiteral, nil
		}
		return MatchLiteral, []Glob{{Id: id, Metric: query, IsLeaf: true}}
	}

	var response []Glob
	var matchType MatchType

	if star == len(query)-1 {
		// only one trailing star
		query = query[:len(query)-1]
		response = i.QueryRadix(org, query)
		matchType = MatchPrefix
	} else {
		// at least one interior star
		response = i.QueryPath(org, query)
		matchType = MatchTrigram
	}

	sort.Sort(globByName(response))

	return matchType, response
}

func extractTrigrams(query string) []trigram.T {

	if len(query) < 3 {
		return nil
	}

	var start int
	var i int

	var trigrams []trigram.T

	for i < len(query) {
		if query[i] == '[' || query[i] == '*' || query[i] == '?' {
			trigrams = trigram.Extract(query[start:i], trigrams)

			if query[i] == '[' {
				for i < len(query) && query[i] != ']' {
					i++
				}
			}

			start = i + 1
		}
		i++
	}

	trigrams = trigram.Extract(query[start:i], trigrams)

	return trigrams
}

// Prune is a custom pruning function for the trigram index
// it removes all trigrams that are present in more than the specified percentage of the documents.
// However, it makes sure that neither tAllDocIDs,
// nor any of the org specific trigrams used for access control are ever pruned.
func (i *Idx) Prune(pct float64) {
	maxDocs := int(pct * float64(len(i.Pathidx[trigram.TAllDocIDs])))
	for k, v := range i.Pathidx {
		// all values >= 1<<31 are special trigrams that should be kept
		if uint32(k) < uint32(2147483648) && len(v) > maxDocs {
			i.Pathidx[k] = nil
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metrics

import (
	"encoding/json"
	"fmt"
	"strconv"
)

const timeFormat = "2006/01/02-15:04:00-MST"

type Float struct {
	Value float64
	IsNaN bool
}

func (f *Float) UnmarshalJSON(b []byte) error {
	var source string
	json.Unmarshal(b, &source)

	switch source {
	case "NaN":
		f.Value = 0
		f.IsNaN = true
	default:
		json.Unmarshal(b, &f.Value)
		f.IsNaN = false
	}

	return nil
}

func (f Float) MarshalJSON() ([]byte, error) {
	if f.IsNaN {
		return json.Marshal("NaN")
	}

	value := strconv.FormatFloat(f.Value, 'E', -1, 64)
	return json.Marshal(value)
}

type V2Datapoint []float64

func (dp V2Datapoint) Timestamp() float64 {
	if len(dp) < 1 {
		return 0
	}
	return dp[0]
}

func (dp V2Datapoint) Value() float64 {
	if len(dp) < 2 {
		return 0
	}
	return dp[1]
}

// PerformanceOptions is the request object for doing a performance query.
type V2PerformanceOptions struct {
	Start     string            `json:"start,omitempty"`
	End       string            `json:"end,omitempty"`
	Returnset string            `json:"returnset,omitempty"`
	Metrics   []V2MetricOptions `json:"queries,omitempty"`
}

// MetricOptions are the options for receiving metrics for a set of data.
type V2MetricOptions struct {
	Metric      string              `json:"metric,omitempty"`
	Aggregator  string              `json:"aggregator,omitempty"`
	Rate        bool                `json:"rate,omitempty"`
	RateOptions V2RateOptions       `json:"rateOptions,omitempty"`
	Expression  string              `json:"expression,omitempty"`
	Tags        map[string][]string `json:"tags,omitempty"`
	Downsample  string              `json:"downsample,omitempty"`
}

// RateOptions are the options for collecting performance data.
type V2RateOptions struct {
	Counter        bool `json:"counter,omitempty"`
	CounterMax     int  `json:"counterMax,omitempty"`
	ResetThreshold int  `json:"resetThreshold,omitempty"`
}

// PerformanceData is the resulting object from a performance query.
type V2PerformanceData struct {
	Series   []V2ResultData `json:"series,omitempty"`
	Statuses []V2Status     `json:"statuses,omitempty"`
}

type V2ResultData struct {
	Datapoints []V2Datapoint     `json:"datapoints"`
	Metric     string            `json:"metric, omitempty"`
	Tags       map[string]string `json:"tags,omitempty"`
}

type V2Status struct {
	Message string `json:"message"`
	Status  string `json:"status"`
}

// PerformanceOptions is the request object for doing a performance query.
type PerformanceOptions struct {
	Start                string              `json:"start,omitempty"`
	End                  string              `json:"end,omitempty"`
	Returnset            string              `json:"returnset,omitempty"`
	Downsample           string              `json:"downsample,omitempty"`
	DownsampleMultiplier string              `json:"downsampleMultiplier,omitempty"`
	Tags                 map[string][]string `json:"tags,omitempty"`
	Metrics              []MetricOptions     `json:"metrics,omitempty"`
}

// MetricOptions are the options for receiving metrics for a set of data.
type MetricOptions struct {
	Metric       string              `json:"metric,omitempty"`
	Name         string              `json:"name,omitempty"`
	ID           string              `json:"id,omitempty"`
	Aggregator   string              `json:"aggregator,omitempty"`
	Interpolator string              `json:"interpolator,omitempty"`
	Rate         bool                `json:"rate,omitempty"`
	RateOptions  RateOptions         `json:"rateOptions,omitempty"`
	Expression   string              `json:"expression,omitempty"`
	Tags         map[string][]string `json:"tags,omitempty"`
}

// RateOptions are the options for collecting performance data.
type RateOptions struct {
	Counter        bool `json:"counter,omitempty"`
	CounterMax     int  `json:"counterMax,omitempty"`
	ResetThreshold int  `json:"resetThreshold,omitempty"`
}

// PerformanceData is the resulting object from a performance query.
type PerformanceData struct {
	ClientID        string       `json:"clientId,omitempty"`
	Source          string       `json:"source,omitempty"`
	StartTime       string       `json:"startTime,omitempty"`
	StartTimeActual int64        `json:"startTimeActual"`
	EndTime         string       `json:"endTime,omitempty"`
	EndTimeActual   int64        `json:"endTimeActual"`
	ExactTimeWindow bool         `json:"exactTimeWindow,omitempty"`
	Results         []ResultData `json:"results,omitempty"`
}

// ResultData is actual resulting data from the query per metric and tag
type ResultData struct {
	Datapoints []Datapoint         `json:"datapoints,omitempty"`
	Metric     string              `json:"metric,omitempty"`
	Tags       map[string][]string `json:"tags,omitempty"`
}

// Datapoint is a single numerical datapoint.
type Datapoint struct {
	Timestamp int64 `json:"timestamp"`
	Value     Float `json:"value,omitempty"`
}

func (c *Client) performanceQuery(opts PerformanceOptions) (*PerformanceData, error) {
	path := "/api/performance/query"
	body, _, err := c.do("POST", path, opts)
	if err != nil {
		return nil, err
	}
	var perfdata PerformanceData
	if err = json.Unmarshal(body, &perfdata); err != nil {
		return nil, err
	}
	return &perfdata, nil
}

func (c *Client) v2performanceQuery(opts V2PerformanceOptions) (*V2PerformanceData, error) {
	path := "/api/v2/performance/query"
	body, _, err := c.do("POST", path, opts)
	if err != nil {
		return nil, err
	}
	var perfdata V2PerformanceData
	if err = json.Unmarshal(body, &perfdata); err != nil {
		return nil, err
	}
	for _, status := range perfdata.Statuses {
		if status.Status == "ERROR" {
			return nil, fmt.Errorf("received error requesting performance data: %s", status.Message)
		}
	}
	return &perfdata, nil
}
<commit_msg>Remove space in struct field tag.<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metrics

import (
	"encoding/json"
	"fmt"
	"strconv"
)

const timeFormat = "2006/01/02-15:04:00-MST"

type Float struct {
	Value float64
	IsNaN bool
}

func (f *Float) UnmarshalJSON(b []byte) error {
	var source string
	json.Unmarshal(b, &source)

	switch source {
	case "NaN":
		f.Value = 0
		f.IsNaN = true
	default:
		json.Unmarshal(b, &f.Value)
		f.IsNaN = false
	}

	return nil
}

func (f Float) MarshalJSON() ([]byte, error) {
	if f.IsNaN {
		return json.Marshal("NaN")
	}

	value := strconv.FormatFloat(f.Value, 'E', -1, 64)
	return json.Marshal(value)
}

type V2Datapoint []float64

func (dp V2Datapoint) Timestamp() float64 {
	if len(dp) < 1 {
		return 0
	}
	return dp[0]
}

func (dp V2Datapoint) Value() float64 {
	if len(dp) < 2 {
		return 0
	}
	return dp[1]
}

// PerformanceOptions is the request object for doing a performance query.
type V2PerformanceOptions struct {
	Start     string            `json:"start,omitempty"`
	End       string            `json:"end,omitempty"`
	Returnset string            `json:"returnset,omitempty"`
	Metrics   []V2MetricOptions `json:"queries,omitempty"`
}

// MetricOptions are the options for receiving metrics for a set of data.
type V2MetricOptions struct {
	Metric      string              `json:"metric,omitempty"`
	Aggregator  string              `json:"aggregator,omitempty"`
	Rate        bool                `json:"rate,omitempty"`
	RateOptions V2RateOptions       `json:"rateOptions,omitempty"`
	Expression  string              `json:"expression,omitempty"`
	Tags        map[string][]string `json:"tags,omitempty"`
	Downsample  string              `json:"downsample,omitempty"`
}

// RateOptions are the options for collecting performance data.
type V2RateOptions struct {
	Counter        bool `json:"counter,omitempty"`
	CounterMax     int  `json:"counterMax,omitempty"`
	ResetThreshold int  `json:"resetThreshold,omitempty"`
}

// PerformanceData is the resulting object from a performance query.
type V2PerformanceData struct {
	Series   []V2ResultData `json:"series,omitempty"`
	Statuses []V2Status     `json:"statuses,omitempty"`
}

type V2ResultData struct {
	Datapoints []V2Datapoint     `json:"datapoints"`
	Metric     string            `json:"metric,omitempty"`
	Tags       map[string]string `json:"tags,omitempty"`
}

type V2Status struct {
	Message string `json:"message"`
	Status  string `json:"status"`
}

// PerformanceOptions is the request object for doing a performance query.
type PerformanceOptions struct {
	Start                string              `json:"start,omitempty"`
	End                  string              `json:"end,omitempty"`
	Returnset            string              `json:"returnset,omitempty"`
	Downsample           string              `json:"downsample,omitempty"`
	DownsampleMultiplier string              `json:"downsampleMultiplier,omitempty"`
	Tags                 map[string][]string `json:"tags,omitempty"`
	Metrics              []MetricOptions     `json:"metrics,omitempty"`
}

// MetricOptions are the options for receiving metrics for a set of data.
type MetricOptions struct {
	Metric       string              `json:"metric,omitempty"`
	Name         string              `json:"name,omitempty"`
	ID           string              `json:"id,omitempty"`
	Aggregator   string              `json:"aggregator,omitempty"`
	Interpolator string              `json:"interpolator,omitempty"`
	Rate         bool                `json:"rate,omitempty"`
	RateOptions  RateOptions         `json:"rateOptions,omitempty"`
	Expression   string              `json:"expression,omitempty"`
	Tags         map[string][]string `json:"tags,omitempty"`
}

// RateOptions are the options for collecting performance data.
type RateOptions struct {
	Counter        bool `json:"counter,omitempty"`
	CounterMax     int  `json:"counterMax,omitempty"`
	ResetThreshold int  `json:"resetThreshold,omitempty"`
}

// PerformanceData is the resulting object from a performance query.
type PerformanceData struct {
	ClientID        string       `json:"clientId,omitempty"`
	Source          string       `json:"source,omitempty"`
	StartTime       string       `json:"startTime,omitempty"`
	StartTimeActual int64        `json:"startTimeActual"`
	EndTime         string       `json:"endTime,omitempty"`
	EndTimeActual   int64        `json:"endTimeActual"`
	ExactTimeWindow bool         `json:"exactTimeWindow,omitempty"`
	Results         []ResultData `json:"results,omitempty"`
}

// ResultData is actual resulting data from the query per metric and tag
type ResultData struct {
	Datapoints []Datapoint         `json:"datapoints,omitempty"`
	Metric     string              `json:"metric,omitempty"`
	Tags       map[string][]string `json:"tags,omitempty"`
}

// Datapoint is a single numerical datapoint.
type Datapoint struct {
	Timestamp int64 `json:"timestamp"`
	Value     Float `json:"value,omitempty"`
}

func (c *Client) performanceQuery(opts PerformanceOptions) (*PerformanceData, error) {
	path := "/api/performance/query"
	body, _, err := c.do("POST", path, opts)
	if err != nil {
		return nil, err
	}
	var perfdata PerformanceData
	if err = json.Unmarshal(body, &perfdata); err != nil {
		return nil, err
	}
	return &perfdata, nil
}

func (c *Client) v2performanceQuery(opts V2PerformanceOptions) (*V2PerformanceData, error) {
	path := "/api/v2/performance/query"
	body, _, err := c.do("POST", path, opts)
	if err != nil {
		return nil, err
	}
	var perfdata V2PerformanceData
	if err = json.Unmarshal(body, &perfdata); err != nil {
		return nil, err
	}
	for _, status := range perfdata.Statuses {
		if status.Status == "ERROR" {
			return nil, fmt.Errorf("received error requesting performance data: %s", status.Message)
		}
	}
	return &perfdata, nil
}
<|endoftext|>