<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/smtp"
	"net/url"
	"os"
	"os/user"
)

const (
	name   = "trafficjam"
	apiURL = "https://maps.googleapis.com/maps/api/distancematrix/json"
)

type config struct {
	Origins      string `json:"origins"`
	Destinations string `json:"destinations"`
	APIKey       string `json:"api_key"`
	Mode         string `json:"mode"`
	Avoid        string `json:"avoid"`
	TrafficModel string `json:"traffic_model"`
	MaxDuration  int    `json:"max_duration"`
	SMTP         struct {
		Host string `json:"host"`
		Port int    `json:"port"`
		User string `json:"user"`
		Pass string `json:"pass"`
	} `json:"smtp"`
	Recipient string `json:"recipient"`
}

type apiResponse struct {
	Rows []struct {
		Elements []struct {
			DurationInTraffic struct {
				Value int `json:"value"`
			} `json:"duration_in_traffic"`
			Status string `json:"status"`
		} `json:"elements"`
	} `json:"rows"`
	Status string `json:"status"`
}

func main() {
	log.SetPrefix(name + ": ")
	log.SetFlags(0)

	if len(os.Args) != 2 {
		log.Fatalf("usage: %s config.json", name)
	}

	conf, err := readConfig(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}

	params := map[string]string{
		"origins":        conf.Origins,
		"destinations":   conf.Destinations,
		"key":            conf.APIKey,
		"mode":           conf.Mode,
		"avoid":          conf.Avoid,
		"departure_time": "now",
		"traffic_model":  conf.TrafficModel,
	}

	apiResp, err := queryMapsAPI(params)
	if err != nil {
		log.Fatal(err)
	}

	duration := apiResp.Rows[0].Elements[0].DurationInTraffic.Value
	durationMins := int(float64(duration)/60.0 + 0.5) // round

	if durationMins > conf.MaxDuration {
		sendMail(conf, fmt.Sprintf("%d minutes", durationMins))
	}
}

func readConfig(filename string) (*config, error) {
	var conf config

	confData, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	if err := json.Unmarshal(confData, &conf); err != nil {
		return nil, err
	}

	return &conf, nil
}

func queryMapsAPI(params map[string]string) (*apiResponse, error) {
	urlBuf := bytes.NewBufferString(apiURL)
	urlBuf.WriteByte('?')

	for key, val := range params {
		urlBuf.WriteString(url.QueryEscape(key))
		urlBuf.WriteByte('=')
		urlBuf.WriteString(url.QueryEscape(val))
		urlBuf.WriteByte('&')
	}

	resp, err := http.Get(urlBuf.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var apiResp apiResponse
	if err := json.Unmarshal(body, &apiResp); err != nil {
		return nil, err
	}

	if apiResp.Status != "OK" {
		return nil, fmt.Errorf("%s: bad response status: %s\n", name, apiResp.Status)
	}
	if len(apiResp.Rows) != 1 {
		return nil, fmt.Errorf("%s: response row count is not 1\n", name)
	}
	if len(apiResp.Rows[0].Elements) != 1 {
		return nil, fmt.Errorf("%s: response first row element count is not 1\n", name)
	}
	if apiResp.Rows[0].Elements[0].Status != "OK" {
		return nil, fmt.Errorf("%s: bad response first row first element status: %s\n", name, apiResp.Rows[0].Elements[0].Status)
	}

	return &apiResp, nil
}

func sendMail(conf *config, body string) error {
	user, err := user.Current()
	if err != nil {
		return err
	}

	hostname, err := os.Hostname()
	if err != nil {
		return err
	}

	auth := smtp.PlainAuth("", conf.SMTP.User, conf.SMTP.Pass, conf.SMTP.Host)
	sender := user.Username + "@" + hostname
	to := []string{conf.Recipient}
	msg := []byte("To: " + conf.Recipient + "\r\n" +
		"Subject: " + name + " alert\r\n" +
		"\r\n" +
		body + "\r\n")

	return smtp.SendMail(fmt.Sprintf("%s:%d", conf.SMTP.Host, conf.SMTP.Port), auth, sender, to, msg)
}
<commit_msg>Build URL with url package<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/smtp"
	"net/url"
	"os"
	"os/user"
)

const (
	name   = "trafficjam"
	apiURL = "https://maps.googleapis.com/maps/api/distancematrix/json"
)

type config struct {
	Origins      string `json:"origins"`
	Destinations string `json:"destinations"`
	APIKey       string `json:"api_key"`
	Mode         string `json:"mode"`
	Avoid        string `json:"avoid"`
	TrafficModel string `json:"traffic_model"`
	MaxDuration  int    `json:"max_duration"`
	SMTP         struct {
		Host string `json:"host"`
		Port int    `json:"port"`
		User string `json:"user"`
		Pass string `json:"pass"`
	} `json:"smtp"`
	Recipient string `json:"recipient"`
}

type apiResponse struct {
	Rows []struct {
		Elements []struct {
			DurationInTraffic struct {
				Value int `json:"value"`
			} `json:"duration_in_traffic"`
			Status string `json:"status"`
		} `json:"elements"`
	} `json:"rows"`
	Status string `json:"status"`
}

func main() {
	log.SetPrefix(name + ": ")
	log.SetFlags(0)

	if len(os.Args) != 2 {
		log.Fatalf("usage: %s config.json", name)
	}

	conf, err := readConfig(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}

	params := map[string]string{
		"origins":        conf.Origins,
		"destinations":   conf.Destinations,
		"key":            conf.APIKey,
		"mode":           conf.Mode,
		"avoid":          conf.Avoid,
		"departure_time": "now",
		"traffic_model":  conf.TrafficModel,
	}

	apiResp, err := queryMapsAPI(params)
	if err != nil {
		log.Fatal(err)
	}

	duration := apiResp.Rows[0].Elements[0].DurationInTraffic.Value
	durationMins := int(float64(duration)/60.0 + 0.5) // round

	if durationMins > conf.MaxDuration {
		sendMail(conf, fmt.Sprintf("%d minutes", durationMins))
	}
}

func readConfig(filename string) (*config, error) {
	var conf config

	confData, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	if err := json.Unmarshal(confData, &conf); err != nil {
		return nil, err
	}

	return &conf, nil
}

func queryMapsAPI(params map[string]string) (*apiResponse, error) {
	query := url.Values{}
	for key, val := range params {
		if val != "" {
			query.Set(key, val)
		}
	}

	uri, err := url.Parse(apiURL)
	if err != nil {
		return nil, err
	}
	uri.RawQuery = query.Encode()

	resp, err := http.Get(uri.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var apiResp apiResponse
	if err := json.Unmarshal(body, &apiResp); err != nil {
		return nil, err
	}

	if apiResp.Status != "OK" {
		return nil, fmt.Errorf("%s: bad response status: %s\n", name, apiResp.Status)
	}
	if len(apiResp.Rows) != 1 {
		return nil, fmt.Errorf("%s: response row count is not 1\n", name)
	}
	if len(apiResp.Rows[0].Elements) != 1 {
		return nil, fmt.Errorf("%s: response first row element count is not 1\n", name)
	}
	if apiResp.Rows[0].Elements[0].Status != "OK" {
		return nil, fmt.Errorf("%s: bad response first row first element status: %s\n", name, apiResp.Rows[0].Elements[0].Status)
	}

	return &apiResp, nil
}

func sendMail(conf *config, body string) error {
	user, err := user.Current()
	if err != nil {
		return err
	}

	hostname, err := os.Hostname()
	if err != nil {
		return err
	}

	auth := smtp.PlainAuth("", conf.SMTP.User, conf.SMTP.Pass, conf.SMTP.Host)
	sender := user.Username + "@" + hostname
	to := []string{conf.Recipient}
	msg := []byte("To: " + conf.Recipient + "\r\n" +
		"Subject: " + name + " alert\r\n" +
		"\r\n" +
		body + "\r\n")

	return smtp.SendMail(fmt.Sprintf("%s:%d", conf.SMTP.Host, conf.SMTP.Port), auth, sender, to, msg)
}
<|endoftext|>


<commit_before>package process

// Logic for this file is largely based on:
// https://github.com/jarib/childprocess/blob/783f7a00a1678b5d929062564ef5ae76822dfd62/lib/childprocess/unix/process.rb

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/buildkite/agent/logger"
)

type Process struct {
	Pid        int
	PTY        bool
	Script     string
	Env        []string
	ExitStatus string

	buffer bytes.Buffer

	command *exec.Cmd

	// This callback is called when the process offically starts
	StartCallback func()

	// For every line in the process output, this callback will be called
	// with the contents of the line
	LineCallback func(string)

	// Running is stored as an int32 so we can use atomic operations to
	// set/get it (it's accessed by multiple goroutines)
	running int32
}

func (p Process) Create() *Process {
	// Find the script to run
	absolutePath, _ := filepath.Abs(p.Script)
	scriptDirectory := filepath.Dir(absolutePath)

	// If the command is a file on the file system, just run it normally,
	// otherwise, execute it via a shell.
	if _, err := os.Stat(absolutePath); !os.IsNotExist(err) {
		p.command = exec.Command(absolutePath)
		p.command.Dir = scriptDirectory
	} else {
		p.command = exec.Command("/bin/bash", "-c", p.Script)
	}

	// Copy the current processes ENV and merge in the new ones. We do this
	// so the sub process gets PATH and stuff. We merge our path in over
	// the top of the current one so the ENV from Buildkite and the agent
	// take precedence over the agent
	currentEnv := os.Environ()
	p.command.Env = append(currentEnv, p.Env...)

	return &p
}

func (p *Process) Start() error {
	var waitGroup sync.WaitGroup

	lineReaderPipe, lineWriterPipe := io.Pipe()

	multiWriter := io.MultiWriter(&p.buffer, lineWriterPipe)

	logger.Info("Starting to run: %s", strings.Join(p.command.Args, " "))

	// Toggle between running in a pty
	if p.PTY {
		pty, err := StartPTY(p.command)
		if err != nil {
			p.ExitStatus = "1"
			return err
		}

		p.Pid = p.command.Process.Pid
		p.setRunning(true)

		waitGroup.Add(1)

		go func() {
			logger.Debug("[Process] Starting to copy PTY to the buffer")

			// Copy the pty to our buffer. This will block until it
			// EOF's or something breaks.
			_, err = io.Copy(multiWriter, pty)
			if e, ok := err.(*os.PathError); ok && e.Err == syscall.EIO {
				// We can safely ignore this error, because
				// it's just the PTY telling us that it closed
				// successfully.  See:
				// https://github.com/buildkite/agent/pull/34#issuecomment-46080419
				err = nil
			}

			if err != nil {
				logger.Error("[Process] PTY output copy failed with error: %T: %v", err, err)
			} else {
				logger.Debug("[Process] PTY has finished being copied to the buffer")
			}

			waitGroup.Done()
		}()
	} else {
		p.command.Stdout = multiWriter
		p.command.Stderr = multiWriter

		err := p.command.Start()
		if err != nil {
			p.ExitStatus = "1"
			return err
		}

		p.Pid = p.command.Process.Pid
		p.setRunning(true)
	}

	logger.Info("[Process] Process is running with PID: %d", p.Pid)

	// Add the line callback routine to the waitGroup
	waitGroup.Add(1)

	go func() {
		logger.Debug("[LineScanner] Starting to read lines")

		reader := bufio.NewReader(lineReaderPipe)

		var appending []byte

		for {
			line, isPrefix, err := reader.ReadLine()
			if err != nil {
				if err == io.EOF {
					logger.Debug("[LineScanner] Encountered EOF")
					break
				}

				logger.Error("[LineScanner] Failed to read: (%T: %v)", err, err)
			}

			// If isPrefix is true, that means we've got a really
			// long line incoming, and we'll keep appending to it
			// until isPrefix is false (which means the long line
			// has ended.
			if isPrefix && appending == nil {
				logger.Debug("[LineScanner] Line is too long to read, going to buffer it until it finishes")
				appending = line

				continue
			}

			// Should we be appending?
			if appending != nil {
				appending = append(appending, line...)

				// No more isPrefix! Line is finished!
				if !isPrefix {
					logger.Debug("[LineScanner] Finished buffering long line")
					line = appending

					// Reset appending back to nil
					appending = nil
				} else {
					continue
				}
			}

			go p.LineCallback(string(line))
		}

		logger.Debug("[LineScanner] Finished")

		waitGroup.Done()
	}()

	// Call the StartCallback
	go p.StartCallback()

	// Wait until the process has finished. The returned error is nil if the command runs,
	// has no problems copying stdin, stdout, and stderr, and exits with a zero exit status.
	waitResult := p.command.Wait()

	// Close the line writer pipe
	lineWriterPipe.Close()

	// The process is no longer running at this point
	p.setRunning(false)

	// Find the exit status of the script
	p.ExitStatus = getExitStatus(waitResult)

	logger.Info("Process with PID: %d finished with Exit Status: %s", p.Pid, p.ExitStatus)

	// Sometimes (in docker containers) io.Copy never seems to finish. This is a mega
	// hack around it. If it doesn't finish after 1 second, just continue.
	logger.Debug("[Process] Waiting for routines to finish")
	err := timeoutWait(&waitGroup)
	if err != nil {
		logger.Debug("[Process] Timed out waiting for wait group: (%T: %v)", err, err)
	}

	// No error occurred so we can return nil
	return nil
}

func (p *Process) Output() string {
	return p.buffer.String()
}

func (p *Process) Kill() error {
	// Send a sigterm
	err := p.signal(syscall.SIGTERM)
	if err != nil {
		return err
	}

	// Make a channel that we'll use as a timeout
	c := make(chan int, 1)
	checking := true

	// Start a routine that checks to see if the process
	// is still alive.
	go func() {
		for checking {
			logger.Debug("[Process] Checking to see if PID: %d is still alive", p.Pid)

			foundProcess, err := os.FindProcess(p.Pid)

			// Can't find the process at all
			if err != nil {
				logger.Debug("[Process] Could not find process with PID: %d", p.Pid)

				break
			}

			// We have some information about the process
			if foundProcess != nil {
				processState, err := foundProcess.Wait()

				if err != nil || processState.Exited() {
					logger.Debug("[Process] Process with PID: %d has exited.", p.Pid)

					break
				}
			}

			// Retry in a moment
			sleepTime := time.Duration(1 * time.Second)
			time.Sleep(sleepTime)
		}

		c <- 1
	}()

	// Timeout this process after 3 seconds
	select {
	case _ = <-c:
		// Was successfully terminated
	case <-time.After(10 * time.Second):
		// Stop checking in the routine above
		checking = false

		// Forcefully kill the thing
		err = p.signal(syscall.SIGKILL)

		if err != nil {
			return err
		}
	}

	return nil
}

func (p *Process) signal(sig os.Signal) error {
	if p.command != nil && p.command.Process != nil {
		logger.Debug("[Process] Sending signal: %s to PID: %d", sig.String(), p.Pid)

		err := p.command.Process.Signal(syscall.SIGTERM)
		if err != nil {
			logger.Error("[Process] Failed to send signal: %s to PID: %d (%T: %v)", sig.String(), p.Pid, err, err)
			return err
		}
	} else {
		logger.Debug("[Process] No process to signal yet")
	}

	return nil
}

// Returns whether or not the process is running
func (p *Process) IsRunning() bool {
	return atomic.LoadInt32(&p.running) != 0
}

// Sets the running flag of the process
func (p *Process) setRunning(r bool) {
	// Use the atomic package to avoid race conditions when setting the
	// `running` value from multiple routines
	if r {
		atomic.StoreInt32(&p.running, 1)
	} else {
		atomic.StoreInt32(&p.running, 0)
	}
}

// https://github.com/hnakamur/commango/blob/fe42b1cf82bf536ce7e24dceaef6656002e03743/os/executil/executil.go#L29
// TODO: Can this be better?
func getExitStatus(waitResult error) string {
	exitStatus := -1

	if waitResult != nil {
		if err, ok := waitResult.(*exec.ExitError); ok {
			if s, ok := err.Sys().(syscall.WaitStatus); ok {
				exitStatus = s.ExitStatus()
			} else {
				logger.Error("[Process] Unimplemented for system where exec.ExitError.Sys() is not syscall.WaitStatus.")
			}
		}
	} else {
		exitStatus = 0
	}

	return fmt.Sprintf("%d", exitStatus)
}

func timeoutWait(waitGroup *sync.WaitGroup) error {
	// Make a chanel that we'll use as a timeout
	c := make(chan int, 1)

	// Start waiting for the routines to finish
	go func() {
		waitGroup.Wait()
		c <- 1
	}()

	select {
	case _ = <-c:
		return nil
	case <-time.After(10 * time.Second):
		return errors.New("Timeout")
	}

	return nil
}
<commit_msg>Fixed running the bootstrap command on windows with the process/process gear<commit_after>package process

// Logic for this file is largely based on:
// https://github.com/jarib/childprocess/blob/783f7a00a1678b5d929062564ef5ae76822dfd62/lib/childprocess/unix/process.rb

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/buildkite/agent/logger"
)

type Process struct {
	Pid        int
	PTY        bool
	Script     string
	Env        []string
	ExitStatus string

	buffer bytes.Buffer

	command *exec.Cmd

	// This callback is called when the process offically starts
	StartCallback func()

	// For every line in the process output, this callback will be called
	// with the contents of the line
	LineCallback func(string)

	// Running is stored as an int32 so we can use atomic operations to
	// set/get it (it's accessed by multiple goroutines)
	running int32
}

func (p Process) Create() *Process {
	// Find the script to run
	absolutePath, _ := filepath.Abs(p.Script)
	scriptDirectory := filepath.Dir(absolutePath)

	// If the command is a file on the file system, just run it normally,
	// otherwise, execute it via a shell.
	if _, err := os.Stat(absolutePath); !os.IsNotExist(err) {
		p.command = exec.Command(absolutePath)
		p.command.Dir = scriptDirectory
	} else {
		if runtime.GOOS == "windows" {
			p.command = exec.Command("cmd", "/c", p.Script)
		} else {
			p.command = exec.Command("/bin/bash", "-c", p.Script)
		}
	}

	// Copy the current processes ENV and merge in the new ones. We do this
	// so the sub process gets PATH and stuff. We merge our path in over
	// the top of the current one so the ENV from Buildkite and the agent
	// take precedence over the agent
	currentEnv := os.Environ()
	p.command.Env = append(currentEnv, p.Env...)

	return &p
}

func (p *Process) Start() error {
	var waitGroup sync.WaitGroup

	lineReaderPipe, lineWriterPipe := io.Pipe()

	multiWriter := io.MultiWriter(&p.buffer, lineWriterPipe)

	logger.Info("Starting to run: %s", strings.Join(p.command.Args, " "))

	// Toggle between running in a pty
	if p.PTY {
		pty, err := StartPTY(p.command)
		if err != nil {
			p.ExitStatus = "1"
			return err
		}

		p.Pid = p.command.Process.Pid
		p.setRunning(true)

		waitGroup.Add(1)

		go func() {
			logger.Debug("[Process] Starting to copy PTY to the buffer")

			// Copy the pty to our buffer. This will block until it
			// EOF's or something breaks.
			_, err = io.Copy(multiWriter, pty)
			if e, ok := err.(*os.PathError); ok && e.Err == syscall.EIO {
				// We can safely ignore this error, because
				// it's just the PTY telling us that it closed
				// successfully.  See:
				// https://github.com/buildkite/agent/pull/34#issuecomment-46080419
				err = nil
			}

			if err != nil {
				logger.Error("[Process] PTY output copy failed with error: %T: %v", err, err)
			} else {
				logger.Debug("[Process] PTY has finished being copied to the buffer")
			}

			waitGroup.Done()
		}()
	} else {
		p.command.Stdout = multiWriter
		p.command.Stderr = multiWriter

		err := p.command.Start()
		if err != nil {
			p.ExitStatus = "1"
			return err
		}

		p.Pid = p.command.Process.Pid
		p.setRunning(true)
	}

	logger.Info("[Process] Process is running with PID: %d", p.Pid)

	// Add the line callback routine to the waitGroup
	waitGroup.Add(1)

	go func() {
		logger.Debug("[LineScanner] Starting to read lines")

		reader := bufio.NewReader(lineReaderPipe)

		var appending []byte

		for {
			line, isPrefix, err := reader.ReadLine()
			if err != nil {
				if err == io.EOF {
					logger.Debug("[LineScanner] Encountered EOF")
					break
				}

				logger.Error("[LineScanner] Failed to read: (%T: %v)", err, err)
			}

			// If isPrefix is true, that means we've got a really
			// long line incoming, and we'll keep appending to it
			// until isPrefix is false (which means the long line
			// has ended.
			if isPrefix && appending == nil {
				logger.Debug("[LineScanner] Line is too long to read, going to buffer it until it finishes")
				appending = line

				continue
			}

			// Should we be appending?
			if appending != nil {
				appending = append(appending, line...)

				// No more isPrefix! Line is finished!
				if !isPrefix {
					logger.Debug("[LineScanner] Finished buffering long line")
					line = appending

					// Reset appending back to nil
					appending = nil
				} else {
					continue
				}
			}

			go p.LineCallback(string(line))
		}

		logger.Debug("[LineScanner] Finished")

		waitGroup.Done()
	}()

	// Call the StartCallback
	go p.StartCallback()

	// Wait until the process has finished. The returned error is nil if the command runs,
	// has no problems copying stdin, stdout, and stderr, and exits with a zero exit status.
	waitResult := p.command.Wait()

	// Close the line writer pipe
	lineWriterPipe.Close()

	// The process is no longer running at this point
	p.setRunning(false)

	// Find the exit status of the script
	p.ExitStatus = getExitStatus(waitResult)

	logger.Info("Process with PID: %d finished with Exit Status: %s", p.Pid, p.ExitStatus)

	// Sometimes (in docker containers) io.Copy never seems to finish. This is a mega
	// hack around it. If it doesn't finish after 1 second, just continue.
	logger.Debug("[Process] Waiting for routines to finish")
	err := timeoutWait(&waitGroup)
	if err != nil {
		logger.Debug("[Process] Timed out waiting for wait group: (%T: %v)", err, err)
	}

	// No error occurred so we can return nil
	return nil
}

func (p *Process) Output() string {
	return p.buffer.String()
}

func (p *Process) Kill() error {
	// Send a sigterm
	err := p.signal(syscall.SIGTERM)
	if err != nil {
		return err
	}

	// Make a channel that we'll use as a timeout
	c := make(chan int, 1)
	checking := true

	// Start a routine that checks to see if the process
	// is still alive.
	go func() {
		for checking {
			logger.Debug("[Process] Checking to see if PID: %d is still alive", p.Pid)

			foundProcess, err := os.FindProcess(p.Pid)

			// Can't find the process at all
			if err != nil {
				logger.Debug("[Process] Could not find process with PID: %d", p.Pid)

				break
			}

			// We have some information about the process
			if foundProcess != nil {
				processState, err := foundProcess.Wait()

				if err != nil || processState.Exited() {
					logger.Debug("[Process] Process with PID: %d has exited.", p.Pid)

					break
				}
			}

			// Retry in a moment
			sleepTime := time.Duration(1 * time.Second)
			time.Sleep(sleepTime)
		}

		c <- 1
	}()

	// Timeout this process after 3 seconds
	select {
	case _ = <-c:
		// Was successfully terminated
	case <-time.After(10 * time.Second):
		// Stop checking in the routine above
		checking = false

		// Forcefully kill the thing
		err = p.signal(syscall.SIGKILL)

		if err != nil {
			return err
		}
	}

	return nil
}

func (p *Process) signal(sig os.Signal) error {
	if p.command != nil && p.command.Process != nil {
		logger.Debug("[Process] Sending signal: %s to PID: %d", sig.String(), p.Pid)

		err := p.command.Process.Signal(syscall.SIGTERM)
		if err != nil {
			logger.Error("[Process] Failed to send signal: %s to PID: %d (%T: %v)", sig.String(), p.Pid, err, err)
			return err
		}
	} else {
		logger.Debug("[Process] No process to signal yet")
	}

	return nil
}

// Returns whether or not the process is running
func (p *Process) IsRunning() bool {
	return atomic.LoadInt32(&p.running) != 0
}

// Sets the running flag of the process
func (p *Process) setRunning(r bool) {
	// Use the atomic package to avoid race conditions when setting the
	// `running` value from multiple routines
	if r {
		atomic.StoreInt32(&p.running, 1)
	} else {
		atomic.StoreInt32(&p.running, 0)
	}
}

// https://github.com/hnakamur/commango/blob/fe42b1cf82bf536ce7e24dceaef6656002e03743/os/executil/executil.go#L29
// TODO: Can this be better?
func getExitStatus(waitResult error) string {
	exitStatus := -1

	if waitResult != nil {
		if err, ok := waitResult.(*exec.ExitError); ok {
			if s, ok := err.Sys().(syscall.WaitStatus); ok {
				exitStatus = s.ExitStatus()
			} else {
				logger.Error("[Process] Unimplemented for system where exec.ExitError.Sys() is not syscall.WaitStatus.")
			}
		}
	} else {
		exitStatus = 0
	}

	return fmt.Sprintf("%d", exitStatus)
}

func timeoutWait(waitGroup *sync.WaitGroup) error {
	// Make a chanel that we'll use as a timeout
	c := make(chan int, 1)

	// Start waiting for the routines to finish
	go func() {
		waitGroup.Wait()
		c <- 1
	}()

	select {
	case _ = <-c:
		return nil
	case <-time.After(10 * time.Second):
		return errors.New("Timeout")
	}

	return nil
}
<|endoftext|>


<commit_before>package treap

import (
	"bytes"
	"testing"
)

func stringCompare(a, b Item) int {
	return bytes.Compare([]byte(a.(string)), []byte(b.(string)))
}

func TestTreap(t *testing.T) {
	x := NewTreap(stringCompare)
	if x == nil {
		t.Errorf("expected NewTreap to work")
	}
	tests := []struct {
		op string
		val string
		pri int
		exp string
	}{
		{"get", "not-there", -1, "NIL"},
		{"ups", "a", 1, ""},
		{"get", "a", -1, "a"},
		{"ups", "b", 2, ""},
		{"get", "a", -1, "a"},
		{"get", "b", -1, "b"},
		{"ups", "c", 3, ""},
		{"get", "a", -1, "a"},
		{"get", "b", -1, "b"},
		{"get", "c", -1, "c"},
		{"get", "not-there", -1, "NIL"},
	}

	for testIdx, test := range tests {
		switch test.op {
		case "get":
			i := x.Get(test.val)
			if i != test.exp && !(i == nil && test.exp == "NIL") {
				t.Errorf("test: %v, on Get, expected: %v, got: %v", testIdx, test.exp, i)
			}
		case "ups":
			x = x.Upsert(test.val, test.pri)
		}
	}
}
<commit_msg>First test on an update.<commit_after>package treap

import (
	"bytes"
	"testing"
)

func stringCompare(a, b Item) int {
	return bytes.Compare([]byte(a.(string)), []byte(b.(string)))
}

func TestTreap(t *testing.T) {
	x := NewTreap(stringCompare)
	if x == nil {
		t.Errorf("expected NewTreap to work")
	}
	tests := []struct {
		op string
		val string
		pri int
		exp string
	}{
		{"get", "not-there", -1, "NIL"},
		{"ups", "a", 100, ""},
		{"get", "a", -1, "a"},
		{"ups", "b", 200, ""},
		{"get", "a", -1, "a"},
		{"get", "b", -1, "b"},
		{"ups", "c", 300, ""},
		{"get", "a", -1, "a"},
		{"get", "b", -1, "b"},
		{"get", "c", -1, "c"},
		{"get", "not-there", -1, "NIL"},
		{"ups", "a", 400, ""},
		{"get", "a", -1, "a"},
		{"get", "b", -1, "b"},
		{"get", "c", -1, "c"},
		{"get", "not-there", -1, "NIL"},
	}

	for testIdx, test := range tests {
		switch test.op {
		case "get":
			i := x.Get(test.val)
			if i != test.exp && !(i == nil && test.exp == "NIL") {
				t.Errorf("test: %v, on Get, expected: %v, got: %v", testIdx, test.exp, i)
			}
		case "ups":
			x = x.Upsert(test.val, test.pri)
		}
	}
}
<|endoftext|>


<commit_before>package program

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.21.14 Zinc 2018-03-21"
<commit_msg>Bump version: v0.21.15 Zinc 2018-03-26<commit_after>package program

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.21.15 Zinc 2018-03-26"
<|endoftext|>


<commit_before>// TestLiteIDE
package main

import (
	"fmt"
	//"golang.org/x/tour/pic"
	"math"
	"strings"
	"time"
)

type Vertex struct {
	X, Y int
}

func passage(x int) {
	x += 3
}
func passage_ptr(x *int) {
	*x += 3
}

func printSlice(s string, x []int) {
	fmt.Printf("%s len=%d cap=%d %v",
		s, len(x), cap(x), x)
	if x == nil {
		fmt.Println(" nil!")
	} else {
		fmt.Println()
	}
}

func Pic(dx, dy int) [][]uint8 {

	ret := make([][]uint8, dy)

	for i := 0; i < dy; i++ {
		ret[i] = make([]uint8, dx)
	}

	for x := 0; x < dx; x++ {
		for y := 0; y < dy; y++ {
			//ret[y][x] = uint8((x+y)/2)
			ret[y][x] = uint8((x * y))
			//ret[y][x] = uint8(math.Pow(float64(x), float64(y)))
		}
	}

	return ret

}

// Pour les maps
func WordCount(s string) map[string]int {
	ret := make(map[string]int)

	decoupe := strings.Fields(s)

	for _, mot := range decoupe {
		if _, exist := ret[mot]; exist {
			ret[mot] += 1
		} else {
			ret[mot] = 1
		}
	}

	return ret
}

// Pour fonction closure
func adder() func(int) int {
	sum := 0
	return func(x int) int {
		sum += x
		return sum
	}
}

// Pour les methodes
func (v *Vertex) Abs() float64 {
	return math.Sqrt(float64(v.X*v.X + v.Y*v.Y))
}

type MyFloat float64

func (f MyFloat) Abs() float64 {
	if f < 0 {
		return float64(-f)
	}
	return float64(f)
}

func (v *Vertex) Scale(f int) {
	v.X = v.X * f
	v.Y = v.Y * f
}

// Pour les interfaces
type Abser interface {
	Abs() float64
}

type Person struct {
	Name string
	Age  int
}

// Interface stringer
func (p Person) String() string {
	return fmt.Sprintf("%v (%v years)", p.Name, p.Age)
}

// Pour les errors.
type MyError struct {
	When time.Time
	What string
}

func (e *MyError) Error() string { // Fonction repondant à l'interface Error
	return fmt.Sprintf("at %v, %s",
		e.When, e.What)
}

func run() error {
	return &MyError{
		time.Now(),
		"it didn't work",
	}
}

func main() {
	fmt.Println("Hello World!")
	fmt.Println("Hello World!")

	val := 8
	passage(val)
	fmt.Println(val)

	ptr := &val
	passage_ptr(ptr)
	fmt.Println(val)

	/// Struct ///
	v1 := Vertex{}
	v2 := Vertex{1, 2}
	v3 := Vertex{X: 33}
	v4 := &Vertex{Y: 6}

	fmt.Println(v1, v2, v3, v4, *v4)

	/// ¤¤ Conteneurs ¤¤ ///
	// -- Array --
	// constante size array
	var tab [10]int
	fmt.Println(tab, len(tab))

	// -- Slice --
	// tableau a taille variable
	tab2 := []int{1, 2, 3, 45, 6, 789}
	fmt.Println(tab2)

	for i := 0; i < len(tab2); i++ {
		fmt.Print(tab2[i], " ")
	}
	fmt.Println()

	// Il est possible de slicer les tableaux et les slices
	// missing low index implies 0
	fmt.Println("tab2[:3] ==", tab2[:3])

	// missing high index implies len(tab2)
	fmt.Println("tab2[4:] ==", tab2[4:])

	fmt.Println(tab2[4:][0])

	a := make([]int, 5) // len(a)=5
	printSlice("a", a)
	b := make([]int, 0, 5) // len(b)=0, cap(b)=5
	printSlice("b", b)
	var z []int // Slice vide
	printSlice("z", z)

	// Append to slice
	// the slice grows as needed.
	a = append(a, 1)
	printSlice("a", a)

	// we can add more than one element at a time.
	a = append(a, 2, 3, 4)
	printSlice("a", a)

	b = append(b, 1)
	printSlice("b", b)
	z = append(z, 1)
	printSlice("z", z)

	// Parcours de slice avec for
	// Fonctionne avec les slices et maps
	for i, v := range tab2 {
		fmt.Printf("%d : %d\n", i, v)
	}
	// Pour n'avoir que l'index
	pow := make([]int, 10)
	for i := range pow {
		pow[i] = 1 << uint(i)
	}
	// Pour ne pas s'occuper de l'index
	for _, value := range pow {
		fmt.Printf("%d ", value)
	}
	fmt.Println()

	//pic.Show(Pic)

	// -- Map --
	// conteneur clef -> valeur
	var m map[string]Vertex     // A ce moment la map est 'nil'.
	m = make(map[string]Vertex) // Elle doit etre alloue avec make
	// ou directement m:= make(map[string]Vertex)
	m["Bell Labs"] = Vertex{40, -74}
	fmt.Println(m)
	fmt.Println(m["Bell Labs"])

	// Peut etre instancie directement
	var m2 = map[string]Vertex{
		"Bell Labs": Vertex{
			40, -74,
		},
		"Google": Vertex{
			37, -122,
		},
	}
	fmt.Println(m2)

	// If the top-level type is just a type name, you can omit it from the
	// elements of the literal.
	var m3 = map[string]Vertex{
		"Bell Labs": {40, -74},
		"Google":    {37, -122},
	}
	fmt.Println(m3, len(m3))

	// Suppresion d'un element
	delete(m2, "Google")

	// Test si un element est present
	v, ok := m3["Google"]
	fmt.Println("The value:", v, "Present?", ok)
	v, ok = m3["Chopek"]
	fmt.Println("The value:", v, "Present?", ok)

	// Attention si un element n'est pas dans une map il y aura quand meme
	// une variable de retour representant un objet nul.

	// Exmple avec fonction WordCount
	phrase := "I am learning Go!"
	fmt.Println(phrase)
	fmt.Println(WordCount(phrase))

	phrase = "I ate a donut. Then I ate another donut."
	fmt.Println(phrase)
	fmt.Println(WordCount(phrase))

	// ¤¤ Fonctions ¤¤
	// Les fonctione peuvent etre des valeurs
	hypot := func(x, y float64) float64 {
		return math.Sqrt(x*x + y*y)
	}
	fmt.Println(hypot(3, 4))

	// Elles peuvent etre des closures
	// A closure is a function value that references variables from outside its
	// body. The function may access and assign to the referenced variables;
	// in this sense the function is "bound" to the variables.
	// cf adder, the adder function returns a closure.
	// Each closure is bound to its own sum variable.
	pos, neg := adder(), adder()
	for i := 0; i < 10; i++ {
		fmt.Println(
			pos(i),
			neg(-2*i),
		)
	}

	// ¤¤ Methodes ¤¤
	// Il est possible d'ajouter des méthodes aux classes
	// Go does not have classes. However, you can define methods on struct types.
	// The method receiver appears in its own argument list between
	// the func keyword and the method name.
	// cf fonction (method) Abs
	vMeth := &Vertex{3, 4}
	fmt.Println(vMeth.Abs())
	// You can declare a method on any type that is declared in your package,
	// not just struct types.
	// However, you cannot define a method on a type from another package
	// (including built in types).
	// cf autre fonction Abs
	f := MyFloat(-math.Sqrt2)
	fmt.Println(f.Abs())

	// Si la methode s'attache a un pointeur de strucure il est possible de
	// modifier ses parametres
	// cf func Scale
	fmt.Println(vMeth)
	vMeth.Scale(3)
	fmt.Println(vMeth)

	// ¤¤ Interface ¤¤
	// Permet une sorte de polymorphisme
	// An interface type is defined by a set of methods.
	// A value of interface type can hold any value that implements those methods.
	// cf Abser
	var abser Abser
	abser = f  // a MyFloat implements Abser
	abser = &v // a *Vertex implements Abser

	// In the following line, v is a Vertex (not *Vertex)
	// and does NOT implement Abser.
	//abser = v
	fmt.Println(abser)

	// Pour la description des elements il est possible d'utilsier l'interface
	// Stringer utilise par les print de fmt.
	// Pour cela il faut redefinir String pour un objet ex avec Personne
	p1 := Person{"Arthur Dent", 42}
	p2 := Person{"Zaphod Beeblebrox", 9001}
	fmt.Println(p1, p2)

	// ¤¤ Errors ¤¤
	// Les errors sont des interfaces :
	//type error interface {
	//Error() string
	//}

	err := run()
	if err != nil {
		fmt.Println(err)
	}

}
<commit_msg>Fin errors et readers<commit_after>// TestLiteIDE
package main

import (
	"fmt"
	//"golang.org/x/tour/pic"
	"io"
	"math"
	"strings"
	"time"
)

type Vertex struct {
	X, Y int
}

func passage(x int) {
	x += 3
}
func passage_ptr(x *int) {
	*x += 3
}

func printSlice(s string, x []int) {
	fmt.Printf("%s len=%d cap=%d %v",
		s, len(x), cap(x), x)
	if x == nil {
		fmt.Println(" nil!")
	} else {
		fmt.Println()
	}
}

func Pic(dx, dy int) [][]uint8 {

	ret := make([][]uint8, dy)

	for i := 0; i < dy; i++ {
		ret[i] = make([]uint8, dx)
	}

	for x := 0; x < dx; x++ {
		for y := 0; y < dy; y++ {
			//ret[y][x] = uint8((x+y)/2)
			ret[y][x] = uint8((x * y))
			//ret[y][x] = uint8(math.Pow(float64(x), float64(y)))
		}
	}

	return ret

}

// Pour les maps
func WordCount(s string) map[string]int {
	ret := make(map[string]int)

	decoupe := strings.Fields(s)

	for _, mot := range decoupe {
		if _, exist := ret[mot]; exist {
			ret[mot] += 1
		} else {
			ret[mot] = 1
		}
	}

	return ret
}

// Pour fonction closure
func adder() func(int) int {
	sum := 0
	return func(x int) int {
		sum += x
		return sum
	}
}

// Pour les methodes
func (v *Vertex) Abs() float64 {
	return math.Sqrt(float64(v.X*v.X + v.Y*v.Y))
}

type MyFloat float64

func (f MyFloat) Abs() float64 {
	if f < 0 {
		return float64(-f)
	}
	return float64(f)
}

func (v *Vertex) Scale(f int) {
	v.X = v.X * f
	v.Y = v.Y * f
}

// Pour les interfaces
type Abser interface {
	Abs() float64
}

type Person struct {
	Name string
	Age  int
}

// Interface stringer
func (p Person) String() string {
	return fmt.Sprintf("%v (%v years)", p.Name, p.Age)
}

// Pour les errors.
type MyError struct {
	When time.Time
	What string
}

func (e *MyError) Error() string { // Fonction repondant à l'interface Error
	return fmt.Sprintf("at %v, %s",
		e.When, e.What)
}

func run() error {
	return &MyError{
		time.Now(),
		"it didn't work",
	}
}

func main() {
	fmt.Println("Hello World!")
	fmt.Println("Hello World!")

	val := 8
	passage(val)
	fmt.Println(val)

	ptr := &val
	passage_ptr(ptr)
	fmt.Println(val)

	/// Struct ///
	v1 := Vertex{}
	v2 := Vertex{1, 2}
	v3 := Vertex{X: 33}
	v4 := &Vertex{Y: 6}

	fmt.Println(v1, v2, v3, v4, *v4)

	/// ¤¤ Conteneurs ¤¤ ///
	// -- Array --
	// constante size array
	var tab [10]int
	fmt.Println(tab, len(tab))

	// -- Slice --
	// tableau a taille variable
	tab2 := []int{1, 2, 3, 45, 6, 789}
	fmt.Println(tab2)

	for i := 0; i < len(tab2); i++ {
		fmt.Print(tab2[i], " ")
	}
	fmt.Println()

	// Il est possible de slicer les tableaux et les slices
	// missing low index implies 0
	fmt.Println("tab2[:3] ==", tab2[:3])

	// missing high index implies len(tab2)
	fmt.Println("tab2[4:] ==", tab2[4:])

	fmt.Println(tab2[4:][0])

	a := make([]int, 5) // len(a)=5
	printSlice("a", a)
	b := make([]int, 0, 5) // len(b)=0, cap(b)=5
	printSlice("b", b)
	var z []int // Slice vide
	printSlice("z", z)

	// Append to slice
	// the slice grows as needed.
	a = append(a, 1)
	printSlice("a", a)

	// we can add more than one element at a time.
	a = append(a, 2, 3, 4)
	printSlice("a", a)

	b = append(b, 1)
	printSlice("b", b)
	z = append(z, 1)
	printSlice("z", z)

	// Parcours de slice avec for
	// Fonctionne avec les slices et maps
	for i, v := range tab2 {
		fmt.Printf("%d : %d\n", i, v)
	}
	// Pour n'avoir que l'index
	pow := make([]int, 10)
	for i := range pow {
		pow[i] = 1 << uint(i)
	}
	// Pour ne pas s'occuper de l'index
	for _, value := range pow {
		fmt.Printf("%d ", value)
	}
	fmt.Println()

	//pic.Show(Pic)

	// -- Map --
	// conteneur clef -> valeur
	var m map[string]Vertex     // A ce moment la map est 'nil'.
	m = make(map[string]Vertex) // Elle doit etre alloue avec make
	// ou directement m:= make(map[string]Vertex)
	m["Bell Labs"] = Vertex{40, -74}
	fmt.Println(m)
	fmt.Println(m["Bell Labs"])

	// Peut etre instancie directement
	var m2 = map[string]Vertex{
		"Bell Labs": Vertex{
			40, -74,
		},
		"Google": Vertex{
			37, -122,
		},
	}
	fmt.Println(m2)

	// If the top-level type is just a type name, you can omit it from the
	// elements of the literal.
	var m3 = map[string]Vertex{
		"Bell Labs": {40, -74},
		"Google":    {37, -122},
	}
	fmt.Println(m3, len(m3))

	// Suppresion d'un element
	delete(m2, "Google")

	// Test si un element est present
	v, ok := m3["Google"]
	fmt.Println("The value:", v, "Present?", ok)
	v, ok = m3["Chopek"]
	fmt.Println("The value:", v, "Present?", ok)

	// Attention si un element n'est pas dans une map il y aura quand meme
	// une variable de retour representant un objet nul.

	// Exmple avec fonction WordCount
	phrase := "I am learning Go!"
	fmt.Println(phrase)
	fmt.Println(WordCount(phrase))

	phrase = "I ate a donut. Then I ate another donut."
	fmt.Println(phrase)
	fmt.Println(WordCount(phrase))

	// ¤¤ Fonctions ¤¤
	// Les fonctione peuvent etre des valeurs
	hypot := func(x, y float64) float64 {
		return math.Sqrt(x*x + y*y)
	}
	fmt.Println(hypot(3, 4))

	// Elles peuvent etre des closures
	// A closure is a function value that references variables from outside its
	// body. The function may access and assign to the referenced variables;
	// in this sense the function is "bound" to the variables.
	// cf adder, the adder function returns a closure.
	// Each closure is bound to its own sum variable.
	pos, neg := adder(), adder()
	for i := 0; i < 10; i++ {
		fmt.Println(
			pos(i),
			neg(-2*i),
		)
	}

	// ¤¤ Methodes ¤¤
	// Il est possible d'ajouter des méthodes aux classes
	// Go does not have classes. However, you can define methods on struct types.
	// The method receiver appears in its own argument list between
	// the func keyword and the method name.
	// cf fonction (method) Abs
	vMeth := &Vertex{3, 4}
	fmt.Println(vMeth.Abs())
	// You can declare a method on any type that is declared in your package,
	// not just struct types.
	// However, you cannot define a method on a type from another package
	// (including built in types).
	// cf autre fonction Abs
	f := MyFloat(-math.Sqrt2)
	fmt.Println(f.Abs())

	// Si la methode s'attache a un pointeur de strucure il est possible de
	// modifier ses parametres
	// cf func Scale
	fmt.Println(vMeth)
	vMeth.Scale(3)
	fmt.Println(vMeth)

	// ¤¤ Interface ¤¤
	// Permet une sorte de polymorphisme
	// An interface type is defined by a set of methods.
	// A value of interface type can hold any value that implements those methods.
	// cf Abser
	var abser Abser
	abser = f  // a MyFloat implements Abser
	abser = &v // a *Vertex implements Abser

	// In the following line, v is a Vertex (not *Vertex)
	// and does NOT implement Abser.
	//abser = v
	fmt.Println(abser)

	// Pour la description des elements il est possible d'utilsier l'interface
	// Stringer utilise par les print de fmt.
	// Pour cela il faut redefinir String pour un objet ex avec Personne
	p1 := Person{"Arthur Dent", 42}
	p2 := Person{"Zaphod Beeblebrox", 9001}
	fmt.Println(p1, p2)

	// ¤¤ Errors ¤¤
	// Les errors sont des interfaces :
	//type error interface {
	//Error() string
	//}

	err := run()
	if err != nil {
		fmt.Println(err)
	}

	// ¤¤ Reader ¤¤
	// Interface pour la lecture de stream, texte, fichiers ...
	//func (T) Read(b []byte) (n int, err error)
	// Rmplit le tableau en entree et retourne le nombre de byte ecrit
	// et EOF si la fin du stream.
	r := strings.NewReader("Hello, Reader!")

	taby := make([]byte, 10)

	for {
		n, err := r.Read(taby)
		fmt.Printf("n = %v err = %v taby = %v\n", n, err, taby)
		fmt.Printf("taby[:n] = %q\n", taby[:n])
		if err == io.EOF {
			break
		}
	}

}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package render

import (
	htmlTemplate "html/template"
	"io"
	"io/ioutil"
	"path/filepath"
	"sort"
	textTemplate "text/template"

	"github.com/googlecodelabs/tools/claat/types"
)

// Context is a template context during execution.
type Context struct {
	Env      string
	Prefix   string
	GlobalGA string
	Meta     *types.Meta
	Steps    []*types.Step
	Extra    map[string]string // Extra variables passed from the command line.
}

// Execute renders a template of the fmt format into w.
//
// The fmt argument can also be a path to a local file.
//
// Template execution context data is expected to be of type *Context
// but can be an arbitrary struct, as long as it contains at least Context's fields
// for the built-in templates to be successfully executed.
func Execute(w io.Writer, fmt string, data interface{}) error {
	t, err := parseTemplate(fmt)
	if err != nil {
		return err
	}
	if ctx, ok := data.(*Context); ok {
		sort.Strings(ctx.Meta.Tags)
	}
	return t.Execute(w, data)
}

// executer satisfies both html/template and text/template.
type executer interface {
	Execute(io.Writer, interface{}) error
}

// funcMap are exposted to the templates.
var funcMap = map[string]interface{}{
	"renderHTML": HTML,
	"renderMD":   MD,
	"matchEnv": func(tags []string, t string) bool {
		if len(tags) == 0 || t == "" {
			return true
		}
		i := sort.SearchStrings(tags, t)
		return i < len(tags) && tags[i] == t
	},
}

//go:generate go run gen-tmpldata.go

type template struct {
	bytes []byte
	html  bool
}

// parseTemplate parses template name defined either in tmpldata
// or a local file.
//
// A local file template is parsed as HTML if file extension is ".html",
// text otherwise.
func parseTemplate(name string) (executer, error) {
	tmpl := tmpldata[name] // defined in pre-generated tmpldata.go
	if tmpl == nil {
		// TODO: add templates in-mem caching
		var err error
		if tmpl, err = readTemplate(name); err != nil {
			return nil, err
		}
	}
	if tmpl.html {
		return htmlTemplate.New(name).
			Funcs(funcMap).
			Parse(string(tmpl.bytes))
	}
	return textTemplate.New(name).
		Funcs(funcMap).
		Parse(string(tmpl.bytes))
}

func readTemplate(name string) (*template, error) {
	b, err := ioutil.ReadFile(name)
	if err != nil {
		return nil, err
	}
	return &template{
		bytes: b,
		html:  filepath.Ext(name) == ".html",
	}, nil
}
<commit_msg>render: optional arguments for Execute<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package render

import (
	htmlTemplate "html/template"
	"io"
	"io/ioutil"
	"path/filepath"
	"sort"
	textTemplate "text/template"

	"github.com/googlecodelabs/tools/claat/types"
)

// Context is a template context during execution.
type Context struct {
	Env      string
	Prefix   string
	GlobalGA string
	Meta     *types.Meta
	Steps    []*types.Step
	Extra    map[string]string // Extra variables passed from the command line.
}

// Execute renders a template of the fmt format into w.
//
// The fmt argument can also be a path to a local file.
//
// Template execution context data is expected to be of type *Context
// but can be an arbitrary struct, as long as it contains at least Context's fields
// for the built-in templates to be successfully executed.
func Execute(w io.Writer, fmt string, data interface{}, opt ...Option) error {
	var funcs map[string]interface{}
	for _, o := range opt {
		switch o := o.(type) {
		case optFuncMap:
			funcs = o
		}
	}
	t, err := parseTemplate(fmt, funcs)
	if err != nil {
		return err
	}
	if ctx, ok := data.(*Context); ok {
		sort.Strings(ctx.Meta.Tags)
	}
	return t.Execute(w, data)
}

// executer satisfies both html/template and text/template.
type executer interface {
	Execute(io.Writer, interface{}) error
}

// funcMap are exposted to the templates.
var funcMap = map[string]interface{}{
	"renderHTML": HTML,
	"renderMD":   MD,
	"matchEnv": func(tags []string, t string) bool {
		if len(tags) == 0 || t == "" {
			return true
		}
		i := sort.SearchStrings(tags, t)
		return i < len(tags) && tags[i] == t
	},
}

//go:generate go run gen-tmpldata.go

type template struct {
	bytes []byte
	html  bool
}

// parseTemplate parses template name defined either in tmpldata
// or a local file.
//
// A local file template is parsed as HTML if file extension is ".html",
// text otherwise.
func parseTemplate(name string, fmap map[string]interface{}) (executer, error) {
	tmpl := tmpldata[name] // defined in pre-generated tmpldata.go
	if tmpl == nil {
		// TODO: add templates in-mem caching
		var err error
		if tmpl, err = readTemplate(name); err != nil {
			return nil, err
		}
	}

	funcs := make(map[string]interface{}, len(funcMap))
	for k, v := range funcMap {
		funcs[k] = v
	}
	for k, v := range fmap {
		funcs[k] = v
	}

	if tmpl.html {
		return htmlTemplate.New(name).
			Funcs(funcs).
			Parse(string(tmpl.bytes))
	}
	return textTemplate.New(name).
		Funcs(funcs).
		Parse(string(tmpl.bytes))
}

func readTemplate(name string) (*template, error) {
	b, err := ioutil.ReadFile(name)
	if err != nil {
		return nil, err
	}
	return &template{
		bytes: b,
		html:  filepath.Ext(name) == ".html",
	}, nil
}

// Option is the type of optional arguments for Execute.
type Option interface {
	option()
}

// WithFuncMap creates a user-supplied template functions option.
func WithFuncMap(fm map[string]interface{}) Option {
	return optFuncMap(fm)
}

type optFuncMap map[string]interface{}

func (o optFuncMap) option() {}
<|endoftext|>


<commit_before>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package main

import (
	"errors"
	"fmt"
	"strconv"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

// SimpleChaincode example simple Chaincode implementation
type SimpleChaincode struct {
}

func (t *SimpleChaincode) Init(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	//var Name string    // Entities
	var err error

	if len(args) == 0 {
		return nil, errors.New("Incorrect number of arguments. At least one Meter's name is required.")
	}

	for _,name := range args {
		if len(name) == 0{
			continue
		}
		err = stub.PutState(name, 0);
		if err != nil {
			return nil, errors.New("Meter cannot be created")
		}
	}

	return nil, nil
}

// Transaction makes payment of X units from A to B
func (t *SimpleChaincode) Invoke(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	if function == "delete" {
		// Deletes an entity from its state
		return t.delete(stub, args)
	}

	var A, B string    // Entities
	var Aval, Bval int // Asset holdings
	var X int          // Transaction value
	var err error

	if len(args) != 3 {
		return nil, errors.New("Incorrect number of arguments. Expecting 3")
	}

	A = args[0]
	B = args[1]

	// Get the state from the ledger
	// TODO: will be nice to have a GetAllState call to ledger
	Avalbytes, err := stub.GetState(A)
	if err != nil {
		return nil, errors.New("Failed to get state")
	}
	if Avalbytes == nil {
		return nil, errors.New("Entity not found")
	}
	Aval, _ = strconv.Atoi(string(Avalbytes))

	Bvalbytes, err := stub.GetState(B)
	if err != nil {
		return nil, errors.New("Failed to get state")
	}
	if Bvalbytes == nil {
		return nil, errors.New("Entity not found")
	}
	Bval, _ = strconv.Atoi(string(Bvalbytes))

	// Perform the execution
	X, err = strconv.Atoi(args[2])
	Aval = Aval - X
	Bval = Bval + X
	fmt.Printf("Aval = %d, Bval = %d\n", Aval, Bval)

	// Write the state back to the ledger
	err = stub.PutState(A, []byte(strconv.Itoa(Aval)))
	if err != nil {
		return nil, err
	}

	err = stub.PutState(B, []byte(strconv.Itoa(Bval)))
	if err != nil {
		return nil, err
	}

	return nil, nil
}

// Deletes an entity from state
func (t *SimpleChaincode) delete(stub *shim.ChaincodeStub, args []string) ([]byte, error) {
	if len(args) != 1 {
		return nil, errors.New("Incorrect number of arguments. Expecting 1")
	}

	A := args[0]

	// Delete the key from the state in ledger
	err := stub.DelState(A)
	if err != nil {
		return nil, errors.New("Failed to delete state")
	}

	return nil, nil
}

// Query callback representing the query of a chaincode
func (t *SimpleChaincode) Query(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	if function != "query" {
		return nil, errors.New("Invalid query function name. Expecting \"query\"")
	}
	var A string // Entities
	var err error

	if len(args) != 1 {
		return nil, errors.New("Incorrect number of arguments. Expecting name of the person to query")
	}

	A = args[0]

	// Get the state from the ledger
	Avalbytes, err := stub.GetState(A)
	if err != nil {
		jsonResp := "{\"Error\":\"Failed to get state for " + A + "\"}"
		return nil, errors.New(jsonResp)
	}

	if Avalbytes == nil {
		jsonResp := "{\"Error\":\"Nil amount for " + A + "\"}"
		return nil, errors.New(jsonResp)
	}

	jsonResp := "{\"Name\":\"" + A + "\",\"Amount\":\"" + string(Avalbytes) + "\"}"
	fmt.Printf("Query Response:%s\n", jsonResp)
	return Avalbytes, nil
}

func main() {
	err := shim.Start(new(SimpleChaincode))
	if err != nil {
		fmt.Printf("Error starting Simple chaincode: %s", err)
	}
}
<commit_msg>revert back<commit_after>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package main

import (
	"errors"
	"fmt"
	"strconv"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

// SimpleChaincode example simple Chaincode implementation
type SimpleChaincode struct {
}

func (t *SimpleChaincode) Init(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	var A, B string    // Entities
	var Aval, Bval int // Asset holdings
	var err error

	if len(args) != 4 {
		return nil, errors.New("Incorrect number of arguments. Expecting 4")
	}

	// Initialize the chaincode
	A = args[0]
	Aval, err = strconv.Atoi(args[1])
	if err != nil {
		return nil, errors.New("Expecting integer value for asset holding")
	}
	B = args[2]
	Bval, err = strconv.Atoi(args[3])
	if err != nil {
		return nil, errors.New("Expecting integer value for asset holding")
	}
	fmt.Printf("Aval = %d, Bval = %d\n", Aval, Bval)

	// Write the state to the ledger
	err = stub.PutState(A, []byte(strconv.Itoa(Aval)))
	if err != nil {
		return nil, err
	}

	err = stub.PutState(B, []byte(strconv.Itoa(Bval)))
	if err != nil {
		return nil, err
	}

	return nil, nil
}

// Transaction makes payment of X units from A to B
func (t *SimpleChaincode) Invoke(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	if function == "delete" {
		// Deletes an entity from its state
		return t.delete(stub, args)
	}

	var A, B string    // Entities
	var Aval, Bval int // Asset holdings
	var X int          // Transaction value
	var err error

	if len(args) != 3 {
		return nil, errors.New("Incorrect number of arguments. Expecting 3")
	}

	A = args[0]
	B = args[1]

	// Get the state from the ledger
	// TODO: will be nice to have a GetAllState call to ledger
	Avalbytes, err := stub.GetState(A)
	if err != nil {
		return nil, errors.New("Failed to get state")
	}
	if Avalbytes == nil {
		return nil, errors.New("Entity not found")
	}
	Aval, _ = strconv.Atoi(string(Avalbytes))

	Bvalbytes, err := stub.GetState(B)
	if err != nil {
		return nil, errors.New("Failed to get state")
	}
	if Bvalbytes == nil {
		return nil, errors.New("Entity not found")
	}
	Bval, _ = strconv.Atoi(string(Bvalbytes))

	// Perform the execution
	X, err = strconv.Atoi(args[2])
	Aval = Aval - X
	Bval = Bval + X
	fmt.Printf("Aval = %d, Bval = %d\n", Aval, Bval)

	// Write the state back to the ledger
	err = stub.PutState(A, []byte(strconv.Itoa(Aval)))
	if err != nil {
		return nil, err
	}

	err = stub.PutState(B, []byte(strconv.Itoa(Bval)))
	if err != nil {
		return nil, err
	}

	return nil, nil
}

// Deletes an entity from state
func (t *SimpleChaincode) delete(stub *shim.ChaincodeStub, args []string) ([]byte, error) {
	if len(args) != 1 {
		return nil, errors.New("Incorrect number of arguments. Expecting 1")
	}

	A := args[0]

	// Delete the key from the state in ledger
	err := stub.DelState(A)
	if err != nil {
		return nil, errors.New("Failed to delete state")
	}

	return nil, nil
}

// Query callback representing the query of a chaincode
func (t *SimpleChaincode) Query(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	if function != "query" {
		return nil, errors.New("Invalid query function name. Expecting \"query\"")
	}
	var A string // Entities
	var err error

	if len(args) != 1 {
		return nil, errors.New("Incorrect number of arguments. Expecting name of the person to query")
	}

	A = args[0]

	// Get the state from the ledger
	Avalbytes, err := stub.GetState(A)
	if err != nil {
		jsonResp := "{\"Error\":\"Failed to get state for " + A + "\"}"
		return nil, errors.New(jsonResp)
	}

	if Avalbytes == nil {
		jsonResp := "{\"Error\":\"Nil amount for " + A + "\"}"
		return nil, errors.New(jsonResp)
	}

	jsonResp := "{\"Name\":\"" + A + "\",\"Amount\":\"" + string(Avalbytes) + "\"}"
	fmt.Printf("Query Response:%s\n", jsonResp)
	return Avalbytes, nil
}

func main() {
	err := shim.Start(new(SimpleChaincode))
	if err != nil {
		fmt.Printf("Error starting Simple chaincode: %s", err)
	}
}
<|endoftext|>


<commit_before>package trust

import (
	"github.com/docker/cli/cli"
	"github.com/docker/cli/cli/command"
	"github.com/spf13/cobra"
)

// NewTrustCommand returns a cobra command for `trust` subcommands
func NewTrustCommand(dockerCli command.Cli) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "trust",
		Short: "Manage trust on Docker images",
		Args:  cli.NoArgs,
		RunE:  command.ShowHelp(dockerCli.Err()),
	}
	cmd.AddCommand(
		newKeyLoadCommand(dockerCli),
		newInspectCommand(dockerCli),
		newRevokeCommand(dockerCli),
		newSignCommand(dockerCli),
		newSignerAddCommand(dockerCli),
		newSignerRemoveCommand(dockerCli),
		newInspectCommand(dockerCli),
		newRevokeCommand(dockerCli),
		newSignCommand(dockerCli),
	)
	return cmd
}
<commit_msg>cmd: fixup double commands from rebasing<commit_after>package trust

import (
	"github.com/docker/cli/cli"
	"github.com/docker/cli/cli/command"
	"github.com/spf13/cobra"
)

// NewTrustCommand returns a cobra command for `trust` subcommands
func NewTrustCommand(dockerCli command.Cli) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "trust",
		Short: "Manage trust on Docker images",
		Args:  cli.NoArgs,
		RunE:  command.ShowHelp(dockerCli.Err()),
	}
	cmd.AddCommand(
		newKeyLoadCommand(dockerCli),
		newInspectCommand(dockerCli),
		newRevokeCommand(dockerCli),
		newSignCommand(dockerCli),
		newSignerAddCommand(dockerCli),
		newSignerRemoveCommand(dockerCli),
	)
	return cmd
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/aristanetworks/goeapi"
	"github.com/aristanetworks/goeapi/module"
)

func main() {
	node, err := goeapi.ConnectTo("dut")
	if err != nil {
		panic(err)
	}
	conf := node.RunningConfig()
	fmt.Println(conf)

	var showversion module.ShowVersion
	handle, _ := node.GetHandle("json")
	handle.Enable(&showversion)
	fmt.Println("\nVersion:", showversion.Version)

	s := module.Show(node)
	showData := s.ShowVersion()
	fmt.Printf("\nModelname         : %s\n", showData.ModelName)
	fmt.Printf("Internal Version  : %s\n", showData.InternalVersion)
	fmt.Printf("System MAC        : %s\n", showData.SystemMacAddress)
	fmt.Printf("Serial Number     : %s\n", showData.SerialNumber)
	fmt.Printf("Mem Total         : %d\n", showData.MemTotal)
	fmt.Printf("Bootup Timestamp  : %.2f\n", showData.BootupTimestamp)
	fmt.Printf("Mem Free          : %d\n", showData.MemFree)
	fmt.Printf("Version           : %s\n", showData.Version)
	fmt.Printf("Architecture      : %s\n", showData.Architecture)
	fmt.Printf("Internal Build ID : %s\n", showData.InternalBuildID)
	fmt.Printf("Hardware Revision : %s\n", showData.HardwareRevision)

	sys := module.System(node)
	if ok := sys.SetHostname("Ladie"); !ok {
		fmt.Printf("SetHostname Failed\n")
	}
	sysInfo := sys.Get()
	fmt.Printf("\nSysinfo: %#v\n", sysInfo.HostName())
}
<commit_msg>rename example.go to example1.go<commit_after><|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package estimator

import (
	"sort"

	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/autoscaler/cluster-autoscaler/simulator"
	schedulerUtils "k8s.io/autoscaler/cluster-autoscaler/utils/scheduler"
	schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
)

// podInfo contains Pod and score that corresponds to how important it is to handle the pod first.
type podInfo struct {
	score float64
	pod   *apiv1.Pod
}

// BinpackingNodeEstimator estimates the number of needed nodes to handle the given amount of pods.
type BinpackingNodeEstimator struct {
	predicateChecker simulator.PredicateChecker
	clusterSnapshot  simulator.ClusterSnapshot
}

// NewBinpackingNodeEstimator builds a new BinpackingNodeEstimator.
func NewBinpackingNodeEstimator(
	predicateChecker simulator.PredicateChecker,
	clusterSnapshot simulator.ClusterSnapshot) *BinpackingNodeEstimator {
	return &BinpackingNodeEstimator{
		predicateChecker: predicateChecker,
		clusterSnapshot:  clusterSnapshot,
	}
}

// Estimate implements First Fit Decreasing bin-packing approximation algorithm.
// See https://en.wikipedia.org/wiki/Bin_packing_problem for more details.
// While it is a multi-dimensional bin packing (cpu, mem, ports) in most cases the main dimension
// will be cpu thus the estimated overprovisioning of 11/9 * optimal + 6/9 should be
// still be maintained.
// It is assumed that all pods from the given list can fit to nodeTemplate.
// Returns the number of nodes needed to accommodate all pods from the list.
func (estimator *BinpackingNodeEstimator) Estimate(
	pods []*apiv1.Pod,
	nodeTemplate *schedulernodeinfo.NodeInfo,
	upcomingNodes []*schedulernodeinfo.NodeInfo) int {
	podInfos := calculatePodScore(pods, nodeTemplate)
	sort.Slice(podInfos, func(i, j int) bool { return podInfos[i].score > podInfos[j].score })

	newNodes := make([]*schedulernodeinfo.NodeInfo, 0)

	for _, podInfo := range podInfos {
		found := false
		for i, nodeInfo := range newNodes {
			if err := estimator.predicateChecker.CheckPredicates(nil, podInfo.pod, nodeInfo); err == nil {
				found = true
				newNodes[i] = schedulerUtils.NodeWithPod(nodeInfo, podInfo.pod)
				break
			}
		}
		if !found {
			newNodes = append(newNodes, schedulerUtils.NodeWithPod(nodeTemplate, podInfo.pod))
		}
	}
	return len(newNodes)
}

// Calculates score for all pods and returns podInfo structure.
// Score is defined as cpu_sum/node_capacity + mem_sum/node_capacity.
// Pods that have bigger requirements should be processed first, thus have higher scores.
func calculatePodScore(pods []*apiv1.Pod, nodeTemplate *schedulernodeinfo.NodeInfo) []*podInfo {
	podInfos := make([]*podInfo, 0, len(pods))

	for _, pod := range pods {
		cpuSum := resource.Quantity{}
		memorySum := resource.Quantity{}

		for _, container := range pod.Spec.Containers {
			if request, ok := container.Resources.Requests[apiv1.ResourceCPU]; ok {
				cpuSum.Add(request)
			}
			if request, ok := container.Resources.Requests[apiv1.ResourceMemory]; ok {
				memorySum.Add(request)
			}
		}
		score := float64(0)
		if cpuAllocatable, ok := nodeTemplate.Node().Status.Allocatable[apiv1.ResourceCPU]; ok && cpuAllocatable.MilliValue() > 0 {
			score += float64(cpuSum.MilliValue()) / float64(cpuAllocatable.MilliValue())
		}
		if memAllocatable, ok := nodeTemplate.Node().Status.Allocatable[apiv1.ResourceMemory]; ok && memAllocatable.Value() > 0 {
			score += float64(memorySum.Value()) / float64(memAllocatable.Value())
		}

		podInfos = append(podInfos, &podInfo{
			score: score,
			pod:   pod,
		})
	}
	return podInfos
}
<commit_msg>Use ClusterSnapshot in BinpackingNodeEstimator<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package estimator

import (
	"fmt"
	"sort"
	"time"

	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/autoscaler/cluster-autoscaler/simulator"
	"k8s.io/klog"
	schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
)

// podInfo contains Pod and score that corresponds to how important it is to handle the pod first.
type podInfo struct {
	score float64
	pod   *apiv1.Pod
}

// BinpackingNodeEstimator estimates the number of needed nodes to handle the given amount of pods.
type BinpackingNodeEstimator struct {
	predicateChecker simulator.PredicateChecker
	clusterSnapshot  simulator.ClusterSnapshot
}

// NewBinpackingNodeEstimator builds a new BinpackingNodeEstimator.
func NewBinpackingNodeEstimator(
	predicateChecker simulator.PredicateChecker,
	clusterSnapshot simulator.ClusterSnapshot) *BinpackingNodeEstimator {
	return &BinpackingNodeEstimator{
		predicateChecker: predicateChecker,
		clusterSnapshot:  clusterSnapshot,
	}
}

// Estimate implements First Fit Decreasing bin-packing approximation algorithm.
// See https://en.wikipedia.org/wiki/Bin_packing_problem for more details.
// While it is a multi-dimensional bin packing (cpu, mem, ports) in most cases the main dimension
// will be cpu thus the estimated overprovisioning of 11/9 * optimal + 6/9 should be
// still be maintained.
// It is assumed that all pods from the given list can fit to nodeTemplate.
// Returns the number of nodes needed to accommodate all pods from the list.
func (estimator *BinpackingNodeEstimator) Estimate(
	pods []*apiv1.Pod,
	nodeTemplate *schedulernodeinfo.NodeInfo,
	upcomingNodes []*schedulernodeinfo.NodeInfo) int {
	podInfos := calculatePodScore(pods, nodeTemplate)
	sort.Slice(podInfos, func(i, j int) bool { return podInfos[i].score > podInfos[j].score })

	newNodeNames := make([]string, 0)

	if err := estimator.clusterSnapshot.Fork(); err != nil {
		klog.Errorf("Error while calling ClusterSnapshot.Fork; %v", err)
		return 0
	}
	defer func() {
		if err := estimator.clusterSnapshot.Revert(); err != nil {
			klog.Errorf("Error while calling ClusterSnapshot.Revert; %v", err)
		}
	}()

	newNodeNameTimestamp := time.Now()
	newNodeNameIndex := 0

	for _, podInfo := range podInfos {
		found := false
		for _, nodeName := range newNodeNames {
			if err := estimator.predicateChecker.CheckPredicates(estimator.clusterSnapshot, podInfo.pod, simulator.FakeNodeInfoForNodeName(nodeName)); err == nil {
				found = true
				if err := estimator.clusterSnapshot.AddPod(podInfo.pod, nodeName); err != nil {
					klog.Errorf("Error adding pod %v.%v to node %v in ClusterSnapshot; %v", podInfo.pod.Namespace, podInfo.pod.Name, nodeName, err)
					return 0
				}
				break
			}
		}
		if !found {
			// Add new node
			newNodeName, err := estimator.addNewNodeToSnapshot(nodeTemplate, newNodeNameTimestamp, newNodeNameIndex)
			if err != nil {
				klog.Errorf("Error while adding new node for template to ClusterSnapshot; %v", err)
				return 0
			}
			newNodeNameIndex++
			// And schedule pod to it
			if err := estimator.clusterSnapshot.AddPod(podInfo.pod, newNodeName); err != nil {
				klog.Errorf("Error adding pod %v.%v to node %v in ClusterSnapshot; %v", podInfo.pod.Namespace, podInfo.pod.Name, newNodeName, err)
				return 0
			}
			newNodeNames = append(newNodeNames, newNodeName)
		}
	}
	return len(newNodeNames)
}

func (estimator *BinpackingNodeEstimator) addNewNodeToSnapshot(
	template *schedulernodeinfo.NodeInfo,
	nameTimestamp time.Time,
	nameIndex int) (string, error) {

	newNode := template.Node().DeepCopy()
	newNode.Name = fmt.Sprintf("%s-%d-%d", newNode.Name, nameTimestamp.Unix(), nameIndex)
	if err := estimator.clusterSnapshot.AddNodeWithPods(newNode, template.Pods()); err != nil {
		return "", err
	}
	return newNode.Name, nil
}

// Calculates score for all pods and returns podInfo structure.
// Score is defined as cpu_sum/node_capacity + mem_sum/node_capacity.
// Pods that have bigger requirements should be processed first, thus have higher scores.
func calculatePodScore(pods []*apiv1.Pod, nodeTemplate *schedulernodeinfo.NodeInfo) []*podInfo {
	podInfos := make([]*podInfo, 0, len(pods))

	for _, pod := range pods {
		cpuSum := resource.Quantity{}
		memorySum := resource.Quantity{}

		for _, container := range pod.Spec.Containers {
			if request, ok := container.Resources.Requests[apiv1.ResourceCPU]; ok {
				cpuSum.Add(request)
			}
			if request, ok := container.Resources.Requests[apiv1.ResourceMemory]; ok {
				memorySum.Add(request)
			}
		}
		score := float64(0)
		if cpuAllocatable, ok := nodeTemplate.Node().Status.Allocatable[apiv1.ResourceCPU]; ok && cpuAllocatable.MilliValue() > 0 {
			score += float64(cpuSum.MilliValue()) / float64(cpuAllocatable.MilliValue())
		}
		if memAllocatable, ok := nodeTemplate.Node().Status.Allocatable[apiv1.ResourceMemory]; ok && memAllocatable.Value() > 0 {
			score += float64(memorySum.Value()) / float64(memAllocatable.Value())
		}

		podInfos = append(podInfos, &podInfo{
			score: score,
			pod:   pod,
		})
	}
	return podInfos
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013-2017, Jeremy Bingham (<jeremy@goiardi.gl>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package search provides search and index capabilities for goiardi.
package search

import (
	"fmt"
	"reflect"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/ctdk/goiardi/client"
	"github.com/ctdk/goiardi/databag"
	"github.com/ctdk/goiardi/environment"
	"github.com/ctdk/goiardi/indexer"
	"github.com/ctdk/goiardi/node"
	"github.com/ctdk/goiardi/role"
	"github.com/ctdk/goiardi/util"
	"github.com/tideland/golib/logger"
)

// Searcher is an interface that any search backend needs to implement. It's
// up to the Searcher to use whatever backend it wants to return the desired
// results.
type Searcher interface {
	Search(string, string, int, string, int, map[string]interface{}) ([]map[string]interface{}, error)
	GetEndpoints() []string
}

type results struct {
	res     []map[string]interface{}
	sortKey string
}

func (r results) Len() int      { return len(r.res) }
func (r results) Swap(i, j int) { r.res[i], r.res[j] = r.res[j], r.res[i] }
func (r results) Less(i, j int) bool {
	ibase := r.res[i][r.sortKey]
	jbase := r.res[j][r.sortKey]
	ival := reflect.ValueOf(ibase)
	jval := reflect.ValueOf(jbase)
	if (!ival.IsValid() && !jval.IsValid()) || ival.IsValid() && !jval.IsValid() {
		return true
	} else if !ival.IsValid() && jval.IsValid() {
		return false
	}
	// don't try and compare different types for now. If this ever becomes
	// an issue in practice, though, it should be revisited
	if ival.Type() == jval.Type() {
		switch ibase.(type) {
		case int, int8, int32, int64:
			return ival.Int() < jval.Int()
		case uint, uint8, uint32, uint64:
			return ival.Uint() < jval.Uint()
		case float32, float64:
			return ival.Float() < jval.Float()
		case string:
			return ival.String() < jval.String()
		}
	}

	return false
}

// SolrQuery holds a parsed query and query chain to run against the index. It's
// called SolrQuery because the search queries use a subset of Solr's syntax.
type SolrQuery struct {
	queryChain Queryable
	idxName    string
	docs       map[string]indexer.Document
	parentOp   Op
}

var m *sync.Mutex

func init() {
	m = new(sync.Mutex)
}

type TrieSearch struct {
}

// Search parses the given query string and search the given index for any
// matching results.
func (t *TrieSearch) Search(idx string, query string, rows int, sortOrder string, start int, partialData map[string]interface{}) ([]map[string]interface{}, error) {
	defer trackSearchTiming(time.Now(), query, inMemSearchTimings)
	m.Lock()
	defer m.Unlock()
	qq := &Tokenizer{Buffer: query}
	qq.Init()
	if err := qq.Parse(); err != nil {
		return nil, err
	}
	qq.Execute()
	qchain := qq.Evaluate()
	d := make(map[string]indexer.Document)
	solrQ := &SolrQuery{queryChain: qchain, idxName: idx, docs: d}

	_, err := solrQ.execute()
	if err != nil {
		return nil, err
	}
	qresults := solrQ.results()
	objs := getResults(idx, qresults)
	res := make([]map[string]interface{}, len(objs))
	for i, r := range objs {
		switch r := r.(type) {
		case *client.Client:
			jc := map[string]interface{}{
				"name":       r.Name,
				"chef_type":  r.ChefType,
				"json_class": r.JSONClass,
				"admin":      r.Admin,
				"public_key": r.PublicKey(),
				"validator":  r.Validator,
			}
			res[i] = jc
		default:
			res[i] = util.MapifyObject(r)
		}
	}

	/* If we're doing partial search, tease out the fields we want. */
	if partialData != nil {
		res, err = formatPartials(res, objs, partialData)
		if err != nil {
			return nil, err
		}
	}

	// and at long last, sort
	ss := strings.Split(sortOrder, " ")
	sortKey := ss[0]
	if sortKey == "id" {
		sortKey = "name"
	}
	var ordering string
	if len(ss) > 1 {
		ordering = strings.ToLower(ss[1])
	} else {
		ordering = "asc"
	}
	sortResults := results{res, sortKey}
	if ordering == "desc" {
		sort.Sort(sort.Reverse(sortResults))
	} else {
		sort.Sort(sortResults)
	}
	res = sortResults.res

	end := start + rows
	if end > len(res) {
		end = len(res)
	}
	res = res[start:end]
	return res, nil
}

func (sq *SolrQuery) execute() (map[string]indexer.Document, error) {
	s := sq.queryChain
	curOp := OpNotAnOp

	// set op for subqueries
	if sq.parentOp != OpNotAnOp {
		curOp = sq.parentOp
	}

	for s != nil {
		var r map[string]indexer.Document
		var err error

		switch c := s.(type) {
		case *SubQuery:
			_ = c
			newq, nend, nerr := extractSubQuery(s)
			if nerr != nil {
				return nil, nerr
			}
			s = nend
			var d map[string]indexer.Document
			if curOp == OpBinAnd {
				d = sq.docs
			} else {
				d = make(map[string]indexer.Document)
			}
			nsq := &SolrQuery{queryChain: newq, idxName: sq.idxName, docs: d, parentOp: curOp}
			r, err = nsq.execute()
		case *NotQuery:
			s.AddOp(curOp)
		default:
			if curOp == OpBinAnd {
				r, err = s.SearchResults(sq.docs)
			} else {
				r, err = s.SearchIndex(sq.idxName)
			}
		}
		if err != nil {
			return nil, err
		}

		if len(sq.docs) == 0 || curOp == OpBinAnd { // nothing in place yet
			sq.docs = r
		} else if curOp == OpBinOr {
			for k, v := range r {
				sq.docs[k] = v
			}
		} else {
			logger.Debugf("Somehow we got to what should have been an impossible state with search - sq.docs len was %d, op was %s", len(sq.docs), opMap[curOp])
		}

		curOp = s.Op()
		s = s.Next()
	}
	return sq.docs, nil
}

func extractSubQuery(s Queryable) (Queryable, Queryable, error) {
	n := 1
	prev := s
	s = s.Next()
	top := s
	for {
		logger.Debugf("n: %d s: %T %+v", n, s, s)
		switch q := s.(type) {
		case *SubQuery:
			if q.start {
				n++
			} else {
				n--
			}
		}
		if n == 0 {
			// we've followed this subquery chain to its end
			prev.SetNext(nil) // snip this chain off at the end
			return top, s, nil
		}
		prev = s
		s = s.Next()
		if s == nil {
			break
		}
	}
	err := fmt.Errorf("Yikes! Somehow we weren't able to finish the subquery.")
	return nil, nil, err
}

func (sq *SolrQuery) results() []string {
	results := make([]string, len(sq.docs))
	n := 0
	for k := range sq.docs {
		results[n] = k
		n++
	}
	return results
}

// GetEndpoints gets a list from the indexer of all the endpoints available to
// search, namely the defaults (node, role, client, environment) and any data
// bags.
func (t *TrieSearch) GetEndpoints() []string {
	// TODO: deal with possible errors
	endpoints, _ := indexer.Endpoints()
	return endpoints
}

func getResults(variety string, toGet []string) []indexer.Indexable {
	var results []indexer.Indexable
	if len(toGet) > 0 {
		switch variety {
		case "node":
			ns, _ := node.GetMulti(toGet)
			// ....
			results = make([]indexer.Indexable, 0, len(ns))
			for _, n := range ns {
				results = append(results, n)
			}
		case "role":
			rs, _ := role.GetMulti(toGet)
			results = make([]indexer.Indexable, 0, len(rs))
			for _, r := range rs {
				results = append(results, r)
			}
		case "client":
			cs, _ := client.GetMulti(toGet)
			results = make([]indexer.Indexable, 0, len(cs))
			for _, c := range cs {
				results = append(results, c)
			}
		case "environment":
			es, _ := environment.GetMulti(toGet)
			results = make([]indexer.Indexable, 0, len(es))
			for _, e := range es {
				results = append(results, e)
			}
		default: // It's a data bag
			/* These may require further processing later. */
			dbag, _ := databag.Get(variety)
			if dbag != nil {
				ds, _ := dbag.GetMultiDBItems(toGet)
				results = make([]indexer.Indexable, 0, len(ds))
				for _, d := range ds {
					results = append(results, d)
				}
			}
		}
	}
	return results
}

func partialSearchFormat(results []map[string]interface{}, partialFormat map[string]interface{}) ([]map[string]interface{}, error) {
	/* regularize partial search keys */
	psearchKeys := make(map[string][]string, len(partialFormat))
	for k, v := range partialFormat {
		switch v := v.(type) {
		case []interface{}:
			psearchKeys[k] = make([]string, len(v))
			for i, j := range v {
				switch j := j.(type) {
				case string:
					psearchKeys[k][i] = j
				default:
					err := fmt.Errorf("Partial search key %s badly formatted: %T %v", k, j, j)
					return nil, err
				}
			}
		case []string:
			psearchKeys[k] = make([]string, len(v))
			for i, j := range v {
				psearchKeys[k][i] = j
			}
		default:
			err := fmt.Errorf("Partial search key %s badly formatted: %T %v", k, v, v)
			return nil, err
		}
	}
	newResults := make([]map[string]interface{}, len(results))

	for i, j := range results {
		newResults[i] = make(map[string]interface{})
		for key, vals := range psearchKeys {
			var pval interface{}
			/* The first key can either be top or first level.
			 * Annoying, but that's how it is. */
			if len(vals) > 0 {
				if step, found := j[vals[0]]; found {
					if len(vals) > 1 {
						pval = walk(step, vals[1:])
					} else {
						pval = step
					}
				} else {
					if len(vals) > 0 {
						// bear in mind precedence. We need to
						// overwrite later values with earlier
						// ones.
						keyRange := []string{"raw_data", "default", "default_attributes", "normal", "override", "override_attributes", "automatic"}
						for _, r := range keyRange {
							tval := walk(j[r], vals[0:])
							if tval != nil {
								switch pv := pval.(type) {
								case map[string]interface{}:
									// only merge if tval is also a map[string]interface{}
									switch tval := tval.(type) {
									case map[string]interface{}:
										for g, h := range tval {
											pv[g] = h
										}
										pval = pv
									}
								default:
									pval = tval
								}
							}
						}
					}
				}
			}
			newResults[i][key] = pval
		}
	}
	return newResults, nil
}

func walk(v interface{}, keys []string) interface{} {
	switch v := v.(type) {
	case map[string]interface{}:
		if _, found := v[keys[0]]; found {
			if len(keys) > 1 {
				return walk(v[keys[0]], keys[1:])
			}
			return v[keys[0]]
		}
		return nil
	case map[string]string:
		return v[keys[0]]
	case map[string][]string:
		return v[keys[0]]
	default:
		if len(keys) == 1 {
			return v
		}
		return nil
	}
}

func formatPartials(results []map[string]interface{}, objs []indexer.Indexable, partialData map[string]interface{}) ([]map[string]interface{}, error) {
	var err error
	results, err = partialSearchFormat(results, partialData)
	if err != nil {
		return nil, err
	}
	for x, z := range results {
		tmpRes := make(map[string]interface{})
		switch ro := objs[x].(type) {
		case *databag.DataBagItem:
			dbiURL := fmt.Sprintf("/data/%s/%s", ro.DataBagName, ro.RawData["id"].(string))
			tmpRes["url"] = util.CustomURL(dbiURL)
		default:
			tmpRes["url"] = util.ObjURL(objs[x].(util.GoiardiObj))
		}
		tmpRes["data"] = z

		results[x] = tmpRes
	}
	return results, nil
}
<commit_msg>in-mem search index NOT queries broke for a bit, fixing<commit_after>/*
 * Copyright (c) 2013-2017, Jeremy Bingham (<jeremy@goiardi.gl>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package search provides search and index capabilities for goiardi.
package search

import (
	"fmt"
	"reflect"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/ctdk/goiardi/client"
	"github.com/ctdk/goiardi/databag"
	"github.com/ctdk/goiardi/environment"
	"github.com/ctdk/goiardi/indexer"
	"github.com/ctdk/goiardi/node"
	"github.com/ctdk/goiardi/role"
	"github.com/ctdk/goiardi/util"
	"github.com/tideland/golib/logger"
)

// Searcher is an interface that any search backend needs to implement. It's
// up to the Searcher to use whatever backend it wants to return the desired
// results.
type Searcher interface {
	Search(string, string, int, string, int, map[string]interface{}) ([]map[string]interface{}, error)
	GetEndpoints() []string
}

type results struct {
	res     []map[string]interface{}
	sortKey string
}

func (r results) Len() int      { return len(r.res) }
func (r results) Swap(i, j int) { r.res[i], r.res[j] = r.res[j], r.res[i] }
func (r results) Less(i, j int) bool {
	ibase := r.res[i][r.sortKey]
	jbase := r.res[j][r.sortKey]
	ival := reflect.ValueOf(ibase)
	jval := reflect.ValueOf(jbase)
	if (!ival.IsValid() && !jval.IsValid()) || ival.IsValid() && !jval.IsValid() {
		return true
	} else if !ival.IsValid() && jval.IsValid() {
		return false
	}
	// don't try and compare different types for now. If this ever becomes
	// an issue in practice, though, it should be revisited
	if ival.Type() == jval.Type() {
		switch ibase.(type) {
		case int, int8, int32, int64:
			return ival.Int() < jval.Int()
		case uint, uint8, uint32, uint64:
			return ival.Uint() < jval.Uint()
		case float32, float64:
			return ival.Float() < jval.Float()
		case string:
			return ival.String() < jval.String()
		}
	}

	return false
}

// SolrQuery holds a parsed query and query chain to run against the index. It's
// called SolrQuery because the search queries use a subset of Solr's syntax.
type SolrQuery struct {
	queryChain Queryable
	idxName    string
	docs       map[string]indexer.Document
	parentOp   Op
}

var m *sync.Mutex

func init() {
	m = new(sync.Mutex)
}

type TrieSearch struct {
}

// Search parses the given query string and search the given index for any
// matching results.
func (t *TrieSearch) Search(idx string, query string, rows int, sortOrder string, start int, partialData map[string]interface{}) ([]map[string]interface{}, error) {
	defer trackSearchTiming(time.Now(), query, inMemSearchTimings)
	m.Lock()
	defer m.Unlock()
	qq := &Tokenizer{Buffer: query}
	qq.Init()
	if err := qq.Parse(); err != nil {
		return nil, err
	}
	qq.Execute()
	qchain := qq.Evaluate()
	d := make(map[string]indexer.Document)
	solrQ := &SolrQuery{queryChain: qchain, idxName: idx, docs: d}

	_, err := solrQ.execute()
	if err != nil {
		return nil, err
	}
	qresults := solrQ.results()
	objs := getResults(idx, qresults)
	res := make([]map[string]interface{}, len(objs))
	for i, r := range objs {
		switch r := r.(type) {
		case *client.Client:
			jc := map[string]interface{}{
				"name":       r.Name,
				"chef_type":  r.ChefType,
				"json_class": r.JSONClass,
				"admin":      r.Admin,
				"public_key": r.PublicKey(),
				"validator":  r.Validator,
			}
			res[i] = jc
		default:
			res[i] = util.MapifyObject(r)
		}
	}

	/* If we're doing partial search, tease out the fields we want. */
	if partialData != nil {
		res, err = formatPartials(res, objs, partialData)
		if err != nil {
			return nil, err
		}
	}

	// and at long last, sort
	ss := strings.Split(sortOrder, " ")
	sortKey := ss[0]
	if sortKey == "id" {
		sortKey = "name"
	}
	var ordering string
	if len(ss) > 1 {
		ordering = strings.ToLower(ss[1])
	} else {
		ordering = "asc"
	}
	sortResults := results{res, sortKey}
	if ordering == "desc" {
		sort.Sort(sort.Reverse(sortResults))
	} else {
		sort.Sort(sortResults)
	}
	res = sortResults.res

	end := start + rows
	if end > len(res) {
		end = len(res)
	}
	res = res[start:end]
	return res, nil
}

func (sq *SolrQuery) execute() (map[string]indexer.Document, error) {
	s := sq.queryChain
	curOp := OpNotAnOp

	// set op for subqueries
	if sq.parentOp != OpNotAnOp {
		curOp = sq.parentOp
	}

	for s != nil {
		var r map[string]indexer.Document
		var err error

		switch c := s.(type) {
		case *SubQuery:
			_ = c
			newq, nend, nerr := extractSubQuery(s)
			if nerr != nil {
				return nil, nerr
			}
			s = nend
			var d map[string]indexer.Document
			if curOp == OpBinAnd {
				d = sq.docs
			} else {
				d = make(map[string]indexer.Document)
			}
			nsq := &SolrQuery{queryChain: newq, idxName: sq.idxName, docs: d, parentOp: curOp}
			r, err = nsq.execute()
		case *NotQuery:
			s = s.Next()
			continue
		default:
			if curOp == OpBinAnd {
				r, err = s.SearchResults(sq.docs)
			} else {
				r, err = s.SearchIndex(sq.idxName)
			}
		}
		if err != nil {
			return nil, err
		}

		if len(sq.docs) == 0 || curOp == OpBinAnd { // nothing in place yet
			sq.docs = r
		} else if curOp == OpBinOr {
			for k, v := range r {
				sq.docs[k] = v
			}
		} else {
			logger.Debugf("Somehow we got to what should have been an impossible state with search - sq.docs len was %d, op was %s", len(sq.docs), opMap[curOp])
		}

		curOp = s.Op()
		s = s.Next()
	}
	return sq.docs, nil
}

func extractSubQuery(s Queryable) (Queryable, Queryable, error) {
	n := 1
	prev := s
	s = s.Next()
	top := s
	for {
		logger.Debugf("n: %d s: %T %+v", n, s, s)
		switch q := s.(type) {
		case *SubQuery:
			if q.start {
				n++
			} else {
				n--
			}
		}
		if n == 0 {
			// we've followed this subquery chain to its end
			prev.SetNext(nil) // snip this chain off at the end
			return top, s, nil
		}
		prev = s
		s = s.Next()
		if s == nil {
			break
		}
	}
	err := fmt.Errorf("Yikes! Somehow we weren't able to finish the subquery.")
	return nil, nil, err
}

func (sq *SolrQuery) results() []string {
	results := make([]string, len(sq.docs))
	n := 0
	for k := range sq.docs {
		results[n] = k
		n++
	}
	return results
}

// GetEndpoints gets a list from the indexer of all the endpoints available to
// search, namely the defaults (node, role, client, environment) and any data
// bags.
func (t *TrieSearch) GetEndpoints() []string {
	// TODO: deal with possible errors
	endpoints, _ := indexer.Endpoints()
	return endpoints
}

func getResults(variety string, toGet []string) []indexer.Indexable {
	var results []indexer.Indexable
	if len(toGet) > 0 {
		switch variety {
		case "node":
			ns, _ := node.GetMulti(toGet)
			// ....
			results = make([]indexer.Indexable, 0, len(ns))
			for _, n := range ns {
				results = append(results, n)
			}
		case "role":
			rs, _ := role.GetMulti(toGet)
			results = make([]indexer.Indexable, 0, len(rs))
			for _, r := range rs {
				results = append(results, r)
			}
		case "client":
			cs, _ := client.GetMulti(toGet)
			results = make([]indexer.Indexable, 0, len(cs))
			for _, c := range cs {
				results = append(results, c)
			}
		case "environment":
			es, _ := environment.GetMulti(toGet)
			results = make([]indexer.Indexable, 0, len(es))
			for _, e := range es {
				results = append(results, e)
			}
		default: // It's a data bag
			/* These may require further processing later. */
			dbag, _ := databag.Get(variety)
			if dbag != nil {
				ds, _ := dbag.GetMultiDBItems(toGet)
				results = make([]indexer.Indexable, 0, len(ds))
				for _, d := range ds {
					results = append(results, d)
				}
			}
		}
	}
	return results
}

func partialSearchFormat(results []map[string]interface{}, partialFormat map[string]interface{}) ([]map[string]interface{}, error) {
	/* regularize partial search keys */
	psearchKeys := make(map[string][]string, len(partialFormat))
	for k, v := range partialFormat {
		switch v := v.(type) {
		case []interface{}:
			psearchKeys[k] = make([]string, len(v))
			for i, j := range v {
				switch j := j.(type) {
				case string:
					psearchKeys[k][i] = j
				default:
					err := fmt.Errorf("Partial search key %s badly formatted: %T %v", k, j, j)
					return nil, err
				}
			}
		case []string:
			psearchKeys[k] = make([]string, len(v))
			for i, j := range v {
				psearchKeys[k][i] = j
			}
		default:
			err := fmt.Errorf("Partial search key %s badly formatted: %T %v", k, v, v)
			return nil, err
		}
	}
	newResults := make([]map[string]interface{}, len(results))

	for i, j := range results {
		newResults[i] = make(map[string]interface{})
		for key, vals := range psearchKeys {
			var pval interface{}
			/* The first key can either be top or first level.
			 * Annoying, but that's how it is. */
			if len(vals) > 0 {
				if step, found := j[vals[0]]; found {
					if len(vals) > 1 {
						pval = walk(step, vals[1:])
					} else {
						pval = step
					}
				} else {
					if len(vals) > 0 {
						// bear in mind precedence. We need to
						// overwrite later values with earlier
						// ones.
						keyRange := []string{"raw_data", "default", "default_attributes", "normal", "override", "override_attributes", "automatic"}
						for _, r := range keyRange {
							tval := walk(j[r], vals[0:])
							if tval != nil {
								switch pv := pval.(type) {
								case map[string]interface{}:
									// only merge if tval is also a map[string]interface{}
									switch tval := tval.(type) {
									case map[string]interface{}:
										for g, h := range tval {
											pv[g] = h
										}
										pval = pv
									}
								default:
									pval = tval
								}
							}
						}
					}
				}
			}
			newResults[i][key] = pval
		}
	}
	return newResults, nil
}

func walk(v interface{}, keys []string) interface{} {
	switch v := v.(type) {
	case map[string]interface{}:
		if _, found := v[keys[0]]; found {
			if len(keys) > 1 {
				return walk(v[keys[0]], keys[1:])
			}
			return v[keys[0]]
		}
		return nil
	case map[string]string:
		return v[keys[0]]
	case map[string][]string:
		return v[keys[0]]
	default:
		if len(keys) == 1 {
			return v
		}
		return nil
	}
}

func formatPartials(results []map[string]interface{}, objs []indexer.Indexable, partialData map[string]interface{}) ([]map[string]interface{}, error) {
	var err error
	results, err = partialSearchFormat(results, partialData)
	if err != nil {
		return nil, err
	}
	for x, z := range results {
		tmpRes := make(map[string]interface{})
		switch ro := objs[x].(type) {
		case *databag.DataBagItem:
			dbiURL := fmt.Sprintf("/data/%s/%s", ro.DataBagName, ro.RawData["id"].(string))
			tmpRes["url"] = util.CustomURL(dbiURL)
		default:
			tmpRes["url"] = util.ObjURL(objs[x].(util.GoiardiObj))
		}
		tmpRes["data"] = z

		results[x] = tmpRes
	}
	return results, nil
}
<|endoftext|>