<commit_before>package main

import (
	"bufio"
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	// "io"
	"flag"
	"io/ioutil"
	"strings"
	"time"
)

func get(hostname string, port int, path string, auth string, urls bool, verbose bool, timeout int) (rv bool, err error) {

	defer func() {
		if err := recover(); err != nil {
			return
		}
	}()

	rv = true

	if verbose {
		fmt.Fprintf(os.Stderr, "fetching:hostname:%s:\n", hostname)
	}

	res := &http.Response{}

	if urls {

		url := hostname
		res, err = http.Head(url)
		defer res.Body.Close()

		if err != nil {
			fmt.Println(err.Error())
			rv = false
			return
		}

	} else {

		client := &http.Client{Timeout: time.Duration(timeout) * time.Second}

		// had to allocate this or the SetBasicAuth will panic
		headers := make(map[string][]string)
		hostPort := fmt.Sprintf("%s:%d", hostname, port)

		if verbose {

			fmt.Fprintf(os.Stderr, "adding hostPort:%s:%d:path:%s:\n", hostname, port, path)

		}
		req := &http.Request{
			Method: "HEAD",
			// Host:  hostPort,
			URL: &url.URL{
				Host:   hostPort,
				Scheme: "http",
				Opaque: path,
			},
			Header: headers,
		}

		if auth != "" {

			up := strings.SplitN(auth, ":", 2)

			if verbose {

				fmt.Fprintf(os.Stderr, "Doing auth with:username:%s:password:%s:", up[0], up[1])

			}
			req.SetBasicAuth(up[0], up[1])

		}

		if verbose {

			dump, _ := httputil.DumpRequestOut(req, true)
			fmt.Fprintf(os.Stderr, "%s", dump)

		}

		res, err = client.Do(req)
		if err != nil {
			fmt.Println(err.Error())
			rv = false
			return
		}

		defer res.Body.Close()
		_, err = ioutil.ReadAll(res.Body)

	}

	if verbose {

		fmt.Println(res.Status)
		for k, v := range res.Header {
			fmt.Println(k+":", v)
		}

	}

	if res.StatusCode != http.StatusOK {
		rv = false
	}

	return
}

func main() {

	status := "OK"
	rv := 0
	name := "Bulk HTTP Check"
	bad := 0
	total := 0

	// this needs improvement. the number of spaces here has to equal the number of chars in the badHosts append line suffix
	badHosts := []byte("  ")

	verbose := flag.Bool("v", false, "verbose output")
	warn := flag.Int("w", 10, "warning level - number of non-200s or percentage of non-200s (default is numeric not percentage)")
	crit := flag.Int("c", 20, "critical level - number of non-200s or percentage of non-200s (default is numeric not percentage)")
	timeout := flag.Int("t", 2, "timeout in seconds - don't wait.  Do Head requests and don't wait.")
	pct := flag.Bool("pct", false, "interpret warming and critical levels are percentages")
	path := flag.String("path", "", "optional path to append to the input lines including the leading slash - these will not be urlencoded. This is ignored is the urls option is given.")
	file := flag.String("file", "", "input data source: a filename or '-' for STDIN.")
	port := flag.Int("port", 80, "optional port for the http request - ignored if urls is specified")
	urls := flag.Bool("urls", false, "Assume the input data is full urls - its normally a list of hostnames")
	auth := flag.String("auth", "", "Do basic auth with this username:passwd - ignored if urls is specified - make this use .netrc instead")
	checkName := flag.String("name", "", "a name to be included in the check output to distinguish the check output")

	flag.Usage = func() {

		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, `
	Read hostnames from a file or STDIN and do a single nagios check over
	them all.  Just check for 200s.  Warning and Critical are either
	percentages of the total, or a regular numeric thresholds.

	The output contains the hostname of any non-200 reporting hosts.

	Skip input lines that are commented out with shell style comments
	like /^#/.

	Do Head requests since we don't care about the content.  Make this
	optional some day.

	The -path is appended to the hostnames to make full URLs for the checks.

	If the -urls option is specified, then the input is assumed a complete URLs, like http://$hostname:$port/$path.

	Examples:

	./someCommand |  ./check_http_bulk  -w 1 -c 2 -path '/api/aliveness-test/%%2F/' -port 15672 -file - -auth zup:nuch

	./check_http_bulk -urls -file urls.txt

`)

		flag.PrintDefaults()
	}

	flag.Parse()

	if len(flag.Args()) > 0 {

		flag.Usage()
		os.Exit(3)

	}

	// it urls is specified, the input is full urls to be used enmasse and to be url encoded
	if *urls {
		*path = ""
	}

	if *checkName != "" {
		name = *checkName
	}

	defer func() {
		if err := recover(); err != nil {
			fmt.Println(name+" Unknown: ", err)
			os.Exit(3)
		}
	}()

	if file == nil || *file == "" {
		flag.Usage()
		os.Exit(3)
	}

	inputSource := os.Stdin

	if (*file)[0] != "-"[0] {

		var err error

		inputSource, err = os.Open(*file)

		if err != nil {

			fmt.Printf("Couldn't open the specified input file:%s:error:%v:\n\n", name, err)
			flag.Usage()
			os.Exit(3)

		}

	}

	scanner := bufio.NewScanner(inputSource)
	for scanner.Scan() {

		total++

		hostname := scanner.Text()

		if hostname[0] == "#"[0] {

			if *verbose {

				fmt.Printf("skipping:%s:\n", hostname)

			}

			continue
		}

		if *verbose {

			fmt.Printf("working on:%s:\n", hostname)

		}

		goodCheck, err := get(hostname, *port, *path, *auth, *urls, *verbose, *timeout)
		if err != nil {

			fmt.Printf("%s get error: %T %s %#v\n", name, err, err, err)
			badHosts = append(badHosts, hostname...)
			badHosts = append(badHosts, ", "...)
			bad++

			continue

		}

		if !goodCheck {
			badHosts = append(badHosts, hostname...)
			badHosts = append(badHosts, ", "...)
			bad++
		}

	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
		status = "Unknown"
		rv = 3
	}

	if *pct {

		ratio := int(float64(bad)/float64(total)*100)

		if *verbose {

			fmt.Fprintf(os.Stderr, "ratio:%d:\n", ratio)

		}

		if ratio >= *crit {
			status = "Critical"
			rv = 1
		} else if ratio >= *warn {
			status = "Warning"
			rv = 2
		}

	} else {

		if bad >= *crit {
			status = "Critical"
			rv = 1
		} else if bad >= *warn {
			status = "Warning"
			rv = 2
		}

	}

	fmt.Printf("%s %s: %d of %d |%s\n", name, status, bad, total, badHosts[:len(badHosts)-2])
	os.Exit(rv)
}
<commit_msg>fix bug where i had the rc for warning and critical reversed<commit_after>package main

import (
	"bufio"
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	// "io"
	"flag"
	"io/ioutil"
	"strings"
	"time"
)

func get(hostname string, port int, path string, auth string, urls bool, verbose bool, timeout int) (rv bool, err error) {

	defer func() {
		if err := recover(); err != nil {
			return
		}
	}()

	rv = true

	if verbose {
		fmt.Fprintf(os.Stderr, "fetching:hostname:%s:\n", hostname)
	}

	res := &http.Response{}

	if urls {

		url := hostname
		res, err = http.Head(url)
		defer res.Body.Close()

		if err != nil {
			fmt.Println(err.Error())
			rv = false
			return
		}

	} else {

		client := &http.Client{Timeout: time.Duration(timeout) * time.Second}

		// had to allocate this or the SetBasicAuth will panic
		headers := make(map[string][]string)
		hostPort := fmt.Sprintf("%s:%d", hostname, port)

		if verbose {

			fmt.Fprintf(os.Stderr, "adding hostPort:%s:%d:path:%s:\n", hostname, port, path)

		}
		req := &http.Request{
			Method: "HEAD",
			// Host:  hostPort,
			URL: &url.URL{
				Host:   hostPort,
				Scheme: "http",
				Opaque: path,
			},
			Header: headers,
		}

		if auth != "" {

			up := strings.SplitN(auth, ":", 2)

			if verbose {

				fmt.Fprintf(os.Stderr, "Doing auth with:username:%s:password:%s:", up[0], up[1])

			}
			req.SetBasicAuth(up[0], up[1])

		}

		if verbose {

			dump, _ := httputil.DumpRequestOut(req, true)
			fmt.Fprintf(os.Stderr, "%s", dump)

		}

		res, err = client.Do(req)
		if err != nil {
			fmt.Println(err.Error())
			rv = false
			return
		}

		defer res.Body.Close()
		_, err = ioutil.ReadAll(res.Body)

	}

	if verbose {

		fmt.Println(res.Status)
		for k, v := range res.Header {
			fmt.Println(k+":", v)
		}

	}

	if res.StatusCode != http.StatusOK {
		rv = false
	}

	return
}

func main() {

	status := "OK"
	rv := 0
	name := "Bulk HTTP Check"
	bad := 0
	total := 0

	// this needs improvement. the number of spaces here has to equal the number of chars in the badHosts append line suffix
	badHosts := []byte("  ")

	verbose := flag.Bool("v", false, "verbose output")
	warn := flag.Int("w", 10, "warning level - number of non-200s or percentage of non-200s (default is numeric not percentage)")
	crit := flag.Int("c", 20, "critical level - number of non-200s or percentage of non-200s (default is numeric not percentage)")
	timeout := flag.Int("t", 2, "timeout in seconds - don't wait.  Do Head requests and don't wait.")
	pct := flag.Bool("pct", false, "interpret warming and critical levels are percentages")
	path := flag.String("path", "", "optional path to append to the input lines including the leading slash - these will not be urlencoded. This is ignored is the urls option is given.")
	file := flag.String("file", "", "input data source: a filename or '-' for STDIN.")
	port := flag.Int("port", 80, "optional port for the http request - ignored if urls is specified")
	urls := flag.Bool("urls", false, "Assume the input data is full urls - its normally a list of hostnames")
	auth := flag.String("auth", "", "Do basic auth with this username:passwd - ignored if urls is specified - make this use .netrc instead")
	checkName := flag.String("name", "", "a name to be included in the check output to distinguish the check output")

	flag.Usage = func() {

		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, `
	Read hostnames from a file or STDIN and do a single nagios check over
	them all.  Just check for 200s.  Warning and Critical are either
	percentages of the total, or a regular numeric thresholds.

	The output contains the hostname of any non-200 reporting hosts.

	Skip input lines that are commented out with shell style comments
	like /^#/.

	Do Head requests since we don't care about the content.  Make this
	optional some day.

	The -path is appended to the hostnames to make full URLs for the checks.

	If the -urls option is specified, then the input is assumed a complete URLs, like http://$hostname:$port/$path.

	Examples:

	./someCommand |  ./check_http_bulk  -w 1 -c 2 -path '/api/aliveness-test/%%2F/' -port 15672 -file - -auth zup:nuch

	./check_http_bulk -urls -file urls.txt

`)

		flag.PrintDefaults()
	}

	flag.Parse()

	if len(flag.Args()) > 0 {

		flag.Usage()
		os.Exit(3)

	}

	// it urls is specified, the input is full urls to be used enmasse and to be url encoded
	if *urls {
		*path = ""
	}

	if *checkName != "" {
		name = *checkName
	}

	defer func() {
		if err := recover(); err != nil {
			fmt.Println(name+" Unknown: ", err)
			os.Exit(3)
		}
	}()

	if file == nil || *file == "" {
		flag.Usage()
		os.Exit(3)
	}

	inputSource := os.Stdin

	if (*file)[0] != "-"[0] {

		var err error

		inputSource, err = os.Open(*file)

		if err != nil {

			fmt.Printf("Couldn't open the specified input file:%s:error:%v:\n\n", name, err)
			flag.Usage()
			os.Exit(3)

		}

	}

	scanner := bufio.NewScanner(inputSource)
	for scanner.Scan() {

		total++

		hostname := scanner.Text()

		if hostname[0] == "#"[0] {

			if *verbose {

				fmt.Printf("skipping:%s:\n", hostname)

			}

			continue
		}

		if *verbose {

			fmt.Printf("working on:%s:\n", hostname)

		}

		goodCheck, err := get(hostname, *port, *path, *auth, *urls, *verbose, *timeout)
		if err != nil {

			fmt.Printf("%s get error: %T %s %#v\n", name, err, err, err)
			badHosts = append(badHosts, hostname...)
			badHosts = append(badHosts, ", "...)
			bad++

			continue

		}

		if !goodCheck {
			badHosts = append(badHosts, hostname...)
			badHosts = append(badHosts, ", "...)
			bad++
		}

	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
		status = "Unknown"
		rv = 3
	}

	if *pct {

		ratio := int(float64(bad)/float64(total)*100)

		if *verbose {

			fmt.Fprintf(os.Stderr, "ratio:%d:\n", ratio)

		}

		if ratio >= *crit {
			status = "Critical"
			rv = 2
		} else if ratio >= *warn {
			status = "Warning"
			rv = 1
		}

	} else {

		if bad >= *crit {
			status = "Critical"
			rv = 2
		} else if bad >= *warn {
			status = "Warning"
			rv = 1
		}

	}

	fmt.Printf("%s %s: %d of %d |%s\n", name, status, bad, total, badHosts[:len(badHosts)-2])
	os.Exit(rv)
}
<|endoftext|>


<commit_before>package check_http_bulk

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"strings"
	"time"
)

type BulkCheck struct {
	Silence bool
	Urls    bool
	Path    string
	Port    int
	Auth    string
	Verbose bool
	results  chan getReply
	requests chan checkRequest
	BadHosts []byte
	Bad      int
	Total    int
	Timeout  int
	received int
	done     chan string
	Workers int
}

type checkRequest struct {
	path     string
	port     int
	hostname string
}

type getReply struct {
	hostname string
	err      error
	rv       bool
}

func (this BulkCheck) vLogger(msg string, args ...interface{}) {

	if this.Verbose {
		fmt.Fprintf(os.Stderr, msg, args...)
	}

}

func (this *BulkCheck) get(request chan checkRequest, client *http.Client) {

	var err error

	for args := range request {

		this.vLogger("fetching:hostname:%s:\n", args.hostname)

		res := &http.Response{}

		if this.Urls {

			_url := args.hostname

			u, err := url.Parse(_url)
			if err != nil {
				this.results <- getReply{hostname: args.hostname, rv: false, err: err}
				continue
			}

			req := &http.Request{
				Method: "HEAD",
				URL:    u,
			}

			res, err = client.Do(req)

		} else {

			// had to allocate this or the SetBasicAuth will panic
			headers := make(map[string][]string)
			hostPort := fmt.Sprintf("%s:%d", args.hostname, this.Port)

			this.vLogger("adding hostPort:%s:%d:path:%s:\n", args.hostname, this.Port, this.Path)

			req := &http.Request{
				Method: "HEAD",
				// Host:  hostPort,
				URL: &url.URL{
					Host:   hostPort,
					Scheme: "http",
					Opaque: this.Path,
				},
				Header: headers,
			}

			if this.Auth != "" {

				up := strings.SplitN(this.Auth, ":", 2)

				this.vLogger("Doing auth with:username:%s:password:%s:", up[0], up[1])
				req.SetBasicAuth(up[0], up[1])

			}

			if this.Verbose {

				dump, _ := httputil.DumpRequestOut(req, true)
				this.vLogger("%s", dump)

			}

			res, err = client.Do(req)

		}

		if this.Verbose && res != nil {

			fmt.Println(res.Status)
			for k, v := range res.Header {
				fmt.Println(k+":", v)
			}

		}

		if err != nil {
			fmt.Println(err)
			this.results <- getReply{hostname: args.hostname, rv: false, err: err}
		} else if res != nil && res.StatusCode == http.StatusOK {
			this.results <- getReply{hostname: args.hostname, rv: true}
			io.Copy(ioutil.Discard, res.Body)
			res.Body.Close()
		} else {
			this.results <- getReply{hostname: args.hostname, rv: false}
		}

	}

}

func (this *BulkCheck) readAll() {
	for {
		select {

		case result := <-this.results:

			this.received++
			err := result.err
			goodCheck := result.rv

			if err != nil {

				if !this.Silence {
					this.BadHosts = append(this.BadHosts, result.hostname...)
					this.BadHosts = append(this.BadHosts, ", "...)
				}
				this.Bad++

			} else if !goodCheck {
				if !this.Silence {
					this.BadHosts = append(this.BadHosts, result.hostname...)
					this.BadHosts = append(this.BadHosts, ", "...)
				}
				this.Bad++
			}

		}

		this.vLogger("checking if done:total:%d:received:%d:\n", this.Total, this.received)
		if this.Total == this.received {
			this.done <- "done"
		}
	}

}

func (this *BulkCheck) DoChecks(scanner *bufio.Scanner) (err error) {

	for scanner.Scan() {

		hostname := scanner.Text()

		if len(hostname) == 0 {

			this.vLogger("skipping blank:\n")

			continue
		}

		if hostname[0] == "#"[0] {

			this.vLogger("skipping:%s:\n", hostname)

			continue
		}

		this.Total++

		this.vLogger("working on:%s:\n", hostname)

		//send the request off to the workers
		this.requests <- checkRequest{hostname: hostname, port: this.Port, path: this.Path}

	}
	if err = scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}

	//wait for all responses
	this.vLogger("waiting for done:total:%d:received:%d:\n", this.Total, this.received)
	select {
	case <-this.done:
	}

	//fix badHosts
	if len(this.BadHosts) > 2 {
		this.BadHosts = this.BadHosts[:len(this.BadHosts)-2]
	}
	return

}

func (this *BulkCheck) workerPool() chan checkRequest {
	requests := make(chan checkRequest)

	client := &http.Client{Timeout: time.Duration(this.Timeout) * time.Second}

	for i := 0; i < this.Workers; i++ {
		go this.get(requests, client)
	}

	go this.readAll()

	this.vLogger("exec'd:%d:workers:\n", this.Workers)

	return requests
}

func NewCheckHttpBulk(checker *BulkCheck) *BulkCheck {

	badHosts := []byte{}

	defer func() {
		if err := recover(); err != nil {
			fmt.Println("Unknown err NewCheckHttpBulk: ", err)
			os.Exit(3)
		}
	}()

	repliesChannel := make(chan getReply)

	checker.BadHosts = badHosts
	checker.requests = checker.workerPool()
	checker.results = repliesChannel
	checker.done = make(chan string)

	return checker

}
<commit_msg>godocs for the lib and chose better names for some hidden vars<commit_after>// library to do nagios-style http checks from a list of ips
//
// to be embedded in programs that can access a cloud provisioning api
//

package check_http_bulk

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"strings"
	"time"
)

// Silence: bool determining is string of failing hosts should be returned
//
// Urls: bool indicating the incoming data stream is full urls, not just IPs
//
// Path: the path to append to IPs if the input is just IPs
//
// Port: the target port for the checks
//
// Auth: an auth string is authentication is needed
//
// Verbose: display details about its operation as it runs
//
// Bad: the number of ips/urls that failed the check
//
// Total: the total number of ips or urls
//
// Timeout: http timeout so it won't get stuck for too long
//
// Workers: how many workers to use when doing checks in parallel
type BulkCheck struct {
	Silence  bool
	Urls     bool
	Path     string
	Port     int
	Auth     string
	Verbose  bool
	replies  chan getReply
	requests chan getRequest
	BadHosts []byte
	Bad      int
	Total    int
	Timeout  int
	received int
	done     chan string
	Workers  int
}

type getRequest struct {
	path     string
	port     int
	hostname string
}

type getReply struct {
	hostname string
	err      error
	rv       bool
}

func (this BulkCheck) vLogger(msg string, args ...interface{}) {

	if this.Verbose {
		fmt.Fprintf(os.Stderr, msg, args...)
	}

}

// the worker func that does the gets
// these run async according to the Workers int
func (this *BulkCheck) get(request chan getRequest, client *http.Client) {

	var err error

	for args := range request {

		this.vLogger("fetching:hostname:%s:\n", args.hostname)

		res := &http.Response{}

		if this.Urls {

			_url := args.hostname

			u, err := url.Parse(_url)
			if err != nil {
				this.replies <- getReply{hostname: args.hostname, rv: false, err: err}
				continue
			}

			req := &http.Request{
				Method: "HEAD",
				URL:    u,
			}

			res, err = client.Do(req)

		} else {

			// had to allocate this or the SetBasicAuth will panic
			headers := make(map[string][]string)
			hostPort := fmt.Sprintf("%s:%d", args.hostname, this.Port)

			this.vLogger("adding hostPort:%s:%d:path:%s:\n", args.hostname, this.Port, this.Path)

			req := &http.Request{
				Method: "HEAD",
				// Host:  hostPort,
				URL: &url.URL{
					Host:   hostPort,
					Scheme: "http",
					Opaque: this.Path,
				},
				Header: headers,
			}

			if this.Auth != "" {

				up := strings.SplitN(this.Auth, ":", 2)

				this.vLogger("Doing auth with:username:%s:password:%s:", up[0], up[1])
				req.SetBasicAuth(up[0], up[1])

			}

			if this.Verbose {

				dump, _ := httputil.DumpRequestOut(req, true)
				this.vLogger("%s", dump)

			}

			res, err = client.Do(req)

		}

		if this.Verbose && res != nil {

			fmt.Println(res.Status)
			for k, v := range res.Header {
				fmt.Println(k+":", v)
			}

		}

		if err != nil {
			fmt.Println(err)
			this.replies <- getReply{hostname: args.hostname, rv: false, err: err}
		} else if res != nil && res.StatusCode == http.StatusOK {
			this.replies <- getReply{hostname: args.hostname, rv: true}
			io.Copy(ioutil.Discard, res.Body)
			res.Body.Close()
		} else {
			this.replies <- getReply{hostname: args.hostname, rv: false}
		}

	}

}

// this reads all the responses from the get workers
func (this *BulkCheck) readAll() {
	for {
		select {

		case result := <-this.replies:

			this.received++
			err := result.err
			goodCheck := result.rv

			if err != nil {

				if !this.Silence {
					this.BadHosts = append(this.BadHosts, result.hostname...)
					this.BadHosts = append(this.BadHosts, ", "...)
				}
				this.Bad++

			} else if !goodCheck {
				if !this.Silence {
					this.BadHosts = append(this.BadHosts, result.hostname...)
					this.BadHosts = append(this.BadHosts, ", "...)
				}
				this.Bad++
			}

		}

		this.vLogger("checking if done:total:%d:received:%d:\n", this.Total, this.received)
		if this.Total == this.received {
			this.done <- "done"
		}
	}

}

// this is the exposed access point for the library
//
// users create a BulkCheck and call DoChecks on it, passing it a Scanner
//
// eg. scanner := bufio.NewScanner(inputSource)
//
// The scanner provides the list of urls or IPs
func (this *BulkCheck) DoChecks(scanner *bufio.Scanner) (err error) {

	for scanner.Scan() {

		hostname := scanner.Text()

		if len(hostname) == 0 {

			this.vLogger("skipping blank:\n")

			continue
		}

		if hostname[0] == "#"[0] {

			this.vLogger("skipping:%s:\n", hostname)

			continue
		}

		this.Total++

		this.vLogger("working on:%s:\n", hostname)

		//send the request off to the workers
		this.requests <- getRequest{hostname: hostname, port: this.Port, path: this.Path}

	}
	if err = scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}

	//wait for all responses
	this.vLogger("waiting for done:total:%d:received:%d:\n", this.Total, this.received)
	select {
	case <-this.done:
	}

	//fix badHosts
	if len(this.BadHosts) > 2 {
		this.BadHosts = this.BadHosts[:len(this.BadHosts)-2]
	}
	return

}

// start up the workers
//
// the number of workers is specified by BulkCheck.Workers
//
// default value is 1
func (this *BulkCheck) workerPool() chan getRequest {
	requests := make(chan getRequest)

	client := &http.Client{Timeout: time.Duration(this.Timeout) * time.Second}

	for i := 0; i < this.Workers; i++ {
		go this.get(requests, client)
	}

	go this.readAll()

	this.vLogger("exec'd:%d:workers:\n", this.Workers)

	return requests
}

// initializer for using a BulkCheck
//
//
// call it like this
//
// check_http_bulk.NewCheckHttpBulk(&check_http_bulk.BulkCheck{Silence: *silence,
//     Urls: *urls,
//     Path: *path,
//     Port: *port,
//     Auth: *auth,
//     Verbose: *verbose,
//     Timeout: *timeout,
//     Workers: *workers
// })
//
// then call DoCheck on it passing it a scanner.  See bufio.Scanner
//
// it creates the channels and a slice for a result string
func NewCheckHttpBulk(checker *BulkCheck) *BulkCheck {

	badHosts := []byte{}

	defer func() {
		if err := recover(); err != nil {
			fmt.Println("Unknown err NewCheckHttpBulk: ", err)
			os.Exit(3)
		}
	}()

	repliesChannel := make(chan getReply)

	checker.BadHosts = badHosts
	checker.requests = checker.workerPool()
	checker.replies = repliesChannel
	checker.done = make(chan string)

	return checker

}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"flag"
	"io/ioutil"
	"strings"
	"time"
)

var verbose *bool

type getArgs struct {
	hostname string
	port int
	path string
	auth string
	urls bool
	verbose bool
	timeout int
	resultChan chan getReply
}

type getReply struct {
	err interface{}
	rv bool
}

func vLogger(msg string, args ...interface{}) {

	if *verbose {
		fmt.Fprintf(os.Stderr, msg, args...)
	}

}

func get(request chan *getArgs) {

	var err error

	for args := range request {

		// defer func() {
		// 	if err := recover(); err != nil {
		// 		args.resultChan <- getReply{err: err}

		// 		//something bad happened
		// 		return
		// 	}
		// }()

		vLogger("fetching:hostname:%s:\n", args.hostname)

		res := &http.Response{}

		if args.urls {

			url := args.hostname
			res, err = http.Head(url)

			if err != nil {
				fmt.Println(err.Error())
				args.resultChan <- getReply{rv: false, err: err}
				_, err = ioutil.ReadAll(res.Body)
				res.Body.Close()
				continue
			}

			_, err = ioutil.ReadAll(res.Body)
			res.Body.Close()

		} else {

			client := &http.Client{Timeout: time.Duration(args.timeout) * time.Second}

			// had to allocate this or the SetBasicAuth will panic
			headers := make(map[string][]string)
			hostPort := fmt.Sprintf("%s:%d", args.hostname, args.port)

			vLogger("adding hostPort:%s:%d:path:%s:\n", args.hostname, args.port, args.path)

			req := &http.Request{
				Method: "HEAD",
				// Host:  hostPort,
				URL: &url.URL{
					Host:   hostPort,
					Scheme: "http",
					Opaque: args.path,
				},
				Header: headers,
			}

			if args.auth != "" {

				up := strings.SplitN(args.auth, ":", 2)

				vLogger("Doing auth with:username:%s:password:%s:", up[0], up[1])
				req.SetBasicAuth(up[0], up[1])

			}

			if args.verbose {

				dump, _ := httputil.DumpRequestOut(req, true)
				vLogger("%s", dump)

			}

			res, err = client.Do(req)
			if err != nil {
				fmt.Println(err.Error())
				args.resultChan <- getReply{rv: false, err: err}
				_, err = ioutil.ReadAll(res.Body)
				res.Body.Close()
				continue
			}

			_, err = ioutil.ReadAll(res.Body)
			res.Body.Close()

		}

		if args.verbose {

			fmt.Println(res.Status)
			for k, v := range res.Header {
				fmt.Println(k+":", v)
			}

		}

		if res.StatusCode != http.StatusOK {
			args.resultChan <- getReply{rv: false}
		}

		args.resultChan <- getReply{rv: true}

	}

}

func WorkerPool(n int) chan *getArgs {
    requests := make(chan *getArgs)

    for i:=0; i<n; i++ {
        go get(requests)
    }

    return requests
}

func main() {

	status := "OK"
	rv := 0
	name := "Bulk HTTP Check"
	bad := 0
	total := 0

	// this needs improvement. the number of spaces here has to equal the number of chars in the badHosts append line suffix
	badHosts := []byte("  ")

	//verbose := flag.Bool("v", false, "verbose output")
	verbose = flag.Bool("v", false, "verbose output")
	warn := flag.Int("w", 10, "warning level - number of non-200s or percentage of non-200s (default is numeric not percentage)")
	crit := flag.Int("c", 20, "critical level - number of non-200s or percentage of non-200s (default is numeric not percentage)")
	timeout := flag.Int("t", 2, "timeout in seconds - don't wait.  Do Head requests and don't wait.")
	pct := flag.Bool("pct", false, "interpret warming and critical levels are percentages")
	path := flag.String("path", "", "optional path to append to the input lines including the leading slash - these will not be urlencoded. This is ignored is the urls option is given.")
	file := flag.String("file", "", "input data source: a filename or '-' for STDIN.")
	port := flag.Int("port", 80, "optional port for the http request - ignored if urls is specified")
	urls := flag.Bool("urls", false, "Assume the input data is full urls - its normally a list of hostnames")
	auth := flag.String("auth", "", "Do basic auth with this username:passwd - ignored if urls is specified - make this use .netrc instead")
	checkName := flag.String("name", "", "a name to be included in the check output to distinguish the check output")
	silence := flag.Bool("silence", false, "don't make a huge list of all failing checks")
	workers := flag.Int("workers", 5, "how many workers to do the requests")

	flag.Usage = func() {

		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, `
	Read hostnames from a file or STDIN and do a single nagios check over
	them all.  Just check for 200s.  Warning and Critical are either
	percentages of the total, or a regular numeric thresholds.

	The output contains the hostname of any non-200 reporting hosts (see -silence).

	Skip input lines that are commented out with shell style comments
	like /^#/.

	Do Head requests since we don't care about the content.  Make this
	optional some day.

	The -path is appended to the hostnames to make full URLs for the checks.

	If the -urls option is specified, then the input is assumed a complete URL, like http://$hostname:$port/$path.

	Examples:

	./someCommand |  ./check_http_bulk  -w 1 -c 2 -path '/api/aliveness-test/%%2F/' -port 15672 -file - -auth zup:nuch

	./check_http_bulk -urls -file urls.txt

`)

		flag.PrintDefaults()
	}

	flag.Parse()

	if len(flag.Args()) > 0 {

		flag.Usage()
		os.Exit(3)

	}

	// it urls is specified, the input is full urls to be used enmasse and to be url encoded
	if *urls {
		*path = ""
	}

	if *checkName != "" {
		name = *checkName
	}

	defer func() {
		if err := recover(); err != nil {
			fmt.Println(name+" Unknown: ", err)
			os.Exit(3)
		}
	}()

	if file == nil || *file == "" {
		flag.Usage()
		os.Exit(3)
	}

	inputSource := os.Stdin

	if (*file)[0] != "-"[0] {

		var err error

		inputSource, err = os.Open(*file)

		if err != nil {

			fmt.Printf("Couldn't open the specified input file:%s:error:%v:\n\n", name, err)
			flag.Usage()
			os.Exit(3)

		}

	}

    requests := WorkerPool(*workers)
	scanner := bufio.NewScanner(inputSource)

	//leave some room since we start sending to this before we read from it
	repliesChannel := make(chan chan getReply, 1)
	for scanner.Scan() {

		hostname := scanner.Text()

		if len(hostname) == 0 {

			vLogger("skipping blank:\n" )

			continue
		}

		if hostname[0] == "#"[0] {

			vLogger("skipping:%s:\n", hostname)

			continue
		}

		total++

		vLogger("working on:%s:\n", hostname)

		thisReplyChan := make(chan getReply)

		//put the reply chan into the chan of reply chans
		repliesChannel <- thisReplyChan
		request := &getArgs{hostname: hostname, port: *port, path: *path, auth: *auth, urls: *urls, verbose: *verbose, timeout: *timeout, resultChan: thisReplyChan}

		//send the request off so the workers can go
		requests <- request

		//get a replyChannel that's ready
		readyReplyChan := <- repliesChannel

		//read a reply
		result := <- readyReplyChan

		err := result.err
		goodCheck := result.rv

		if err != nil {

			fmt.Printf("%s get error: %T %s %#v\n", name, err, err, err)
			if ! *silence {
				badHosts = append(badHosts, hostname...)
				badHosts = append(badHosts, ", "...)
			}
			bad++

			continue

		}

		if !goodCheck {
			if ! *silence {
				badHosts = append(badHosts, hostname...)
				badHosts = append(badHosts, ", "...)
			}
			bad++
		}

	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
		status = "Unknown"
		rv = 3
	}

	if *pct {

		ratio := int(float64(bad)/float64(total)*100)

		vLogger("ratio:%d:\n", ratio)

		if ratio >= *crit {
			status = "Critical"
			rv = 2
		} else if ratio >= *warn {
			status = "Warning"
			rv = 1
		}

	} else {

		if bad >= *crit {
			status = "Critical"
			rv = 2
		} else if bad >= *warn {
			status = "Warning"
			rv = 1
		}

	}

	fmt.Printf("%s %s: %d of %d failed|%s\n", name, status, bad, total, badHosts[:len(badHosts)-2])
	os.Exit(rv)
}
<commit_msg>go fmt<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"strings"
	"time"
)

var verbose *bool

type getArgs struct {
	hostname   string
	port       int
	path       string
	auth       string
	urls       bool
	verbose    bool
	timeout    int
	resultChan chan getReply
}

type getReply struct {
	err interface{}
	rv  bool
}

func vLogger(msg string, args ...interface{}) {

	if *verbose {
		fmt.Fprintf(os.Stderr, msg, args...)
	}

}

func get(request chan *getArgs) {

	var err error

	for args := range request {

		// defer func() {
		// 	if err := recover(); err != nil {
		// 		args.resultChan <- getReply{err: err}

		// 		//something bad happened
		// 		return
		// 	}
		// }()

		vLogger("fetching:hostname:%s:\n", args.hostname)

		res := &http.Response{}

		if args.urls {

			url := args.hostname
			res, err = http.Head(url)

			if err != nil {
				fmt.Println(err.Error())
				args.resultChan <- getReply{rv: false, err: err}
				_, err = ioutil.ReadAll(res.Body)
				res.Body.Close()
				continue
			}

			_, err = ioutil.ReadAll(res.Body)
			res.Body.Close()

		} else {

			client := &http.Client{Timeout: time.Duration(args.timeout) * time.Second}

			// had to allocate this or the SetBasicAuth will panic
			headers := make(map[string][]string)
			hostPort := fmt.Sprintf("%s:%d", args.hostname, args.port)

			vLogger("adding hostPort:%s:%d:path:%s:\n", args.hostname, args.port, args.path)

			req := &http.Request{
				Method: "HEAD",
				// Host:  hostPort,
				URL: &url.URL{
					Host:   hostPort,
					Scheme: "http",
					Opaque: args.path,
				},
				Header: headers,
			}

			if args.auth != "" {

				up := strings.SplitN(args.auth, ":", 2)

				vLogger("Doing auth with:username:%s:password:%s:", up[0], up[1])
				req.SetBasicAuth(up[0], up[1])

			}

			if args.verbose {

				dump, _ := httputil.DumpRequestOut(req, true)
				vLogger("%s", dump)

			}

			res, err = client.Do(req)
			if err != nil {
				fmt.Println(err.Error())
				args.resultChan <- getReply{rv: false, err: err}
				_, err = ioutil.ReadAll(res.Body)
				res.Body.Close()
				continue
			}

			_, err = ioutil.ReadAll(res.Body)
			res.Body.Close()

		}

		if args.verbose {

			fmt.Println(res.Status)
			for k, v := range res.Header {
				fmt.Println(k+":", v)
			}

		}

		if res.StatusCode != http.StatusOK {
			args.resultChan <- getReply{rv: false}
		}

		args.resultChan <- getReply{rv: true}

	}

}

func WorkerPool(n int) chan *getArgs {
	requests := make(chan *getArgs)

	for i := 0; i < n; i++ {
		go get(requests)
	}

	return requests
}

func main() {

	status := "OK"
	rv := 0
	name := "Bulk HTTP Check"
	bad := 0
	total := 0

	// this needs improvement. the number of spaces here has to equal the number of chars in the badHosts append line suffix
	badHosts := []byte("  ")

	//verbose := flag.Bool("v", false, "verbose output")
	verbose = flag.Bool("v", false, "verbose output")
	warn := flag.Int("w", 10, "warning level - number of non-200s or percentage of non-200s (default is numeric not percentage)")
	crit := flag.Int("c", 20, "critical level - number of non-200s or percentage of non-200s (default is numeric not percentage)")
	timeout := flag.Int("t", 2, "timeout in seconds - don't wait.  Do Head requests and don't wait.")
	pct := flag.Bool("pct", false, "interpret warming and critical levels are percentages")
	path := flag.String("path", "", "optional path to append to the input lines including the leading slash - these will not be urlencoded. This is ignored is the urls option is given.")
	file := flag.String("file", "", "input data source: a filename or '-' for STDIN.")
	port := flag.Int("port", 80, "optional port for the http request - ignored if urls is specified")
	urls := flag.Bool("urls", false, "Assume the input data is full urls - its normally a list of hostnames")
	auth := flag.String("auth", "", "Do basic auth with this username:passwd - ignored if urls is specified - make this use .netrc instead")
	checkName := flag.String("name", "", "a name to be included in the check output to distinguish the check output")
	silence := flag.Bool("silence", false, "don't make a huge list of all failing checks")
	workers := flag.Int("workers", 5, "how many workers to do the requests")

	flag.Usage = func() {

		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, `
	Read hostnames from a file or STDIN and do a single nagios check over
	them all.  Just check for 200s.  Warning and Critical are either
	percentages of the total, or a regular numeric thresholds.

	The output contains the hostname of any non-200 reporting hosts (see -silence).

	Skip input lines that are commented out with shell style comments
	like /^#/.

	Do Head requests since we don't care about the content.  Make this
	optional some day.

	The -path is appended to the hostnames to make full URLs for the checks.

	If the -urls option is specified, then the input is assumed a complete URL, like http://$hostname:$port/$path.

	Examples:

	./someCommand |  ./check_http_bulk  -w 1 -c 2 -path '/api/aliveness-test/%%2F/' -port 15672 -file - -auth zup:nuch

	./check_http_bulk -urls -file urls.txt

`)

		flag.PrintDefaults()
	}

	flag.Parse()

	if len(flag.Args()) > 0 {

		flag.Usage()
		os.Exit(3)

	}

	// it urls is specified, the input is full urls to be used enmasse and to be url encoded
	if *urls {
		*path = ""
	}

	if *checkName != "" {
		name = *checkName
	}

	defer func() {
		if err := recover(); err != nil {
			fmt.Println(name+" Unknown: ", err)
			os.Exit(3)
		}
	}()

	if file == nil || *file == "" {
		flag.Usage()
		os.Exit(3)
	}

	inputSource := os.Stdin

	if (*file)[0] != "-"[0] {

		var err error

		inputSource, err = os.Open(*file)

		if err != nil {

			fmt.Printf("Couldn't open the specified input file:%s:error:%v:\n\n", name, err)
			flag.Usage()
			os.Exit(3)

		}

	}

	requests := WorkerPool(*workers)
	scanner := bufio.NewScanner(inputSource)

	//leave some room since we start sending to this before we read from it
	repliesChannel := make(chan chan getReply, 1)
	for scanner.Scan() {

		hostname := scanner.Text()

		if len(hostname) == 0 {

			vLogger("skipping blank:\n")

			continue
		}

		if hostname[0] == "#"[0] {

			vLogger("skipping:%s:\n", hostname)

			continue
		}

		total++

		vLogger("working on:%s:\n", hostname)

		thisReplyChan := make(chan getReply)

		//put the reply chan into the chan of reply chans
		repliesChannel <- thisReplyChan
		request := &getArgs{hostname: hostname, port: *port, path: *path, auth: *auth, urls: *urls, verbose: *verbose, timeout: *timeout, resultChan: thisReplyChan}

		//send the request off so the workers can go
		requests <- request

		//get a replyChannel that's ready
		readyReplyChan := <-repliesChannel

		//read a reply
		result := <-readyReplyChan

		err := result.err
		goodCheck := result.rv

		if err != nil {

			fmt.Printf("%s get error: %T %s %#v\n", name, err, err, err)
			if !*silence {
				badHosts = append(badHosts, hostname...)
				badHosts = append(badHosts, ", "...)
			}
			bad++

			continue

		}

		if !goodCheck {
			if !*silence {
				badHosts = append(badHosts, hostname...)
				badHosts = append(badHosts, ", "...)
			}
			bad++
		}

	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
		status = "Unknown"
		rv = 3
	}

	if *pct {

		ratio := int(float64(bad) / float64(total) * 100)

		vLogger("ratio:%d:\n", ratio)

		if ratio >= *crit {
			status = "Critical"
			rv = 2
		} else if ratio >= *warn {
			status = "Warning"
			rv = 1
		}

	} else {

		if bad >= *crit {
			status = "Critical"
			rv = 2
		} else if bad >= *warn {
			status = "Warning"
			rv = 1
		}

	}

	fmt.Printf("%s %s: %d of %d failed|%s\n", name, status, bad, total, badHosts[:len(badHosts)-2])
	os.Exit(rv)
}
<|endoftext|>


<commit_before>package gorocksdb

import (
	"github.com/facebookgo/ensure"
	"io/ioutil"
	"os"
	"testing"
)

func TestCheckpoint(t *testing.T) {

	suffix := "checkpoint"
	dir, err := ioutil.TempDir("", "gorocksdb-"+suffix)
	ensure.Nil(t, err)
	err = os.RemoveAll(dir)
	ensure.Nil(t, err)

	db := newTestDB(t, "TestCheckpoint", nil)
	defer db.Close()

	// insert keys
	givenKeys := [][]byte{[]byte("key1"), []byte("key2"), []byte("key3")}
	givenVal := []byte("val")
	wo := NewDefaultWriteOptions()
	for _, k := range givenKeys {
		ensure.Nil(t, db.Put(wo, k, givenVal))
	}

	var dbCheck *DB
	var checkpoint *Checkpoint

	checkpoint, err = db.NewCheckpoint()
	defer checkpoint.Destroy()
	ensure.NotNil(t, checkpoint)
	ensure.Nil(t, err)

	err = checkpoint.CreateCheckpoint(dir, 0)
	ensure.Nil(t, err)

	opts := NewDefaultOptions()
	opts.SetCreateIfMissing(true)
	dbCheck, err = OpenDb(opts, dir)
	ensure.Nil(t, err)

	// test keys
	var value *Slice
	ro := NewDefaultReadOptions()
	for _, k := range givenKeys {
		value, err = dbCheck.Get(ro, k)
		defer value.Free()
		ensure.Nil(t, err)
		ensure.DeepEqual(t, value.Data(), givenVal)
	}

}
<commit_msg>naming and checkpoint test<commit_after>package gorocksdb

import (
	"github.com/facebookgo/ensure"
	"io/ioutil"
	"os"
	"testing"
)

func TestCheckpoint(t *testing.T) {

	suffix := "checkpoint"
	dir, err := ioutil.TempDir("", "gorocksdb-"+suffix)
	ensure.Nil(t, err)
	err = os.RemoveAll(dir)
	ensure.Nil(t, err)

	db := newTestDB(t, "TestCheckpoint", nil)
	defer db.Close()

	// insert keys
	givenKeys := [][]byte{[]byte("key1"), []byte("key2"), []byte("key3")}
	givenVal := []byte("val")
	wo := NewDefaultWriteOptions()
	for _, k := range givenKeys {
		ensure.Nil(t, db.Put(wo, k, givenVal))
	}

	var dbCheck *DB
	var checkpoint *Checkpoint

	checkpoint, err = db.NewCheckpoint()
	defer checkpoint.Destroy()
	ensure.NotNil(t, checkpoint)
	ensure.Nil(t, err)

	err = checkpoint.CreateCheckpoint(dir, 0)
	ensure.Nil(t, err)

	opts := NewDefaultOptions()
	opts.SetCreateIfMissing(true)
	dbCheck, err = OpenDb(opts, dir)
	defer dbCheck.Close()
	ensure.Nil(t, err)

	// test keys
	var value *Slice
	ro := NewDefaultReadOptions()
	for _, k := range givenKeys {
		value, err = dbCheck.Get(ro, k)
		defer value.Free()
		ensure.Nil(t, err)
		ensure.DeepEqual(t, value.Data(), givenVal)
	}

}
<|endoftext|>


<commit_before>package electionday

// NewVoteCounter returns a new vote counter with
// a given number of inital votes.
func NewVoteCounter(initialVotes int) *int {
	panic("Please implement the NewVoteCounter() function")
}

// VoteCount extracts the number of votes from a counter.
func VoteCount(counter *int) int {
	panic("Please implement the VoteCount() function")
}

// IncrementVoteCount increments the value in a vote counter
func IncrementVoteCount(counter *int, increment int) {
	panic("Please implement the IncrementVoteCount() function")
}

// NewElectionResult creates a new election result
func NewElectionResult(candidateName string, votes int) *ElectionResult {
	panic("Please implement the NewElectionResult() function")
}

// DisplayResult creates a message with the result to be displayed
func DisplayResult(result *ElectionResult) string {
	panic("Please implement the DisplayResult() function")
}

// DecrementVotesOfCandidate decrements by one the vote count of a candidate in a map
func DecrementVotesOfCandidate(results map[string]int, candidate string) {
	panic("Please implement the DecrementVotesOfCandidate() function")
}
<commit_msg>Solve election day<commit_after>package electionday

import "fmt"

// NewVoteCounter returns a new vote counter with
// a given number of inital votes.
func NewVoteCounter(initialVotes int) *int {
	return &initialVotes
}

// VoteCount extracts the number of votes from a counter.
func VoteCount(counter *int) int {
	if counter == nil {
		return 0
	}
	return *counter
}

// IncrementVoteCount increments the value in a vote counter
func IncrementVoteCount(counter *int, increment int) {
	*counter += increment
}

// NewElectionResult creates a new election result
func NewElectionResult(candidateName string, votes int) *ElectionResult {
	return &ElectionResult{
		Name:  candidateName,
		Votes: votes,
	}
}

// DisplayResult creates a message with the result to be displayed
func DisplayResult(result *ElectionResult) string {
	return fmt.Sprintf("%s (%d)", result.Name, result.Votes)
}

// DecrementVotesOfCandidate decrements by one the vote count of a candidate in a map
func DecrementVotesOfCandidate(results map[string]int, candidate string) {
	results[candidate] -= 1
}
<|endoftext|>


<commit_before>package google

import (
	"errors"
	"fmt"

	"time"

	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
)

const readyStatus string = "Ready"

type Condition struct {
	Type    string
	Status  string
	Reason  string
	Message string
}

// KnativeStatus is a struct that can contain a Knative style resource's Status block. It is not
// intended to be used for anything other than polling for the success of the given resource.
type KnativeStatus struct {
	Metadata struct {
		Name      string
		Namespace string
		SelfLink  string
	}
	Status struct {
		Conditions []Condition
	}
}

// ConditionByType is a helper method for extracting a given condition
func (s KnativeStatus) ConditionByType(typ string) *Condition {
	for _, condition := range s.Status.Conditions {
		if condition.Type == typ {
			c := condition
			return &c
		}
	}
	return nil
}

// LatestMessage will return a human consumable status of the resource. This can
// be used to determine the human actionable error the GET doesn't return an explicit
// error but the resource is in an error state.
func (s KnativeStatus) LatestMessage() string {
	c := s.ConditionByType(readyStatus)
	if c != nil {
		return fmt.Sprintf("%s - %s", c.Reason, c.Message)
	}

	return ""
}

// State will return a string representing the status of the Ready condition.
// No other conditions are currently returned as part of the state.
func (s KnativeStatus) State(res interface{}) string {
	for _, condition := range s.Status.Conditions {
		if condition.Type == "Ready" {
			return fmt.Sprintf("%s:%s", condition.Type, condition.Status)
		}
	}
	return "Empty"
}

// CloudRunPolling allows for polling against a cloud run resource that implements the
// Kubernetes style status schema.
type CloudRunPolling struct {
	Config  *Config
	WaitURL string
}

func (p *CloudRunPolling) PendingStates() []string {
	return []string{"Ready:Unknown", "Empty"}
}
func (p *CloudRunPolling) TargetStates() []string {
	return []string{"Ready:True"}
}
func (p *CloudRunPolling) ErrorStates() []string {
	return []string{"Ready:False"}
}

func cloudRunPollingWaitTime(config *Config, res map[string]interface{}, project, url, activity string, timeoutMinutes int) error {
	w := &CloudRunPolling{}

	scc := &resource.StateChangeConf{
		Pending: w.PendingStates(),
		Target:  w.TargetStates(),
		Refresh: func() (interface{}, string, error) {
			res, err := sendRequest(config, "GET", project, url, nil)
			if err != nil {
				return res, "", err
			}

			status := KnativeStatus{}
			err = Convert(res, &status)
			if err != nil {
				return res, "", err
			}

			for _, errState := range w.ErrorStates() {
				if status.State(res) == errState {
					err = errors.New(status.LatestMessage())
				}
			}

			return res, status.State(res), err
		},
		Timeout: time.Duration(timeoutMinutes) * time.Minute,
	}

	_, err := scc.WaitForState()
	return err
}
<commit_msg>Allow domain mapping to succeed if DNS is pending<commit_after>package google

import (
	"errors"
	"fmt"

	"time"

	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
)

const readyStatus string = "Ready"

type Condition struct {
	Type    string
	Status  string
	Reason  string
	Message string
}

// KnativeStatus is a struct that can contain a Knative style resource's Status block. It is not
// intended to be used for anything other than polling for the success of the given resource.
type KnativeStatus struct {
	Metadata struct {
		Name      string
		Namespace string
		SelfLink  string
	}
	Status struct {
		Conditions []Condition
	}
}

// ConditionByType is a helper method for extracting a given condition
func (s KnativeStatus) ConditionByType(typ string) *Condition {
	for _, condition := range s.Status.Conditions {
		if condition.Type == typ {
			c := condition
			return &c
		}
	}
	return nil
}

// LatestMessage will return a human consumable status of the resource. This can
// be used to determine the human actionable error the GET doesn't return an explicit
// error but the resource is in an error state.
func (s KnativeStatus) LatestMessage() string {
	c := s.ConditionByType(readyStatus)
	if c != nil {
		return fmt.Sprintf("%s - %s", c.Reason, c.Message)
	}

	return ""
}

// State will return a string representing the status of the Ready condition.
// No other conditions are currently returned as part of the state.
func (s KnativeStatus) State(res interface{}) string {
	for _, condition := range s.Status.Conditions {
		if condition.Type == "Ready" {
			// DomainMapping can enter a 'terminal' state of waiting for external verification
			// of DNS records.
			if condition.Reason == "CertificatePending" {
				return "Ready:CertificatePending"
			}
			return fmt.Sprintf("%s:%s", condition.Type, condition.Status)
		}
	}
	return "Empty"
}

// CloudRunPolling allows for polling against a cloud run resource that implements the
// Kubernetes style status schema.
type CloudRunPolling struct {
	Config  *Config
	WaitURL string
}

func (p *CloudRunPolling) PendingStates() []string {
	return []string{"Ready:Unknown", "Empty"}
}
func (p *CloudRunPolling) TargetStates() []string {
	return []string{"Ready:True", "Ready:CertificatePending"}
}
func (p *CloudRunPolling) ErrorStates() []string {
	return []string{"Ready:False"}
}

func cloudRunPollingWaitTime(config *Config, res map[string]interface{}, project, url, activity string, timeoutMinutes int) error {
	w := &CloudRunPolling{}

	scc := &resource.StateChangeConf{
		Pending: w.PendingStates(),
		Target:  w.TargetStates(),
		Refresh: func() (interface{}, string, error) {
			res, err := sendRequest(config, "GET", project, url, nil)
			if err != nil {
				return res, "", err
			}

			status := KnativeStatus{}
			err = Convert(res, &status)
			if err != nil {
				return res, "", err
			}

			for _, errState := range w.ErrorStates() {
				if status.State(res) == errState {
					err = errors.New(status.LatestMessage())
				}
			}

			return res, status.State(res), err
		},
		Timeout: time.Duration(timeoutMinutes) * time.Minute,
	}

	_, err := scc.WaitForState()
	return err
}
<|endoftext|>


<commit_before>package cluster

import (
	log "code.google.com/p/log4go"
	"parser"
	"protocol"
	"time"
	"wal"
)

// A shard imements an interface for writing and querying data.
// It can be copied to multiple servers.
// Shard contains data from [startTime, endTime)
// Ids are unique across the cluster
type Shard interface {
	Id() uint32
	StartTime() time.Time
	EndTime() time.Time
	Write(*protocol.Request) error
	Query(*parser.Query, chan *protocol.Response) error
}

type ShardData struct {
	id             uint32
	startTime      time.Time
	endTime        time.Time
	wal            WAL
	clusterServers []*ClusterServer
	servers        []wal.Server
	store          LocalShardStore
	localWrites    chan *protocol.Request
	serverWrites   map[*ClusterServer]chan *protocol.Request
}

func NewShard(id uint32, startTime, endTime time.Time) *ShardData {
	return &ShardData{id: id, startTime: startTime, endTime: endTime}
}

const (
	PER_SERVER_BUFFER_SIZE  = 10
	LOCAL_WRITE_BUFFER_SIZE = 10
)

type LocalShardDb interface {
	Write(database string, series *protocol.Series) error
	Query(*parser.Query, chan *protocol.Response) error
}

type LocalShardStore interface {
	GetOrCreateShard(id uint32) (LocalShardDb, error)
}

func (self *ShardData) Id() uint32 {
	return self.id
}

func (self *ShardData) StartTime() time.Time {
	return self.startTime
}

func (self *ShardData) EndTime() time.Time {
	return self.endTime
}

func (self *ShardData) SetServers(servers []*ClusterServer) {
	self.clusterServers = servers

	// Doing this thing with the serveres because wal.AssignSequenceNumbersAndLog expects a slice of
	// wal.Server interface and go won't do the conversion to *ClusterServer automatically.
	// See: http://stackoverflow.com/questions/12994679/golang-slice-of-struct-slice-of-interface-it-implements
	self.servers = make([]wal.Server, len(servers), len(servers))
	self.serverWrites = make(map[*ClusterServer]chan *protocol.Request)
	for i, server := range servers {
		self.servers[i] = server
		writeBuffer := make(chan *protocol.Request, PER_SERVER_BUFFER_SIZE)
		go self.handleWritesToServer(server, writeBuffer)
		self.serverWrites[server] = writeBuffer
	}
}

func (self *ShardData) SetLocalStore(store LocalShardStore) {
	self.store = store
	self.localWrites = make(chan *protocol.Request, LOCAL_WRITE_BUFFER_SIZE)
	go self.handleLocalWrites()
}

func (self *ShardData) Write(request *protocol.Request) error {
	requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self, self.servers)
	if err != nil {
		return err
	}
	request.RequestNumber = &requestNumber
	if self.store != nil {
		self.localWrites <- request
	}
	for _, writeBuffer := range self.serverWrites {
		writeBuffer <- request
	}
	return nil
}

func (self *ShardData) Query(*parser.Query, chan *protocol.Response) error {
	return nil
}

func (self *ShardData) handleWritesToServer(server *ClusterServer, writeBuffer chan *protocol.Request) {
	responseStream := make(chan *protocol.Response)
	for {
		request := <-writeBuffer
		requestNumber := *request.RequestNumber
		// this doens't need to be sent to the remote server, we just keep it around for the WAL commit
		request.SequenceNumber = nil

		// TODO: make some sort of timeout for this response along with a replay from the WAL.
		// Basically, if the server is in a timeout state do the following:
		// * keep pulling requests from the writeBuffer and throw them on the ground. Or keep some in memory and then toss
		// * check periodically for the server to come back. when it has:
		// * self.WAL.RecoverServerFromRequestNumber(requestNumber, server, yield func(request *protocol.Request, shard wal.Shard) error)
		// * once all those have been sent to the server, resume sending requests
		server.MakeRequest(request, responseStream)
		response := <-responseStream
		if *response.Type == protocol.Response_WRITE_OK {
			self.wal.Commit(requestNumber, server)
		} else {
			// TODO: retry logic for failed request
			log.Error("REQUEST to server %s failed:: ", server.ProtobufConnectionString, response.GetErrorMessage())
		}
	}
}

func (self *ShardData) handleLocalWrites() {
	for {
		request := <-self.localWrites
		shard, err := self.store.GetOrCreateShard(self.id)
		if err != nil {
			log.Error("Creating shard %d: %s", self.id, err)
		}
		err = shard.Write(*request.Database, request.Series)
		if err != nil {
			log.Error("Writing to local shard: ", err)
		}
	}
}
<commit_msg>Remove cluster servers from Shard since it's not used<commit_after>package cluster

import (
	log "code.google.com/p/log4go"
	"parser"
	"protocol"
	"time"
	"wal"
)

// A shard imements an interface for writing and querying data.
// It can be copied to multiple servers.
// Shard contains data from [startTime, endTime)
// Ids are unique across the cluster
type Shard interface {
	Id() uint32
	StartTime() time.Time
	EndTime() time.Time
	Write(*protocol.Request) error
	Query(*parser.Query, chan *protocol.Response) error
}

type ShardData struct {
	id           uint32
	startTime    time.Time
	endTime      time.Time
	wal          WAL
	servers      []wal.Server
	store        LocalShardStore
	localWrites  chan *protocol.Request
	serverWrites map[*ClusterServer]chan *protocol.Request
}

func NewShard(id uint32, startTime, endTime time.Time) *ShardData {
	return &ShardData{id: id, startTime: startTime, endTime: endTime}
}

const (
	PER_SERVER_BUFFER_SIZE  = 10
	LOCAL_WRITE_BUFFER_SIZE = 10
)

type LocalShardDb interface {
	Write(database string, series *protocol.Series) error
	Query(*parser.Query, chan *protocol.Response) error
}

type LocalShardStore interface {
	GetOrCreateShard(id uint32) (LocalShardDb, error)
}

func (self *ShardData) Id() uint32 {
	return self.id
}

func (self *ShardData) StartTime() time.Time {
	return self.startTime
}

func (self *ShardData) EndTime() time.Time {
	return self.endTime
}

func (self *ShardData) SetServers(servers []*ClusterServer) {
	self.servers = make([]wal.Server, len(servers), len(servers))
	self.serverWrites = make(map[*ClusterServer]chan *protocol.Request)
	for i, server := range servers {
		self.servers[i] = server
		writeBuffer := make(chan *protocol.Request, PER_SERVER_BUFFER_SIZE)
		go self.handleWritesToServer(server, writeBuffer)
		self.serverWrites[server] = writeBuffer
	}
}

func (self *ShardData) SetLocalStore(store LocalShardStore) {
	self.store = store
	self.localWrites = make(chan *protocol.Request, LOCAL_WRITE_BUFFER_SIZE)
	go self.handleLocalWrites()
}

func (self *ShardData) Write(request *protocol.Request) error {
	requestNumber, err := self.wal.AssignSequenceNumbersAndLog(request, self, self.servers)
	if err != nil {
		return err
	}
	request.RequestNumber = &requestNumber
	if self.store != nil {
		self.localWrites <- request
	}
	for _, writeBuffer := range self.serverWrites {
		writeBuffer <- request
	}
	return nil
}

func (self *ShardData) Query(*parser.Query, chan *protocol.Response) error {
	return nil
}

func (self *ShardData) handleWritesToServer(server *ClusterServer, writeBuffer chan *protocol.Request) {
	responseStream := make(chan *protocol.Response)
	for {
		request := <-writeBuffer
		requestNumber := *request.RequestNumber
		// this doens't need to be sent to the remote server, we just keep it around for the WAL commit
		request.SequenceNumber = nil

		// TODO: make some sort of timeout for this response along with a replay from the WAL.
		// Basically, if the server is in a timeout state do the following:
		// * keep pulling requests from the writeBuffer and throw them on the ground. Or keep some in memory and then toss
		// * check periodically for the server to come back. when it has:
		// * self.WAL.RecoverServerFromRequestNumber(requestNumber, server, yield func(request *protocol.Request, shard wal.Shard) error)
		// * once all those have been sent to the server, resume sending requests
		server.MakeRequest(request, responseStream)
		response := <-responseStream
		if *response.Type == protocol.Response_WRITE_OK {
			self.wal.Commit(requestNumber, server)
		} else {
			// TODO: retry logic for failed request
			log.Error("REQUEST to server %s failed:: ", server.ProtobufConnectionString, response.GetErrorMessage())
		}
	}
}

func (self *ShardData) handleLocalWrites() {
	for {
		request := <-self.localWrites
		shard, err := self.store.GetOrCreateShard(self.id)
		if err != nil {
			log.Error("Creating shard %d: %s", self.id, err)
		}
		err = shard.Write(*request.Database, request.Series)
		if err != nil {
			log.Error("Writing to local shard: ", err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/fsouza/go-dockerclient"

	"go.pedge.io/env"
	"go.pedge.io/proto/server"

	"go.pachyderm.com/pachyderm"
	"go.pachyderm.com/pachyderm/src/pfs"
	"go.pachyderm.com/pachyderm/src/pkg/container"
	"go.pachyderm.com/pachyderm/src/pps"
	"go.pachyderm.com/pachyderm/src/pps/persist"
	persistserver "go.pachyderm.com/pachyderm/src/pps/persist/server"
	"go.pachyderm.com/pachyderm/src/pps/server"
	"go.pachyderm.com/pachyderm/src/pps/watch"
	watchserver "go.pachyderm.com/pachyderm/src/pps/watch/server"
	"google.golang.org/grpc"
)

var (
	defaultEnv = map[string]string{
		"PPS_ADDRESS":       "0.0.0.0",
		"PPS_PORT":          "651",
		"PPS_TRACE_PORT":    "1051",
		"PPS_DATABASE_NAME": "pachyderm",
	}
)

type appEnv struct {
	PachydermPfsd1Port string `env:"PACHYDERM_PFSD_1_PORT"`
	PfsAddress         string `env:"PFS_ADDRESS"`
	Address            string `env:"PPS_ADDRESS"`
	Port               int    `env:"PPS_PORT"`
	DatabaseAddress    string `env:"PPS_DATABASE_ADDRESS"`
	DatabaseName       string `env:"PPS_DATABASE_NAME"`
	DebugPort          int    `env:"PPS_TRACE_PORT"`
}

func main() {
	env.Main(do, &appEnv{}, defaultEnv)
}

func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	containerClient, err := getContainerClient()
	if err != nil {
		return err
	}
	rethinkAPIClient, err := getRethinkAPIClient(appEnv.DatabaseAddress, appEnv.DatabaseName)
	if err != nil {
		return err
	}
	pfsAddress := appEnv.PachydermPfsd1Port
	if pfsAddress == "" {
		pfsAddress = appEnv.PfsAddress
	} else {
		pfsAddress = strings.Replace(pfsAddress, "tcp://", "", -1)
	}
	clientConn, err := grpc.Dial(pfsAddress, grpc.WithInsecure())
	if err != nil {
		return err
	}
	pfsAPIClient := pfs.NewApiClient(clientConn)
	watchAPIServer := watchserver.NewAPIServer(pfsAPIClient, rethinkAPIClient)
	watchAPIClient := watch.NewLocalAPIClient(watchAPIServer)
	return protoserver.Serve(
		uint16(appEnv.Port),
		func(s *grpc.Server) {
			pps.RegisterAPIServer(s, server.NewAPIServer(rethinkAPIClient, watchAPIClient, containerClient))
		},
		protoserver.ServeOptions{
			DebugPort: uint16(appEnv.DebugPort),
			Version:   pachyderm.Version,
		},
	)
}

func getContainerClient() (container.Client, error) {
	client, err := docker.NewClientFromEnv()
	if err != nil {
		return nil, err
	}
	return container.NewDockerClient(client), nil
}

func getRethinkAPIClient(address string, databaseName string) (persist.APIClient, error) {
	var err error
	if address == "" {
		address, err = getRethinkAddress()
		if err != nil {
			return nil, err
		}
	}
	if err := persistserver.InitDBs(address, databaseName); err != nil {
		return nil, err
	}
	rethinkAPIServer, err := persistserver.NewRethinkAPIServer(address, databaseName)
	if err != nil {
		return nil, err
	}
	return persist.NewLocalAPIClient(rethinkAPIServer), nil
}

func getRethinkAddress() (string, error) {
	rethinkAddr := os.Getenv("RETHINK_PORT_28015_TCP_ADDR")
	if rethinkAddr == "" {
		return "", errors.New("RETHINK_PORT_28015_TCP_ADDR not set")
	}
	return fmt.Sprintf("%s:28015", rethinkAddr), nil
}
<commit_msg>start watch api client from ppsd<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"golang.org/x/net/context"

	"github.com/fsouza/go-dockerclient"

	"go.pedge.io/env"
	"go.pedge.io/google-protobuf"
	"go.pedge.io/proto/server"

	"go.pachyderm.com/pachyderm"
	"go.pachyderm.com/pachyderm/src/pfs"
	"go.pachyderm.com/pachyderm/src/pkg/container"
	"go.pachyderm.com/pachyderm/src/pps"
	"go.pachyderm.com/pachyderm/src/pps/persist"
	persistserver "go.pachyderm.com/pachyderm/src/pps/persist/server"
	"go.pachyderm.com/pachyderm/src/pps/server"
	"go.pachyderm.com/pachyderm/src/pps/watch"
	watchserver "go.pachyderm.com/pachyderm/src/pps/watch/server"
	"google.golang.org/grpc"
)

var (
	defaultEnv = map[string]string{
		"PPS_ADDRESS":       "0.0.0.0",
		"PPS_PORT":          "651",
		"PPS_TRACE_PORT":    "1051",
		"PPS_DATABASE_NAME": "pachyderm",
	}
)

type appEnv struct {
	PachydermPfsd1Port string `env:"PACHYDERM_PFSD_1_PORT"`
	PfsAddress         string `env:"PFS_ADDRESS"`
	Address            string `env:"PPS_ADDRESS"`
	Port               int    `env:"PPS_PORT"`
	DatabaseAddress    string `env:"PPS_DATABASE_ADDRESS"`
	DatabaseName       string `env:"PPS_DATABASE_NAME"`
	DebugPort          int    `env:"PPS_TRACE_PORT"`
}

func main() {
	env.Main(do, &appEnv{}, defaultEnv)
}

func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	containerClient, err := getContainerClient()
	if err != nil {
		return err
	}
	rethinkAPIClient, err := getRethinkAPIClient(appEnv.DatabaseAddress, appEnv.DatabaseName)
	if err != nil {
		return err
	}
	pfsAddress := appEnv.PachydermPfsd1Port
	if pfsAddress == "" {
		pfsAddress = appEnv.PfsAddress
	} else {
		pfsAddress = strings.Replace(pfsAddress, "tcp://", "", -1)
	}
	clientConn, err := grpc.Dial(pfsAddress, grpc.WithInsecure())
	if err != nil {
		return err
	}
	pfsAPIClient := pfs.NewApiClient(clientConn)
	watchAPIServer := watchserver.NewAPIServer(pfsAPIClient, rethinkAPIClient)
	watchAPIClient := watch.NewLocalAPIClient(watchAPIServer)
	if _, err := watchAPIClient.Start(context.Background(), &google_protobuf.Empty{}); err != nil {
		return err
	}
	return protoserver.Serve(
		uint16(appEnv.Port),
		func(s *grpc.Server) {
			pps.RegisterAPIServer(s, server.NewAPIServer(rethinkAPIClient, watchAPIClient, containerClient))
		},
		protoserver.ServeOptions{
			DebugPort: uint16(appEnv.DebugPort),
			Version:   pachyderm.Version,
		},
	)
}

func getContainerClient() (container.Client, error) {
	client, err := docker.NewClientFromEnv()
	if err != nil {
		return nil, err
	}
	return container.NewDockerClient(client), nil
}

func getRethinkAPIClient(address string, databaseName string) (persist.APIClient, error) {
	var err error
	if address == "" {
		address, err = getRethinkAddress()
		if err != nil {
			return nil, err
		}
	}
	if err := persistserver.InitDBs(address, databaseName); err != nil {
		return nil, err
	}
	rethinkAPIServer, err := persistserver.NewRethinkAPIServer(address, databaseName)
	if err != nil {
		return nil, err
	}
	return persist.NewLocalAPIClient(rethinkAPIServer), nil
}

func getRethinkAddress() (string, error) {
	rethinkAddr := os.Getenv("RETHINK_PORT_28015_TCP_ADDR")
	if rethinkAddr == "" {
		return "", errors.New("RETHINK_PORT_28015_TCP_ADDR not set")
	}
	return fmt.Sprintf("%s:28015", rethinkAddr), nil
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains the printf-checker.

package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/token"
	"strings"
	"unicode/utf8"
)

var printfuncs = flag.String("printfuncs", "", "comma-separated list of print function names to check")

// printfList records the formatted-print functions. The value is the location
// of the format parameter. Names are lower-cased so the lookup is
// case insensitive.
var printfList = map[string]int{
	"errorf":  0,
	"fatalf":  0,
	"fprintf": 1,
	"panicf":  0,
	"printf":  0,
	"sprintf": 0,
}

// printList records the unformatted-print functions. The value is the location
// of the first parameter to be printed.  Names are lower-cased so the lookup is
// case insensitive.
var printList = map[string]int{
	"error":  0,
	"fatal":  0,
	"fprint": 1, "fprintln": 1,
	"panic": 0, "panicln": 0,
	"print": 0, "println": 0,
	"sprint": 0, "sprintln": 0,
}

// checkCall triggers the print-specific checks if the call invokes a print function.
func (f *File) checkFmtPrintfCall(call *ast.CallExpr, Name string) {
	if !*vetPrintf && !*vetAll {
		return
	}
	name := strings.ToLower(Name)
	if skip, ok := printfList[name]; ok {
		f.checkPrintf(call, Name, skip)
		return
	}
	if skip, ok := printList[name]; ok {
		f.checkPrint(call, Name, skip)
		return
	}
}

// literal returns the literal value represented by the expression, or nil if it is not a literal.
func (f *File) literal(value ast.Expr) *ast.BasicLit {
	switch v := value.(type) {
	case *ast.BasicLit:
		return v
	case *ast.Ident:
		// See if it's a constant or initial value (we can't tell the difference).
		if v.Obj == nil || v.Obj.Decl == nil {
			return nil
		}
		valueSpec, ok := v.Obj.Decl.(*ast.ValueSpec)
		if ok && len(valueSpec.Names) == len(valueSpec.Values) {
			// Find the index in the list of names
			var i int
			for i = 0; i < len(valueSpec.Names); i++ {
				if valueSpec.Names[i].Name == v.Name {
					if lit, ok := valueSpec.Values[i].(*ast.BasicLit); ok {
						return lit
					}
					return nil
				}
			}
		}
	}
	return nil
}

// checkPrintf checks a call to a formatted print routine such as Printf.
// The skip argument records how many arguments to ignore; that is,
// call.Args[skip] is (well, should be) the format argument.
func (f *File) checkPrintf(call *ast.CallExpr, name string, skip int) {
	if len(call.Args) <= skip {
		return
	}
	lit := f.literal(call.Args[skip])
	if lit == nil {
		if *verbose {
			f.Warn(call.Pos(), "can't check non-literal format in call to", name)
		}
		return
	}
	if lit.Kind != token.STRING {
		f.Badf(call.Pos(), "literal %v not a string in call to", lit.Value, name)
	}
	format := lit.Value
	if !strings.Contains(format, "%") {
		if len(call.Args) > skip+1 {
			f.Badf(call.Pos(), "no formatting directive in %s call", name)
		}
		return
	}
	// Hard part: check formats against args.
	// Trivial but useful test: count.
	numArgs := 0
	for i, w := 0, 0; i < len(format); i += w {
		w = 1
		if format[i] == '%' {
			nbytes, nargs := f.parsePrintfVerb(call, format[i:])
			w = nbytes
			numArgs += nargs
		}
	}
	expect := len(call.Args) - (skip + 1)
	// Don't be too strict on dotdotdot.
	if call.Ellipsis.IsValid() && numArgs >= expect {
		return
	}
	if numArgs != expect {
		f.Badf(call.Pos(), "wrong number of args in %s call: %d needed but %d args", name, numArgs, expect)
	}
}

// parsePrintfVerb returns the number of bytes and number of arguments
// consumed by the Printf directive that begins s, including its percent sign
// and verb.
func (f *File) parsePrintfVerb(call *ast.CallExpr, s string) (nbytes, nargs int) {
	// There's guaranteed a percent sign.
	flags := make([]byte, 0, 5)
	nbytes = 1
	end := len(s)
	// There may be flags.
FlagLoop:
	for nbytes < end {
		switch s[nbytes] {
		case '#', '0', '+', '-', ' ':
			flags = append(flags, s[nbytes])
			nbytes++
		default:
			break FlagLoop
		}
	}
	getNum := func() {
		if nbytes < end && s[nbytes] == '*' {
			nbytes++
			nargs++
		} else {
			for nbytes < end && '0' <= s[nbytes] && s[nbytes] <= '9' {
				nbytes++
			}
		}
	}
	// There may be a width.
	getNum()
	// If there's a period, there may be a precision.
	if nbytes < end && s[nbytes] == '.' {
		flags = append(flags, '.') // Treat precision as a flag.
		nbytes++
		getNum()
	}
	// Now a verb.
	c, w := utf8.DecodeRuneInString(s[nbytes:])
	nbytes += w
	if c != '%' {
		nargs++
		f.checkPrintfVerb(call, c, flags)
	}
	return
}

type printVerb struct {
	verb  rune
	flags string // known flags are all ASCII
}

// Common flag sets for printf verbs.
const (
	numFlag      = " -+.0"
	sharpNumFlag = " -+.0#"
	allFlags     = " -+.0#"
)

// printVerbs identifies which flags are known to printf for each verb.
// TODO: A type that implements Formatter may do what it wants, and vet
// will complain incorrectly.
var printVerbs = []printVerb{
	// '-' is a width modifier, always valid.
	// '.' is a precision for float, max width for strings.
	// '+' is required sign for numbers, Go format for %v.
	// '#' is alternate format for several verbs.
	// ' ' is spacer for numbers
	{'b', numFlag},
	{'c', "-"},
	{'d', numFlag},
	{'e', numFlag},
	{'E', numFlag},
	{'f', numFlag},
	{'F', numFlag},
	{'g', numFlag},
	{'G', numFlag},
	{'o', sharpNumFlag},
	{'p', "-#"},
	{'q', " -+.0#"},
	{'s', " -+.0"},
	{'t', "-"},
	{'T', "-"},
	{'U', "-#"},
	{'v', allFlags},
	{'x', sharpNumFlag},
	{'X', sharpNumFlag},
}

const printfVerbs = "bcdeEfFgGopqstTvxUX"

func (f *File) checkPrintfVerb(call *ast.CallExpr, verb rune, flags []byte) {
	// Linear scan is fast enough for a small list.
	for _, v := range printVerbs {
		if v.verb == verb {
			for _, flag := range flags {
				if !strings.ContainsRune(v.flags, rune(flag)) {
					f.Badf(call.Pos(), "unrecognized printf flag for verb %q: %q", verb, flag)
				}
			}
			return
		}
	}
	f.Badf(call.Pos(), "unrecognized printf verb %q", verb)
}

// checkPrint checks a call to an unformatted print routine such as Println.
// The skip argument records how many arguments to ignore; that is,
// call.Args[skip] is the first argument to be printed.
func (f *File) checkPrint(call *ast.CallExpr, name string, skip int) {
	isLn := strings.HasSuffix(name, "ln")
	isF := strings.HasPrefix(name, "F")
	args := call.Args
	// check for Println(os.Stderr, ...)
	if skip == 0 && !isF && len(args) > 0 {
		if sel, ok := args[0].(*ast.SelectorExpr); ok {
			if x, ok := sel.X.(*ast.Ident); ok {
				if x.Name == "os" && strings.HasPrefix(sel.Sel.Name, "Std") {
					f.Warnf(call.Pos(), "first argument to %s is %s.%s", name, x.Name, sel.Sel.Name)
				}
			}
		}
	}
	if len(args) <= skip {
		if *verbose && !isLn {
			f.Badf(call.Pos(), "no args in %s call", name)
		}
		return
	}
	arg := args[skip]
	if lit, ok := arg.(*ast.BasicLit); ok && lit.Kind == token.STRING {
		if strings.Contains(lit.Value, "%") {
			f.Badf(call.Pos(), "possible formatting directive in %s call", name)
		}
	}
	if isLn {
		// The last item, if a string, should not have a newline.
		arg = args[len(call.Args)-1]
		if lit, ok := arg.(*ast.BasicLit); ok && lit.Kind == token.STRING {
			if strings.HasSuffix(lit.Value, `\n"`) {
				f.Badf(call.Pos(), "%s call ends with newline", name)
			}
		}
	}
}

// This function never executes, but it serves as a simple test for the program.
// Test with make test.
func BadFunctionUsedInTests() {
	fmt.Println()                      // not an error
	fmt.Println("%s", "hi")            // ERROR "possible formatting directive in Println call"
	fmt.Printf("%s", "hi", 3)          // ERROR "wrong number of args in Printf call"
	fmt.Printf("%s%%%d", "hi", 3)      // correct
	fmt.Printf("%08s", "woo")          // correct
	fmt.Printf("% 8s", "woo")          // correct
	fmt.Printf("%.*d", 3, 3)           // correct
	fmt.Printf("%.*d", 3, 3, 3)        // ERROR "wrong number of args in Printf call"
	fmt.Printf("%q %q", multi()...)    // ok
	fmt.Printf("%#q", `blah`)          // ok
	printf("now is the time", "buddy") // ERROR "no formatting directive"
	Printf("now is the time", "buddy") // ERROR "no formatting directive"
	Printf("hi")                       // ok
	const format = "%s %s\n"
	Printf(format, "hi", "there")
	Printf(format, "hi") // ERROR "wrong number of args in Printf call"
	f := new(File)
	f.Warn(0, "%s", "hello", 3)  // ERROR "possible formatting directive in Warn call"
	f.Warnf(0, "%s", "hello", 3) // ERROR "wrong number of args in Warnf call"
	f.Warnf(0, "%r", "hello")    // ERROR "unrecognized printf verb"
	f.Warnf(0, "%#s", "hello")   // ERROR "unrecognized printf flag"
}

// printf is used by the test.
func printf(format string, args ...interface{}) {
	panic("don't call - testing only")
}

// multi is used by the test.
func multi() []interface{} {
	panic("don't call - testing only")
}
<commit_msg>cmd/vet: don't complain about Error() Fixes issue 4598.<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains the printf-checker.

package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/token"
	"strings"
	"unicode/utf8"
)

var printfuncs = flag.String("printfuncs", "", "comma-separated list of print function names to check")

// printfList records the formatted-print functions. The value is the location
// of the format parameter. Names are lower-cased so the lookup is
// case insensitive.
var printfList = map[string]int{
	"errorf":  0,
	"fatalf":  0,
	"fprintf": 1,
	"panicf":  0,
	"printf":  0,
	"sprintf": 0,
}

// printList records the unformatted-print functions. The value is the location
// of the first parameter to be printed.  Names are lower-cased so the lookup is
// case insensitive.
var printList = map[string]int{
	"error":  0,
	"fatal":  0,
	"fprint": 1, "fprintln": 1,
	"panic": 0, "panicln": 0,
	"print": 0, "println": 0,
	"sprint": 0, "sprintln": 0,
}

// checkCall triggers the print-specific checks if the call invokes a print function.
func (f *File) checkFmtPrintfCall(call *ast.CallExpr, Name string) {
	if !*vetPrintf && !*vetAll {
		return
	}
	name := strings.ToLower(Name)
	if skip, ok := printfList[name]; ok {
		f.checkPrintf(call, Name, skip)
		return
	}
	if skip, ok := printList[name]; ok {
		f.checkPrint(call, Name, skip)
		return
	}
}

// literal returns the literal value represented by the expression, or nil if it is not a literal.
func (f *File) literal(value ast.Expr) *ast.BasicLit {
	switch v := value.(type) {
	case *ast.BasicLit:
		return v
	case *ast.Ident:
		// See if it's a constant or initial value (we can't tell the difference).
		if v.Obj == nil || v.Obj.Decl == nil {
			return nil
		}
		valueSpec, ok := v.Obj.Decl.(*ast.ValueSpec)
		if ok && len(valueSpec.Names) == len(valueSpec.Values) {
			// Find the index in the list of names
			var i int
			for i = 0; i < len(valueSpec.Names); i++ {
				if valueSpec.Names[i].Name == v.Name {
					if lit, ok := valueSpec.Values[i].(*ast.BasicLit); ok {
						return lit
					}
					return nil
				}
			}
		}
	}
	return nil
}

// checkPrintf checks a call to a formatted print routine such as Printf.
// The skip argument records how many arguments to ignore; that is,
// call.Args[skip] is (well, should be) the format argument.
func (f *File) checkPrintf(call *ast.CallExpr, name string, skip int) {
	if len(call.Args) <= skip {
		return
	}
	lit := f.literal(call.Args[skip])
	if lit == nil {
		if *verbose {
			f.Warn(call.Pos(), "can't check non-literal format in call to", name)
		}
		return
	}
	if lit.Kind != token.STRING {
		f.Badf(call.Pos(), "literal %v not a string in call to", lit.Value, name)
	}
	format := lit.Value
	if !strings.Contains(format, "%") {
		if len(call.Args) > skip+1 {
			f.Badf(call.Pos(), "no formatting directive in %s call", name)
		}
		return
	}
	// Hard part: check formats against args.
	// Trivial but useful test: count.
	numArgs := 0
	for i, w := 0, 0; i < len(format); i += w {
		w = 1
		if format[i] == '%' {
			nbytes, nargs := f.parsePrintfVerb(call, format[i:])
			w = nbytes
			numArgs += nargs
		}
	}
	expect := len(call.Args) - (skip + 1)
	// Don't be too strict on dotdotdot.
	if call.Ellipsis.IsValid() && numArgs >= expect {
		return
	}
	if numArgs != expect {
		f.Badf(call.Pos(), "wrong number of args in %s call: %d needed but %d args", name, numArgs, expect)
	}
}

// parsePrintfVerb returns the number of bytes and number of arguments
// consumed by the Printf directive that begins s, including its percent sign
// and verb.
func (f *File) parsePrintfVerb(call *ast.CallExpr, s string) (nbytes, nargs int) {
	// There's guaranteed a percent sign.
	flags := make([]byte, 0, 5)
	nbytes = 1
	end := len(s)
	// There may be flags.
FlagLoop:
	for nbytes < end {
		switch s[nbytes] {
		case '#', '0', '+', '-', ' ':
			flags = append(flags, s[nbytes])
			nbytes++
		default:
			break FlagLoop
		}
	}
	getNum := func() {
		if nbytes < end && s[nbytes] == '*' {
			nbytes++
			nargs++
		} else {
			for nbytes < end && '0' <= s[nbytes] && s[nbytes] <= '9' {
				nbytes++
			}
		}
	}
	// There may be a width.
	getNum()
	// If there's a period, there may be a precision.
	if nbytes < end && s[nbytes] == '.' {
		flags = append(flags, '.') // Treat precision as a flag.
		nbytes++
		getNum()
	}
	// Now a verb.
	c, w := utf8.DecodeRuneInString(s[nbytes:])
	nbytes += w
	if c != '%' {
		nargs++
		f.checkPrintfVerb(call, c, flags)
	}
	return
}

type printVerb struct {
	verb  rune
	flags string // known flags are all ASCII
}

// Common flag sets for printf verbs.
const (
	numFlag      = " -+.0"
	sharpNumFlag = " -+.0#"
	allFlags     = " -+.0#"
)

// printVerbs identifies which flags are known to printf for each verb.
// TODO: A type that implements Formatter may do what it wants, and vet
// will complain incorrectly.
var printVerbs = []printVerb{
	// '-' is a width modifier, always valid.
	// '.' is a precision for float, max width for strings.
	// '+' is required sign for numbers, Go format for %v.
	// '#' is alternate format for several verbs.
	// ' ' is spacer for numbers
	{'b', numFlag},
	{'c', "-"},
	{'d', numFlag},
	{'e', numFlag},
	{'E', numFlag},
	{'f', numFlag},
	{'F', numFlag},
	{'g', numFlag},
	{'G', numFlag},
	{'o', sharpNumFlag},
	{'p', "-#"},
	{'q', " -+.0#"},
	{'s', " -+.0"},
	{'t', "-"},
	{'T', "-"},
	{'U', "-#"},
	{'v', allFlags},
	{'x', sharpNumFlag},
	{'X', sharpNumFlag},
}

const printfVerbs = "bcdeEfFgGopqstTvxUX"

func (f *File) checkPrintfVerb(call *ast.CallExpr, verb rune, flags []byte) {
	// Linear scan is fast enough for a small list.
	for _, v := range printVerbs {
		if v.verb == verb {
			for _, flag := range flags {
				if !strings.ContainsRune(v.flags, rune(flag)) {
					f.Badf(call.Pos(), "unrecognized printf flag for verb %q: %q", verb, flag)
				}
			}
			return
		}
	}
	f.Badf(call.Pos(), "unrecognized printf verb %q", verb)
}

// checkPrint checks a call to an unformatted print routine such as Println.
// The skip argument records how many arguments to ignore; that is,
// call.Args[skip] is the first argument to be printed.
func (f *File) checkPrint(call *ast.CallExpr, name string, skip int) {
	isLn := strings.HasSuffix(name, "ln")
	isF := strings.HasPrefix(name, "F")
	args := call.Args
	// check for Println(os.Stderr, ...)
	if skip == 0 && !isF && len(args) > 0 {
		if sel, ok := args[0].(*ast.SelectorExpr); ok {
			if x, ok := sel.X.(*ast.Ident); ok {
				if x.Name == "os" && strings.HasPrefix(sel.Sel.Name, "Std") {
					f.Warnf(call.Pos(), "first argument to %s is %s.%s", name, x.Name, sel.Sel.Name)
				}
			}
		}
	}
	if len(args) <= skip {
		// TODO: check that the receiver of Error() is of type error.
		if !isLn && name != "Error" {
			f.Badf(call.Pos(), "no args in %s call", name)
		}
		return
	}
	arg := args[skip]
	if lit, ok := arg.(*ast.BasicLit); ok && lit.Kind == token.STRING {
		if strings.Contains(lit.Value, "%") {
			f.Badf(call.Pos(), "possible formatting directive in %s call", name)
		}
	}
	if isLn {
		// The last item, if a string, should not have a newline.
		arg = args[len(call.Args)-1]
		if lit, ok := arg.(*ast.BasicLit); ok && lit.Kind == token.STRING {
			if strings.HasSuffix(lit.Value, `\n"`) {
				f.Badf(call.Pos(), "%s call ends with newline", name)
			}
		}
	}
}

// This function never executes, but it serves as a simple test for the program.
// Test with make test.
func BadFunctionUsedInTests() {
	fmt.Println()                      // not an error
	fmt.Println("%s", "hi")            // ERROR "possible formatting directive in Println call"
	fmt.Printf("%s", "hi", 3)          // ERROR "wrong number of args in Printf call"
	fmt.Printf("%s%%%d", "hi", 3)      // correct
	fmt.Printf("%08s", "woo")          // correct
	fmt.Printf("% 8s", "woo")          // correct
	fmt.Printf("%.*d", 3, 3)           // correct
	fmt.Printf("%.*d", 3, 3, 3)        // ERROR "wrong number of args in Printf call"
	fmt.Printf("%q %q", multi()...)    // ok
	fmt.Printf("%#q", `blah`)          // ok
	printf("now is the time", "buddy") // ERROR "no formatting directive"
	Printf("now is the time", "buddy") // ERROR "no formatting directive"
	Printf("hi")                       // ok
	const format = "%s %s\n"
	Printf(format, "hi", "there")
	Printf(format, "hi") // ERROR "wrong number of args in Printf call"
	f := new(File)
	f.Warn(0, "%s", "hello", 3)  // ERROR "possible formatting directive in Warn call"
	f.Warnf(0, "%s", "hello", 3) // ERROR "wrong number of args in Warnf call"
	f.Warnf(0, "%r", "hello")    // ERROR "unrecognized printf verb"
	f.Warnf(0, "%#s", "hello")   // ERROR "unrecognized printf flag"
	var e error
	fmt.Println(e.Error()) // correct, used to trigger "no args in Error call"
}

// printf is used by the test.
func printf(format string, args ...interface{}) {
	panic("don't call - testing only")
}

// multi is used by the test.
func multi() []interface{} {
	panic("don't call - testing only")
}
<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package newtutil

import (
	"fmt"
	"os/user"
	"sort"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cast"

	"mynewt.apache.org/newt/newt/interfaces"
	"mynewt.apache.org/newt/util"
	"mynewt.apache.org/newt/viper"
)

var NewtVersionStr string = "Apache Newt (incubating) version: 1.0.0.b2"
var NewtBlinkyTag string = "mynewt_1_0_0_b2_rc1_tag"
var NewtNumJobs int

const NEWTRC_DIR string = ".newt"
const REPOS_FILENAME string = "repos.yml"

const CORE_REPO_NAME string = "apache-mynewt-core"
const ARDUINO_ZERO_REPO_NAME string = "mynewt_arduino_zero"

type RepoCommitEntry struct {
	Version     string
	Hash        string
	Description string
}

// A warning is displayed if newt requires a newer version of a repo.
var RepoMinCommits = map[string]*RepoCommitEntry{
	// Newt no longer cd's to a source directory when it compiles its contents.
	// Consequently, package include flags need to be relative to the project
	// directory, not the package source directory.
	CORE_REPO_NAME: &RepoCommitEntry{
		Version:     "develop",
		Hash:        "cd99344df197d5b9e372b93142184a39ec078f69",
		Description: "Include paths now relative to project base.",
	},
	ARDUINO_ZERO_REPO_NAME: &RepoCommitEntry{
		Version:     "develop",
		Hash:        "a6348961fef56dbfe09a1b9418d3add3ad22eaf2",
		Description: "Include paths now relative to project base.",
	},
}

// Contains general newt settings read from $HOME/.newt
var newtrc *viper.Viper

func readNewtrc() *viper.Viper {
	usr, err := user.Current()
	if err != nil {
		log.Warn("Failed to obtain user name")
		return viper.New()
	}

	dir := usr.HomeDir + "/" + NEWTRC_DIR
	v, err := util.ReadConfig(dir, strings.TrimSuffix(REPOS_FILENAME, ".yml"))
	if err != nil {
		log.Debugf("Failed to read %s/%s file", dir, REPOS_FILENAME)
		return viper.New()
	}

	return v
}

func Newtrc() *viper.Viper {
	if newtrc != nil {
		return newtrc
	}

	newtrc = readNewtrc()
	return newtrc
}

func GetSliceFeatures(v *viper.Viper, features map[string]bool,
	key string) []interface{} {

	val := v.Get(key)
	vals := []interface{}{val}

	// Process the features in alphabetical order to ensure consistent
	// results across repeated runs.
	featureKeys := make([]string, 0, len(features))
	for feature, _ := range features {
		featureKeys = append(featureKeys, feature)
	}
	sort.Strings(featureKeys)

	for _, feature := range featureKeys {
		overwriteVal := v.Get(key + "." + feature + ".OVERWRITE")
		if overwriteVal != nil {
			return []interface{}{overwriteVal}
		}

		appendVal := v.Get(key + "." + feature)
		if appendVal != nil {
			vals = append(vals, appendVal)
		}
	}

	return vals
}

func GetStringMapFeatures(v *viper.Viper, features map[string]bool,
	key string) map[string]interface{} {

	result := map[string]interface{}{}

	slice := GetSliceFeatures(v, features, key)
	for _, itf := range slice {
		sub := cast.ToStringMap(itf)
		for k, v := range sub {
			result[k] = v
		}
	}

	return result
}

func GetStringFeatures(v *viper.Viper, features map[string]bool,
	key string) string {
	val := v.GetString(key)

	// Process the features in alphabetical order to ensure consistent
	// results across repeated runs.
	var featureKeys []string
	for feature, _ := range features {
		featureKeys = append(featureKeys, feature)
	}
	sort.Strings(featureKeys)

	for _, feature := range featureKeys {
		overwriteVal := v.GetString(key + "." + feature + ".OVERWRITE")
		if overwriteVal != "" {
			val = strings.Trim(overwriteVal, "\n")
			break
		}

		appendVal := v.GetString(key + "." + feature)
		if appendVal != "" {
			val += " " + strings.Trim(appendVal, "\n")
		}
	}
	return strings.TrimSpace(val)
}

func GetBoolFeaturesDflt(v *viper.Viper, features map[string]bool,
	key string, dflt bool) (bool, error) {

	s := GetStringFeatures(v, features, key)
	if s == "" {
		return dflt, nil
	}

	b, err := strconv.ParseBool(s)
	if err != nil {
		return dflt, util.FmtNewtError("invalid bool value for %s: %s",
			key, s)
	}

	return b, nil
}

func GetBoolFeatures(v *viper.Viper, features map[string]bool,
	key string) (bool, error) {

	return GetBoolFeaturesDflt(v, features, key, false)
}

func GetStringSliceFeatures(v *viper.Viper, features map[string]bool,
	key string) []string {

	vals := GetSliceFeatures(v, features, key)

	strVals := []string{}
	for _, v := range vals {
		subVals := cast.ToStringSlice(v)
		strVals = append(strVals, subVals...)
	}

	return strVals
}

// Parses a string of the following form:
//     [@repo]<path/to/package>
//
// @return string               repo name ("" if no repo)
//         string               package name
//         error                if invalid package string
func ParsePackageString(pkgStr string) (string, string, error) {
	if strings.HasPrefix(pkgStr, "@") {
		nameParts := strings.SplitN(pkgStr[1:], "/", 2)
		if len(nameParts) == 1 {
			return "", "", util.NewNewtError(fmt.Sprintf("Invalid package "+
				"string; contains repo but no package name: %s", pkgStr))
		} else {
			return nameParts[0], nameParts[1], nil
		}
	} else {
		return "", pkgStr, nil
	}
}

func FindRepoDesignator(s string) (int, int) {
	start := strings.Index(s, "@")
	if start == -1 {
		return -1, -1
	}

	len := strings.Index(s[start:], "/")
	if len == -1 {
		return -1, -1
	}

	return start, len
}

func ReplaceRepoDesignators(s string) (string, bool) {
	start, len := FindRepoDesignator(s)
	if start == -1 {
		return s, false
	}
	repoName := s[start+1 : start+len]

	proj := interfaces.GetProject()
	repoPath := proj.FindRepoPath(repoName)
	if repoPath == "" {
		return s, false
	}

	// Trim common project base from repo path.
	relRepoPath := strings.TrimPrefix(repoPath, proj.Path()+"/")

	return s[:start] + relRepoPath + s[start+len:], true
}

func BuildPackageString(repoName string, pkgName string) string {
	if repoName != "" {
		return "@" + repoName + "/" + pkgName
	} else {
		return pkgName
	}
}

func GeneratedPreamble() string {
	return fmt.Sprintf("/**\n * This file was generated by %s\n */\n\n",
		NewtVersionStr)
}
<commit_msg>Should not have touched the develop branch for the 1.0.0.b2 release"<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package newtutil

import (
	"fmt"
	"os/user"
	"sort"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/spf13/cast"

	"mynewt.apache.org/newt/newt/interfaces"
	"mynewt.apache.org/newt/util"
	"mynewt.apache.org/newt/viper"
)

var NewtVersionStr string = "Apache Newt (incubating) version: 1.0.0-dev"
var NewtBlinkyTag string = "develop"
var NewtNumJobs int

const NEWTRC_DIR string = ".newt"
const REPOS_FILENAME string = "repos.yml"

const CORE_REPO_NAME string = "apache-mynewt-core"
const ARDUINO_ZERO_REPO_NAME string = "mynewt_arduino_zero"

type RepoCommitEntry struct {
	Version     string
	Hash        string
	Description string
}

// A warning is displayed if newt requires a newer version of a repo.
var RepoMinCommits = map[string]*RepoCommitEntry{
	// Newt no longer cd's to a source directory when it compiles its contents.
	// Consequently, package include flags need to be relative to the project
	// directory, not the package source directory.
	CORE_REPO_NAME: &RepoCommitEntry{
		Version:     "develop",
		Hash:        "cd99344df197d5b9e372b93142184a39ec078f69",
		Description: "Include paths now relative to project base.",
	},
	ARDUINO_ZERO_REPO_NAME: &RepoCommitEntry{
		Version:     "develop",
		Hash:        "a6348961fef56dbfe09a1b9418d3add3ad22eaf2",
		Description: "Include paths now relative to project base.",
	},
}

// Contains general newt settings read from $HOME/.newt
var newtrc *viper.Viper

func readNewtrc() *viper.Viper {
	usr, err := user.Current()
	if err != nil {
		log.Warn("Failed to obtain user name")
		return viper.New()
	}

	dir := usr.HomeDir + "/" + NEWTRC_DIR
	v, err := util.ReadConfig(dir, strings.TrimSuffix(REPOS_FILENAME, ".yml"))
	if err != nil {
		log.Debugf("Failed to read %s/%s file", dir, REPOS_FILENAME)
		return viper.New()
	}

	return v
}

func Newtrc() *viper.Viper {
	if newtrc != nil {
		return newtrc
	}

	newtrc = readNewtrc()
	return newtrc
}

func GetSliceFeatures(v *viper.Viper, features map[string]bool,
	key string) []interface{} {

	val := v.Get(key)
	vals := []interface{}{val}

	// Process the features in alphabetical order to ensure consistent
	// results across repeated runs.
	featureKeys := make([]string, 0, len(features))
	for feature, _ := range features {
		featureKeys = append(featureKeys, feature)
	}
	sort.Strings(featureKeys)

	for _, feature := range featureKeys {
		overwriteVal := v.Get(key + "." + feature + ".OVERWRITE")
		if overwriteVal != nil {
			return []interface{}{overwriteVal}
		}

		appendVal := v.Get(key + "." + feature)
		if appendVal != nil {
			vals = append(vals, appendVal)
		}
	}

	return vals
}

func GetStringMapFeatures(v *viper.Viper, features map[string]bool,
	key string) map[string]interface{} {

	result := map[string]interface{}{}

	slice := GetSliceFeatures(v, features, key)
	for _, itf := range slice {
		sub := cast.ToStringMap(itf)
		for k, v := range sub {
			result[k] = v
		}
	}

	return result
}

func GetStringFeatures(v *viper.Viper, features map[string]bool,
	key string) string {
	val := v.GetString(key)

	// Process the features in alphabetical order to ensure consistent
	// results across repeated runs.
	var featureKeys []string
	for feature, _ := range features {
		featureKeys = append(featureKeys, feature)
	}
	sort.Strings(featureKeys)

	for _, feature := range featureKeys {
		overwriteVal := v.GetString(key + "." + feature + ".OVERWRITE")
		if overwriteVal != "" {
			val = strings.Trim(overwriteVal, "\n")
			break
		}

		appendVal := v.GetString(key + "." + feature)
		if appendVal != "" {
			val += " " + strings.Trim(appendVal, "\n")
		}
	}
	return strings.TrimSpace(val)
}

func GetBoolFeaturesDflt(v *viper.Viper, features map[string]bool,
	key string, dflt bool) (bool, error) {

	s := GetStringFeatures(v, features, key)
	if s == "" {
		return dflt, nil
	}

	b, err := strconv.ParseBool(s)
	if err != nil {
		return dflt, util.FmtNewtError("invalid bool value for %s: %s",
			key, s)
	}

	return b, nil
}

func GetBoolFeatures(v *viper.Viper, features map[string]bool,
	key string) (bool, error) {

	return GetBoolFeaturesDflt(v, features, key, false)
}

func GetStringSliceFeatures(v *viper.Viper, features map[string]bool,
	key string) []string {

	vals := GetSliceFeatures(v, features, key)

	strVals := []string{}
	for _, v := range vals {
		subVals := cast.ToStringSlice(v)
		strVals = append(strVals, subVals...)
	}

	return strVals
}

// Parses a string of the following form:
//     [@repo]<path/to/package>
//
// @return string               repo name ("" if no repo)
//         string               package name
//         error                if invalid package string
func ParsePackageString(pkgStr string) (string, string, error) {
	if strings.HasPrefix(pkgStr, "@") {
		nameParts := strings.SplitN(pkgStr[1:], "/", 2)
		if len(nameParts) == 1 {
			return "", "", util.NewNewtError(fmt.Sprintf("Invalid package "+
				"string; contains repo but no package name: %s", pkgStr))
		} else {
			return nameParts[0], nameParts[1], nil
		}
	} else {
		return "", pkgStr, nil
	}
}

func FindRepoDesignator(s string) (int, int) {
	start := strings.Index(s, "@")
	if start == -1 {
		return -1, -1
	}

	len := strings.Index(s[start:], "/")
	if len == -1 {
		return -1, -1
	}

	return start, len
}

func ReplaceRepoDesignators(s string) (string, bool) {
	start, len := FindRepoDesignator(s)
	if start == -1 {
		return s, false
	}
	repoName := s[start+1 : start+len]

	proj := interfaces.GetProject()
	repoPath := proj.FindRepoPath(repoName)
	if repoPath == "" {
		return s, false
	}

	// Trim common project base from repo path.
	relRepoPath := strings.TrimPrefix(repoPath, proj.Path()+"/")

	return s[:start] + relRepoPath + s[start+len:], true
}

func BuildPackageString(repoName string, pkgName string) string {
	if repoName != "" {
		return "@" + repoName + "/" + pkgName
	} else {
		return pkgName
	}
}

func GeneratedPreamble() string {
	return fmt.Sprintf("/**\n * This file was generated by %s\n */\n\n",
		NewtVersionStr)
}
<|endoftext|>