<commit_before>package elasticthought

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"sync"

	"github.com/couchbaselabs/cbfs/client"
	"github.com/couchbaselabs/logg"
)

// Worker job that splits a dataset into training/test set
type DatasetSplitter struct {
	Configuration Configuration
	Dataset       Dataset
}

func (d DatasetSplitter) Run() {

	logg.LogTo("DATASET_SPLITTER", "Datasetsplitter.run()!.  Config: %+v Dataset: %+v", d.Configuration, d.Dataset)

	// Find the datafile object associated with dataset
	db := d.Configuration.DbConnection()
	datafile, err := d.Dataset.GetDatafile(db)
	if err != nil {
		errMsg := fmt.Errorf("Error looking up datafile with id: %v.  Error: %v", d.Dataset.DatafileID, err)
		logg.LogError(errMsg)
		return
	}

	// Open the url -- content type should be application/x-gzip
	tr1, tr2, err := d.openTwoTarGzStreams(datafile.Url)
	if err != nil {
		errMsg := fmt.Errorf("Error opening tar.gz streams: %v", err)
		logg.LogError(errMsg)
		return
	}

	logg.LogTo("DATASET_SPLITTER", "Creating io pipe")

	cbfsReaderTesting, wTesting := io.Pipe()
	cbfsReaderTraining, wTraining := io.Pipe()

	tarWriterTesting := tar.NewWriter(wTesting)
	tarWriterTraining := tar.NewWriter(wTraining)

	cbfs, err := cbfsclient.New(d.Configuration.CbfsUrl)
	logg.LogTo("DATASET_SPLITTER", "Created cbfs client: %v", cbfs)
	options := cbfsclient.PutOptions{
		ContentType: "application/x-gzip",
	}
	logg.LogTo("DATASET_SPLITTER", "options: %v", options)

	destTraining := fmt.Sprintf("%v/training.tar.gz", d.Dataset.Id)
	destTesting := fmt.Sprintf("%v/testing.tar.gz", d.Dataset.Id)
	logg.LogTo("DATASET_SPLITTER", " %v %v ", destTesting, destTraining)

	wg := &sync.WaitGroup{}
	wg.Add(1)
	wg.Add(1)

	testingBuffer := &bytes.Buffer{}
	trainingBuffer := &bytes.Buffer{}

	go func() {
		defer wg.Done()
		io.Copy(testingBuffer, cbfsReaderTesting)
		logg.LogTo("DATASET_SPLITTER", "Saved testing data to buffer")
	}()

	go func() {
		defer wg.Done()
		io.Copy(trainingBuffer, cbfsReaderTraining)
		logg.LogTo("DATASET_SPLITTER", "Saved training data to buffer")
	}()

	logg.LogTo("DATASET_SPLITTER", "Calling transform")
	err = d.transform(tr1, tr2, tarWriterTraining, tarWriterTesting)
	if err != nil {
		errMsg := fmt.Errorf("Error transforming tar stream: %v", err)
		logg.LogError(errMsg)
		return
	}

	logg.LogTo("DATASET_SPLITTER", "waiting for reading buffer")

	wg.Wait()

	logg.LogTo("DATASET_SPLITTER", "Done waiting for reading buffer")

	// close writers
	logg.LogTo("DATASET_SPLITTER", "Closing writers")
	if err := tarWriterTesting.Close(); err != nil {
		errMsg := fmt.Errorf("Error closing tar writer: %v", err)
		logg.LogError(errMsg)
		return
	}
	if err := tarWriterTraining.Close(); err != nil {
		errMsg := fmt.Errorf("Error closing tar reader: %v", err)
		logg.LogError(errMsg)
		return
	}
	logg.LogTo("DATASET_SPLITTER", "Closed writers")

	logg.LogTo("DATASET_SPLITTER", "closing cbfsReaders")

	cbfsReaderTesting.Close()
	cbfsReaderTraining.Close()

	// Read from the stream and open tar archive

	// Walk the directories and split the files

	// Write to training and test tar archive

	// Save both training and test tar archive to cbfs (wrapped in gzip stream)

}

// Opens to tar.gz streams to the same url.  The reason this is done twice is due to
// ugly hack, which is documented in the transform() method.
func (d DatasetSplitter) openTwoTarGzStreams(url string) (*tar.Reader, *tar.Reader, error) {

	resp1, err := http.Get(url)
	if err != nil {
		return nil, nil, err
	}
	gzipReader1, err := gzip.NewReader(resp1.Body)
	if err != nil {
		return nil, nil, err
	}
	tarReader1 := tar.NewReader(gzipReader1)

	resp2, err := http.Get(url)
	if err != nil {
		return nil, nil, err
	}
	gzipReader2, err := gzip.NewReader(resp2.Body)
	if err != nil {
		return nil, nil, err
	}
	tarReader2 := tar.NewReader(gzipReader2)

	return tarReader1, tarReader2, nil

}

// Read from source tar stream and write training and test to given tar writers
//
// TODO: fix ugly hack.  Since I'm trying to read from the source stream *twice*, which
// doesn't work, the workaround is to expect *two* source streams: source1 and source2.
// That way after source1 is read, source2 is ready for reading from the beginning
func (d DatasetSplitter) transform(source1, source2 *tar.Reader, train, test *tar.Writer) error {

	// build a map from the source
	sourceMap, err := d.createMap(source1)
	if err != nil {
		return err
	}
	logg.Log("sourceMap: %+v", sourceMap)

	// split the map into training and test
	trainMap, testMap, err := d.splitMap(sourceMap)
	if err != nil {
		return err
	}

	// iterate over the source
	logg.LogTo("DATASET_SPLITTER", "iterate over source")
	for {
		hdr, err := source2.Next()

		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			return err
		}

		// distribute to writers based on training and test maps
		var twToAdd *tar.Writer

		// if strings.HasPrefix(hdr.Name, "foo") {
		if trainMap.hasPath(hdr.Name) {
			// add to training tar writer
			twToAdd = train
		} else if testMap.hasPath(hdr.Name) {
			// add to testing tar writer
			twToAdd = test
		} else {
			logg.LogPanic("File not present in either test/train: %v", hdr.Name)
		}

		// TODO: is there a more efficient way to do this?
		bytes, err := ioutil.ReadAll(source2)
		if err != nil {
			return err
		}

		hdrToAdd := &tar.Header{
			Name: hdr.Name,
			Size: int64(len(bytes)),
		}
		if err := twToAdd.WriteHeader(hdrToAdd); err != nil {
			return err
		}
		if _, err := twToAdd.Write(bytes); err != nil {
			return err
		}

	}

	logg.LogTo("DATASET_SPLITTER", "done iterating over source")

	return nil
}

// Validate that the source tar stream conforms to expected specs
func (d DatasetSplitter) validate(source *tar.Reader) (bool, error) {

	// validation rules:
	// 1. has at least 2 files
	// 2. the depth of each file is 2 (folder/filename.xxx)

	numFiles := 0
	for {
		hdr, err := source.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			return false, err
		}
		numFiles += 1

		pathComponents := strings.Split(hdr.Name, "/")
		if len(pathComponents) != 2 {
			return false, fmt.Errorf("Path does not have 2 components: %v", hdr.Name)
		}

	}

	if numFiles < 2 {
		return false, fmt.Errorf("Archive must contain at least 2 files")
	}

	return true, nil
}

// Create a map of folder -> []filename for all entries in the archive
func (d DatasetSplitter) createMap(source *tar.Reader) (filemap, error) {

	resultMap := filemap{}
	for {
		hdr, err := source.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			return nil, err
		}

		pathComponents := strings.Split(hdr.Name, "/")

		if len(pathComponents) != 2 {
			return nil, fmt.Errorf("Path does not have 2 components: %v", hdr.Name)
		}

		directory := pathComponents[0]
		filename := pathComponents[1]

		resultMap.addFileToDirectory(directory, filename)

	}

	return resultMap, nil

}

// Split map into training and testing disjoint subsets based on values
// of DatasetSplitter's Dataset
func (d DatasetSplitter) splitMap(source filemap) (training filemap, testing filemap, err error) {

	training = filemap{}
	testing = filemap{}

	// iterate over source keys
	for directory, files := range source {

		numTraining := int(float64(len(files)) * d.Dataset.TrainingDataset.SplitPercentage)

		numTest := len(files) - int(numTraining)

		// split files into subsets based on ratios in dataset
		trainingFiles, testFiles, err := splitFiles(files, numTraining, numTest)

		if err != nil {
			return nil, nil, err
		}

		// add to respective maps
		training[directory] = trainingFiles
		testing[directory] = testFiles

	}

	return training, testing, nil

}

func splitFiles(files []string, numTraining, numTest int) (training []string, test []string, err error) {
	for i, file := range files {
		if i < numTraining {
			training = append(training, file)
		} else {
			test = append(test, file)
		}
	}
	return
}
<commit_msg>stores everything a buffer, but at least successfully saves to cbfs<commit_after>package elasticthought

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/couchbaselabs/cbfs/client"
	"github.com/couchbaselabs/logg"
)

// Worker job that splits a dataset into training/test set
type DatasetSplitter struct {
	Configuration Configuration
	Dataset       Dataset
}

// TODO: this is currently reading the entire tar stream into a buffer before
// handing it to cbfs.  Should be using an io.Pipe.
func (d DatasetSplitter) Run() {

	logg.LogTo("DATASET_SPLITTER", "Datasetsplitter.run()!.  Config: %+v Dataset: %+v", d.Configuration, d.Dataset)

	// Find the datafile object associated with dataset
	db := d.Configuration.DbConnection()
	datafile, err := d.Dataset.GetDatafile(db)
	if err != nil {
		errMsg := fmt.Errorf("Error looking up datafile with id: %v.  Error: %v", d.Dataset.DatafileID, err)
		logg.LogError(errMsg)
		return
	}

	// Open the url -- content type should be application/x-gzip
	tr1, tr2, err := d.openTwoTarGzStreams(datafile.Url)
	if err != nil {
		errMsg := fmt.Errorf("Error opening tar.gz streams: %v", err)
		logg.LogError(errMsg)
		return
	}

	bufferTesting := &bytes.Buffer{}
	bufferTraining := &bytes.Buffer{}

	tarWriterTesting := tar.NewWriter(bufferTesting)
	tarWriterTraining := tar.NewWriter(bufferTraining)

	cbfs, err := cbfsclient.New(d.Configuration.CbfsUrl)
	logg.LogTo("DATASET_SPLITTER", "Created cbfs client: %v", cbfs)
	options := cbfsclient.PutOptions{
		ContentType: "application/x-gzip",
	}
	logg.LogTo("DATASET_SPLITTER", "options: %v", options)

	destTraining := fmt.Sprintf("%v/training.tar.gz", d.Dataset.Id)
	destTesting := fmt.Sprintf("%v/testing.tar.gz", d.Dataset.Id)
	logg.LogTo("DATASET_SPLITTER", " %v %v ", destTesting, destTraining)

	logg.LogTo("DATASET_SPLITTER", "Calling transform")
	err = d.transform(tr1, tr2, tarWriterTraining, tarWriterTesting)
	if err != nil {
		errMsg := fmt.Errorf("Error transforming tar stream: %v", err)
		logg.LogError(errMsg)
		return
	}

	// close writers
	logg.LogTo("DATASET_SPLITTER", "Closing writers")
	if err := tarWriterTesting.Close(); err != nil {
		errMsg := fmt.Errorf("Error closing tar writer: %v", err)
		logg.LogError(errMsg)
		return
	}
	if err := tarWriterTraining.Close(); err != nil {
		errMsg := fmt.Errorf("Error closing tar reader: %v", err)
		logg.LogError(errMsg)
		return
	}
	logg.LogTo("DATASET_SPLITTER", "Closed writers")

	// At this point bufferTesting has all data
	logg.LogTo("DATASET_SPLITTER", "bufferTesting size: %d", bufferTesting.Len())
	logg.LogTo("DATASET_SPLITTER", "bufferTraining size: %d", bufferTraining.Len())

	if err := cbfs.Put("", destTesting, bufferTesting, options); err != nil {
		errMsg := fmt.Errorf("Error writing %v to cbfs: %v", destTesting, err)
		logg.LogError(errMsg)
		return

	}
	logg.LogTo("DATASET_SPLITTER", "Wrote %v to cbfs", destTesting)

	if err := cbfs.Put("", destTraining, bufferTraining, options); err != nil {
		errMsg := fmt.Errorf("Error writing %v to cbfs: %v", destTraining, err)
		logg.LogError(errMsg)
		return
	}
	logg.LogTo("DATASET_SPLITTER", "Wrote %v to cbfs", destTraining)

}

// Opens to tar.gz streams to the same url.  The reason this is done twice is due to
// ugly hack, which is documented in the transform() method.
func (d DatasetSplitter) openTwoTarGzStreams(url string) (*tar.Reader, *tar.Reader, error) {

	resp1, err := http.Get(url)
	if err != nil {
		return nil, nil, err
	}
	gzipReader1, err := gzip.NewReader(resp1.Body)
	if err != nil {
		return nil, nil, err
	}
	tarReader1 := tar.NewReader(gzipReader1)

	resp2, err := http.Get(url)
	if err != nil {
		return nil, nil, err
	}
	gzipReader2, err := gzip.NewReader(resp2.Body)
	if err != nil {
		return nil, nil, err
	}
	tarReader2 := tar.NewReader(gzipReader2)

	return tarReader1, tarReader2, nil

}

// Read from source tar stream and write training and test to given tar writers
//
// TODO: fix ugly hack.  Since I'm trying to read from the source stream *twice*, which
// doesn't work, the workaround is to expect *two* source streams: source1 and source2.
// That way after source1 is read, source2 is ready for reading from the beginning
func (d DatasetSplitter) transform(source1, source2 *tar.Reader, train, test *tar.Writer) error {

	// build a map from the source
	sourceMap, err := d.createMap(source1)
	if err != nil {
		return err
	}
	logg.Log("sourceMap: %+v", sourceMap)

	// split the map into training and test
	trainMap, testMap, err := d.splitMap(sourceMap)
	if err != nil {
		return err
	}

	// iterate over the source
	logg.LogTo("DATASET_SPLITTER", "iterate over source")
	for {
		hdr, err := source2.Next()

		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			return err
		}

		// distribute to writers based on training and test maps
		var twToAdd *tar.Writer

		// if strings.HasPrefix(hdr.Name, "foo") {
		if trainMap.hasPath(hdr.Name) {
			// add to training tar writer
			twToAdd = train
		} else if testMap.hasPath(hdr.Name) {
			// add to testing tar writer
			twToAdd = test
		} else {
			logg.LogPanic("File not present in either test/train: %v", hdr.Name)
		}

		// TODO: is there a more efficient way to do this?
		bytes, err := ioutil.ReadAll(source2)
		if err != nil {
			return err
		}

		hdrToAdd := &tar.Header{
			Name: hdr.Name,
			Size: int64(len(bytes)),
		}
		if err := twToAdd.WriteHeader(hdrToAdd); err != nil {
			return err
		}
		if _, err := twToAdd.Write(bytes); err != nil {
			return err
		}

	}

	logg.LogTo("DATASET_SPLITTER", "done iterating over source")

	return nil
}

// Validate that the source tar stream conforms to expected specs
func (d DatasetSplitter) validate(source *tar.Reader) (bool, error) {

	// validation rules:
	// 1. has at least 2 files
	// 2. the depth of each file is 2 (folder/filename.xxx)

	numFiles := 0
	for {
		hdr, err := source.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			return false, err
		}
		numFiles += 1

		pathComponents := strings.Split(hdr.Name, "/")
		if len(pathComponents) != 2 {
			return false, fmt.Errorf("Path does not have 2 components: %v", hdr.Name)
		}

	}

	if numFiles < 2 {
		return false, fmt.Errorf("Archive must contain at least 2 files")
	}

	return true, nil
}

// Create a map of folder -> []filename for all entries in the archive
func (d DatasetSplitter) createMap(source *tar.Reader) (filemap, error) {

	resultMap := filemap{}
	for {
		hdr, err := source.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			return nil, err
		}

		pathComponents := strings.Split(hdr.Name, "/")

		if len(pathComponents) != 2 {
			return nil, fmt.Errorf("Path does not have 2 components: %v", hdr.Name)
		}

		directory := pathComponents[0]
		filename := pathComponents[1]

		resultMap.addFileToDirectory(directory, filename)

	}

	return resultMap, nil

}

// Split map into training and testing disjoint subsets based on values
// of DatasetSplitter's Dataset
func (d DatasetSplitter) splitMap(source filemap) (training filemap, testing filemap, err error) {

	training = filemap{}
	testing = filemap{}

	// iterate over source keys
	for directory, files := range source {

		numTraining := int(float64(len(files)) * d.Dataset.TrainingDataset.SplitPercentage)

		numTest := len(files) - int(numTraining)

		// split files into subsets based on ratios in dataset
		trainingFiles, testFiles, err := splitFiles(files, numTraining, numTest)

		if err != nil {
			return nil, nil, err
		}

		// add to respective maps
		training[directory] = trainingFiles
		testing[directory] = testFiles

	}

	return training, testing, nil

}

func splitFiles(files []string, numTraining, numTest int) (training []string, test []string, err error) {
	for i, file := range files {
		if i < numTraining {
			training = append(training, file)
		} else {
			test = append(test, file)
		}
	}
	return
}
<|endoftext|>


<commit_before>package pingdom

import (
	"encoding/json"
	"fmt"
)

// PingdomResponse represents a general response from the Pingdom API.
type PingdomResponse struct {
	Message string `json:"message"`
}

// PingdomError represents an error response from the Pingdom API.
type PingdomError struct {
	StatusCode int    `json:"statuscode"`
	StatusDesc string `json:"statusdesc"`
	Message    string `json:"errormessage"`
}

// CheckResponse represents the JSON response for a check from the Pingdom API.
type CheckResponse struct {
	ID                       int                 `json:"id"`
	Name                     string              `json:"name"`
	Resolution               int                 `json:"resolution,omitempty"`
	SendNotificationWhenDown int                 `json:"sendnotificationwhendown,omitempty"`
	NotifyAgainEvery         int                 `json:"notifyagainevery,omitempty"`
	NotifyWhenBackup         bool                `json:"notifywhenbackup,omitempty"`
	Created                  int64               `json:"created,omitempty"`
	Hostname                 string              `json:"hostname,omitempty"`
	Status                   string              `json:"status,omitempty"`
	LastErrorTime            int64               `json:"lasterrortime,omitempty"`
	LastTestTime             int64               `json:"lasttesttime,omitempty"`
	LastResponseTime         int64               `json:"lastresponsetime,omitempty"`
	Paused                   bool                `json:"paused,omitempty"`
	IntegrationIds           []int               `json:"integrationids,omitempty"`
	Type                     CheckResponseType   `json:"type,omitempty"`
	Tags                     []CheckResponseTag  `json:"tags,omitempty"`
	UserIds                  []int               `json:"userids,omitempty"`
	Teams                    []CheckTeamResponse `json:"teams,omitempty"`
	ResponseTimeThreshold    int                 `json:"responsetime_threshold,omitempty"`

	// Legacy; this is not returned by the API, we backfill the value from the
	// Teams field.
	TeamIds []int
}

// CheckTeamResponse is a Team returned inside of a Check instance. (We can't
// use TeamResponse because the ID returned here is an int, not a string).
type CheckTeamResponse struct {
	ID   int    `json:"id"`
	Name string `json:"name"`
}

// CheckResponseType is the type of the Pingdom check.
type CheckResponseType struct {
	Name string                    `json:"-"`
	HTTP *CheckResponseHTTPDetails `json:"http,omitempty"`
	TCP  *CheckResponseTCPDetails  `json:"tcp,omitempty"`
}

// CheckResponseTag is an optional tag that can be added to checks.
type CheckResponseTag struct {
	Name  string      `json:"name"`
	Type  string      `json:"type"`
	Count interface{} `json:"count"`
}

// MaintenanceResponse represents the JSON response for a maintenance from the Pingdom API.
type MaintenanceResponse struct {
	ID             int                      `json:"id"`
	Description    string                   `json:"description"`
	From           int64                    `json:"from"`
	To             int64                    `json:"to"`
	RecurrenceType string                   `json:"recurrencetype"`
	RepeatEvery    int                      `json:"repeatevery"`
	EffectiveTo    int64                    `json:"effectiveto"`
	Checks         MaintenanceCheckResponse `json:"checks"`
}

// MaintenanceCheckResponse represents Check reply in json MaintenanceResponse.
type MaintenanceCheckResponse struct {
	Uptime []int `json:"uptime"`
	Tms    []int `json:"tms"`
}

// ProbeResponse represents the JSON response for probes from the Pingdom API.
type ProbeResponse struct {
	ID         int    `json:"id"`
	Country    string `json:"country"`
	City       string `json:"city"`
	Name       string `json:"name"`
	Active     bool   `json:"active"`
	Hostname   string `json:"hostname"`
	IP         string `json:"ip"`
	IPv6       string `json:"ipv6"`
	CountryISO string `json:"countryiso"`
	Region     string `json:"region"`
}

// TeamResponse represents the JSON response for teams from the Pingdom API.
type TeamResponse struct {
	ID    string `json:"id"`
	Name  string `json:"name"`
	Users []TeamUserResponse
}

// TeamUserResponse represents the JSON response for users in teams from the Pingdom API.
type TeamUserResponse struct {
	ID    string `json:"id"`
	Email string `json:"email"`
	Name  string `json:"name"`
}

// TeamDeleteResponse represents the JSON response for delete team from the Pingdom API.
type TeamDeleteResponse struct {
	Success bool `json:"success"`
}

// PublicReportResponse represents the JSON response for a public report from the Pingdom API.
type PublicReportResponse struct {
	ID        int    `json:"checkid"`
	Name      string `json:"checkname"`
	ReportURL string `json:"reporturl"`
}

// SummaryPerformanceResponse represents the JSON response for a summary performance from the Pingdom API.
type SummaryPerformanceResponse struct {
	Summary SummaryPerformanceMap `json:"summary"`
}

// SummaryPerformanceMap is the performance broken down over different time intervals.
type SummaryPerformanceMap struct {
	Hours []SummaryPerformanceSummary `json:"hours,omitempty"`
	Days  []SummaryPerformanceSummary `json:"days,omitempty"`
	Weeks []SummaryPerformanceSummary `json:"weeks,omitempty"`
}

// SummaryPerformanceSummary is the metrics for a performance summary.
type SummaryPerformanceSummary struct {
	AvgResponse int `json:"avgresponse"`
	Downtime    int `json:"downtime"`
	StartTime   int `json:"starttime"`
	Unmonitored int `json:"unmonitored"`
	Uptime      int `json:"uptime"`
}

// UserSmsResponse represents the JSON response for a user SMS contact.
type UserSmsResponse struct {
	Id          int    `json:"id"`
	Severity    string `json:"severity"`
	CountryCode string `json:"country_code"`
	Number      string `json:"number"`
	Provider    string `json:"provider"`
}

// UserEmailResponse represents the JSON response for a user email contact.
type UserEmailResponse struct {
	Id       int    `json:"id"`
	Severity string `json:"severity"`
	Address  string `json:"address"`
}

// CreateUserContactResponse represents the JSON response for a user contact.
type CreateUserContactResponse struct {
	Id int `json:"id"`
}

// UsersResponse represents the JSON response for a Pingom User.
type UsersResponse struct {
	Id       int                 `json:"id"`
	Paused   string              `json:"paused,omitempty"`
	Username string              `json:"name,omitempty"`
	Sms      []UserSmsResponse   `json:"sms,omitempty"`
	Email    []UserEmailResponse `json:"email,omitempty"`
}

// UnmarshalJSON converts a byte array into a CheckResponseType.
func (c *CheckResponseType) UnmarshalJSON(b []byte) error {
	var raw interface{}

	err := json.Unmarshal(b, &raw)
	if err != nil {
		return err
	}

	switch v := raw.(type) {
	case string:
		c.Name = v
	case map[string]interface{}:
		if len(v) != 1 {
			return fmt.Errorf("Check detailed response `check.type` contains more than one object: %+v", v)
		}
		for k := range v {
			c.Name = k
		}

		// Allow continue use json.Unmarshall using a type != Unmarshaller
		// This avoid enter in a infinite loop
		type t CheckResponseType
		var rawCheckDetails t

		err := json.Unmarshal(b, &rawCheckDetails)
		if err != nil {
			return err
		}
		c.HTTP = rawCheckDetails.HTTP
		c.TCP = rawCheckDetails.TCP
	}
	return nil
}

// CheckResponseHTTPDetails represents the details specific to HTTP checks.
type CheckResponseHTTPDetails struct {
	Url              string            `json:"url,omitempty"`
	Encryption       bool              `json:"encryption,omitempty"`
	Port             int               `json:"port,omitempty"`
	Username         string            `json:"username,omitempty"`
	Password         string            `json:"password,omitempty"`
	ShouldContain    string            `json:"shouldcontain,omitempty"`
	ShouldNotContain string            `json:"shouldnotcontain,omitempty"`
	PostData         string            `json:"postdata,omitempty"`
	RequestHeaders   map[string]string `json:"requestheaders,omitempty"`
}

// CheckResponseTCPDetails represents the details specific to TCP checks.
type CheckResponseTCPDetails struct {
	Port           int    `json:"port,omitempty"`
	StringToSend   string `json:"stringtosend,omitempty"`
	StringToExpect string `json:"stringtoexpect,omitempty"`
}

// Return string representation of the PingdomError.
func (r *PingdomError) Error() string {
	return fmt.Sprintf("%d %v: %v", r.StatusCode, r.StatusDesc, r.Message)
}

// private types used to unmarshall JSON responses from Pingdom.

type listChecksJSONResponse struct {
	Checks []CheckResponse `json:"checks"`
}

type listMaintenanceJSONResponse struct {
	Maintenances []MaintenanceResponse `json:"maintenance"`
}

type listProbesJSONResponse struct {
	Probes []ProbeResponse `json:"probes"`
}

type listTeamsJSONResponse struct {
	Teams []TeamResponse `json:"teams"`
}

type listPublicReportsJSONResponse struct {
	Checks []PublicReportResponse `json:"public"`
}

type checkDetailsJSONResponse struct {
	Check *CheckResponse `json:"check"`
}

type maintenanceDetailsJSONResponse struct {
	Maintenance *MaintenanceResponse `json:"maintenance"`
}

type teamDetailsJSONResponse struct {
	Team *TeamResponse `json:"team"`
}

type createUserContactJSONResponse struct {
	Contact *CreateUserContactResponse `json:"contact_target"`
}

type createUserJSONResponse struct {
	User *UsersResponse `json:"user"`
}

type listUsersJSONResponse struct {
	Users []UsersResponse `json:"users"`
}

type errorJSONResponse struct {
	Error *PingdomError `json:"error"`
}
<commit_msg>Update to add probeFilters field to API response data<commit_after>package pingdom

import (
	"encoding/json"
	"fmt"
)

// PingdomResponse represents a general response from the Pingdom API.
type PingdomResponse struct {
	Message string `json:"message"`
}

// PingdomError represents an error response from the Pingdom API.
type PingdomError struct {
	StatusCode int    `json:"statuscode"`
	StatusDesc string `json:"statusdesc"`
	Message    string `json:"errormessage"`
}

// CheckResponse represents the JSON response for a check from the Pingdom API.
type CheckResponse struct {
	ID                       int                 `json:"id"`
	Name                     string              `json:"name"`
	Resolution               int                 `json:"resolution,omitempty"`
	SendNotificationWhenDown int                 `json:"sendnotificationwhendown,omitempty"`
	NotifyAgainEvery         int                 `json:"notifyagainevery,omitempty"`
	NotifyWhenBackup         bool                `json:"notifywhenbackup,omitempty"`
	Created                  int64               `json:"created,omitempty"`
	Hostname                 string              `json:"hostname,omitempty"`
	Status                   string              `json:"status,omitempty"`
	LastErrorTime            int64               `json:"lasterrortime,omitempty"`
	LastTestTime             int64               `json:"lasttesttime,omitempty"`
	LastResponseTime         int64               `json:"lastresponsetime,omitempty"`
	Paused                   bool                `json:"paused,omitempty"`
	IntegrationIds           []int               `json:"integrationids,omitempty"`
	Type                     CheckResponseType   `json:"type,omitempty"`
	Tags                     []CheckResponseTag  `json:"tags,omitempty"`
	UserIds                  []int               `json:"userids,omitempty"`
	Teams                    []CheckTeamResponse `json:"teams,omitempty"`
	ResponseTimeThreshold    int                 `json:"responsetime_threshold,omitempty"`
	ProbeFilters             []string            `json:"probe_filters,omitempty"`

	// Legacy; this is not returned by the API, we backfill the value from the
	// Teams field.
	TeamIds []int
}

// CheckTeamResponse is a Team returned inside of a Check instance. (We can't
// use TeamResponse because the ID returned here is an int, not a string).
type CheckTeamResponse struct {
	ID   int    `json:"id"`
	Name string `json:"name"`
}

// CheckResponseType is the type of the Pingdom check.
type CheckResponseType struct {
	Name string                    `json:"-"`
	HTTP *CheckResponseHTTPDetails `json:"http,omitempty"`
	TCP  *CheckResponseTCPDetails  `json:"tcp,omitempty"`
}

// CheckResponseTag is an optional tag that can be added to checks.
type CheckResponseTag struct {
	Name  string      `json:"name"`
	Type  string      `json:"type"`
	Count interface{} `json:"count"`
}

// MaintenanceResponse represents the JSON response for a maintenance from the Pingdom API.
type MaintenanceResponse struct {
	ID             int                      `json:"id"`
	Description    string                   `json:"description"`
	From           int64                    `json:"from"`
	To             int64                    `json:"to"`
	RecurrenceType string                   `json:"recurrencetype"`
	RepeatEvery    int                      `json:"repeatevery"`
	EffectiveTo    int64                    `json:"effectiveto"`
	Checks         MaintenanceCheckResponse `json:"checks"`
}

// MaintenanceCheckResponse represents Check reply in json MaintenanceResponse.
type MaintenanceCheckResponse struct {
	Uptime []int `json:"uptime"`
	Tms    []int `json:"tms"`
}

// ProbeResponse represents the JSON response for probes from the Pingdom API.
type ProbeResponse struct {
	ID         int    `json:"id"`
	Country    string `json:"country"`
	City       string `json:"city"`
	Name       string `json:"name"`
	Active     bool   `json:"active"`
	Hostname   string `json:"hostname"`
	IP         string `json:"ip"`
	IPv6       string `json:"ipv6"`
	CountryISO string `json:"countryiso"`
	Region     string `json:"region"`
}

// TeamResponse represents the JSON response for teams from the Pingdom API.
type TeamResponse struct {
	ID    string `json:"id"`
	Name  string `json:"name"`
	Users []TeamUserResponse
}

// TeamUserResponse represents the JSON response for users in teams from the Pingdom API.
type TeamUserResponse struct {
	ID    string `json:"id"`
	Email string `json:"email"`
	Name  string `json:"name"`
}

// TeamDeleteResponse represents the JSON response for delete team from the Pingdom API.
type TeamDeleteResponse struct {
	Success bool `json:"success"`
}

// PublicReportResponse represents the JSON response for a public report from the Pingdom API.
type PublicReportResponse struct {
	ID        int    `json:"checkid"`
	Name      string `json:"checkname"`
	ReportURL string `json:"reporturl"`
}

// SummaryPerformanceResponse represents the JSON response for a summary performance from the Pingdom API.
type SummaryPerformanceResponse struct {
	Summary SummaryPerformanceMap `json:"summary"`
}

// SummaryPerformanceMap is the performance broken down over different time intervals.
type SummaryPerformanceMap struct {
	Hours []SummaryPerformanceSummary `json:"hours,omitempty"`
	Days  []SummaryPerformanceSummary `json:"days,omitempty"`
	Weeks []SummaryPerformanceSummary `json:"weeks,omitempty"`
}

// SummaryPerformanceSummary is the metrics for a performance summary.
type SummaryPerformanceSummary struct {
	AvgResponse int `json:"avgresponse"`
	Downtime    int `json:"downtime"`
	StartTime   int `json:"starttime"`
	Unmonitored int `json:"unmonitored"`
	Uptime      int `json:"uptime"`
}

// UserSmsResponse represents the JSON response for a user SMS contact.
type UserSmsResponse struct {
	Id          int    `json:"id"`
	Severity    string `json:"severity"`
	CountryCode string `json:"country_code"`
	Number      string `json:"number"`
	Provider    string `json:"provider"`
}

// UserEmailResponse represents the JSON response for a user email contact.
type UserEmailResponse struct {
	Id       int    `json:"id"`
	Severity string `json:"severity"`
	Address  string `json:"address"`
}

// CreateUserContactResponse represents the JSON response for a user contact.
type CreateUserContactResponse struct {
	Id int `json:"id"`
}

// UsersResponse represents the JSON response for a Pingom User.
type UsersResponse struct {
	Id       int                 `json:"id"`
	Paused   string              `json:"paused,omitempty"`
	Username string              `json:"name,omitempty"`
	Sms      []UserSmsResponse   `json:"sms,omitempty"`
	Email    []UserEmailResponse `json:"email,omitempty"`
}

// UnmarshalJSON converts a byte array into a CheckResponseType.
func (c *CheckResponseType) UnmarshalJSON(b []byte) error {
	var raw interface{}

	err := json.Unmarshal(b, &raw)
	if err != nil {
		return err
	}

	switch v := raw.(type) {
	case string:
		c.Name = v
	case map[string]interface{}:
		if len(v) != 1 {
			return fmt.Errorf("Check detailed response `check.type` contains more than one object: %+v", v)
		}
		for k := range v {
			c.Name = k
		}

		// Allow continue use json.Unmarshall using a type != Unmarshaller
		// This avoid enter in a infinite loop
		type t CheckResponseType
		var rawCheckDetails t

		err := json.Unmarshal(b, &rawCheckDetails)
		if err != nil {
			return err
		}
		c.HTTP = rawCheckDetails.HTTP
		c.TCP = rawCheckDetails.TCP
	}
	return nil
}

// CheckResponseHTTPDetails represents the details specific to HTTP checks.
type CheckResponseHTTPDetails struct {
	Url              string            `json:"url,omitempty"`
	Encryption       bool              `json:"encryption,omitempty"`
	Port             int               `json:"port,omitempty"`
	Username         string            `json:"username,omitempty"`
	Password         string            `json:"password,omitempty"`
	ShouldContain    string            `json:"shouldcontain,omitempty"`
	ShouldNotContain string            `json:"shouldnotcontain,omitempty"`
	PostData         string            `json:"postdata,omitempty"`
	RequestHeaders   map[string]string `json:"requestheaders,omitempty"`
}

// CheckResponseTCPDetails represents the details specific to TCP checks.
type CheckResponseTCPDetails struct {
	Port           int    `json:"port,omitempty"`
	StringToSend   string `json:"stringtosend,omitempty"`
	StringToExpect string `json:"stringtoexpect,omitempty"`
}

// Return string representation of the PingdomError.
func (r *PingdomError) Error() string {
	return fmt.Sprintf("%d %v: %v", r.StatusCode, r.StatusDesc, r.Message)
}

// private types used to unmarshall JSON responses from Pingdom.

type listChecksJSONResponse struct {
	Checks []CheckResponse `json:"checks"`
}

type listMaintenanceJSONResponse struct {
	Maintenances []MaintenanceResponse `json:"maintenance"`
}

type listProbesJSONResponse struct {
	Probes []ProbeResponse `json:"probes"`
}

type listTeamsJSONResponse struct {
	Teams []TeamResponse `json:"teams"`
}

type listPublicReportsJSONResponse struct {
	Checks []PublicReportResponse `json:"public"`
}

type checkDetailsJSONResponse struct {
	Check *CheckResponse `json:"check"`
}

type maintenanceDetailsJSONResponse struct {
	Maintenance *MaintenanceResponse `json:"maintenance"`
}

type teamDetailsJSONResponse struct {
	Team *TeamResponse `json:"team"`
}

type createUserContactJSONResponse struct {
	Contact *CreateUserContactResponse `json:"contact_target"`
}

type createUserJSONResponse struct {
	User *UsersResponse `json:"user"`
}

type listUsersJSONResponse struct {
	Users []UsersResponse `json:"users"`
}

type errorJSONResponse struct {
	Error *PingdomError `json:"error"`
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"sync"

	"github.com/gorilla/mux"
)

// gorilla_server implements a REST API service that lets you reserve and release
// physical servers in a cloud computing system

type ServerStatus struct {
	NumReservations int  `json:"total_num_reservations"`
	Reserved        bool `json:"currently_reserved"`
}

// global variables are bad, using them here for brevity

// each server and the # times they've been reserved
var servers = map[string]*ServerStatus{
	"alice": &ServerStatus{},
	"bob":   &ServerStatus{},
	"carol": &ServerStatus{},
}

// the mutex to protect against concurrent access to servers
var mx sync.RWMutex

func main() {
	// creates a new top level mux.Router. since a mux.Router implements the http.Handler interface,
	// we can pass it to http.ListenAndServe below
	router := mux.NewRouter()

	// configure the router to always run this handler when it couldn't match a request to any other handler
	router.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte(fmt.Sprintf("%s not found\n", r.URL)))
	})

	// create a subrouter just for standard API calls. subrouters are convenient ways to
	// group similar functionality together.
	apiRouter := router.Headers("Content-Type", "application/json").Subrouter()
	apiRouter.HandleFunc("/api/{name}", getServer).Methods("GET")
	apiRouter.HandleFunc("/api/{name}", reserveServer).Methods("POST")
	apiRouter.HandleFunc("/api/{name}", releaseServer).Methods("DELETE")

	adminAPIRouter := router.Headers("Content-Type", "application/json").MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
		adminToken := r.Header.Get("X-ADMIN-TOKEN")
		if adminToken == "" {
			return false
		}
		// in a production setting, this could be a DB lookup to determine whether the token
		// is valid for admin usage
		return adminToken == "SuperAdmin"
	}).Subrouter()
	adminAPIRouter.HandleFunc("/api/admin/servers", getAllServers).Methods("GET")
	adminAPIRouter.HandleFunc("/api/admin/servers", releaseAllServers).Methods("DELETE")

	log.Printf("serving on port 8080")
	http.ListenAndServe(":8080", router)
}
<commit_msg>more comments<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"sync"

	"github.com/gorilla/mux"
)

// gorilla_server implements a REST API service that lets you reserve and release
// physical servers in a cloud computing system

type ServerStatus struct {
	NumReservations int  `json:"total_num_reservations"`
	Reserved        bool `json:"currently_reserved"`
}

// global variables are bad, using them here for brevity

// each server and the # times they've been reserved
var servers = map[string]*ServerStatus{
	"alice": &ServerStatus{},
	"bob":   &ServerStatus{},
	"carol": &ServerStatus{},
}

// the mutex to protect against concurrent access to servers
var mx sync.RWMutex

func main() {
	// creates a new top level mux.Router. since a mux.Router implements the http.Handler interface,
	// we can pass it to http.ListenAndServe below
	router := mux.NewRouter()

	// configure the router to always run this handler when it couldn't match a request to any other handler
	router.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte(fmt.Sprintf("%s not found\n", r.URL)))
	})

	// create a subrouter just for standard API calls. subrouters are convenient ways to
	// group similar functionality together. this subrouter also verifies that the Content-Type
	// header is correct for a JSON API.
	apiRouter := router.Headers("Content-Type", "application/json").Subrouter()
	apiRouter.HandleFunc("/api/{name}", getServer).Methods("GET")
	apiRouter.HandleFunc("/api/{name}", reserveServer).Methods("POST")
	apiRouter.HandleFunc("/api/{name}", releaseServer).Methods("DELETE")

	// create a subrouter just for admin API calls. this router not only verifies the Content-Type
	// header as above, it also looks up and validates the admin token. if the token is invalid,
	// the server will 404, just as if the path didn't match.
	adminAPIRouter := router.Headers("Content-Type", "application/json").MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
		adminToken := r.Header.Get("X-ADMIN-TOKEN")
		if adminToken == "" {
			return false
		}
		// in a production setting, this could be a DB lookup to determine whether the token
		// is valid for admin usage
		return adminToken == "SuperAdmin"
	}).Subrouter()
	adminAPIRouter.HandleFunc("/api/admin/servers", getAllServers).Methods("GET")
	adminAPIRouter.HandleFunc("/api/admin/servers", releaseAllServers).Methods("DELETE")

	log.Printf("serving on port 8080")
	http.ListenAndServe(":8080", router)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1

import (
	"fmt"
	"reflect"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/conversion"
)

func addConversionFuncs() {
	// Add non-generated conversion functions
	err := api.Scheme.AddConversionFuncs(
		convert_v1_ReplicationControllerSpec_To_api_ReplicationControllerSpec,
		convert_api_ReplicationControllerSpec_To_v1_ReplicationControllerSpec,
	)
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}

	// Add field conversion funcs.
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "Pod",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name",
				"metadata.namespace",
				"status.phase",
				"spec.nodeName":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "Node",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name":
				return label, value, nil
			case "spec.unschedulable":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "ReplicationController",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name",
				"status.replicas":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "Event",
		func(label, value string) (string, string, error) {
			switch label {
			case "involvedObject.kind",
				"involvedObject.namespace",
				"involvedObject.name",
				"involvedObject.uid",
				"involvedObject.apiVersion",
				"involvedObject.resourceVersion",
				"involvedObject.fieldPath",
				"reason",
				"source":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "Namespace",
		func(label, value string) (string, string, error) {
			switch label {
			case "status.phase":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "Secret",
		func(label, value string) (string, string, error) {
			switch label {
			case "type":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "ServiceAccount",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "Endpoints",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
}

func convert_api_ReplicationControllerSpec_To_v1_ReplicationControllerSpec(in *api.ReplicationControllerSpec, out *ReplicationControllerSpec, s conversion.Scope) error {
	if defaulting, found := s.DefaultingInterface(reflect.TypeOf(*in)); found {
		defaulting.(func(*api.ReplicationControllerSpec))(in)
	}
	out.Replicas = new(int)
	*out.Replicas = in.Replicas
	if in.Selector != nil {
		out.Selector = make(map[string]string)
		for key, val := range in.Selector {
			out.Selector[key] = val
		}
	} else {
		out.Selector = nil
	}
	//if in.TemplateRef != nil {
	//	out.TemplateRef = new(ObjectReference)
	//	if err := convert_api_ObjectReference_To_v1_ObjectReference(in.TemplateRef, out.TemplateRef, s); err != nil {
	//		return err
	//	}
	//} else {
	//	out.TemplateRef = nil
	//}
	if in.Template != nil {
		out.Template = new(PodTemplateSpec)
		if err := convert_api_PodTemplateSpec_To_v1_PodTemplateSpec(in.Template, out.Template, s); err != nil {
			return err
		}
	} else {
		out.Template = nil
	}
	return nil
}

func convert_v1_ReplicationControllerSpec_To_api_ReplicationControllerSpec(in *ReplicationControllerSpec, out *api.ReplicationControllerSpec, s conversion.Scope) error {
	if defaulting, found := s.DefaultingInterface(reflect.TypeOf(*in)); found {
		defaulting.(func(*ReplicationControllerSpec))(in)
	}
	out.Replicas = *in.Replicas
	if in.Selector != nil {
		out.Selector = make(map[string]string)
		for key, val := range in.Selector {
			out.Selector[key] = val
		}
	} else {
		out.Selector = nil
	}
	//if in.TemplateRef != nil {
	//	out.TemplateRef = new(api.ObjectReference)
	//	if err := convert_v1_ObjectReference_To_api_ObjectReference(in.TemplateRef, out.TemplateRef, s); err != nil {
	//		return err
	//	}
	//} else {
	//	out.TemplateRef = nil
	//}
	if in.Template != nil {
		out.Template = new(api.PodTemplateSpec)
		if err := convert_v1_PodTemplateSpec_To_api_PodTemplateSpec(in.Template, out.Template, s); err != nil {
			return err
		}
	} else {
		out.Template = nil
	}
	return nil
}
<commit_msg>add spec.host as a synonym for spec.nodeName in v1<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1

import (
	"fmt"
	"reflect"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/conversion"
)

func addConversionFuncs() {
	// Add non-generated conversion functions
	err := api.Scheme.AddConversionFuncs(
		convert_v1_ReplicationControllerSpec_To_api_ReplicationControllerSpec,
		convert_api_ReplicationControllerSpec_To_v1_ReplicationControllerSpec,
	)
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}

	// Add field conversion funcs.
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "Pod",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name",
				"metadata.namespace",
				"status.phase",
				"spec.nodeName":
				return label, value, nil
		    // This is for backwards compatability with old v1 clients which send spec.host
			case "spec.host":
				return "spec.nodeName", value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "Node",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name":
				return label, value, nil
			case "spec.unschedulable":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "ReplicationController",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name",
				"status.replicas":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "Event",
		func(label, value string) (string, string, error) {
			switch label {
			case "involvedObject.kind",
				"involvedObject.namespace",
				"involvedObject.name",
				"involvedObject.uid",
				"involvedObject.apiVersion",
				"involvedObject.resourceVersion",
				"involvedObject.fieldPath",
				"reason",
				"source":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "Namespace",
		func(label, value string) (string, string, error) {
			switch label {
			case "status.phase":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "Secret",
		func(label, value string) (string, string, error) {
			switch label {
			case "type":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "ServiceAccount",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = api.Scheme.AddFieldLabelConversionFunc("v1", "Endpoints",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
}

func convert_api_ReplicationControllerSpec_To_v1_ReplicationControllerSpec(in *api.ReplicationControllerSpec, out *ReplicationControllerSpec, s conversion.Scope) error {
	if defaulting, found := s.DefaultingInterface(reflect.TypeOf(*in)); found {
		defaulting.(func(*api.ReplicationControllerSpec))(in)
	}
	out.Replicas = new(int)
	*out.Replicas = in.Replicas
	if in.Selector != nil {
		out.Selector = make(map[string]string)
		for key, val := range in.Selector {
			out.Selector[key] = val
		}
	} else {
		out.Selector = nil
	}
	//if in.TemplateRef != nil {
	//	out.TemplateRef = new(ObjectReference)
	//	if err := convert_api_ObjectReference_To_v1_ObjectReference(in.TemplateRef, out.TemplateRef, s); err != nil {
	//		return err
	//	}
	//} else {
	//	out.TemplateRef = nil
	//}
	if in.Template != nil {
		out.Template = new(PodTemplateSpec)
		if err := convert_api_PodTemplateSpec_To_v1_PodTemplateSpec(in.Template, out.Template, s); err != nil {
			return err
		}
	} else {
		out.Template = nil
	}
	return nil
}

func convert_v1_ReplicationControllerSpec_To_api_ReplicationControllerSpec(in *ReplicationControllerSpec, out *api.ReplicationControllerSpec, s conversion.Scope) error {
	if defaulting, found := s.DefaultingInterface(reflect.TypeOf(*in)); found {
		defaulting.(func(*ReplicationControllerSpec))(in)
	}
	out.Replicas = *in.Replicas
	if in.Selector != nil {
		out.Selector = make(map[string]string)
		for key, val := range in.Selector {
			out.Selector[key] = val
		}
	} else {
		out.Selector = nil
	}
	//if in.TemplateRef != nil {
	//	out.TemplateRef = new(api.ObjectReference)
	//	if err := convert_v1_ObjectReference_To_api_ObjectReference(in.TemplateRef, out.TemplateRef, s); err != nil {
	//		return err
	//	}
	//} else {
	//	out.TemplateRef = nil
	//}
	if in.Template != nil {
		out.Template = new(api.PodTemplateSpec)
		if err := convert_v1_PodTemplateSpec_To_api_PodTemplateSpec(in.Template, out.Template, s); err != nil {
			return err
		}
	} else {
		out.Template = nil
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package autoscaler

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"knative.dev/serving/pkg/apis/autoscaling"

	corev1 "k8s.io/api/core/v1"
)

const (
	// ConfigName is the name of the config map of the autoscaler.
	ConfigName = "config-autoscaler"
)

// Config defines the tunable autoscaler parameters
// +k8s:deepcopy-gen=true
type Config struct {
	// Feature flags.
	EnableScaleToZero bool

	// Target concurrency knobs for different container concurrency configurations.
	ContainerConcurrencyTargetFraction float64
	ContainerConcurrencyTargetDefault  float64
	// NB: most of our computations are in floats, so this is float to avoid casting.
	TargetBurstCapacity float64

	// General autoscaler algorithm configuration.
	MaxScaleUpRate           float64
	StableWindow             time.Duration
	PanicWindowPercentage    float64
	PanicThresholdPercentage float64
	// Deprecated in favor of PanicWindowPercentage.
	PanicWindow  time.Duration
	TickInterval time.Duration

	ScaleToZeroGracePeriod time.Duration
}

// NewConfigFromMap creates a Config from the supplied map
func NewConfigFromMap(data map[string]string) (*Config, error) {
	lc := &Config{}

	// Process bool fields.
	for _, b := range []struct {
		key          string
		field        *bool
		defaultValue bool
	}{{
		key:          "enable-scale-to-zero",
		field:        &lc.EnableScaleToZero,
		defaultValue: true,
	}} {
		if raw, ok := data[b.key]; !ok {
			*b.field = b.defaultValue
		} else {
			*b.field = strings.ToLower(raw) == "true"
		}
	}

	// Process Float64 fields
	for _, f64 := range []struct {
		key   string
		field *float64
		// specified exactly when optional
		defaultValue float64
	}{{
		key:          "max-scale-up-rate",
		field:        &lc.MaxScaleUpRate,
		defaultValue: 1000.0,
	}, {
		key:   "container-concurrency-target-percentage",
		field: &lc.ContainerConcurrencyTargetFraction,
		// TODO(#1956): Tune target usage based on empirical data.
		defaultValue: 0.7,
	}, {
		key:          "container-concurrency-target-default",
		field:        &lc.ContainerConcurrencyTargetDefault,
		defaultValue: 100.0,
	}, {
		key:          "target-burst-capacity",
		field:        &lc.TargetBurstCapacity,
		defaultValue: 0,
	}, {
		key:          "panic-window-percentage",
		field:        &lc.PanicWindowPercentage,
		defaultValue: 10.0,
	}, {
		key:          "panic-threshold-percentage",
		field:        &lc.PanicThresholdPercentage,
		defaultValue: 200.0,
	}} {
		if raw, ok := data[f64.key]; !ok {
			*f64.field = f64.defaultValue
		} else if val, err := strconv.ParseFloat(raw, 64); err != nil {
			return nil, err
		} else {
			*f64.field = val
		}
	}

	// Adjust % ⇒ fractions: for legacy reasons we allow values in the
	// (0, 1] interval, so minimal percentage must be greater than 1.0.
	// Internally we want to have fractions, since otherwise we'll have
	// to perform division on each computation.
	if lc.ContainerConcurrencyTargetFraction > 1.0 {
		lc.ContainerConcurrencyTargetFraction /= 100.0
	}

	// Process Duration fields
	for _, dur := range []struct {
		key          string
		field        *time.Duration
		defaultValue time.Duration
	}{{
		key:          "stable-window",
		field:        &lc.StableWindow,
		defaultValue: 60 * time.Second,
	}, {
		key:          "panic-window",
		field:        &lc.PanicWindow,
		defaultValue: 6 * time.Second,
	}, {
		key:          "scale-to-zero-grace-period",
		field:        &lc.ScaleToZeroGracePeriod,
		defaultValue: 30 * time.Second,
	}, {
		key:          "tick-interval",
		field:        &lc.TickInterval,
		defaultValue: 2 * time.Second,
	}} {
		if raw, ok := data[dur.key]; !ok {
			*dur.field = dur.defaultValue
		} else if val, err := time.ParseDuration(raw); err != nil {
			return nil, err
		} else {
			*dur.field = val
		}
	}

	return validate(lc)
}

func validate(lc *Config) (*Config, error) {
	if lc.ScaleToZeroGracePeriod < 30*time.Second {
		return nil, fmt.Errorf("scale-to-zero-grace-period must be at least 30s, got %v", lc.ScaleToZeroGracePeriod)
	}
	if lc.TargetBurstCapacity < 0 && lc.TargetBurstCapacity != -1 {
		return nil, fmt.Errorf("target-burst-capacity must be non-negative, got %f", lc.TargetBurstCapacity)
	}

	if lc.ContainerConcurrencyTargetFraction <= 0 || lc.ContainerConcurrencyTargetFraction > 1 {
		return nil, fmt.Errorf("container-concurrency-target-percentage = %f is outside of valid range of (0, 100]", lc.ContainerConcurrencyTargetFraction)
	}

	if x := lc.ContainerConcurrencyTargetFraction * lc.ContainerConcurrencyTargetDefault; x < 1.0 {
		return nil, fmt.Errorf("container-concurrency-target-percentage and container-concurrency-target-default yield target concurrency of %f, can't be less than 1", x)
	}

	// We can't permit stable window be less than our aggregation window for correctness.
	if lc.StableWindow < autoscaling.WindowMin {
		return nil, fmt.Errorf("stable-window = %v, must be at least %v", lc.StableWindow, BucketSize)
	}

	if lc.PanicWindow < BucketSize || lc.PanicWindow > lc.StableWindow {
		return nil, fmt.Errorf("panic-window = %v, must be in [%v, %v] interval", lc.PanicWindow, BucketSize, lc.StableWindow)
	}

	effPW := time.Duration(lc.PanicWindowPercentage / 100 * float64(lc.StableWindow))
	if effPW < BucketSize || effPW > lc.StableWindow {
		return nil, fmt.Errorf("panic-window-percentage = %v, must be in [%v, 100] interval", lc.PanicWindow, 100*float64(BucketSize)/float64(lc.StableWindow))
	}

	return lc, nil
}

// NewConfigFromConfigMap creates a Config from the supplied ConfigMap
func NewConfigFromConfigMap(configMap *corev1.ConfigMap) (*Config, error) {
	return NewConfigFromMap(configMap.Data)
}
<commit_msg>Fix panic-window-percentage error (#4817)<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package autoscaler

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"knative.dev/serving/pkg/apis/autoscaling"

	corev1 "k8s.io/api/core/v1"
)

const (
	// ConfigName is the name of the config map of the autoscaler.
	ConfigName = "config-autoscaler"
)

// Config defines the tunable autoscaler parameters
// +k8s:deepcopy-gen=true
type Config struct {
	// Feature flags.
	EnableScaleToZero bool

	// Target concurrency knobs for different container concurrency configurations.
	ContainerConcurrencyTargetFraction float64
	ContainerConcurrencyTargetDefault  float64
	// NB: most of our computations are in floats, so this is float to avoid casting.
	TargetBurstCapacity float64

	// General autoscaler algorithm configuration.
	MaxScaleUpRate           float64
	StableWindow             time.Duration
	PanicWindowPercentage    float64
	PanicThresholdPercentage float64
	// Deprecated in favor of PanicWindowPercentage.
	PanicWindow  time.Duration
	TickInterval time.Duration

	ScaleToZeroGracePeriod time.Duration
}

// NewConfigFromMap creates a Config from the supplied map
func NewConfigFromMap(data map[string]string) (*Config, error) {
	lc := &Config{}

	// Process bool fields.
	for _, b := range []struct {
		key          string
		field        *bool
		defaultValue bool
	}{{
		key:          "enable-scale-to-zero",
		field:        &lc.EnableScaleToZero,
		defaultValue: true,
	}} {
		if raw, ok := data[b.key]; !ok {
			*b.field = b.defaultValue
		} else {
			*b.field = strings.ToLower(raw) == "true"
		}
	}

	// Process Float64 fields
	for _, f64 := range []struct {
		key   string
		field *float64
		// specified exactly when optional
		defaultValue float64
	}{{
		key:          "max-scale-up-rate",
		field:        &lc.MaxScaleUpRate,
		defaultValue: 1000.0,
	}, {
		key:   "container-concurrency-target-percentage",
		field: &lc.ContainerConcurrencyTargetFraction,
		// TODO(#1956): Tune target usage based on empirical data.
		defaultValue: 0.7,
	}, {
		key:          "container-concurrency-target-default",
		field:        &lc.ContainerConcurrencyTargetDefault,
		defaultValue: 100.0,
	}, {
		key:          "target-burst-capacity",
		field:        &lc.TargetBurstCapacity,
		defaultValue: 0,
	}, {
		key:          "panic-window-percentage",
		field:        &lc.PanicWindowPercentage,
		defaultValue: 10.0,
	}, {
		key:          "panic-threshold-percentage",
		field:        &lc.PanicThresholdPercentage,
		defaultValue: 200.0,
	}} {
		if raw, ok := data[f64.key]; !ok {
			*f64.field = f64.defaultValue
		} else if val, err := strconv.ParseFloat(raw, 64); err != nil {
			return nil, err
		} else {
			*f64.field = val
		}
	}

	// Adjust % ⇒ fractions: for legacy reasons we allow values in the
	// (0, 1] interval, so minimal percentage must be greater than 1.0.
	// Internally we want to have fractions, since otherwise we'll have
	// to perform division on each computation.
	if lc.ContainerConcurrencyTargetFraction > 1.0 {
		lc.ContainerConcurrencyTargetFraction /= 100.0
	}

	// Process Duration fields
	for _, dur := range []struct {
		key          string
		field        *time.Duration
		defaultValue time.Duration
	}{{
		key:          "stable-window",
		field:        &lc.StableWindow,
		defaultValue: 60 * time.Second,
	}, {
		key:          "panic-window",
		field:        &lc.PanicWindow,
		defaultValue: 6 * time.Second,
	}, {
		key:          "scale-to-zero-grace-period",
		field:        &lc.ScaleToZeroGracePeriod,
		defaultValue: 30 * time.Second,
	}, {
		key:          "tick-interval",
		field:        &lc.TickInterval,
		defaultValue: 2 * time.Second,
	}} {
		if raw, ok := data[dur.key]; !ok {
			*dur.field = dur.defaultValue
		} else if val, err := time.ParseDuration(raw); err != nil {
			return nil, err
		} else {
			*dur.field = val
		}
	}

	return validate(lc)
}

func validate(lc *Config) (*Config, error) {
	if lc.ScaleToZeroGracePeriod < 30*time.Second {
		return nil, fmt.Errorf("scale-to-zero-grace-period must be at least 30s, got %v", lc.ScaleToZeroGracePeriod)
	}
	if lc.TargetBurstCapacity < 0 && lc.TargetBurstCapacity != -1 {
		return nil, fmt.Errorf("target-burst-capacity must be non-negative, got %f", lc.TargetBurstCapacity)
	}

	if lc.ContainerConcurrencyTargetFraction <= 0 || lc.ContainerConcurrencyTargetFraction > 1 {
		return nil, fmt.Errorf("container-concurrency-target-percentage = %f is outside of valid range of (0, 100]", lc.ContainerConcurrencyTargetFraction)
	}

	if x := lc.ContainerConcurrencyTargetFraction * lc.ContainerConcurrencyTargetDefault; x < 1.0 {
		return nil, fmt.Errorf("container-concurrency-target-percentage and container-concurrency-target-default yield target concurrency of %f, can't be less than 1", x)
	}

	// We can't permit stable window be less than our aggregation window for correctness.
	if lc.StableWindow < autoscaling.WindowMin {
		return nil, fmt.Errorf("stable-window = %v, must be at least %v", lc.StableWindow, BucketSize)
	}

	if lc.PanicWindow < BucketSize || lc.PanicWindow > lc.StableWindow {
		return nil, fmt.Errorf("panic-window = %v, must be in [%v, %v] interval", lc.PanicWindow, BucketSize, lc.StableWindow)
	}

	effPW := time.Duration(lc.PanicWindowPercentage / 100 * float64(lc.StableWindow))
	if effPW < BucketSize || effPW > lc.StableWindow {
		return nil, fmt.Errorf("panic-window-percentage = %v, must be in [%v, 100] interval", lc.PanicWindowPercentage, 100*float64(BucketSize)/float64(lc.StableWindow))
	}

	return lc, nil
}

// NewConfigFromConfigMap creates a Config from the supplied ConfigMap
func NewConfigFromConfigMap(configMap *corev1.ConfigMap) (*Config, error) {
	return NewConfigFromMap(configMap.Data)
}
<|endoftext|>


<commit_before>package godbg

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"regexp"
	"runtime"
	"strings"
)

// http://stackoverflow.com/a/23554672/6309 https://vividcortex.com/blog/2013/12/03/go-idiom-package-and-object/
// you design a type with methods as usual, and then you also place matching functions at the package level itself.
// These functions simply delegate to a default instance of the type that’s a private package-level variable, created in an init() function.

// Pdbg allows to print debug message with indent and function name added
type Pdbg struct {
	bout     *bytes.Buffer
	berr     *bytes.Buffer
	sout     *bufio.Writer
	serr     *bufio.Writer
	breaks   []string
	excludes []string
	skips    []string
}

// Out returns a writer for normal messages.
// By default, os.StdOut
func Out() io.Writer {
	return pdbg.Out()
}

// Out returns a writer for normal messages for a given pdbg instance.
// By default, os.StdOut
func (pdbg *Pdbg) Out() io.Writer {
	if pdbg.sout == nil {
		return os.Stdout
	}
	return pdbg.sout
}

// Err returns a writer for error messages.
// By default, os.StdErr
func Err() io.Writer {
	return pdbg.Err()
}

// Err returns a writer for error messages for a given pdbg instance.
// By default, os.StdErr
func (pdbg *Pdbg) Err() io.Writer {
	if pdbg.serr == nil {
		return os.Stderr
	}
	return pdbg.serr
}

// global pdbg used for printing
var pdbg = NewPdbg()

// Option set an option for a Pdbg
// http://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis
type Option func(*Pdbg)

// SetBuffers is an option for replacing stdout and stderr by
// bytes buffers (in a bufio.Writer).
// If apdbg is nil, set for the global pdbg instance
func SetBuffers(apdbg *Pdbg) {
	if apdbg == nil {
		apdbg = pdbg
	}
	apdbg.bout = bytes.NewBuffer(nil)
	apdbg.sout = bufio.NewWriter(apdbg.bout)
	apdbg.berr = bytes.NewBuffer(nil)
	apdbg.serr = bufio.NewWriter(apdbg.berr)
}

// SetExcludes set excludes on a pdbg (nil for global pdbg)
func (pdbg *Pdbg) SetExcludes(excludes []string) {
	pdbg.excludes = excludes
}

// OptExcludes is an option to set excludes at the creation of a pdbg
func OptExcludes(excludes []string) Option {
	return func(apdbg *Pdbg) {
		apdbg.SetExcludes(excludes)
	}
}

// NewPdbg creates a PDbg instance, with options
func NewPdbg(options ...Option) *Pdbg {
	newpdbg := &Pdbg{}
	for _, option := range options {
		option(newpdbg)
	}
	newpdbg.breaks = append(newpdbg.breaks, "smartystreets")
	//newpdbg.breaks = append(newpdbg.breaks, "(*Pdbg).Pdbgf")
	newpdbg.skips = append(newpdbg.breaks, "/gogdb.go'")
	return newpdbg
}

// ResetIOs reset the out and err buffer of global pdbg instance
func ResetIOs() {
	pdbg.ResetIOs()
}

// ResetIOs reset the out and err buffer
// (unless they were the default stdout and stderr,
// in which case it does nothing)
func (pdbg *Pdbg) ResetIOs() {
	if pdbg.sout != nil {
		pdbg.bout = bytes.NewBuffer(nil)
		pdbg.sout.Reset(pdbg.bout)
		pdbg.berr = bytes.NewBuffer(nil)
		pdbg.serr.Reset(pdbg.berr)
	}
}

// OutString returns the string for out messages for the global pdbg instance.
// It flushes the out buffer.
// If out is set to os.Stdout, returns an empty string
func OutString() string {
	return pdbg.OutString()
}

// OutString returns the string for out messages for a given pdbg instance.
// It flushes the out buffer.
// If out is set to os.Stdout, returns an empty string
func (pdbg *Pdbg) OutString() string {
	if pdbg.sout == nil {
		return ""
	}
	pdbg.sout.Flush()
	return pdbg.bout.String()
}

// ErrString returns the string for error messages for the global pdbg instance.
// It flushes the err buffer.
// If err is set to os.StdErr, returns an empty string
func ErrString() string {
	return pdbg.ErrString()
}

// ErrString returns the string for error messages for a given pdbg instance.
// It flushes the err buffer.
// If err is set to os.StdErr, returns an empty string
func (pdbg *Pdbg) ErrString() string {
	if pdbg.serr == nil {
		return ""
	}
	pdbg.serr.Flush()
	return pdbg.berr.String()
}

func (pdbg *Pdbg) pdbgExcluded(dbg string) bool {
	for _, e := range pdbg.excludes {
		if strings.Contains(dbg, e) {
			fmt.Printf("EXCLUDE over '%v' including '%v'\n", dbg, e)
			return true
		}
	}
	return false
}

func (pdbg *Pdbg) pdbgBreak(dbg string) bool {
	for _, b := range pdbg.breaks {
		if strings.Contains(dbg, b) {
			fmt.Printf("BREAK over '%v' including '%v'\n", dbg, b)
			return true
		}
	}
	return false
}

func (pdbg *Pdbg) pdbgSkip(dbg string) bool {
	for _, s := range pdbg.skips {
		if strings.Contains(dbg, s) {
			fmt.Printf("SKIP over '%v' including '%v'\n", dbg, s)
			return true
		}
	}
	return false
}

// Pdbgf uses global Pdbg variable for printing strings, with indent and function name
func Pdbgf(format string, args ...interface{}) string {
	return pdbg.Pdbgf(format, args...)
}

// Pdbgf uses custom Pdbg variable for printing strings, with indent and function name
func (pdbg *Pdbg) Pdbgf(format string, args ...interface{}) string {
	msg := fmt.Sprintf(format+"\n", args...)
	msg = strings.TrimSpace(msg)

	pmsg := ""
	depth := 0
	nbskip := 0
	first := true
	for ok := true; ok; {
		pc, file, line, ok := runtime.Caller(depth)
		if !ok {
			break
		}
		fname := runtime.FuncForPC(pc).Name()
		fline := fmt.Sprintf("Name of function: '%v': '%+x' (line %v): file '%v'\n", fname, fname, line, file)
		fmt.Println(fline)
		if pdbg.pdbgBreak(fline) {
			break
		}
		if pdbg.pdbgSkip(fline) {
			depth = depth + 1
			nbskip = nbskip + 1
			continue
		}
		fnamerx1 := regexp.MustCompile(`.*\.func[^a-zA-Z0-9]`)
		fname = fnamerx1.ReplaceAllString(fname, "func.")
		dbg := fname + ":" + fmt.Sprintf("%d", line)
		if first {
			if pdbg.pdbgExcluded(dbg) {
				return ""
			}
			pmsg = "[" + dbg + "]"
		} else {
			pmsg = pmsg + " (" + dbg + ")"
		}
		first = false
		depth = depth + 1
	}
	depth = depth - nbskip

	spaces := ""
	if depth >= 2 {
		spaces = strings.Repeat(" ", depth-2)
	}
	// fmt.Printf("spaces '%s', depth '%d'\n", spaces, depth)
	res := pmsg
	pmsg = spaces + pmsg
	msg = pmsg + "\n" + spaces + "  " + msg + "\n"
	// fmt.Printf("MSG '%v'\n", msg)
	fmt.Fprint(pdbg.Err(), fmt.Sprint(msg))
	return res
}
<commit_msg>Exclude earlier<commit_after>package godbg

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"regexp"
	"runtime"
	"strings"
)

// http://stackoverflow.com/a/23554672/6309 https://vividcortex.com/blog/2013/12/03/go-idiom-package-and-object/
// you design a type with methods as usual, and then you also place matching functions at the package level itself.
// These functions simply delegate to a default instance of the type that’s a private package-level variable, created in an init() function.

// Pdbg allows to print debug message with indent and function name added
type Pdbg struct {
	bout     *bytes.Buffer
	berr     *bytes.Buffer
	sout     *bufio.Writer
	serr     *bufio.Writer
	breaks   []string
	excludes []string
	skips    []string
}

// Out returns a writer for normal messages.
// By default, os.StdOut
func Out() io.Writer {
	return pdbg.Out()
}

// Out returns a writer for normal messages for a given pdbg instance.
// By default, os.StdOut
func (pdbg *Pdbg) Out() io.Writer {
	if pdbg.sout == nil {
		return os.Stdout
	}
	return pdbg.sout
}

// Err returns a writer for error messages.
// By default, os.StdErr
func Err() io.Writer {
	return pdbg.Err()
}

// Err returns a writer for error messages for a given pdbg instance.
// By default, os.StdErr
func (pdbg *Pdbg) Err() io.Writer {
	if pdbg.serr == nil {
		return os.Stderr
	}
	return pdbg.serr
}

// global pdbg used for printing
var pdbg = NewPdbg()

// Option set an option for a Pdbg
// http://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis
type Option func(*Pdbg)

// SetBuffers is an option for replacing stdout and stderr by
// bytes buffers (in a bufio.Writer).
// If apdbg is nil, set for the global pdbg instance
func SetBuffers(apdbg *Pdbg) {
	if apdbg == nil {
		apdbg = pdbg
	}
	apdbg.bout = bytes.NewBuffer(nil)
	apdbg.sout = bufio.NewWriter(apdbg.bout)
	apdbg.berr = bytes.NewBuffer(nil)
	apdbg.serr = bufio.NewWriter(apdbg.berr)
}

// SetExcludes set excludes on a pdbg (nil for global pdbg)
func (pdbg *Pdbg) SetExcludes(excludes []string) {
	pdbg.excludes = excludes
}

// OptExcludes is an option to set excludes at the creation of a pdbg
func OptExcludes(excludes []string) Option {
	return func(apdbg *Pdbg) {
		apdbg.SetExcludes(excludes)
	}
}

// NewPdbg creates a PDbg instance, with options
func NewPdbg(options ...Option) *Pdbg {
	newpdbg := &Pdbg{}
	for _, option := range options {
		option(newpdbg)
	}
	newpdbg.breaks = append(newpdbg.breaks, "smartystreets")
	//newpdbg.breaks = append(newpdbg.breaks, "(*Pdbg).Pdbgf")
	newpdbg.skips = append(newpdbg.breaks, "/gogdb.go'")
	return newpdbg
}

// ResetIOs reset the out and err buffer of global pdbg instance
func ResetIOs() {
	pdbg.ResetIOs()
}

// ResetIOs reset the out and err buffer
// (unless they were the default stdout and stderr,
// in which case it does nothing)
func (pdbg *Pdbg) ResetIOs() {
	if pdbg.sout != nil {
		pdbg.bout = bytes.NewBuffer(nil)
		pdbg.sout.Reset(pdbg.bout)
		pdbg.berr = bytes.NewBuffer(nil)
		pdbg.serr.Reset(pdbg.berr)
	}
}

// OutString returns the string for out messages for the global pdbg instance.
// It flushes the out buffer.
// If out is set to os.Stdout, returns an empty string
func OutString() string {
	return pdbg.OutString()
}

// OutString returns the string for out messages for a given pdbg instance.
// It flushes the out buffer.
// If out is set to os.Stdout, returns an empty string
func (pdbg *Pdbg) OutString() string {
	if pdbg.sout == nil {
		return ""
	}
	pdbg.sout.Flush()
	return pdbg.bout.String()
}

// ErrString returns the string for error messages for the global pdbg instance.
// It flushes the err buffer.
// If err is set to os.StdErr, returns an empty string
func ErrString() string {
	return pdbg.ErrString()
}

// ErrString returns the string for error messages for a given pdbg instance.
// It flushes the err buffer.
// If err is set to os.StdErr, returns an empty string
func (pdbg *Pdbg) ErrString() string {
	if pdbg.serr == nil {
		return ""
	}
	pdbg.serr.Flush()
	return pdbg.berr.String()
}

func (pdbg *Pdbg) pdbgExcluded(dbg string) bool {
	for _, e := range pdbg.excludes {
		if strings.Contains(dbg, e) {
			fmt.Printf("EXCLUDE over '%v' including '%v'\n", dbg, e)
			return true
		}
	}
	return false
}

func (pdbg *Pdbg) pdbgBreak(dbg string) bool {
	for _, b := range pdbg.breaks {
		if strings.Contains(dbg, b) {
			fmt.Printf("BREAK over '%v' including '%v'\n", dbg, b)
			return true
		}
	}
	return false
}

func (pdbg *Pdbg) pdbgSkip(dbg string) bool {
	for _, s := range pdbg.skips {
		if strings.Contains(dbg, s) {
			fmt.Printf("SKIP over '%v' including '%v'\n", dbg, s)
			return true
		}
	}
	return false
}

// Pdbgf uses global Pdbg variable for printing strings, with indent and function name
func Pdbgf(format string, args ...interface{}) string {
	return pdbg.Pdbgf(format, args...)
}

// Pdbgf uses custom Pdbg variable for printing strings, with indent and function name
func (pdbg *Pdbg) Pdbgf(format string, args ...interface{}) string {
	msg := fmt.Sprintf(format+"\n", args...)
	msg = strings.TrimSpace(msg)

	pmsg := ""
	depth := 0
	nbskip := 0
	first := true
	for ok := true; ok; {
		pc, file, line, ok := runtime.Caller(depth)
		if !ok {
			break
		}
		fname := runtime.FuncForPC(pc).Name()
		fline := fmt.Sprintf("Name of function: '%v': '%+x' (line %v): file '%v'\n", fname, fname, line, file)
		fmt.Println(fline)
		if pdbg.pdbgExcluded(fline) {
			return ""
		}
		if pdbg.pdbgBreak(fline) {
			break
		}
		if pdbg.pdbgSkip(fline) {
			depth = depth + 1
			nbskip = nbskip + 1
			continue
		}
		fnamerx1 := regexp.MustCompile(`.*\.func[^a-zA-Z0-9]`)
		fname = fnamerx1.ReplaceAllString(fname, "func.")
		dbg := fname + ":" + fmt.Sprintf("%d", line)
		if first {
			pmsg = "[" + dbg + "]"
		} else {
			pmsg = pmsg + " (" + dbg + ")"
		}
		first = false
		depth = depth + 1
	}
	depth = depth - nbskip

	spaces := ""
	if depth >= 2 {
		spaces = strings.Repeat(" ", depth-2)
	}
	// fmt.Printf("spaces '%s', depth '%d'\n", spaces, depth)
	res := pmsg
	pmsg = spaces + pmsg
	msg = pmsg + "\n" + spaces + "  " + msg + "\n"
	// fmt.Printf("MSG '%v'\n", msg)
	fmt.Fprint(pdbg.Err(), fmt.Sprint(msg))
	return res
}
<|endoftext|>


<commit_before>// Package goini provides a config parser for .ini files.
//
// In this dialect:
//
//   - Comments are lines starting with either a '#' or a ';'.
//   - A line ending with a '\' continues onto the next line.
//   - It is illegal to have a continuation before a comment or the end of
//     file.
//
package goini

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
)

const (
	UniqueOption = iota
	MultiOption  = iota
)

type DecodeOption struct {
	Kind  int
	Usage string
	Parse func(interface{}, string) error
}

type DecodeOptionSet map[string]*DecodeOption

// Warning: Prefer to use the public methods since the type of RawSection
// might change.
type RawSection map[string][]string

type RawConfig struct {
	GlobalSection RawSection
	sections      map[string]RawSection
}

// An object for parsing config files and building a RawConfig. Can be
// used to parse and merge multiple config files. Uses the "Errors are values"
// pattern -- intermediate steps can return error, but only the final error
// returned from Finish() needs to be checked.
type RawConfigParser struct {
	config         *RawConfig
	currentSection RawSection
	currentLine    string
	err            error
}

func (s RawSection) addProperty(property, value string) {
	s[property] = append(s[property], value)
}

// Returns all the values set for a property or the empty list nil if has
// never been set.
func (s RawSection) GetPropertyValues(property string) []string {
	return s[property]
}

// If the property has been set at least once, returns all values joined
// as a space separated string. Returns true if the propery has been set
// at least once.
func (s RawSection) GetPropertyNumber(property string) (json.Number, bool) {
	vs, ok := s[property]
	if !ok {
		return "", false
	}
	return json.Number(strings.Join(vs, " ")), true
}

// Returns the list of unique properties that have been set at least once.
func (s RawSection) Properties() []string {
	keys := []string{}
	for p := range s {
		keys = append(keys, p)
	}
	return keys
}

func (dos DecodeOptionSet) Decode(dest interface{}, section RawSection) error {
	for _, property := range section.Properties() {
		option, ok := dos[property]
		if !ok {
			return fmt.Errorf("unexpected property %s",
				strconv.Quote(property))
		}
		values := section.GetPropertyValues(property)
		if option.Kind == UniqueOption && len(values) != 1 {
			return fmt.Errorf("property %s cannot be repeated",
				strconv.Quote(property))
		}
		for _, value := range values {
			if e := option.Parse(dest, value); e != nil {
				return fmt.Errorf("error parsing %s: %s",
					strconv.Quote(property), e)
			}
		}
	}
	return nil
}

// Returns the list of unique sections in the config object.
func (c *RawConfig) Sections() map[string]RawSection {
	return c.sections
}

func NewRawConfigParser() *RawConfigParser {
	config := &RawConfig{make(RawSection), make(map[string]RawSection)}
	return &RawConfigParser{config, config.GlobalSection, "", nil}
}

func (cp *RawConfigParser) parseLine(line string) error {
	if cp.err != nil {
		return cp.err
	}

	if len(line) > 0 && (line[0] == ';' || line[0] == '#') {
		if cp.currentLine != "" {
			cp.err = errors.New("Invalid continuation into comment line.")
			return cp.err
		}
		return nil
	}

	if len(line) > 0 && line[len(line)-1] == '\\' {
		cp.currentLine += line[:len(line)-1]
		return nil
	}
	line = cp.currentLine + line
	cp.currentLine = ""

	if len(strings.TrimSpace(line)) == 0 {
		return nil
	}

	if line[0] == '[' {
		if cp.err = cp.parseSectionHeader(line); cp.err != nil {
			return cp.err
		}
	} else if cp.err = cp.parseProperty(line); cp.err != nil {
		return cp.err
	}

	return nil
}

func (cp *RawConfigParser) parseSectionHeader(line string) error {
	if line[0] != '[' {
		cp.err = errors.New("Invalid section header start character")
		return cp.err
	}

	parts := strings.SplitN(line[1:], "]", 2)
	if len(parts) != 2 {
		cp.err = errors.New("No section header end character found")
		return cp.err
	}
	if parts[1] != "" {
		cp.err = errors.New("Trailing characters after section header")
		return cp.err
	}

	return cp.addSection(parts[0])
}

func (cp *RawConfigParser) parseProperty(line string) error {
	parts := strings.SplitN(line, "=", 2)
	if len(parts) != 2 || len(parts[0]) == 0 {
		cp.err = errors.New("Invalid property line")
		return cp.err
	}

	cp.currentSection.addProperty(parts[0], parts[1])
	return nil
}

// Returns the new config object or the first error encountered while parsing.
//
// Also resets the config parser.
func (cp *RawConfigParser) Finish() (*RawConfig, error) {
	retConfig, retError := cp.config, cp.err
	cp.config = &RawConfig{make(RawSection), make(map[string]RawSection)}
	cp.currentSection = cp.config.GlobalSection
	cp.err = nil
	if retError != nil {
		return nil, retError
	}
	return retConfig, nil
}

func (cp *RawConfigParser) addSection(name string) error {
	if _, ok := cp.config.sections[name]; ok {
		cp.err = errors.New(fmt.Sprint("Duplicate section name", strconv.Quote(name)))
		return cp.err
	}

	cp.currentSection = make(map[string][]string)
	cp.config.sections[name] = cp.currentSection
	return nil
}

func (cp *RawConfigParser) Parse(file io.Reader) error {
	if cp.err != nil {
		return cp.err
	}

	line := 0
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line++

		if err := cp.parseLine(scanner.Text()); err != nil {
			return fmt.Errorf("error parsing line %d %v",
				line, err)
		}
	}
	if err := scanner.Err(); err != nil {
		return err
	}

	if cp.currentLine != "" {
		return fmt.Errorf(
			"error parsing line %d: continuation at end of file", line)
	}
	return nil
}

func (cp *RawConfigParser) ParseFile(filename string) error {
	if cp.err != nil {
		return cp.err
	}

	file, err := os.Open(filename)
	if err != nil {
		return fmt.Errorf("error opening file %s: %v",
			strconv.Quote(filename), err)
	}
	return cp.Parse(file)
}

func ParseFile(filename string) (*RawConfig, error) {
	cp := NewRawConfigParser()
	if err := cp.ParseFile(filename); err != nil {
		return nil, err
	}
	return cp.Finish()
}

func Parse(reader io.Reader) (*RawConfig, error) {
	cp := NewRawConfigParser()
	if err := cp.Parse(reader); err != nil {
		return nil, err
	}
	return cp.Finish()
}
<commit_msg>Make type for DecodeOptionKind<commit_after>// Package goini provides a config parser for .ini files.
//
// In this dialect:
//
//   - Comments are lines starting with either a '#' or a ';'.
//   - A line ending with a '\' continues onto the next line.
//   - It is illegal to have a continuation before a comment or the end of
//     file.
//
package goini

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
)

type DecodeOptionKind int

const (
	UniqueOption DecodeOptionKind = iota
	MultiOption                   = iota
)

type DecodeOption struct {
	Kind  DecodeOptionKind
	Usage string
	Parse func(interface{}, string) error
}

type DecodeOptionSet map[string]*DecodeOption

// Warning: Prefer to use the public methods since the type of RawSection
// might change.
type RawSection map[string][]string

type RawConfig struct {
	GlobalSection RawSection
	sections      map[string]RawSection
}

// An object for parsing config files and building a RawConfig. Can be
// used to parse and merge multiple config files. Uses the "Errors are values"
// pattern -- intermediate steps can return error, but only the final error
// returned from Finish() needs to be checked.
type RawConfigParser struct {
	config         *RawConfig
	currentSection RawSection
	currentLine    string
	err            error
}

func (s RawSection) addProperty(property, value string) {
	s[property] = append(s[property], value)
}

// Returns all the values set for a property or the empty list nil if has
// never been set.
func (s RawSection) GetPropertyValues(property string) []string {
	return s[property]
}

// If the property has been set at least once, returns all values joined
// as a space separated string. Returns true if the propery has been set
// at least once.
func (s RawSection) GetPropertyNumber(property string) (json.Number, bool) {
	vs, ok := s[property]
	if !ok {
		return "", false
	}
	return json.Number(strings.Join(vs, " ")), true
}

// Returns the list of unique properties that have been set at least once.
func (s RawSection) Properties() []string {
	keys := []string{}
	for p := range s {
		keys = append(keys, p)
	}
	return keys
}

func (dos DecodeOptionSet) Decode(dest interface{}, section RawSection) error {
	for _, property := range section.Properties() {
		option, ok := dos[property]
		if !ok {
			return fmt.Errorf("unexpected property %s",
				strconv.Quote(property))
		}
		values := section.GetPropertyValues(property)
		if option.Kind == UniqueOption && len(values) != 1 {
			return fmt.Errorf("property %s cannot be repeated",
				strconv.Quote(property))
		}
		for _, value := range values {
			if e := option.Parse(dest, value); e != nil {
				return fmt.Errorf("error parsing %s: %s",
					strconv.Quote(property), e)
			}
		}
	}
	return nil
}

// Returns the list of unique sections in the config object.
func (c *RawConfig) Sections() map[string]RawSection {
	return c.sections
}

func NewRawConfigParser() *RawConfigParser {
	config := &RawConfig{make(RawSection), make(map[string]RawSection)}
	return &RawConfigParser{config, config.GlobalSection, "", nil}
}

func (cp *RawConfigParser) parseLine(line string) error {
	if cp.err != nil {
		return cp.err
	}

	if len(line) > 0 && (line[0] == ';' || line[0] == '#') {
		if cp.currentLine != "" {
			cp.err = errors.New("Invalid continuation into comment line.")
			return cp.err
		}
		return nil
	}

	if len(line) > 0 && line[len(line)-1] == '\\' {
		cp.currentLine += line[:len(line)-1]
		return nil
	}
	line = cp.currentLine + line
	cp.currentLine = ""

	if len(strings.TrimSpace(line)) == 0 {
		return nil
	}

	if line[0] == '[' {
		if cp.err = cp.parseSectionHeader(line); cp.err != nil {
			return cp.err
		}
	} else if cp.err = cp.parseProperty(line); cp.err != nil {
		return cp.err
	}

	return nil
}

func (cp *RawConfigParser) parseSectionHeader(line string) error {
	if line[0] != '[' {
		cp.err = errors.New("Invalid section header start character")
		return cp.err
	}

	parts := strings.SplitN(line[1:], "]", 2)
	if len(parts) != 2 {
		cp.err = errors.New("No section header end character found")
		return cp.err
	}
	if parts[1] != "" {
		cp.err = errors.New("Trailing characters after section header")
		return cp.err
	}

	return cp.addSection(parts[0])
}

func (cp *RawConfigParser) parseProperty(line string) error {
	parts := strings.SplitN(line, "=", 2)
	if len(parts) != 2 || len(parts[0]) == 0 {
		cp.err = errors.New("Invalid property line")
		return cp.err
	}

	cp.currentSection.addProperty(parts[0], parts[1])
	return nil
}

// Returns the new config object or the first error encountered while parsing.
//
// Also resets the config parser.
func (cp *RawConfigParser) Finish() (*RawConfig, error) {
	retConfig, retError := cp.config, cp.err
	cp.config = &RawConfig{make(RawSection), make(map[string]RawSection)}
	cp.currentSection = cp.config.GlobalSection
	cp.err = nil
	if retError != nil {
		return nil, retError
	}
	return retConfig, nil
}

func (cp *RawConfigParser) addSection(name string) error {
	if _, ok := cp.config.sections[name]; ok {
		cp.err = errors.New(fmt.Sprint("Duplicate section name", strconv.Quote(name)))
		return cp.err
	}

	cp.currentSection = make(map[string][]string)
	cp.config.sections[name] = cp.currentSection
	return nil
}

func (cp *RawConfigParser) Parse(file io.Reader) error {
	if cp.err != nil {
		return cp.err
	}

	line := 0
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line++

		if err := cp.parseLine(scanner.Text()); err != nil {
			return fmt.Errorf("error parsing line %d %v",
				line, err)
		}
	}
	if err := scanner.Err(); err != nil {
		return err
	}

	if cp.currentLine != "" {
		return fmt.Errorf(
			"error parsing line %d: continuation at end of file", line)
	}
	return nil
}

func (cp *RawConfigParser) ParseFile(filename string) error {
	if cp.err != nil {
		return cp.err
	}

	file, err := os.Open(filename)
	if err != nil {
		return fmt.Errorf("error opening file %s: %v",
			strconv.Quote(filename), err)
	}
	return cp.Parse(file)
}

func ParseFile(filename string) (*RawConfig, error) {
	cp := NewRawConfigParser()
	if err := cp.ParseFile(filename); err != nil {
		return nil, err
	}
	return cp.Finish()
}

func Parse(reader io.Reader) (*RawConfig, error) {
	cp := NewRawConfigParser()
	if err := cp.Parse(reader); err != nil {
		return nil, err
	}
	return cp.Finish()
}
<|endoftext|>