<commit_before>package regression

import (
	"context"
	"crypto/md5"
	"errors"
	"fmt"
	"math"
	"sync"
	"time"

	"go.skia.org/infra/go/git/gitinfo"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/vec32"
	"go.skia.org/infra/perf/go/cid"
	"go.skia.org/infra/perf/go/clustering2"
	"go.skia.org/infra/perf/go/config"
	"go.skia.org/infra/perf/go/dataframe"
	"go.skia.org/infra/perf/go/shortcut2"
	"go.skia.org/infra/perf/go/types"
)

type ProcessState string

const (
	PROCESS_RUNNING ProcessState = "Running"
	PROCESS_SUCCESS ProcessState = "Success"
	PROCESS_ERROR   ProcessState = "Error"
)

type ClusterRequestType int

const (
	// MAX_FINISHED_PROCESS_AGE is the amount of time to keep a finished
	// ClusterRequestProcess around before deleting it.
	MAX_FINISHED_PROCESS_AGE = time.Minute

	// The following limits are just to prevent excessively large or long-running
	// clusterings from being triggered.

	// MAX_K is the largest K used for clustering.
	MAX_K = 100

	// MAX_RADIUS  is the maximum number of points on either side of a commit
	// that will be included in clustering. This cannot exceed COMMITS_PER_TILE.
	MAX_RADIUS = 50

	// SPARSE_BLOCK_SEARCH_MULT When searching for commits that have data in a
	// sparse data set, we'll request data in chunks of this many commits per
	// point we are looking for.
	SPARSE_BLOCK_SEARCH_MULT = 100

	CLUSTERING_REQUEST_TYPE_SINGLE ClusterRequestType = 0 // Do clustering at a single commit.
	CLUSTERING_REQUEST_TYPE_LAST_N ClusterRequestType = 1 // Do clustering over a range of dense commits.
)

var (
	errorNotFound = errors.New("Process not found.")
)

// ClusterRequest is all the info needed to start a clustering run.
type ClusterRequest struct {
	Source      string             `json:"source"`
	Offset      int                `json:"offset"`
	Radius      int                `json:"radius"`
	Query       string             `json:"query"`
	K           int                `json:"k"`
	TZ          string             `json:"tz"`
	Algo        types.ClusterAlgo  `json:"algo"`
	Interesting float32            `json:"interesting"`
	Sparse      bool               `json:"sparse"`
	Type        ClusterRequestType `json:"type"`
	N           int32              `json:"n"`
	End         int                `json:"end"`
}

func (c *ClusterRequest) Id() string {
	return fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%#v", *c))))
}

// ClusterResponse is the response from running clustering over a ClusterRequest.
type ClusterResponse struct {
	Summary *clustering2.ClusterSummaries `json:"summary"`
	Frame   *dataframe.FrameResponse      `json:"frame"`
}

// ClusterRequestProcess handles the processing of a single ClusterRequest.
type ClusterRequestProcess struct {
	// These members are read-only, should not be modified.
	request *ClusterRequest
	git     *gitinfo.GitInfo
	iter    DataFrameIterator

	// mutex protects access to the remaining struct members.
	mutex      sync.RWMutex
	response   []*ClusterResponse // The response when the clustering is complete.
	lastUpdate time.Time          // The last time this process was updated.
	state      ProcessState       // The current state of the process.
	message    string             // Describes the current state of the process.
}

func newProcess(req *ClusterRequest, git *gitinfo.GitInfo, cidl *cid.CommitIDLookup, dfBuilder dataframe.DataFrameBuilder) *ClusterRequestProcess {
	ret := &ClusterRequestProcess{
		request:    req,
		git:        git,
		response:   []*ClusterResponse{},
		lastUpdate: time.Now(),
		state:      PROCESS_RUNNING,
		message:    "Running",
	}
	// TODO(jcgregorio) This is awkward and should go away in a future CL.
	ret.iter = NewSingleDataFrameIterator(ret.progress, cidl, git, req, dfBuilder)
	return ret
}

func newRunningProcess(ctx context.Context, req *ClusterRequest, git *gitinfo.GitInfo, cidl *cid.CommitIDLookup, dfBuilder dataframe.DataFrameBuilder) *ClusterRequestProcess {
	ret := newProcess(req, git, cidl, dfBuilder)
	go ret.Run(ctx)
	return ret
}

// RunningClusterRequests keeps track of all the ClusterRequestProcess's.
//
// Once a ClusterRequestProcess is complete the results will be kept in memory
// for MAX_FINISHED_PROCESS_AGE before being deleted.
type RunningClusterRequests struct {
	git                *gitinfo.GitInfo
	cidl               *cid.CommitIDLookup
	defaultInteresting float32 // The threshold to control if a cluster is considered interesting.
	dfBuilder          dataframe.DataFrameBuilder

	mutex sync.Mutex
	// inProcess maps a ClusterRequest.Id() of the request to the ClusterRequestProcess
	// handling that request.
	inProcess map[string]*ClusterRequestProcess
}

// NewRunningClusterRequests return a new RunningClusterRequests.
func NewRunningClusterRequests(git *gitinfo.GitInfo, cidl *cid.CommitIDLookup, interesting float32, dfBuilder dataframe.DataFrameBuilder) *RunningClusterRequests {
	fr := &RunningClusterRequests{
		git:                git,
		cidl:               cidl,
		inProcess:          map[string]*ClusterRequestProcess{},
		defaultInteresting: interesting,
		dfBuilder:          dfBuilder,
	}
	go fr.background()
	return fr
}

// step does a single step in cleaning up old ClusterRequestProcess's.
func (fr *RunningClusterRequests) step() {
	fr.mutex.Lock()
	defer fr.mutex.Unlock()
	now := time.Now()
	for k, v := range fr.inProcess {
		v.mutex.Lock()
		if now.Sub(v.lastUpdate) > MAX_FINISHED_PROCESS_AGE {
			delete(fr.inProcess, k)
		}
		v.mutex.Unlock()
	}
}

// background periodically cleans up old ClusterRequestProcess's.
func (fr *RunningClusterRequests) background() {
	fr.step()
	for range time.Tick(time.Minute) {
		fr.step()
	}
}

// Add starts a new running ClusterRequestProcess and returns
// the ID of the process to be used in calls to Status() and
// Response().
func (fr *RunningClusterRequests) Add(ctx context.Context, req *ClusterRequest) string {
	fr.mutex.Lock()
	defer fr.mutex.Unlock()
	if req.Interesting == 0 {
		req.Interesting = fr.defaultInteresting
	}
	id := req.Id()
	if p, ok := fr.inProcess[id]; ok {
		state, _, _ := p.Status()
		if state != PROCESS_RUNNING {
			delete(fr.inProcess, id)
		}
	}
	if _, ok := fr.inProcess[id]; !ok {
		fr.inProcess[id] = newRunningProcess(ctx, req, fr.git, fr.cidl, fr.dfBuilder)
	}
	return id
}

// Status returns the ProcessingState and the message of a
// ClusterRequestProcess of the given 'id'.
func (fr *RunningClusterRequests) Status(id string) (ProcessState, string, error) {
	fr.mutex.Lock()
	defer fr.mutex.Unlock()
	if p, ok := fr.inProcess[id]; !ok {
		return PROCESS_ERROR, "Not Found", errorNotFound
	} else {
		return p.Status()
	}
}

// Response returns the ClusterResponse of the completed ClusterRequestProcess.
func (fr *RunningClusterRequests) Response(id string) (*ClusterResponse, error) {
	fr.mutex.Lock()
	defer fr.mutex.Unlock()
	if p, ok := fr.inProcess[id]; !ok {
		return nil, errorNotFound
	} else {
		return p.Response(), nil
	}
}

// Responses returns the ClusterResponse's of the completed ClusterRequestProcess.
func (fr *RunningClusterRequests) Responses(id string) ([]*ClusterResponse, error) {
	fr.mutex.Lock()
	defer fr.mutex.Unlock()
	if p, ok := fr.inProcess[id]; !ok {
		return nil, errorNotFound
	} else {
		return p.Responses(), nil
	}
}

// reportError records the reason a ClusterRequestProcess failed.
func (p *ClusterRequestProcess) reportError(err error, message string) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	sklog.Warningf("ClusterRequest failed: %#v %s: %s", *(p.request), message, err)
	p.message = fmt.Sprintf("%s: %s", message, err)
	p.state = PROCESS_ERROR
	p.lastUpdate = time.Now()
}

// progress records the progress of a ClusterRequestProcess.
func (p *ClusterRequestProcess) progress(step, totalSteps int) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	p.message = fmt.Sprintf("Querying: %d%%", int(float32(100.0)*float32(step)/float32(totalSteps)))
	p.lastUpdate = time.Now()
}

// clusterProgress records the progress of a ClusterRequestProcess.
func (p *ClusterRequestProcess) clusterProgress(totalError float64) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	p.message = fmt.Sprintf("Clustering Total Error: %0.2f", totalError)
	p.lastUpdate = time.Now()
}

// Response returns the ClusterResponse of the completed ClusterRequestProcess.
func (p *ClusterRequestProcess) Response() *ClusterResponse {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.response[0]
}

// Responses returns all the ClusterResponse's of the ClusterRequestProcess.
func (p *ClusterRequestProcess) Responses() []*ClusterResponse {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.response
}

// Status returns the ProcessingState and the message of a
// ClusterRequestProcess of the given 'id'.
func (p *ClusterRequestProcess) Status() (ProcessState, string, error) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.state, p.message, nil
}

// missing returns true if >50% of the trace is vec32.MISSING_DATA_SENTINEL.
func missing(tr types.Trace) bool {
	count := 0
	for _, x := range tr {
		if x == vec32.MISSING_DATA_SENTINEL {
			count++
		}
	}
	return (100*count)/len(tr) > 50
}

// tooMuchMissingData returns true if a trace has too many
// MISSING_DATA_SENTINEL values.
//
// The criteria is if there is >50% missing data on either side of the target
// commit, which sits at the center of the trace.
func tooMuchMissingData(tr types.Trace) bool {
	if len(tr) < 3 {
		return false
	}
	n := len(tr) / 2
	if tr[n] == vec32.MISSING_DATA_SENTINEL {
		return true
	}
	return missing(tr[:n]) || missing(tr[len(tr)-n:])
}

// ShortcutFromKeys stores a new shortcut for each cluster based on its Keys.
func ShortcutFromKeys(summary *clustering2.ClusterSummaries) error {
	var err error
	for _, cs := range summary.Clusters {
		if cs.Shortcut, err = shortcut2.InsertShortcut(&shortcut2.Shortcut{Keys: cs.Keys}); err != nil {
			return err
		}
	}
	return nil
}

// Run does the work in a ClusterRequestProcess. It does not return until all the
// work is done or the request failed. Should be run as a Go routine.
func (p *ClusterRequestProcess) Run(ctx context.Context) {
	if p.request.Algo == "" {
		p.request.Algo = types.KMEANS_ALGO
	}
	for p.iter.Next() {
		df, err := p.iter.Value(ctx)
		if err != nil {
			p.reportError(err, "Failed to get DataFrame from DataFrameIterator.")
			return
		}
		before := len(df.TraceSet)
		// Filter out Traces with insufficient data. I.e. we need 50% or more data
		// on either side of the target commit.
		df.FilterOut(tooMuchMissingData)
		after := len(df.TraceSet)
		sklog.Infof("Filtered Traces: %d %d %d", before, after, before-after)

		k := p.request.K
		if k <= 0 || k > MAX_K {
			n := len(df.TraceSet)
			// We want K to be around 50 when n = 30000, which has been determined via
			// trial and error to be a good value for the Perf data we are working in. We
			// want K to decrease from  there as n gets smaller, but don't want K to go
			// below 10, so we use a simple linear relation:
			//
			//  k = 40/30000 * n + 10
			//
			k = int(math.Floor((40.0/30000.0)*float64(n) + 10))
		}
		sklog.Infof("Clustering with K=%d", k)

		var summary *clustering2.ClusterSummaries
		switch p.request.Algo {
		case types.KMEANS_ALGO:
			summary, err = clustering2.CalculateClusterSummaries(df, k, config.MIN_STDDEV, p.clusterProgress, p.request.Interesting)
		case types.STEPFIT_ALGO:
			summary, err = StepFit(df, k, config.MIN_STDDEV, p.clusterProgress, p.request.Interesting)
		case types.TAIL_ALGO:
			summary, err = Tail(df, k, config.MIN_STDDEV, p.clusterProgress, p.request.Interesting)

		}
		if err != nil {
			p.reportError(err, "Invalid clustering.")
			return
		}
		if err := ShortcutFromKeys(summary); err != nil {
			p.reportError(err, "Failed to write shortcut for keys.")
			return
		}

		df.TraceSet = types.TraceSet{}
		frame, err := dataframe.ResponseFromDataFrame(ctx, df, p.git, false, p.request.TZ)
		if err != nil {
			p.reportError(err, "Failed to convert DataFrame to FrameResponse.")
			return
		}

		p.mutex.Lock()
		defer p.mutex.Unlock()
		p.state = PROCESS_SUCCESS
		p.message = ""
		p.response = append(p.response, &ClusterResponse{
			Summary: summary,
			Frame:   frame,
		})
	}
}
<commit_msg>Perf - Increase search range for sparse data sets.<commit_after>package regression

import (
	"context"
	"crypto/md5"
	"errors"
	"fmt"
	"math"
	"sync"
	"time"

	"go.skia.org/infra/go/git/gitinfo"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/vec32"
	"go.skia.org/infra/perf/go/cid"
	"go.skia.org/infra/perf/go/clustering2"
	"go.skia.org/infra/perf/go/config"
	"go.skia.org/infra/perf/go/dataframe"
	"go.skia.org/infra/perf/go/shortcut2"
	"go.skia.org/infra/perf/go/types"
)

type ProcessState string

const (
	PROCESS_RUNNING ProcessState = "Running"
	PROCESS_SUCCESS ProcessState = "Success"
	PROCESS_ERROR   ProcessState = "Error"
)

type ClusterRequestType int

const (
	// MAX_FINISHED_PROCESS_AGE is the amount of time to keep a finished
	// ClusterRequestProcess around before deleting it.
	MAX_FINISHED_PROCESS_AGE = time.Minute

	// The following limits are just to prevent excessively large or long-running
	// clusterings from being triggered.

	// MAX_K is the largest K used for clustering.
	MAX_K = 100

	// MAX_RADIUS  is the maximum number of points on either side of a commit
	// that will be included in clustering. This cannot exceed COMMITS_PER_TILE.
	MAX_RADIUS = 50

	// SPARSE_BLOCK_SEARCH_MULT When searching for commits that have data in a
	// sparse data set, we'll request data in chunks of this many commits per
	// point we are looking for.
	SPARSE_BLOCK_SEARCH_MULT = 200

	CLUSTERING_REQUEST_TYPE_SINGLE ClusterRequestType = 0 // Do clustering at a single commit.
	CLUSTERING_REQUEST_TYPE_LAST_N ClusterRequestType = 1 // Do clustering over a range of dense commits.
)

var (
	errorNotFound = errors.New("Process not found.")
)

// ClusterRequest is all the info needed to start a clustering run.
type ClusterRequest struct {
	Source      string             `json:"source"`
	Offset      int                `json:"offset"`
	Radius      int                `json:"radius"`
	Query       string             `json:"query"`
	K           int                `json:"k"`
	TZ          string             `json:"tz"`
	Algo        types.ClusterAlgo  `json:"algo"`
	Interesting float32            `json:"interesting"`
	Sparse      bool               `json:"sparse"`
	Type        ClusterRequestType `json:"type"`
	N           int32              `json:"n"`
	End         int                `json:"end"`
}

func (c *ClusterRequest) Id() string {
	return fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%#v", *c))))
}

// ClusterResponse is the response from running clustering over a ClusterRequest.
type ClusterResponse struct {
	Summary *clustering2.ClusterSummaries `json:"summary"`
	Frame   *dataframe.FrameResponse      `json:"frame"`
}

// ClusterRequestProcess handles the processing of a single ClusterRequest.
type ClusterRequestProcess struct {
	// These members are read-only, should not be modified.
	request *ClusterRequest
	git     *gitinfo.GitInfo
	iter    DataFrameIterator

	// mutex protects access to the remaining struct members.
	mutex      sync.RWMutex
	response   []*ClusterResponse // The response when the clustering is complete.
	lastUpdate time.Time          // The last time this process was updated.
	state      ProcessState       // The current state of the process.
	message    string             // Describes the current state of the process.
}

func newProcess(req *ClusterRequest, git *gitinfo.GitInfo, cidl *cid.CommitIDLookup, dfBuilder dataframe.DataFrameBuilder) *ClusterRequestProcess {
	ret := &ClusterRequestProcess{
		request:    req,
		git:        git,
		response:   []*ClusterResponse{},
		lastUpdate: time.Now(),
		state:      PROCESS_RUNNING,
		message:    "Running",
	}
	// TODO(jcgregorio) This is awkward and should go away in a future CL.
	ret.iter = NewSingleDataFrameIterator(ret.progress, cidl, git, req, dfBuilder)
	return ret
}

func newRunningProcess(ctx context.Context, req *ClusterRequest, git *gitinfo.GitInfo, cidl *cid.CommitIDLookup, dfBuilder dataframe.DataFrameBuilder) *ClusterRequestProcess {
	ret := newProcess(req, git, cidl, dfBuilder)
	go ret.Run(ctx)
	return ret
}

// RunningClusterRequests keeps track of all the ClusterRequestProcess's.
//
// Once a ClusterRequestProcess is complete the results will be kept in memory
// for MAX_FINISHED_PROCESS_AGE before being deleted.
type RunningClusterRequests struct {
	git                *gitinfo.GitInfo
	cidl               *cid.CommitIDLookup
	defaultInteresting float32 // The threshold to control if a cluster is considered interesting.
	dfBuilder          dataframe.DataFrameBuilder

	mutex sync.Mutex
	// inProcess maps a ClusterRequest.Id() of the request to the ClusterRequestProcess
	// handling that request.
	inProcess map[string]*ClusterRequestProcess
}

// NewRunningClusterRequests return a new RunningClusterRequests.
func NewRunningClusterRequests(git *gitinfo.GitInfo, cidl *cid.CommitIDLookup, interesting float32, dfBuilder dataframe.DataFrameBuilder) *RunningClusterRequests {
	fr := &RunningClusterRequests{
		git:                git,
		cidl:               cidl,
		inProcess:          map[string]*ClusterRequestProcess{},
		defaultInteresting: interesting,
		dfBuilder:          dfBuilder,
	}
	go fr.background()
	return fr
}

// step does a single step in cleaning up old ClusterRequestProcess's.
func (fr *RunningClusterRequests) step() {
	fr.mutex.Lock()
	defer fr.mutex.Unlock()
	now := time.Now()
	for k, v := range fr.inProcess {
		v.mutex.Lock()
		if now.Sub(v.lastUpdate) > MAX_FINISHED_PROCESS_AGE {
			delete(fr.inProcess, k)
		}
		v.mutex.Unlock()
	}
}

// background periodically cleans up old ClusterRequestProcess's.
func (fr *RunningClusterRequests) background() {
	fr.step()
	for range time.Tick(time.Minute) {
		fr.step()
	}
}

// Add starts a new running ClusterRequestProcess and returns
// the ID of the process to be used in calls to Status() and
// Response().
func (fr *RunningClusterRequests) Add(ctx context.Context, req *ClusterRequest) string {
	fr.mutex.Lock()
	defer fr.mutex.Unlock()
	if req.Interesting == 0 {
		req.Interesting = fr.defaultInteresting
	}
	id := req.Id()
	if p, ok := fr.inProcess[id]; ok {
		state, _, _ := p.Status()
		if state != PROCESS_RUNNING {
			delete(fr.inProcess, id)
		}
	}
	if _, ok := fr.inProcess[id]; !ok {
		fr.inProcess[id] = newRunningProcess(ctx, req, fr.git, fr.cidl, fr.dfBuilder)
	}
	return id
}

// Status returns the ProcessingState and the message of a
// ClusterRequestProcess of the given 'id'.
func (fr *RunningClusterRequests) Status(id string) (ProcessState, string, error) {
	fr.mutex.Lock()
	defer fr.mutex.Unlock()
	if p, ok := fr.inProcess[id]; !ok {
		return PROCESS_ERROR, "Not Found", errorNotFound
	} else {
		return p.Status()
	}
}

// Response returns the ClusterResponse of the completed ClusterRequestProcess.
func (fr *RunningClusterRequests) Response(id string) (*ClusterResponse, error) {
	fr.mutex.Lock()
	defer fr.mutex.Unlock()
	if p, ok := fr.inProcess[id]; !ok {
		return nil, errorNotFound
	} else {
		return p.Response(), nil
	}
}

// Responses returns the ClusterResponse's of the completed ClusterRequestProcess.
func (fr *RunningClusterRequests) Responses(id string) ([]*ClusterResponse, error) {
	fr.mutex.Lock()
	defer fr.mutex.Unlock()
	if p, ok := fr.inProcess[id]; !ok {
		return nil, errorNotFound
	} else {
		return p.Responses(), nil
	}
}

// reportError records the reason a ClusterRequestProcess failed.
func (p *ClusterRequestProcess) reportError(err error, message string) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	sklog.Warningf("ClusterRequest failed: %#v %s: %s", *(p.request), message, err)
	p.message = fmt.Sprintf("%s: %s", message, err)
	p.state = PROCESS_ERROR
	p.lastUpdate = time.Now()
}

// progress records the progress of a ClusterRequestProcess.
func (p *ClusterRequestProcess) progress(step, totalSteps int) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	p.message = fmt.Sprintf("Querying: %d%%", int(float32(100.0)*float32(step)/float32(totalSteps)))
	p.lastUpdate = time.Now()
}

// clusterProgress records the progress of a ClusterRequestProcess.
func (p *ClusterRequestProcess) clusterProgress(totalError float64) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	p.message = fmt.Sprintf("Clustering Total Error: %0.2f", totalError)
	p.lastUpdate = time.Now()
}

// Response returns the ClusterResponse of the completed ClusterRequestProcess.
func (p *ClusterRequestProcess) Response() *ClusterResponse {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.response[0]
}

// Responses returns all the ClusterResponse's of the ClusterRequestProcess.
func (p *ClusterRequestProcess) Responses() []*ClusterResponse {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.response
}

// Status returns the ProcessingState and the message of a
// ClusterRequestProcess of the given 'id'.
func (p *ClusterRequestProcess) Status() (ProcessState, string, error) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.state, p.message, nil
}

// missing returns true if >50% of the trace is vec32.MISSING_DATA_SENTINEL.
func missing(tr types.Trace) bool {
	count := 0
	for _, x := range tr {
		if x == vec32.MISSING_DATA_SENTINEL {
			count++
		}
	}
	return (100*count)/len(tr) > 50
}

// tooMuchMissingData returns true if a trace has too many
// MISSING_DATA_SENTINEL values.
//
// The criteria is if there is >50% missing data on either side of the target
// commit, which sits at the center of the trace.
func tooMuchMissingData(tr types.Trace) bool {
	if len(tr) < 3 {
		return false
	}
	n := len(tr) / 2
	if tr[n] == vec32.MISSING_DATA_SENTINEL {
		return true
	}
	return missing(tr[:n]) || missing(tr[len(tr)-n:])
}

// ShortcutFromKeys stores a new shortcut for each cluster based on its Keys.
func ShortcutFromKeys(summary *clustering2.ClusterSummaries) error {
	var err error
	for _, cs := range summary.Clusters {
		if cs.Shortcut, err = shortcut2.InsertShortcut(&shortcut2.Shortcut{Keys: cs.Keys}); err != nil {
			return err
		}
	}
	return nil
}

// Run does the work in a ClusterRequestProcess. It does not return until all the
// work is done or the request failed. Should be run as a Go routine.
func (p *ClusterRequestProcess) Run(ctx context.Context) {
	if p.request.Algo == "" {
		p.request.Algo = types.KMEANS_ALGO
	}
	for p.iter.Next() {
		df, err := p.iter.Value(ctx)
		if err != nil {
			p.reportError(err, "Failed to get DataFrame from DataFrameIterator.")
			return
		}
		before := len(df.TraceSet)
		// Filter out Traces with insufficient data. I.e. we need 50% or more data
		// on either side of the target commit.
		df.FilterOut(tooMuchMissingData)
		after := len(df.TraceSet)
		sklog.Infof("Filtered Traces: %d %d %d", before, after, before-after)

		k := p.request.K
		if k <= 0 || k > MAX_K {
			n := len(df.TraceSet)
			// We want K to be around 50 when n = 30000, which has been determined via
			// trial and error to be a good value for the Perf data we are working in. We
			// want K to decrease from  there as n gets smaller, but don't want K to go
			// below 10, so we use a simple linear relation:
			//
			//  k = 40/30000 * n + 10
			//
			k = int(math.Floor((40.0/30000.0)*float64(n) + 10))
		}
		sklog.Infof("Clustering with K=%d", k)

		var summary *clustering2.ClusterSummaries
		switch p.request.Algo {
		case types.KMEANS_ALGO:
			summary, err = clustering2.CalculateClusterSummaries(df, k, config.MIN_STDDEV, p.clusterProgress, p.request.Interesting)
		case types.STEPFIT_ALGO:
			summary, err = StepFit(df, k, config.MIN_STDDEV, p.clusterProgress, p.request.Interesting)
		case types.TAIL_ALGO:
			summary, err = Tail(df, k, config.MIN_STDDEV, p.clusterProgress, p.request.Interesting)

		}
		if err != nil {
			p.reportError(err, "Invalid clustering.")
			return
		}
		if err := ShortcutFromKeys(summary); err != nil {
			p.reportError(err, "Failed to write shortcut for keys.")
			return
		}

		df.TraceSet = types.TraceSet{}
		frame, err := dataframe.ResponseFromDataFrame(ctx, df, p.git, false, p.request.TZ)
		if err != nil {
			p.reportError(err, "Failed to convert DataFrame to FrameResponse.")
			return
		}

		p.mutex.Lock()
		defer p.mutex.Unlock()
		p.state = PROCESS_SUCCESS
		p.message = ""
		p.response = append(p.response, &ClusterResponse{
			Summary: summary,
			Frame:   frame,
		})
	}
}
<|endoftext|>


<commit_before>package kvite

import (
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	log "github.com/Sirupsen/logrus"
	logx "github.com/mistifyio/mistify-logrus-ext"
	"github.com/stretchr/testify/suite"
)

type KViteTestSuite struct {
	suite.Suite
	DB      *DB
	TempDir string
}

func (s *KViteTestSuite) SetupTest() {
	// Create a new db tempdir
	dir, err := ioutil.TempDir("", "kvite-")
	s.NoError(err)
	s.TempDir = dir

	// Open the db with the tempdir
	db, err := Open(filepath.Join(s.TempDir, "kvite.db"), "testing")
	s.NoError(err)
	s.DB = db
}

func (s *KViteTestSuite) TearDownTest() {
	// Close the db
	logx.LogReturnedErr(s.DB.Close, nil, "failed to close database")

	// Clean up the db tempdir
	logx.LogReturnedErr(func() error { return os.RemoveAll(s.TempDir) },
		log.Fields{"tempdir": s.TempDir}, "failed to remove tempdir")
}

func (s *KViteTestSuite) testStoredValue(bucketName, key string, expectedValue []byte) {
	tx, _ := s.DB.Begin()
	b, _ := tx.CreateBucket(bucketName)
	value, getErr := b.Get(key)
	s.NoError(getErr)
	s.Equal(expectedValue, value)
	_ = tx.Rollback()
}

func TestKViteTestSuite(t *testing.T) {
	suite.Run(t, new(KViteTestSuite))
}

func (s *KViteTestSuite) TestDBOpen() {
	// The suite test setup tests a good call to the kvite.Open function

	tests := []struct {
		filename    string
		table       string
		expectedErr bool
		msg         string
	}{
		{"", "", true, "directory as db file"},
		{"open-test-bad-table.db", "1-23aa'1234", true, "invalid table name"},
		{"open-test-no-table.db", "", false, "no supplied table name"},
	}

	for _, test := range tests {
		_, err := Open(filepath.Join(s.TempDir, test.filename), test.table)
		if test.expectedErr {
			s.Error(err, test.msg)
		} else {
			s.NoError(err, test.msg)
		}
	}
}

func (s *KViteTestSuite) TestDBClose() {
	// The suite test setup tests a good call to the kvite.Close function
	// Attempt to close again
	s.NoError(s.DB.Close())
}

func (s *KViteTestSuite) TestDBBegin() {
	tx, err := s.DB.Begin()
	s.NoError(err)
	s.NotNil(tx)
}

func (s *KViteTestSuite) TestTxRollback() {
	tx, _ := s.DB.Begin()
	s.NoError(tx.Rollback())
	// Can't rollback a finished tx
	s.Error(tx.Rollback())
}

func (s *KViteTestSuite) TestTxCommit() {
	tx, _ := s.DB.Begin()
	s.NoError(tx.Commit())
	// Can't commit a finished tx
	s.Error(tx.Commit())
}

func (s *KViteTestSuite) TestTxCreateBucket() {
	tx, _ := s.DB.Begin()
	b, err := tx.CreateBucket("test")
	s.NoError(err)
	s.NotNil(b)
	s.NoError(tx.Commit())
}

func (s *KViteTestSuite) TestTxCreateBucketIfNotExists() {
	tx, _ := s.DB.Begin()
	b, err := tx.CreateBucketIfNotExists("test")
	s.NoError(err)
	s.NotNil(b)
	s.NoError(tx.Commit())
}

func (s *KViteTestSuite) TestBucketPut() {
	tx, _ := s.DB.Begin()
	b, _ := tx.CreateBucket("test")

	s.NoError(b.Put("foo", []byte("bar")))
	s.NoError(tx.Commit())
}

func (s *KViteTestSuite) TestBucketGet() {
	bucketName := "test"
	key := "foo"
	value := []byte("bar")

	tx, err := s.DB.Begin()
	b, err := tx.CreateBucket(bucketName)

	_ = b.Put(key, value)

	// Get value inside tx
	val, err := b.Get(key)
	s.NoError(err)
	s.EqualValues(value, val)

	s.NoError(tx.Commit())

	// Get value after tx
	s.testStoredValue(bucketName, key, value)
	// Get non-existent value
	s.testStoredValue(bucketName, "asdf", []byte(nil))
}

func (s *KViteTestSuite) TestBucketDelete() {
	bucketName := "test"
	key := "foo"
	value := []byte("bar")

	// Put and remove in same tx
	tx, _ := s.DB.Begin()
	b, _ := tx.CreateBucket(bucketName)
	_ = b.Put(key, value)
	s.NoError(b.Delete(key))

	// Check removed in tx
	val, _ := b.Get(key)
	s.Equal([]byte(nil), val)

	s.NoError(tx.Commit())

	// Check removed after tx
	s.testStoredValue(bucketName, key, []byte(nil))

	// Put and remove in different tx
	tx, _ = s.DB.Begin()
	b, _ = tx.CreateBucket(bucketName)
	_ = b.Put(key, value)
	tx.Commit()

	tx, _ = s.DB.Begin()
	b, _ = tx.CreateBucket(bucketName)
	s.NoError(b.Delete(key))
	s.NoError(tx.Commit())

	// Check removed after tx
	s.testStoredValue(bucketName, key, []byte(nil))
}

func (s *KViteTestSuite) TestDBTransaction() {
	bucketName := "test"
	key := "foo"
	value := []byte("bar")

	// No error, tx should commit
	err := s.DB.Transaction(func(tx *Tx) error {
		b, _ := tx.CreateBucket(bucketName)
		_ = b.Put(key, value)
		return nil
	})
	s.NoError(err)
	s.testStoredValue(bucketName, key, value)

	// Error, tx should rollback
	err = s.DB.Transaction(func(tx *Tx) error {
		b, _ := tx.CreateBucket(bucketName)
		_ = b.Put(key, []byte("asdf"))

		// Can't commit or rollback inside a db managed tx
		s.Error(tx.Commit())
		s.Error(tx.Rollback())

		return errors.New("an error")
	})
	s.Error(err)
	// Still original set value
	s.testStoredValue(bucketName, key, value)
}

func (s *KViteTestSuite) TestBucketForEach() {
	tx, _ := s.DB.Begin()
	b, _ := tx.CreateBucket("test")

	_ = b.Put("foo", []byte("bar"))
	_ = b.Put("baz", []byte("stuff"))

	// No error in fn
	var items []string
	err := b.ForEach(func(k string, v []byte) error {
		items = append(items, k)
		return nil
	})
	s.NoError(err)
	s.Len(items, 2)

	// Error in fn
	err = b.ForEach(func(k string, v []byte) error {
		return errors.New("an error")
	})
	s.Error(err)

}

func (s *KViteTestSuite) TestDBBuckets() {
	buckets := []string{"one", "two", "three"}
	_ = s.DB.Transaction(func(tx *Tx) error {
		for _, name := range buckets {
			b, _ := tx.CreateBucket(name)
			_ = b.Put("foo", []byte("bar"))
		}
		return nil
	})

	names, err := s.DB.Buckets()
	s.NoError(err)
	s.Equal(buckets, names)
}

func (s *KViteTestSuite) TestBucketPutUnique() {
	bucketName := "test"
	key := "foo"
	value := []byte("bar")

	tx, _ := s.DB.Begin()
	b, _ := tx.CreateBucket(bucketName)

	// Should only be one key set and should be last set value
	s.NoError(b.Put(key, []byte("baz")))
	s.NoError(b.Put(key, value))

	i := 0
	_ = b.ForEach(func(k string, v []byte) error {
		i++
		s.Equal(value, v)
		return nil
	})
	s.Equal(1, i)
}
<commit_msg>Explicitly ignore tx error in test<commit_after>package kvite

import (
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	log "github.com/Sirupsen/logrus"
	logx "github.com/mistifyio/mistify-logrus-ext"
	"github.com/stretchr/testify/suite"
)

type KViteTestSuite struct {
	suite.Suite
	DB      *DB
	TempDir string
}

func (s *KViteTestSuite) SetupTest() {
	// Create a new db tempdir
	dir, err := ioutil.TempDir("", "kvite-")
	s.NoError(err)
	s.TempDir = dir

	// Open the db with the tempdir
	db, err := Open(filepath.Join(s.TempDir, "kvite.db"), "testing")
	s.NoError(err)
	s.DB = db
}

func (s *KViteTestSuite) TearDownTest() {
	// Close the db
	logx.LogReturnedErr(s.DB.Close, nil, "failed to close database")

	// Clean up the db tempdir
	logx.LogReturnedErr(func() error { return os.RemoveAll(s.TempDir) },
		log.Fields{"tempdir": s.TempDir}, "failed to remove tempdir")
}

func (s *KViteTestSuite) testStoredValue(bucketName, key string, expectedValue []byte) {
	tx, _ := s.DB.Begin()
	b, _ := tx.CreateBucket(bucketName)
	value, getErr := b.Get(key)
	s.NoError(getErr)
	s.Equal(expectedValue, value)
	_ = tx.Rollback()
}

func TestKViteTestSuite(t *testing.T) {
	suite.Run(t, new(KViteTestSuite))
}

func (s *KViteTestSuite) TestDBOpen() {
	// The suite test setup tests a good call to the kvite.Open function

	tests := []struct {
		filename    string
		table       string
		expectedErr bool
		msg         string
	}{
		{"", "", true, "directory as db file"},
		{"open-test-bad-table.db", "1-23aa'1234", true, "invalid table name"},
		{"open-test-no-table.db", "", false, "no supplied table name"},
	}

	for _, test := range tests {
		_, err := Open(filepath.Join(s.TempDir, test.filename), test.table)
		if test.expectedErr {
			s.Error(err, test.msg)
		} else {
			s.NoError(err, test.msg)
		}
	}
}

func (s *KViteTestSuite) TestDBClose() {
	// The suite test setup tests a good call to the kvite.Close function
	// Attempt to close again
	s.NoError(s.DB.Close())
}

func (s *KViteTestSuite) TestDBBegin() {
	tx, err := s.DB.Begin()
	s.NoError(err)
	s.NotNil(tx)
}

func (s *KViteTestSuite) TestTxRollback() {
	tx, _ := s.DB.Begin()
	s.NoError(tx.Rollback())
	// Can't rollback a finished tx
	s.Error(tx.Rollback())
}

func (s *KViteTestSuite) TestTxCommit() {
	tx, _ := s.DB.Begin()
	s.NoError(tx.Commit())
	// Can't commit a finished tx
	s.Error(tx.Commit())
}

func (s *KViteTestSuite) TestTxCreateBucket() {
	tx, _ := s.DB.Begin()
	b, err := tx.CreateBucket("test")
	s.NoError(err)
	s.NotNil(b)
	s.NoError(tx.Commit())
}

func (s *KViteTestSuite) TestTxCreateBucketIfNotExists() {
	tx, _ := s.DB.Begin()
	b, err := tx.CreateBucketIfNotExists("test")
	s.NoError(err)
	s.NotNil(b)
	s.NoError(tx.Commit())
}

func (s *KViteTestSuite) TestBucketPut() {
	tx, _ := s.DB.Begin()
	b, _ := tx.CreateBucket("test")

	s.NoError(b.Put("foo", []byte("bar")))
	s.NoError(tx.Commit())
}

func (s *KViteTestSuite) TestBucketGet() {
	bucketName := "test"
	key := "foo"
	value := []byte("bar")

	tx, err := s.DB.Begin()
	b, err := tx.CreateBucket(bucketName)

	_ = b.Put(key, value)

	// Get value inside tx
	val, err := b.Get(key)
	s.NoError(err)
	s.EqualValues(value, val)

	s.NoError(tx.Commit())

	// Get value after tx
	s.testStoredValue(bucketName, key, value)
	// Get non-existent value
	s.testStoredValue(bucketName, "asdf", []byte(nil))
}

func (s *KViteTestSuite) TestBucketDelete() {
	bucketName := "test"
	key := "foo"
	value := []byte("bar")

	// Put and remove in same tx
	tx, _ := s.DB.Begin()
	b, _ := tx.CreateBucket(bucketName)
	_ = b.Put(key, value)
	s.NoError(b.Delete(key))

	// Check removed in tx
	val, _ := b.Get(key)
	s.Equal([]byte(nil), val)

	s.NoError(tx.Commit())

	// Check removed after tx
	s.testStoredValue(bucketName, key, []byte(nil))

	// Put and remove in different tx
	tx, _ = s.DB.Begin()
	b, _ = tx.CreateBucket(bucketName)
	_ = b.Put(key, value)
	_ = tx.Commit()

	tx, _ = s.DB.Begin()
	b, _ = tx.CreateBucket(bucketName)
	s.NoError(b.Delete(key))
	s.NoError(tx.Commit())

	// Check removed after tx
	s.testStoredValue(bucketName, key, []byte(nil))
}

func (s *KViteTestSuite) TestDBTransaction() {
	bucketName := "test"
	key := "foo"
	value := []byte("bar")

	// No error, tx should commit
	err := s.DB.Transaction(func(tx *Tx) error {
		b, _ := tx.CreateBucket(bucketName)
		_ = b.Put(key, value)
		return nil
	})
	s.NoError(err)
	s.testStoredValue(bucketName, key, value)

	// Error, tx should rollback
	err = s.DB.Transaction(func(tx *Tx) error {
		b, _ := tx.CreateBucket(bucketName)
		_ = b.Put(key, []byte("asdf"))

		// Can't commit or rollback inside a db managed tx
		s.Error(tx.Commit())
		s.Error(tx.Rollback())

		return errors.New("an error")
	})
	s.Error(err)
	// Still original set value
	s.testStoredValue(bucketName, key, value)
}

func (s *KViteTestSuite) TestBucketForEach() {
	tx, _ := s.DB.Begin()
	b, _ := tx.CreateBucket("test")

	_ = b.Put("foo", []byte("bar"))
	_ = b.Put("baz", []byte("stuff"))

	// No error in fn
	var items []string
	err := b.ForEach(func(k string, v []byte) error {
		items = append(items, k)
		return nil
	})
	s.NoError(err)
	s.Len(items, 2)

	// Error in fn
	err = b.ForEach(func(k string, v []byte) error {
		return errors.New("an error")
	})
	s.Error(err)

}

func (s *KViteTestSuite) TestDBBuckets() {
	buckets := []string{"one", "two", "three"}
	_ = s.DB.Transaction(func(tx *Tx) error {
		for _, name := range buckets {
			b, _ := tx.CreateBucket(name)
			_ = b.Put("foo", []byte("bar"))
		}
		return nil
	})

	names, err := s.DB.Buckets()
	s.NoError(err)
	s.Equal(buckets, names)
}

func (s *KViteTestSuite) TestBucketPutUnique() {
	bucketName := "test"
	key := "foo"
	value := []byte("bar")

	tx, _ := s.DB.Begin()
	b, _ := tx.CreateBucket(bucketName)

	// Should only be one key set and should be last set value
	s.NoError(b.Put(key, []byte("baz")))
	s.NoError(b.Put(key, value))

	i := 0
	_ = b.ForEach(func(k string, v []byte) error {
		i++
		s.Equal(value, v)
		return nil
	})
	s.Equal(1, i)
}
<|endoftext|>


<commit_before>// Package sudoku is a generic solver for sudoku type puzzles. It can solve a puzzle of any shape and size.
package sudoku

import "fmt"

var (
	s4 = []Constraint{
		Unique{0, 1, 2, 3},
		Unique{4, 5, 6, 7},
		Unique{8, 9, 10, 11},
		Unique{12, 13, 14, 15},
		Unique{0, 4, 8, 12},
		Unique{1, 5, 9, 13},
		Unique{2, 6, 10, 14},
		Unique{3, 7, 11, 15},
		Unique{0, 1, 4, 5},
		Unique{2, 3, 6, 7},
		Unique{8, 9, 12, 13},
		Unique{10, 11, 14, 15},
	}
	c4 = []int{1, 2, 3, 4}
	s9 = []Constraint{
		Unique{0, 1, 2, 3, 4, 5, 6, 7, 8},
		Unique{9, 10, 11, 12, 13, 14, 15, 16, 17},
		Unique{18, 19, 20, 21, 22, 23, 24, 25, 26},
		Unique{27, 28, 29, 30, 31, 32, 33, 34, 35},
		Unique{36, 37, 38, 39, 40, 41, 42, 43, 44},
		Unique{45, 46, 47, 48, 49, 50, 51, 52, 53},
		Unique{54, 55, 56, 57, 58, 59, 60, 61, 62},
		Unique{63, 64, 65, 66, 67, 68, 69, 70, 71},
		Unique{72, 73, 74, 75, 76, 77, 78, 79, 80},
		Unique{0, 9, 18, 27, 36, 45, 54, 63, 72},
		Unique{1, 10, 19, 28, 37, 46, 55, 64, 73},
		Unique{2, 11, 20, 29, 38, 47, 56, 65, 74},
		Unique{3, 12, 21, 30, 39, 48, 57, 66, 75},
		Unique{4, 13, 22, 31, 40, 49, 58, 67, 76},
		Unique{5, 14, 23, 32, 41, 50, 59, 68, 77},
		Unique{6, 15, 24, 33, 42, 51, 60, 69, 78},
		Unique{7, 16, 25, 34, 43, 52, 61, 70, 79},
		Unique{8, 17, 26, 35, 44, 53, 62, 71, 80},
		Unique{0, 1, 2, 9, 10, 11, 18, 19, 20},
		Unique{3, 4, 5, 12, 13, 14, 21, 22, 23},
		Unique{6, 7, 8, 15, 16, 17, 24, 25, 26},
		Unique{27, 28, 29, 36, 37, 38, 45, 46, 47},
		Unique{30, 31, 32, 39, 40, 41, 48, 49, 50},
		Unique{33, 34, 35, 42, 43, 44, 51, 52, 53},
		Unique{54, 55, 56, 63, 64, 65, 72, 73, 74},
		Unique{57, 58, 59, 66, 67, 68, 75, 76, 77},
		Unique{60, 61, 62, 69, 70, 71, 78, 79, 80},
	}
	c9 = []int{1, 2, 3, 4, 5, 6, 7, 8, 9}
)

// MakeBox is a helper function to make it easier to create the sections in standard rectangular puzzles
func MakeBox(gridWidth, gridHeight, boxWidth, boxHeight int) []Constraint {
	toRet := make([]Constraint, 0, gridWidth/boxWidth*gridHeight/boxHeight)
	for j := 0; j < gridHeight; j += boxHeight {
		for i := 0; i < gridWidth; i += boxWidth {
			thisGrid := make(Unique, 0, boxWidth*boxHeight)
			for y := 0; y < boxHeight; y++ {
				for x := 0; x < boxWidth; x++ {
					thisGrid = append(thisGrid, i+x+(j+y)*gridWidth)
				}
			}
			toRet = append(toRet, thisGrid)
		}
	}
	return toRet
}

// Solve9 is a sudoku puzzle of the standard 9x9 format
func Solve9(data []int) bool {
	if len(data) != 81 {
		return false
	}
	s := Sudoku{
		data:        data,
		chars:       c9,
		constraints: s9,
	}
	return s.Solve()
}

// Solve4 is a sudoku puzzle of the 4x4 format
func Solve4(data []int) bool {
	if len(data) != 16 {
		return false
	}
	s := Sudoku{
		data:        data,
		chars:       c4,
		constraints: s4,
	}
	return s.Solve()
}

// Solve allows the creation of a non-standard Sudoku puzzle and solves it.
//
// data is the puzzle information, layed out left to right, then top to bottom
//
// chars is any valid 'character' the puzzle uses - 0 is used for an unfilled space
//
// structure is a slice of sections, each of which is a slice of positions, len(chars)
// in length, which describes the rows, columns, boxes or other shapes in which there
// can only be one of each character
//
// Will return true if puzzle is solveable and the solution will be stored in the data slice.
// Upon a failure, will return false and the data slice will be as original.
func Solve(data, chars []int, constraints []Constraint) bool {
	s := Sudoku{data, chars, constraints}
	return s.Solve()
}

func slicePos(s []int, p int) int {
	for n, sp := range s {
		if sp == p {
			return n
		}
	}
	return -1
}

type Constraint interface {
	Constrain(*Sudoku, int, []bool) bool
}

type Unique []int

func (u Unique) Constrain(s *Sudoku, pos int, marked []bool) bool {
	if slicePos([]int(u), pos) == -1 {
		return true
	}
	myMark := make([]int, 0, len(u))
	for _, p := range u {
		if mp := s.data[p]; mp != 0 {
			if slicePos(myMark, mp) != -1 {
				fmt.Println(u, pos, myMark, mp)
				return false
			}
			myMark = append(myMark, mp)
			marked[slicePos(s.chars, mp)] = true
		}
	}
	return true
}

/*type Sorted []int

func (s Sorted) Constrain(su *Sudoku, pos int, marked []bool) bool {
	if slicePos([]int(s), pos) == -1 {
		return true
	}
	num := 0
	for _, p := range s {
		if mp := su.data[p]; mp != 0 {
			if mp > num {
				num = mp
			} else {
				return false
			}
		} else if mp == pos {

		}
	}
	return true
}

type UniqueSum struct {
	positions []int
	total     int
}

func (u UniqueSum) Constrain(s *Sudoku, pos int, marked []bool) bool {
	if slicePos(u.positions, pos) == -1 {
		return true
	}
	total := 0
	myMark := make([]int, len(u.positions))
	unmarked := 0
	for _, p := range u.positions {
		if mp := s.data[p]; mp == 0 {
			unmarked++
		} else {
			if slicePos(myMark, mp) != -1 {
				return false
			}
			myMark = append(myMark, mp)
			total += mp
		}
	}
	left := u.total - total
	if left < 0 {
		return false
	}

	return true
}*/

// Sudoku puzzle information
type Sudoku struct {
	data        []int
	chars       []int
	constraints []Constraint
}

func (s *Sudoku) possible(pos int) []int {
	if pos < 0 || pos > len(s.data) || s.data[pos] != 0 {
		return nil
	}
	marked := make([]bool, len(s.chars))
	for _, c := range s.constraints {
		any := c.Constrain(s, pos, marked)
		if !any {
			return []int{}
		}
	}
	toRet := make([]int, 0, len(s.chars))
	for p, m := range marked {
		if !m {
			toRet = append(toRet, s.chars[p])
		}
	}
	return toRet
}

// Solve will solve any solveable puzzle and return whether is was sucessful.
func (s *Sudoku) Solve() bool {
	l := len(s.data)
	possibilities := make([][]int, l)
	var pos int
	for ; pos < l; pos++ {
		if p := s.possible(pos); p != nil {
			possibilities[pos] = p
			break
		}
	}
	for pos < l {
		if len(possibilities[pos]) == 0 {
			s.data[pos] = 0
			for pos--; pos >= 0 && len(possibilities[pos]) == 0; pos-- {
				if possibilities[pos] != nil {
					s.data[pos] = 0
				}
			}
			if pos < 0 {
				return false
			}
			continue
		}
		s.data[pos] = possibilities[pos][0]
		possibilities[pos] = possibilities[pos][1:]
		for pos++; pos < l; pos++ {
			if p := s.possible(pos); p != nil {
				possibilities[pos] = p
				break
			}
		}
	}
	return true
}
<commit_msg>Added comments<commit_after>// Package sudoku is a generic solver for sudoku type puzzles. It can solve a puzzle of any shape and size.
package sudoku

import "fmt"

var (
	s4 = []Constraint{
		Unique{0, 1, 2, 3},
		Unique{4, 5, 6, 7},
		Unique{8, 9, 10, 11},
		Unique{12, 13, 14, 15},
		Unique{0, 4, 8, 12},
		Unique{1, 5, 9, 13},
		Unique{2, 6, 10, 14},
		Unique{3, 7, 11, 15},
		Unique{0, 1, 4, 5},
		Unique{2, 3, 6, 7},
		Unique{8, 9, 12, 13},
		Unique{10, 11, 14, 15},
	}
	c4 = []int{1, 2, 3, 4}
	s9 = []Constraint{
		Unique{0, 1, 2, 3, 4, 5, 6, 7, 8},
		Unique{9, 10, 11, 12, 13, 14, 15, 16, 17},
		Unique{18, 19, 20, 21, 22, 23, 24, 25, 26},
		Unique{27, 28, 29, 30, 31, 32, 33, 34, 35},
		Unique{36, 37, 38, 39, 40, 41, 42, 43, 44},
		Unique{45, 46, 47, 48, 49, 50, 51, 52, 53},
		Unique{54, 55, 56, 57, 58, 59, 60, 61, 62},
		Unique{63, 64, 65, 66, 67, 68, 69, 70, 71},
		Unique{72, 73, 74, 75, 76, 77, 78, 79, 80},
		Unique{0, 9, 18, 27, 36, 45, 54, 63, 72},
		Unique{1, 10, 19, 28, 37, 46, 55, 64, 73},
		Unique{2, 11, 20, 29, 38, 47, 56, 65, 74},
		Unique{3, 12, 21, 30, 39, 48, 57, 66, 75},
		Unique{4, 13, 22, 31, 40, 49, 58, 67, 76},
		Unique{5, 14, 23, 32, 41, 50, 59, 68, 77},
		Unique{6, 15, 24, 33, 42, 51, 60, 69, 78},
		Unique{7, 16, 25, 34, 43, 52, 61, 70, 79},
		Unique{8, 17, 26, 35, 44, 53, 62, 71, 80},
		Unique{0, 1, 2, 9, 10, 11, 18, 19, 20},
		Unique{3, 4, 5, 12, 13, 14, 21, 22, 23},
		Unique{6, 7, 8, 15, 16, 17, 24, 25, 26},
		Unique{27, 28, 29, 36, 37, 38, 45, 46, 47},
		Unique{30, 31, 32, 39, 40, 41, 48, 49, 50},
		Unique{33, 34, 35, 42, 43, 44, 51, 52, 53},
		Unique{54, 55, 56, 63, 64, 65, 72, 73, 74},
		Unique{57, 58, 59, 66, 67, 68, 75, 76, 77},
		Unique{60, 61, 62, 69, 70, 71, 78, 79, 80},
	}
	c9 = []int{1, 2, 3, 4, 5, 6, 7, 8, 9}
)

// MakeBox is a helper function to make it easier to create the sections in standard rectangular puzzles
func MakeBox(gridWidth, gridHeight, boxWidth, boxHeight int) []Constraint {
	toRet := make([]Constraint, 0, gridWidth/boxWidth*gridHeight/boxHeight)
	for j := 0; j < gridHeight; j += boxHeight {
		for i := 0; i < gridWidth; i += boxWidth {
			thisGrid := make(Unique, 0, boxWidth*boxHeight)
			for y := 0; y < boxHeight; y++ {
				for x := 0; x < boxWidth; x++ {
					thisGrid = append(thisGrid, i+x+(j+y)*gridWidth)
				}
			}
			toRet = append(toRet, thisGrid)
		}
	}
	return toRet
}

// Solve9 is a sudoku puzzle of the standard 9x9 format
func Solve9(data []int) bool {
	if len(data) != 81 {
		return false
	}
	s := Sudoku{
		data:        data,
		chars:       c9,
		constraints: s9,
	}
	return s.Solve()
}

// Solve4 is a sudoku puzzle of the 4x4 format
func Solve4(data []int) bool {
	if len(data) != 16 {
		return false
	}
	s := Sudoku{
		data:        data,
		chars:       c4,
		constraints: s4,
	}
	return s.Solve()
}

// Solve allows the creation of a non-standard Sudoku puzzle and solves it.
//
// data is the puzzle information, layed out left to right, then top to bottom
//
// chars is any valid 'character' the puzzle uses - 0 is used for an unfilled space
//
// structure is a slice of sections, each of which is a slice of positions, len(chars)
// in length, which describes the rows, columns, boxes or other shapes in which there
// can only be one of each character
//
// Will return true if puzzle is solveable and the solution will be stored in the data slice.
// Upon a failure, will return false and the data slice will be as original.
func Solve(data, chars []int, constraints []Constraint) bool {
	s := Sudoku{data, chars, constraints}
	return s.Solve()
}

func slicePos(s []int, p int) int {
	for n, sp := range s {
		if sp == p {
			return n
		}
	}
	return -1
}

// Constraint defines the interface through which the character constraints are processed
type Constraint interface {
	Constrain(*Sudoku, int, []bool) bool
}

// Unique represents a section of data where each cell needs to be different
type Unique []int

// Constrain implements the Constraint interface
func (u Unique) Constrain(s *Sudoku, pos int, marked []bool) bool {
	if slicePos([]int(u), pos) == -1 {
		return true
	}
	myMark := make([]int, 0, len(u))
	for _, p := range u {
		if mp := s.data[p]; mp != 0 {
			if slicePos(myMark, mp) != -1 {
				fmt.Println(u, pos, myMark, mp)
				return false
			}
			myMark = append(myMark, mp)
			marked[slicePos(s.chars, mp)] = true
		}
	}
	return true
}

/*type Sorted []int

func (s Sorted) Constrain(su *Sudoku, pos int, marked []bool) bool {
	if slicePos([]int(s), pos) == -1 {
		return true
	}
	num := 0
	for _, p := range s {
		if mp := su.data[p]; mp != 0 {
			if mp > num {
				num = mp
			} else {
				return false
			}
		} else if mp == pos {

		}
	}
	return true
}

type UniqueSum struct {
	positions []int
	total     int
}

func (u UniqueSum) Constrain(s *Sudoku, pos int, marked []bool) bool {
	if slicePos(u.positions, pos) == -1 {
		return true
	}
	total := 0
	myMark := make([]int, len(u.positions))
	unmarked := 0
	for _, p := range u.positions {
		if mp := s.data[p]; mp == 0 {
			unmarked++
		} else {
			if slicePos(myMark, mp) != -1 {
				return false
			}
			myMark = append(myMark, mp)
			total += mp
		}
	}
	left := u.total - total
	if left < 0 {
		return false
	}

	return true
}*/

// Sudoku puzzle information
type Sudoku struct {
	data        []int
	chars       []int
	constraints []Constraint
}

func (s *Sudoku) possible(pos int) []int {
	if pos < 0 || pos > len(s.data) || s.data[pos] != 0 {
		return nil
	}
	marked := make([]bool, len(s.chars))
	for _, c := range s.constraints {
		any := c.Constrain(s, pos, marked)
		if !any {
			return []int{}
		}
	}
	toRet := make([]int, 0, len(s.chars))
	for p, m := range marked {
		if !m {
			toRet = append(toRet, s.chars[p])
		}
	}
	return toRet
}

// Solve will solve any solveable puzzle and return whether is was sucessful.
func (s *Sudoku) Solve() bool {
	l := len(s.data)
	possibilities := make([][]int, l)
	var pos int
	for ; pos < l; pos++ {
		if p := s.possible(pos); p != nil {
			possibilities[pos] = p
			break
		}
	}
	for pos < l {
		if len(possibilities[pos]) == 0 {
			s.data[pos] = 0
			for pos--; pos >= 0 && len(possibilities[pos]) == 0; pos-- {
				if possibilities[pos] != nil {
					s.data[pos] = 0
				}
			}
			if pos < 0 {
				return false
			}
			continue
		}
		s.data[pos] = possibilities[pos][0]
		possibilities[pos] = possibilities[pos][1:]
		for pos++; pos < l; pos++ {
			if p := s.possible(pos); p != nil {
				possibilities[pos] = p
				break
			}
		}
	}
	return true
}
<|endoftext|>


<commit_before>/*

   Copyright since 2017 Wenhui Shen <www.webx.top>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/

package pagination

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html/template"
	"math"
	"strconv"
	"strings"

	"github.com/webx-top/echo"
	"github.com/webx-top/echo/engine"
)

const (
	ModePageNumber = iota + 1
	ModePosition
)

func New(ctx echo.Context) *Pagination {
	return &Pagination{context: ctx, pages: -1, data: echo.H{}, mode: ModePageNumber}
}

type Pagination struct {
	context   echo.Context
	tmpl      string
	urlLayout string
	data      echo.H
	mode      int

	// 按基准位置分页
	position     string
	prevPosition string
	nextPosition string

	// 按页码分页
	page  int
	rows  int //total rows
	limit int
	num   int
	pages int //total pages

}

func (p *Pagination) SetAll(tmpl string, rows int, pnl ...int) *Pagination {
	switch len(pnl) {
	case 3:
		p.limit = pnl[2]
		fallthrough
	case 2:
		p.num = pnl[1]
		fallthrough
	case 1:
		p.page = pnl[0]
	}
	p.rows = rows
	p.tmpl = tmpl
	p.pages = -1
	return p
}

func (p *Pagination) SetPosition(prev string, next string, curr string) *Pagination {
	p.prevPosition = prev
	p.nextPosition = next
	p.position = curr
	p.mode = ModePosition
	return p
}

func (p *Pagination) Set(key string, data interface{}) *Pagination {
	p.data[key] = data
	return p
}

func (p *Pagination) Sets(args ...interface{}) *Pagination {
	var key string
	for i, j := 0, len(args); i < j; i++ {
		if i%2 == 0 {
			key = fmt.Sprint(args[i])
			continue
		}
		p.data[key] = args[i]
	}
	return p
}

func (p *Pagination) Get(key string) interface{} {
	if v, y := p.data[key]; y {
		return v
	}
	return nil
}

func (p *Pagination) Data() echo.H {
	return p.data
}

func (p *Pagination) Position() string {
	return p.position
}

func (p *Pagination) PrevPosition() string {
	return p.prevPosition
}

func (p *Pagination) NextPosition() string {
	return p.nextPosition
}

func (p *Pagination) SetPage(page int) *Pagination {
	p.page = page
	return p
}

func (p *Pagination) Page() int {
	return p.page
}

func (p *Pagination) PrevPage() int {
	if p.page < 2 {
		return 1
	}
	return p.page - 1
}

func (p *Pagination) NextPage() int {
	n := p.page + 1
	if n <= p.pages {
		return n
	}
	return p.pages
}

func (p *Pagination) SetRows(rows int) *Pagination {
	p.pages = -1
	p.rows = rows
	return p
}

func (p *Pagination) Rows() int {
	return p.rows
}

func (p *Pagination) SetLimit(limit int) *Pagination {
	p.pages = -1
	p.limit = limit
	return p
}

func (p *Pagination) Limit() int {
	return p.limit
}

func (p *Pagination) SetNum(num int) *Pagination {
	p.num = num
	return p
}

func (p *Pagination) Num() int {
	return p.num
}

func (p *Pagination) SetTmpl(tmpl string) *Pagination {
	p.tmpl = tmpl
	return p
}

func (p *Pagination) Tmpl() string {
	return p.tmpl
}

func (p *Pagination) Pages() int {
	if p.pages == -1 {
		p.pages = int(math.Ceil(float64(p.rows) / float64(p.limit)))
	}
	return p.pages
}

func (p *Pagination) URL(curr interface{}) (s string) {
	if p.mode == ModePageNumber {
		s = strings.Replace(p.urlLayout, `{page}`, fmt.Sprint(curr), -1)
		s = strings.Replace(s, `{rows}`, strconv.Itoa(p.rows), -1)
		s = strings.Replace(s, `{size}`, strconv.Itoa(p.limit), -1)
		s = strings.Replace(s, `{limit}`, strconv.Itoa(p.limit), -1)
		s = strings.Replace(s, `{pages}`, strconv.Itoa(p.pages), -1)
	} else {
		s = strings.Replace(p.urlLayout, `{curr}`, fmt.Sprint(curr), -1)
		s = strings.Replace(s, `{prev}`, p.prevPosition, -1)
		s = strings.Replace(s, `{next}`, p.nextPosition, -1)
	}
	return s
}

func (p *Pagination) SetURL(s interface{}, delKeys ...string) *Pagination {
	switch v := s.(type) {
	case string:
		p.urlLayout = v
	case map[string]string:
		p.urlLayout = p.RebuildURL(v, delKeys...)
	default:
		panic(`Unsupported type: ` + fmt.Sprintf(`%T`, s))
	}
	return p
}

func (p *Pagination) RebuildQueryString(delKeys ...string) string {
	query := p.context.Request().URL().Query()
	for _, key := range delKeys {
		query.Del(key)
	}
	return query.Encode()
}

func (p *Pagination) RebuildURL(pageVars map[string]string, delKeys ...string) string {
	var (
		pq string
		jn string
	)
	for name, urlVar := range pageVars {
		delKeys = append(delKeys, urlVar)
		pq += jn + urlVar + `={` + name + `}`
		jn = `&`
	}
	q := p.RebuildQueryString(delKeys...)
	if len(q) > 0 {
		q += `&`
	}
	url := p.context.Request().URL().Path() + `?` + q + pq
	return url
}

func (p *Pagination) List(num ...int) []int {
	if len(num) > 0 {
		p.num = num[0]
	}
	pages := p.Pages()
	var (
		pList []int
		start int
		count int
	)
	remainPages := pages - p.num
	if remainPages >= 0 {
		start = remainPages + 1
	} else {
		start = p.Page() - (p.num / 2)
	}
	if start < 1 {
		start = 1
	}
	for page := start; page <= pages; page++ {
		count++
		if count > p.num {
			break
		}
		pList = append(pList, page)
	}
	return pList
}

func (p *Pagination) setDefault() *Pagination {
	if p.mode == ModePageNumber {
		if p.page < 1 {
			p.page = 1
		}
		if p.limit < 1 {
			p.limit = 50
		}
		if p.num < 1 {
			p.num = 10
		}
	}
	return p
}

func (p *Pagination) Render(settings ...string) interface{} {
	p.setDefault()
	switch len(settings) {
	case 1:
		p.tmpl = settings[0]
	}
	if len(p.tmpl) == 0 {
		p.tmpl = `pagination`
	}
	b, e := p.context.Fetch(p.tmpl, p)
	if e != nil {
		return e
	}
	return template.HTML(string(b))
}

// MarshalJSON allows type Pagination to be used with json.Marshal
func (p *Pagination) MarshalJSON() ([]byte, error) {
	b, e := json.Marshal(p.data)
	var s string
	if e != nil {
		s = fmt.Sprintf(`%q`, e.Error())
	} else {
		s = engine.Bytes2str(b)
	}
	if p.mode == ModePageNumber {
		p.setDefault()
		s = fmt.Sprintf(`{"page":%d,"rows":%d,"limit":%d,"pages":%d,"urlLayout":%q,"data":%s}`, p.Page(), p.Rows(), p.Limit(), p.Pages(), p.urlLayout, s)
	} else {
		s = fmt.Sprintf(`{"curr":%q,"prev":%q,"next":%q,"urlLayout":%q,"data":%s}`, p.Position(), p.PrevPosition(), p.NextPosition(), p.urlLayout, s)
	}
	return engine.Str2bytes(s), nil
}

// MarshalXML allows type Pagination to be used with xml.Marshal
func (p *Pagination) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	start.Name.Local = `Pagination`
	if err := e.EncodeToken(start); err != nil {
		return err
	}
	if p.mode == ModePageNumber {
		p.setDefault()
		if err := xmlEncode(e, `page`, p.Page()); err != nil {
			return err
		}
		if err := xmlEncode(e, `rows`, p.Rows()); err != nil {
			return err
		}
		if err := xmlEncode(e, `limit`, p.Limit()); err != nil {
			return err
		}
		if err := xmlEncode(e, `pages`, p.Pages()); err != nil {
			return err
		}
	} else {
		if err := xmlEncode(e, `curr`, p.Position()); err != nil {
			return err
		}
		if err := xmlEncode(e, `prev`, p.PrevPosition()); err != nil {
			return err
		}
		if err := xmlEncode(e, `next`, p.NextPosition()); err != nil {
			return err
		}
	}
	if err := xmlEncode(e, `urlLayout`, p.urlLayout); err != nil {
		return err
	}
	if err := xmlEncode(e, `data`, p.data); err != nil {
		return err
	}
	return e.EncodeToken(xml.EndElement{Name: start.Name})
}

func xmlEncode(e *xml.Encoder, key string, value interface{}) error {
	elem := xml.StartElement{
		Name: xml.Name{Space: ``, Local: key},
		Attr: []xml.Attr{},
	}
	return e.EncodeElement(value, elem)
}
<commit_msg>update<commit_after>/*

   Copyright since 2017 Wenhui Shen <www.webx.top>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/

package pagination

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html/template"
	"math"
	"strconv"
	"strings"

	"github.com/webx-top/echo"
	"github.com/webx-top/echo/engine"
)

const (
	ModePageNumber = iota + 1
	ModePosition
)

func New(ctx echo.Context) *Pagination {
	return &Pagination{context: ctx, pages: -1, data: echo.H{}, mode: ModePageNumber}
}

type Pagination struct {
	context   echo.Context
	tmpl      string
	urlLayout string
	data      echo.H
	mode      int

	// 按基准位置分页
	position     string
	prevPosition string
	nextPosition string

	// 按页码分页
	page  int
	rows  int //total rows
	limit int
	num   int
	pages int //total pages

}

func (p *Pagination) SetAll(tmpl string, rows int, pnl ...int) *Pagination {
	switch len(pnl) {
	case 3:
		p.limit = pnl[2]
		fallthrough
	case 2:
		p.num = pnl[1]
		fallthrough
	case 1:
		p.page = pnl[0]
	}
	p.rows = rows
	p.tmpl = tmpl
	p.pages = -1
	return p
}

func (p *Pagination) SetPosition(prev string, next string, curr string) *Pagination {
	p.prevPosition = prev
	p.nextPosition = next
	p.position = curr
	p.mode = ModePosition
	return p
}

func (p *Pagination) Set(key string, data interface{}) *Pagination {
	p.data[key] = data
	return p
}

func (p *Pagination) Sets(args ...interface{}) *Pagination {
	var key string
	for i, j := 0, len(args); i < j; i++ {
		if i%2 == 0 {
			key = fmt.Sprint(args[i])
			continue
		}
		p.data[key] = args[i]
	}
	return p
}

func (p *Pagination) Get(key string) interface{} {
	if v, y := p.data[key]; y {
		return v
	}
	return nil
}

func (p *Pagination) Data() echo.H {
	return p.data
}

func (p *Pagination) Position() string {
	return p.position
}

func (p *Pagination) PrevPosition() string {
	return p.prevPosition
}

func (p *Pagination) NextPosition() string {
	return p.nextPosition
}

func (p *Pagination) SetPage(page int) *Pagination {
	p.page = page
	return p
}

func (p *Pagination) Page() int {
	return p.page
}

func (p *Pagination) PrevPage() int {
	if p.page < 2 {
		return 1
	}
	return p.page - 1
}

func (p *Pagination) NextPage() int {
	n := p.page + 1
	if n <= p.pages {
		return n
	}
	return p.pages
}

func (p *Pagination) SetRows(rows int) *Pagination {
	p.pages = -1
	p.rows = rows
	return p
}

func (p *Pagination) Rows() int {
	return p.rows
}

func (p *Pagination) SetLimit(limit int) *Pagination {
	p.pages = -1
	p.limit = limit
	return p
}

func (p *Pagination) Limit() int {
	return p.limit
}

func (p *Pagination) SetNum(num int) *Pagination {
	p.num = num
	return p
}

func (p *Pagination) Num() int {
	return p.num
}

func (p *Pagination) SetTmpl(tmpl string) *Pagination {
	p.tmpl = tmpl
	return p
}

func (p *Pagination) Tmpl() string {
	return p.tmpl
}

func (p *Pagination) Pages() int {
	if p.pages == -1 {
		p.pages = int(math.Ceil(float64(p.rows) / float64(p.limit)))
	}
	return p.pages
}

func (p *Pagination) URL(curr interface{}) (s string) {
	if p.mode == ModePageNumber {
		s = strings.Replace(p.urlLayout, `{page}`, fmt.Sprint(curr), -1)
		s = strings.Replace(s, `{rows}`, strconv.Itoa(p.rows), -1)
		s = strings.Replace(s, `{size}`, strconv.Itoa(p.limit), -1)
		s = strings.Replace(s, `{limit}`, strconv.Itoa(p.limit), -1)
		s = strings.Replace(s, `{pages}`, strconv.Itoa(p.pages), -1)
	} else {
		s = strings.Replace(p.urlLayout, `{curr}`, fmt.Sprint(curr), -1)
		s = strings.Replace(s, `{prev}`, p.prevPosition, -1)
		s = strings.Replace(s, `{next}`, p.nextPosition, -1)
	}
	return s
}

func (p *Pagination) SetURL(s interface{}, delKeys ...string) *Pagination {
	switch v := s.(type) {
	case string:
		p.urlLayout = v
	case map[string]string:
		p.urlLayout = p.RebuildURL(v, delKeys...)
	default:
		panic(`Unsupported type: ` + fmt.Sprintf(`%T`, s))
	}
	return p
}

func (p *Pagination) RebuildQueryString(delKeys ...string) string {
	query := p.context.Request().URL().Query()
	for _, key := range delKeys {
		query.Del(key)
	}
	return query.Encode()
}

func (p *Pagination) RebuildURL(pageVars map[string]string, delKeys ...string) string {
	var (
		pq string
		jn string
	)
	for name, urlVar := range pageVars {
		delKeys = append(delKeys, urlVar)
		pq += jn + urlVar + `={` + name + `}`
		jn = `&`
	}
	q := p.RebuildQueryString(delKeys...)
	if len(q) > 0 {
		q += `&`
	}
	url := p.context.Request().URL().Path() + `?` + q + pq
	return url
}

func (p *Pagination) List(num ...int) []int {
	if len(num) > 0 {
		p.num = num[0]
	}
	pages := p.Pages()
	var (
		pList []int
		start int
		count int
	)
	remainPages := pages - p.num
	half := p.num / 2
	start = p.Page() - half
	if start < 1 {
		start = 1
	} else if start+p.num > remainPages {
		start = remainPages
	}
	for page := start; page <= pages; page++ {
		count++
		if count > p.num {
			break
		}
		pList = append(pList, page)
	}
	return pList
}

func (p *Pagination) setDefault() *Pagination {
	if p.mode == ModePageNumber {
		if p.page < 1 {
			p.page = 1
		}
		if p.limit < 1 {
			p.limit = 50
		}
		if p.num < 1 {
			p.num = 10
		}
	}
	return p
}

func (p *Pagination) Render(settings ...string) interface{} {
	p.setDefault()
	switch len(settings) {
	case 1:
		p.tmpl = settings[0]
	}
	if len(p.tmpl) == 0 {
		p.tmpl = `pagination`
	}
	b, e := p.context.Fetch(p.tmpl, p)
	if e != nil {
		return e
	}
	return template.HTML(string(b))
}

// MarshalJSON allows type Pagination to be used with json.Marshal
func (p *Pagination) MarshalJSON() ([]byte, error) {
	b, e := json.Marshal(p.data)
	var s string
	if e != nil {
		s = fmt.Sprintf(`%q`, e.Error())
	} else {
		s = engine.Bytes2str(b)
	}
	if p.mode == ModePageNumber {
		p.setDefault()
		s = fmt.Sprintf(`{"page":%d,"rows":%d,"limit":%d,"pages":%d,"urlLayout":%q,"data":%s}`, p.Page(), p.Rows(), p.Limit(), p.Pages(), p.urlLayout, s)
	} else {
		s = fmt.Sprintf(`{"curr":%q,"prev":%q,"next":%q,"urlLayout":%q,"data":%s}`, p.Position(), p.PrevPosition(), p.NextPosition(), p.urlLayout, s)
	}
	return engine.Str2bytes(s), nil
}

// MarshalXML allows type Pagination to be used with xml.Marshal
func (p *Pagination) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	start.Name.Local = `Pagination`
	if err := e.EncodeToken(start); err != nil {
		return err
	}
	if p.mode == ModePageNumber {
		p.setDefault()
		if err := xmlEncode(e, `page`, p.Page()); err != nil {
			return err
		}
		if err := xmlEncode(e, `rows`, p.Rows()); err != nil {
			return err
		}
		if err := xmlEncode(e, `limit`, p.Limit()); err != nil {
			return err
		}
		if err := xmlEncode(e, `pages`, p.Pages()); err != nil {
			return err
		}
	} else {
		if err := xmlEncode(e, `curr`, p.Position()); err != nil {
			return err
		}
		if err := xmlEncode(e, `prev`, p.PrevPosition()); err != nil {
			return err
		}
		if err := xmlEncode(e, `next`, p.NextPosition()); err != nil {
			return err
		}
	}
	if err := xmlEncode(e, `urlLayout`, p.urlLayout); err != nil {
		return err
	}
	if err := xmlEncode(e, `data`, p.data); err != nil {
		return err
	}
	return e.EncodeToken(xml.EndElement{Name: start.Name})
}

func xmlEncode(e *xml.Encoder, key string, value interface{}) error {
	elem := xml.StartElement{
		Name: xml.Name{Space: ``, Local: key},
		Attr: []xml.Attr{},
	}
	return e.EncodeElement(value, elem)
}
<|endoftext|>


<commit_before>// Package mssql implements reform.Dialect for Microsoft SQL Server (mssql driver).
//
// Deprecated: Use sqlserver dialect instead. https://github.com/denisenkom/go-mssqldb#deprecated
package mssql // import "gopkg.in/reform.v1/dialects/mssql"

import "gopkg.in/reform.v1"

type mssql struct{}

func (mssql) String() string {
	return "mssql"
}

func (mssql) Placeholder(index int) string {
	return "?"
}

func (mssql) Placeholders(start, count int) []string {
	res := make([]string, count)
	for i := 0; i < count; i++ {
		res[i] = "?"
	}
	return res
}

func (mssql) QuoteIdentifier(identifier string) string {
	return "[" + identifier + "]"
}

func (mssql) LastInsertIdMethod() reform.LastInsertIdMethod {
	return reform.OutputInserted
}

func (mssql) SelectLimitMethod() reform.SelectLimitMethod {
	return reform.SelectTop
}

func (mssql) DefaultValuesMethod() reform.DefaultValuesMethod {
	return reform.DefaultValues
}

// Dialect implements reform.Dialect for Microsoft SQL Server.
//
// Deprecated: Use sqlserver.Dialect instead. https://github.com/denisenkom/go-mssqldb#deprecated
var Dialect mssql

// check interface
var _ reform.Dialect = Dialect
<commit_msg>Minor syntax tweak.<commit_after>// Package mssql implements reform.Dialect for Microsoft SQL Server (mssql driver).
//
// Deprecated: Use sqlserver dialect instead. https://github.com/denisenkom/go-mssqldb#deprecated
package mssql // import "gopkg.in/reform.v1/dialects/mssql"

import (
	"gopkg.in/reform.v1"
)

type mssql struct{}

func (mssql) String() string {
	return "mssql"
}

func (mssql) Placeholder(index int) string {
	return "?"
}

func (mssql) Placeholders(start, count int) []string {
	res := make([]string, count)
	for i := 0; i < count; i++ {
		res[i] = "?"
	}
	return res
}

func (mssql) QuoteIdentifier(identifier string) string {
	return "[" + identifier + "]"
}

func (mssql) LastInsertIdMethod() reform.LastInsertIdMethod {
	return reform.OutputInserted
}

func (mssql) SelectLimitMethod() reform.SelectLimitMethod {
	return reform.SelectTop
}

func (mssql) DefaultValuesMethod() reform.DefaultValuesMethod {
	return reform.DefaultValues
}

// Dialect implements reform.Dialect for Microsoft SQL Server.
//
// Deprecated: Use sqlserver.Dialect instead. https://github.com/denisenkom/go-mssqldb#deprecated
var Dialect mssql

// check interface
var _ reform.Dialect = Dialect
<|endoftext|>


<commit_before>package discover
<commit_msg>discoverd: Delete empty server_test.go<commit_after><|endoftext|>


<commit_before>package logfanout

import (
	"fmt"
	"io"
	"sync"
)

type Tracker struct {
	db LogDB

	draining    bool
	connections map[io.Closer]struct{}

	logs    map[string]*LogFanout
	parties map[string]int

	lock *sync.RWMutex
}

func NewTracker(db LogDB) *Tracker {
	return &Tracker{
		db: db,

		logs:        make(map[string]*LogFanout),
		parties:     make(map[string]int),
		connections: make(map[io.Closer]struct{}),

		lock: new(sync.RWMutex),
	}
}

func (tracker *Tracker) Register(job string, id int, conn io.Closer) *LogFanout {
	key := fmt.Sprintf("%s-%d", job, id)

	tracker.lock.Lock()

	tracker.connections[conn] = struct{}{}

	logFanout, found := tracker.logs[key]
	if !found {
		logFanout = NewLogFanout(job, id, tracker.db)
		tracker.logs[key] = logFanout
		tracker.parties[key]++
	}

	draining := tracker.draining

	tracker.lock.Unlock()

	if draining {
		conn.Close()
	}

	return logFanout
}

func (tracker *Tracker) Unregister(job string, id int, conn io.Closer) {
	key := fmt.Sprintf("%s-%d", job, id)

	tracker.lock.Lock()

	tracker.parties[key]--

	delete(tracker.connections, conn)

	if tracker.parties[key] == 0 {
		delete(tracker.logs, key)
		delete(tracker.parties, key)
	}

	tracker.lock.Unlock()
}

func (tracker *Tracker) Drain() {
	tracker.lock.Lock()

	tracker.draining = true

	for _, fanout := range tracker.logs {
		fanout.Close()
	}

	for conn, _ := range tracker.connections {
		conn.Close()
	}

	tracker.lock.Unlock()
}
<commit_msg>don't hold lock while draining connections<commit_after>package logfanout

import (
	"fmt"
	"io"
	"sync"
)

type Tracker struct {
	db LogDB

	draining    bool
	connections map[io.Closer]struct{}

	logs    map[string]*LogFanout
	parties map[string]int

	lock *sync.RWMutex
}

func NewTracker(db LogDB) *Tracker {
	return &Tracker{
		db: db,

		logs:        make(map[string]*LogFanout),
		parties:     make(map[string]int),
		connections: make(map[io.Closer]struct{}),

		lock: new(sync.RWMutex),
	}
}

func (tracker *Tracker) Register(job string, id int, conn io.Closer) *LogFanout {
	key := fmt.Sprintf("%s-%d", job, id)

	tracker.lock.Lock()

	tracker.connections[conn] = struct{}{}

	logFanout, found := tracker.logs[key]
	if !found {
		logFanout = NewLogFanout(job, id, tracker.db)
		tracker.logs[key] = logFanout
		tracker.parties[key]++
	}

	draining := tracker.draining

	tracker.lock.Unlock()

	if draining {
		conn.Close()
	}

	return logFanout
}

func (tracker *Tracker) Unregister(job string, id int, conn io.Closer) {
	key := fmt.Sprintf("%s-%d", job, id)

	tracker.lock.Lock()

	tracker.parties[key]--

	delete(tracker.connections, conn)

	if tracker.parties[key] == 0 {
		delete(tracker.logs, key)
		delete(tracker.parties, key)
	}

	tracker.lock.Unlock()
}

func (tracker *Tracker) Drain() {
	tracker.lock.Lock()

	tracker.draining = true

	logs := tracker.logs
	conns := tracker.connections

	tracker.lock.Unlock()

	for _, fanout := range logs {
		fanout.Close()
	}

	for conn, _ := range conns {
		conn.Close()
	}
}
<|endoftext|>


<commit_before>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sqlparser

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestGen(t *testing.T) {
	_, err := Parse("select :a from a where a in (:b)")
	if err != nil {
		t.Error(err)
	}
}

func TestParse(t *testing.T) {
	sql := "select a from (select * from table1 where table1.a = 'tom') as t1, table2, table3 as t3, table4 left join table5 where t1.k = '1'"
	_, err := Parse(sql)
	if err != nil {
		t.Fatal(err)
	}
}

func TestCreatTable1(t *testing.T) {
	sql := `create table t1 (
	ID int primary key,
	LastName varchar(255),
	FirstName varchar(255)
)`
	tree, err := Parse(sql)
	if err != nil {
		t.Fatal(err)
	}
	s := String(tree)

	assert.Equal(t, sql, s)
}

func TestCreatTable2(t *testing.T) {
	sql := `create table t1 (
	ID int primary key not null auto_increment,
	LastName varchar(255),
	FirstName varchar(255)
)`
	tree, err := Parse(sql)
	if err != nil {
		t.Fatal(err)
	}
	s := String(tree)

	assert.Equal(t, sql, s)
}

func TestCreatTable3(t *testing.T) {
	sql := `create table t1 (
	ID int unique key not null auto_increment,
	LastName varchar(255),
	FirstName varchar(255)
)`
	tree, err := Parse(sql)
	if err != nil {
		t.Fatal(err)
	}
	s := String(tree)

	assert.Equal(t, sql, s)
}

func BenchmarkParse1(b *testing.B) {
	sql := "select 'abcd', 20, 30.0, eid from a where 1=eid and name='3'"
	for i := 0; i < b.N; i++ {
		_, err := Parse(sql)
		if err != nil {
			b.Fatal(err)
		}
	}
}

func BenchmarkParse2(b *testing.B) {
	sql := "select aaaa, bbb, ccc, ddd, eeee, ffff, gggg, hhhh, iiii from tttt, ttt1, ttt3 where aaaa = bbbb and bbbb = cccc and dddd+1 = eeee group by fff, gggg having hhhh = iiii and iiii = jjjj order by kkkk, llll limit 3, 4"
	for i := 0; i < b.N; i++ {
		_, err := Parse(sql)
		if err != nil {
			b.Fatal(err)
		}
	}
}

type testCase struct {
	file   string
	lineno int
	input  string
	output string
}
<commit_msg>add test for insert<commit_after>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sqlparser

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestGen(t *testing.T) {
	_, err := Parse("select :a from a where a in (:b)")
	if err != nil {
		t.Error(err)
	}
}

func TestParse(t *testing.T) {
	sql := "select a from (select * from table1 where table1.a = 'tom') as t1, table2, table3 as t3, table4 left join table5 where t1.k = '1'"
	_, err := Parse(sql)
	if err != nil {
		t.Fatal(err)
	}
}

func TestParseInsert(t *testing.T) {
	sql := "INSERT INTO t3 VALUES (8, 10, 'baz')"
	_, err := Parse(sql)
	assert.Nil(t, err)
}

func TestCreatTable1(t *testing.T) {
	sql := `create table t1 (
	ID int primary key,
	LastName varchar(255),
	FirstName varchar(255)
)`
	tree, err := Parse(sql)
	if err != nil {
		t.Fatal(err)
	}
	s := String(tree)

	assert.Equal(t, sql, s)
}

func TestCreatTable2(t *testing.T) {
	sql := `create table t1 (
	ID int primary key not null auto_increment,
	LastName varchar(255),
	FirstName varchar(255)
)`
	tree, err := Parse(sql)
	if err != nil {
		t.Fatal(err)
	}
	s := String(tree)

	assert.Equal(t, sql, s)
}

func TestCreatTable3(t *testing.T) {
	sql := `create table t1 (
	ID int unique key not null auto_increment,
	LastName varchar(255),
	FirstName varchar(255)
)`
	tree, err := Parse(sql)
	if err != nil {
		t.Fatal(err)
	}
	s := String(tree)

	assert.Equal(t, sql, s)
}

func BenchmarkParse1(b *testing.B) {
	sql := "select 'abcd', 20, 30.0, eid from a where 1=eid and name='3'"
	for i := 0; i < b.N; i++ {
		_, err := Parse(sql)
		if err != nil {
			b.Fatal(err)
		}
	}
}

func BenchmarkParse2(b *testing.B) {
	sql := "select aaaa, bbb, ccc, ddd, eeee, ffff, gggg, hhhh, iiii from tttt, ttt1, ttt3 where aaaa = bbbb and bbbb = cccc and dddd+1 = eeee group by fff, gggg having hhhh = iiii and iiii = jjjj order by kkkk, llll limit 3, 4"
	for i := 0; i < b.N; i++ {
		_, err := Parse(sql)
		if err != nil {
			b.Fatal(err)
		}
	}
}

type testCase struct {
	file   string
	lineno int
	input  string
	output string
}
<|endoftext|>


<commit_before>package pct

import (
	"fmt"
	"github.com/percona/cloud-protocol/proto"
)

type Logger struct {
	logChan chan *proto.LogEntry
	service string
	cmd     *proto.Cmd
}

func NewLogger(logChan chan *proto.LogEntry, service string) *Logger {
	l := &Logger{
		logChan: logChan,
		service: service,
	}
	return l
}

func (l *Logger) LogChan() chan *proto.LogEntry {
	return l.logChan
}

func (l *Logger) InResponseTo(cmd *proto.Cmd) {
	l.cmd = cmd
}

func (l *Logger) Debug(entry ...interface{}) {
	l.log(proto.LOG_DEBUG, entry)
}

func (l *Logger) Info(entry ...interface{}) {
	l.log(proto.LOG_INFO, entry)
}

func (l *Logger) Warn(entry ...interface{}) {
	l.log(proto.LOG_WARNING, entry)
}

func (l *Logger) Error(entry ...interface{}) {
	l.log(proto.LOG_ERROR, entry)
}

func (l *Logger) Fatal(entry ...interface{}) {
	l.log(proto.LOG_CRITICAL, entry)
}

func (l *Logger) log(level int, entry []interface{}) {
	fullMsg := ""
	for i, str := range entry {
		if i > 0 {
			fullMsg += " "
		}
		fullMsg += fmt.Sprintf("%v", str)
	}
	logEntry := &proto.LogEntry{
		Level:   level,
		Service: l.service,
		Msg:     fullMsg,
	}
	if l.cmd != nil {
		logEntry.Cmd = fmt.Sprintf("%s", l.cmd)
	}
	l.logChan <- logEntry
}
<commit_msg>Change log level type from int to byte.<commit_after>package pct

import (
	"fmt"
	"github.com/percona/cloud-protocol/proto"
)

type Logger struct {
	logChan chan *proto.LogEntry
	service string
	cmd     *proto.Cmd
}

func NewLogger(logChan chan *proto.LogEntry, service string) *Logger {
	l := &Logger{
		logChan: logChan,
		service: service,
	}
	return l
}

func (l *Logger) LogChan() chan *proto.LogEntry {
	return l.logChan
}

func (l *Logger) InResponseTo(cmd *proto.Cmd) {
	l.cmd = cmd
}

func (l *Logger) Debug(entry ...interface{}) {
	l.log(proto.LOG_DEBUG, entry)
}

func (l *Logger) Info(entry ...interface{}) {
	l.log(proto.LOG_INFO, entry)
}

func (l *Logger) Warn(entry ...interface{}) {
	l.log(proto.LOG_WARNING, entry)
}

func (l *Logger) Error(entry ...interface{}) {
	l.log(proto.LOG_ERROR, entry)
}

func (l *Logger) Fatal(entry ...interface{}) {
	l.log(proto.LOG_CRITICAL, entry)
}

func (l *Logger) log(level byte, entry []interface{}) {
	fullMsg := ""
	for i, str := range entry {
		if i > 0 {
			fullMsg += " "
		}
		fullMsg += fmt.Sprintf("%v", str)
	}
	logEntry := &proto.LogEntry{
		Level:   level,
		Service: l.service,
		Msg:     fullMsg,
	}
	if l.cmd != nil {
		logEntry.Cmd = fmt.Sprintf("%s", l.cmd)
	}
	l.logChan <- logEntry
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"github.com/buth/stocker/auth"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"strconv"
	"strings"
	"syscall"
)

var Exec = &Command{
	UsageLine: "exec [options] command [argument...]",
	Short:     "execute a command with the given environment",
}

type StringAcumulator []string

func (s *StringAcumulator) Set(value string) error {
	*s = append(*s, value)
	return nil
}

func (s *StringAcumulator) String() string {
	return fmt.Sprintf("%s", *s)
}

var execConfig struct {
	Address, PrivateFilepath, Group, User string
}

func init() {
	Exec.Run = execRun
	Exec.Flag.StringVar(&execConfig.Address, "a", ":2022", "address of the stocker server")
	Exec.Flag.StringVar(&execConfig.Group, "g", "", "group to use for storing and retrieving data")
	Exec.Flag.StringVar(&execConfig.PrivateFilepath, "i", "", "path to an SSH private key")
	Exec.Flag.StringVar(&execConfig.User, "u", "", "user to execute the command as")
}

func execRun(cmd *Command, args []string) {

	// Find the expanded path to cmd.
	command, err := exec.LookPath(args[0])
	if err != nil {
		cmd.Fatal(fmt.Sprintf("%s: command not found", args[0]))
	}

	// Read the private key from disk if a filepath has been provided.
	var privateKey []byte
	if setConfig.PrivateFilepath != "" {
		privateKeyBytes, err := ioutil.ReadFile(setConfig.PrivateFilepath)
		if err != nil {
			cmd.Fatal(err.Error())
		}
		privateKey = privateKeyBytes
	}

	// Get a new client object. If the private key is nil, the method will
	// attempt to use ssh-agent.
	client, err := auth.NewClient(auth.WriterUser, setConfig.Address, privateKey)
	if err != nil {
		cmd.Fatal(err.Error())
	}

	// Create an environment specific to this variable.
	runEnv := map[string]string{
		"GROUP": execConfig.Group,
	}

	stockerEnv, err := client.Run("env", runEnv)
	if err != nil {
		cmd.Fatal(err.Error())
	}

	// Create a map of environment variables to be passed to cmd and
	// initialize it with the current environment.
	env := make(map[string]string)
	for _, variable := range os.Environ() {
		components := strings.Split(variable, "=")
		env[components[0]] = components[1]
	}

	// Parse the stocker environment and save it into the env.
	pairs := strings.Split(stockerEnv, "\n")
	for _, pair := range pairs {
		components := strings.SplitN(pair, "=", 2)
		if len(components) == 2 {
			env[components[0]] = components[1]
		}
	}

	// Create a list of environment key/value pairs and write the
	// flattened environment variables map to it.
	commandEnv := make([]string, 0, len(env))
	for key, value := range env {
		commandEnv = commandEnv[:len(commandEnv)+1]
		commandEnv[len(commandEnv)-1] = fmt.Sprintf("%s=%s", key, value)
	}

	// Handle user.
	if execConfig.User != "" {

		u, err := user.Lookup(execConfig.User)
		if err != nil {
			cmd.Fatal(err.Error())
		}

		uid, err := strconv.Atoi(u.Uid)
		if err != nil {
			cmd.Fatal(err.Error())
		}

		if err := syscall.Setuid(uid); err != nil {
			cmd.Fatal(err.Error())
		}
	}

	// Exec the new command.
	syscall.Exec(command, args, commandEnv)
}
<commit_msg>fix for missing arguments<commit_after>package cmd

import (
	"fmt"
	"github.com/buth/stocker/auth"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"strconv"
	"strings"
	"syscall"
)

var Exec = &Command{
	UsageLine: "exec [options] command [argument...]",
	Short:     "execute a command with the given environment",
}

type StringAcumulator []string

func (s *StringAcumulator) Set(value string) error {
	*s = append(*s, value)
	return nil
}

func (s *StringAcumulator) String() string {
	return fmt.Sprintf("%s", *s)
}

var execConfig struct {
	Address, PrivateFilepath, Group, User string
}

func init() {
	Exec.Run = execRun
	Exec.Flag.StringVar(&execConfig.Address, "a", ":2022", "address of the stocker server")
	Exec.Flag.StringVar(&execConfig.Group, "g", "", "group to use for storing and retrieving data")
	Exec.Flag.StringVar(&execConfig.PrivateFilepath, "i", "", "path to an SSH private key")
	Exec.Flag.StringVar(&execConfig.User, "u", "", "user to execute the command as")
}

func execRun(cmd *Command, args []string) {

	// Check the number of args.
	if len(args) < 1 {
		cmd.Usage(2)
	}

	// Find the expanded path to cmd.
	command, err := exec.LookPath(args[0])
	if err != nil {
		cmd.Fatal(fmt.Sprintf("%s: command not found", args[0]))
	}

	// Read the private key from disk if a filepath has been provided.
	var privateKey []byte
	if setConfig.PrivateFilepath != "" {
		privateKeyBytes, err := ioutil.ReadFile(setConfig.PrivateFilepath)
		if err != nil {
			cmd.Fatal(err.Error())
		}
		privateKey = privateKeyBytes
	}

	// Get a new client object. If the private key is nil, the method will
	// attempt to use ssh-agent.
	client, err := auth.NewClient(auth.WriterUser, setConfig.Address, privateKey)
	if err != nil {
		cmd.Fatal(err.Error())
	}

	// Create an environment specific to this variable.
	runEnv := map[string]string{
		"GROUP": execConfig.Group,
	}

	stockerEnv, err := client.Run("env", runEnv)
	if err != nil {
		cmd.Fatal(err.Error())
	}

	// Create a map of environment variables to be passed to cmd and
	// initialize it with the current environment.
	env := make(map[string]string)
	for _, variable := range os.Environ() {
		components := strings.Split(variable, "=")
		env[components[0]] = components[1]
	}

	// Parse the stocker environment and save it into the env.
	pairs := strings.Split(stockerEnv, "\n")
	for _, pair := range pairs {
		components := strings.SplitN(pair, "=", 2)
		if len(components) == 2 {
			env[components[0]] = components[1]
		}
	}

	// Create a list of environment key/value pairs and write the
	// flattened environment variables map to it.
	commandEnv := make([]string, 0, len(env))
	for key, value := range env {
		commandEnv = commandEnv[:len(commandEnv)+1]
		commandEnv[len(commandEnv)-1] = fmt.Sprintf("%s=%s", key, value)
	}

	// Handle user.
	if execConfig.User != "" {

		u, err := user.Lookup(execConfig.User)
		if err != nil {
			cmd.Fatal(err.Error())
		}

		uid, err := strconv.Atoi(u.Uid)
		if err != nil {
			cmd.Fatal(err.Error())
		}

		if err := syscall.Setuid(uid); err != nil {
			cmd.Fatal(err.Error())
		}
	}

	// Exec the new command.
	syscall.Exec(command, args, commandEnv)
}
<|endoftext|>


<commit_before>// Copyright 2017 The Gop Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/urfave/cli"
)

const (
	defaultMainFile = `package main

func main() {

}
`

	defaultYaml = `targets:
- name: %s
  dir: main
  assets:
  - templates
  - public
`
)

var (
	// Debug indicated whether it is debug mode
	Debug = false
)

// CmdInit represents
var CmdInit = cli.Command{
	Name:        "init",
	Usage:       "Init a new project",
	Description: `Init a new project`,
	Action:      runInit,
	Flags: []cli.Flag{
		cli.BoolTFlag{
			Name:  "verbose, v",
			Usage: "Enables verbose progress and debug output",
		},
	},
}

func runInit(ctx *cli.Context) error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}

	showLog = ctx.IsSet("verbose")

	os.MkdirAll(filepath.Join(wd, "src"), os.ModePerm)
	os.MkdirAll(filepath.Join(wd, "src", "vendor"), os.ModePerm)
	os.MkdirAll(filepath.Join(wd, "src", "main"), os.ModePerm)
	os.MkdirAll(filepath.Join(wd, "bin"), os.ModePerm)

	ymlPath := filepath.Join(wd, "gop.yml")
	_, err = os.Stat(ymlPath)
	if err != nil {
		if os.IsNotExist(err) {
			y, err := os.Create(ymlPath)
			if err != nil {
				return err
			}
			defer y.Close()

			_, err = y.Write([]byte(fmt.Sprintf(defaultYaml, filepath.Base(wd))))
			if err != nil {
				return err
			}
		}
	}

	mainFile := filepath.Join(wd, "src", "main", "main.go")
	_, err = os.Stat(mainFile)
	if err != nil {
		if os.IsNotExist(err) {
			f, err := os.Create(mainFile)
			if err != nil {
				return fmt.Errorf("os.Create: %v", err)
			}
			defer f.Close()
			_, err = f.Write([]byte(defaultMainFile))
			if err != nil {
				return fmt.Errorf("create main file failed: %v", err)
			}
			return nil
		}
		return fmt.Errorf("os.State: %v", err)
	}

	return nil
}
<commit_msg>add editor support for init (#18)<commit_after>// Copyright 2017 The Gop Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/urfave/cli"
)

const (
	defaultMainFile = `package main

func main() {

}
`

	defaultYaml = `targets:
- name: %s
  dir: main
  assets:
  - templates
  - public
`

	defaultVscodeCfgFile = `{
	"go.gopath": "${workspaceRoot}"
}`
)

var (
	// Debug indicated whether it is debug mode
	Debug = false
)

// CmdInit represents
var CmdInit = cli.Command{
	Name:        "init",
	Usage:       "Init a new project",
	Description: `Init a new project`,
	Action:      runInit,
	Flags: []cli.Flag{
		cli.BoolTFlag{
			Name:  "verbose, v",
			Usage: "Enables verbose progress and debug output",
		},
		cli.StringFlag{
			Name:  "editor, e",
			Usage: "Generate specifial editor configuration. Could be vscode or blank with no editor support",
		},
	},
}

func runInit(ctx *cli.Context) error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}

	showLog = ctx.IsSet("verbose")

	os.MkdirAll(filepath.Join(wd, "src"), os.ModePerm)
	os.MkdirAll(filepath.Join(wd, "src", "vendor"), os.ModePerm)
	os.MkdirAll(filepath.Join(wd, "src", "main"), os.ModePerm)
	os.MkdirAll(filepath.Join(wd, "bin"), os.ModePerm)

	ymlPath := filepath.Join(wd, "gop.yml")
	_, err = os.Stat(ymlPath)
	if err != nil {
		if os.IsNotExist(err) {
			y, err := os.Create(ymlPath)
			if err != nil {
				return err
			}
			defer y.Close()

			_, err = y.Write([]byte(fmt.Sprintf(defaultYaml, filepath.Base(wd))))
			if err != nil {
				return err
			}
		}
	}

	mainFile := filepath.Join(wd, "src", "main", "main.go")
	_, err = os.Stat(mainFile)
	if err != nil {
		if !os.IsNotExist(err) {
			return fmt.Errorf("os.State: %v", err)
		}

		f, err := os.Create(mainFile)
		if err != nil {
			return fmt.Errorf("os.Create: %v", err)
		}
		defer f.Close()
		_, err = f.Write([]byte(defaultMainFile))
		if err != nil {
			return fmt.Errorf("create main file failed: %v", err)
		}
	}

	if ctx.IsSet("editor") {
		switch ctx.String("editor") {
		case "vscode":
			os.MkdirAll(filepath.Join(wd, ".vscode"), os.ModePerm)
			cfgFile := filepath.Join(wd, ".vscode", "settings.json")

			_, err = os.Stat(cfgFile)
			if err != nil {
				if !os.IsNotExist(err) {
					return fmt.Errorf("os.State: %v", err)
				}

				f, err := os.Create(cfgFile)
				if err != nil {
					return fmt.Errorf("os.Create: %v", err)
				}
				defer f.Close()
				_, err = f.Write([]byte(defaultVscodeCfgFile))
				if err != nil {
					return fmt.Errorf("create main file failed: %v", err)
				}
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"github.com/markbates/refresh/refresh"
	"github.com/spf13/cobra"
)

func init() {
	RootCmd.AddCommand(initCmd)
}

var initCmd = &cobra.Command{
	Use:   "init",
	Short: "generates a default configuration file for you.",
	Run: func(cmd *cobra.Command, args []string) {
		c := refresh.Configuration{
			AppRoot:            ".",
			IgnoredFolders:     []string{"vendor", "log", "tmp"},
			IncludedExtensions: []string{".go"},
			BuildPath:          "/tmp",
			BuildDelay:         200,
			BinaryName:         "refresh-build",
			CommandFlags:       []string{},
			EnableColors:       true,
		}
		c.Dump(cfgFile)
	},
}
<commit_msg>generate config with os specific temp dir. related to github.com/gobuffalo/buffalo/issues/124<commit_after>package cmd

import (
	"os"

	"github.com/markbates/refresh/refresh"
	"github.com/spf13/cobra"
)

func init() {
	RootCmd.AddCommand(initCmd)
}

var initCmd = &cobra.Command{
	Use:   "init",
	Short: "generates a default configuration file for you.",
	Run: func(cmd *cobra.Command, args []string) {
		c := refresh.Configuration{
			AppRoot:            ".",
			IgnoredFolders:     []string{"vendor", "log", "logs", "tmp", "node_modules", "bin", "templates"},
			IncludedExtensions: []string{".go"},
			BuildPath:          os.TempDir(),
			BuildDelay:         200,
			BinaryName:         "refresh-build",
			CommandFlags:       []string{},
			EnableColors:       true,
		}
		c.Dump(cfgFile)
	},
}
<|endoftext|>


<commit_before>package cmd

import (
	"os"
	"path/filepath"

	"github.com/bitrise-io/bitrise-workflow-editor/apiserver"
	"github.com/bitrise-io/go-utils/log"
	"github.com/bitrise-io/go-utils/pathutil"
	"github.com/spf13/cobra"
)

func failf(format string, v ...interface{}) {
	log.Errorf(format, v...)
	os.Exit(1)
}

// RootCmd ...
var RootCmd = &cobra.Command{
	Use: "workflow-editor",
	Run: func(cmd *cobra.Command, args []string) {
		currentDir, err := filepath.Abs(filepath.Dir(os.Args[0]))
		if err != nil {
			failf("Failed to get current dir, error: %s", err)
		}

		bitriseConfigPth := filepath.Join(currentDir, "bitrise.yml")
		if exist, err := pathutil.IsPathExists(bitriseConfigPth); err != nil {
			failf("Failed to check is bitrise.yml exist, error: %s", err)
		} else if !exist {
			failf("No bitrise config (bitrise.yml) found in the current directory")
		}

		if err := apiserver.LaunchServer(); err != nil {
			failf("Failed to start server, error: %s", err)
		}
	},
}
<commit_msg>Check bitrise config (#182)<commit_after>package cmd

import (
	"os"
	"path/filepath"

	"github.com/bitrise-io/bitrise-workflow-editor/apiserver"
	"github.com/bitrise-io/go-utils/log"
	"github.com/bitrise-io/go-utils/pathutil"
	"github.com/spf13/cobra"
)

func failf(format string, v ...interface{}) {
	log.Errorf(format, v...)
	os.Exit(1)
}

// RootCmd ...
var RootCmd = &cobra.Command{
	Use: "workflow-editor",
	Run: func(cmd *cobra.Command, args []string) {
		currentDir, err := filepath.Abs("./")
		if err != nil {
			failf("Failed to get current dir, error: %s", err)
		}

		bitriseConfigPth := filepath.Join(currentDir, "bitrise.yml")
		if exist, err := pathutil.IsPathExists(bitriseConfigPth); err != nil {
			failf("Failed to check is bitrise.yml exist, error: %s", err)
		} else if !exist {
			failf("No bitrise config (bitrise.yml) found in the current directory")
		}

		if err := apiserver.LaunchServer(); err != nil {
			failf("Failed to start server, error: %s", err)
		}
	},
}
<|endoftext|>


<commit_before>package cmd

import (
	"errors"
	"fmt"
	"os"

	"github.com/coreos/pkg/capnslog"
	"github.com/jgsqware/clairctl/clair"
	"github.com/jgsqware/clairctl/config"
	"github.com/spf13/cobra"
)

var errInternalError = errors.New("client quit unexpectedly")

var log = capnslog.NewPackageLogger("github.com/jgsqware/clairctl", "cmd")

var cfgFile string
var logLevel string
var noClean bool

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "clairctl",
	Short: "Analyze your docker image with Clair, directly from your registry or local images.",
	Long:  ``,
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		err = config.Clean()
		fmt.Println(err)
		os.Exit(-1)
	}

	if err := config.Clean(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)
	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.clairctl.yml)")
	RootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "", "log level [Panic,Fatal,Error,Warn,Info,Debug]")
	RootCmd.PersistentFlags().BoolVar(&noClean, "no-clean", false, "Disable the temporary folder cleaning")
}

func initConfig() {
	config.Init(cfgFile, logLevel, noClean)
	clair.Config()
}
<commit_msg>Fix the help output (#22)<commit_after>package cmd

import (
	"errors"
	"fmt"
	"os"

	"github.com/coreos/pkg/capnslog"
	"github.com/jgsqware/clairctl/clair"
	"github.com/jgsqware/clairctl/config"
	"github.com/spf13/cobra"
)

var errInternalError = errors.New("client quit unexpectedly")

var log = capnslog.NewPackageLogger("github.com/jgsqware/clairctl", "cmd")

var cfgFile string
var logLevel string
var noClean bool

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "clairctl",
	Short: "Analyze your docker image with Clair, directly from your registry or local images.",
	Long:  ``,
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		err = config.Clean()
		fmt.Println(err)
		os.Exit(-1)
	}

	if err := config.Clean(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)
	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/clairctl.yml)")
	RootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "", "log level [Panic,Fatal,Error,Warn,Info,Debug]")
	RootCmd.PersistentFlags().BoolVar(&noClean, "no-clean", false, "Disable the temporary folder cleaning")
}

func initConfig() {
	config.Init(cfgFile, logLevel, noClean)
	clair.Config()
}
<|endoftext|>