<commit_before>package protocol

type Data struct {
	Partition int32
	RecordSet []byte
}

type TopicData struct {
	Topic string
	Data  []*Data
}

type ProduceRequest struct {
	Acks      int16
	Timeout   int32
	TopicData []*TopicData
}

func (r *ProduceRequest) Encode(e PacketEncoder) error {
	e.PutInt16(r.Acks)
	e.PutInt32(r.Timeout)
	e.PutArrayLength(len(r.TopicData))
	for _, td := range r.TopicData {
		e.PutString(td.Topic)
		e.PutArrayLength(len(td.Data))
		for _, d := range td.Data {
			e.PutInt32(d.Partition)
			e.PutBytes(d.RecordSet)
		}
	}
	return nil
}

func (r *ProduceRequest) Decode(d PacketDecoder) error {
	var err error
	r.Acks, err = d.Int16()
	if err != nil {
		return err
	}
	r.Timeout, err = d.Int32()
	if err != nil {
		return err
	}
	tdlen, err := d.ArrayLength()
	r.TopicData = make([]*TopicData, tdlen)
	for i := range r.TopicData {
		td := new(TopicData)
		r.TopicData[i] = td
		td.Topic, err = d.String()
		if err != nil {
			return err
		}
		dlen, err := d.ArrayLength()
		if err != nil {
			return err
		}
		td.Data = make([]*Data, dlen)
		for j := range td.Data {
			data := new(Data)
			td.Data[j] = data
			data.Partition, err = d.Int32()
			if err != nil {
				return err
			}
			data.RecordSet, err = d.Bytes()
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func (r *ProduceRequest) Key() int16 {
	return 0
}

func (r *ProduceRequest) Version() int16 {
	return 2
}
<commit_msg>produce request: check all errors<commit_after>package protocol

type Data struct {
	Partition int32
	RecordSet []byte
}

type TopicData struct {
	Topic string
	Data  []*Data
}

type ProduceRequest struct {
	Acks      int16
	Timeout   int32
	TopicData []*TopicData
}

func (r *ProduceRequest) Encode(e PacketEncoder) (err error) {
	e.PutInt16(r.Acks)
	e.PutInt32(r.Timeout)
	if err = e.PutArrayLength(len(r.TopicData)); err != nil {
		return err
	}
	for _, td := range r.TopicData {
		if err = e.PutString(td.Topic); err != nil {
			return err
		}
		if err = e.PutArrayLength(len(td.Data)); err != nil {
			return err
		}
		for _, d := range td.Data {
			e.PutInt32(d.Partition)
			if err = e.PutBytes(d.RecordSet); err != nil {
				return err
			}
		}
	}
	return nil
}

func (r *ProduceRequest) Decode(d PacketDecoder) error {
	var err error
	r.Acks, err = d.Int16()
	if err != nil {
		return err
	}
	r.Timeout, err = d.Int32()
	if err != nil {
		return err
	}
	tdlen, err := d.ArrayLength()
	r.TopicData = make([]*TopicData, tdlen)
	for i := range r.TopicData {
		td := new(TopicData)
		r.TopicData[i] = td
		td.Topic, err = d.String()
		if err != nil {
			return err
		}
		dlen, err := d.ArrayLength()
		if err != nil {
			return err
		}
		td.Data = make([]*Data, dlen)
		for j := range td.Data {
			data := new(Data)
			td.Data[j] = data
			data.Partition, err = d.Int32()
			if err != nil {
				return err
			}
			data.RecordSet, err = d.Bytes()
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func (r *ProduceRequest) Key() int16 {
	return 0
}

func (r *ProduceRequest) Version() int16 {
	return 2
}
<|endoftext|>


<commit_before><commit_msg>Template Fix<commit_after><|endoftext|>


<commit_before>package workflow

import (
	"context"
	"database/sql"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/group"
	"github.com/ovh/cds/engine/api/observability"
	"github.com/ovh/cds/engine/api/permission"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// loadPrepareGroup returns true if groupsID contains shareInfraGroup
// and list of groups, comma separated
func isSharedInfraGroup(groupsID []int64) bool {
	return sdk.IsInInt64Array(group.SharedInfraGroup.ID, groupsID)
}

// QueueFilter contains all criterias used to fetch queue
type QueueFilter struct {
	ModelType    string
	RatioService *int
	GroupsID     []int64
	User         *sdk.User
	Rights       int
	Since        *time.Time
	Until        *time.Time
	Limit        *int
	Statuses     []string
}

// CountNodeJobRunQueue count all workflow_node_run_job accessible
func CountNodeJobRunQueue(ctx context.Context, db gorp.SqlExecutor, store cache.Store, filter QueueFilter) (sdk.WorkflowNodeJobRunCount, error) {
	c := sdk.WorkflowNodeJobRunCount{}

	queue, err := LoadNodeJobRunQueue(ctx, db, store, filter)
	if err != nil {
		return c, sdk.WrapError(err, "CountNodeJobRunQueue> unable to load queue")
	}

	c.Count = int64(len(queue))
	if filter.Since != nil {
		c.Since = *filter.Since
	}
	if filter.Until != nil {
		c.Until = *filter.Until
	}
	return c, nil
}

// LoadNodeJobRunQueue load all workflow_node_run_job accessible
func LoadNodeJobRunQueue(ctx context.Context, db gorp.SqlExecutor, store cache.Store, filter QueueFilter) ([]sdk.WorkflowNodeJobRun, error) {
	ctx, end := observability.Span(ctx, "LoadNodeJobRunQueue")
	defer end()
	if filter.Since == nil {
		filter.Since = new(time.Time)
	}

	if filter.Until == nil {
		now := time.Now()
		filter.Until = &now
	}

	if len(filter.Statuses) == 0 {
		filter.Statuses = []string{sdk.StatusWaiting.String()}
	}

	containsService := []bool{true, false}
	if filter.RatioService != nil {
		if *filter.RatioService == 100 {
			containsService = []bool{true, true}
		} else if *filter.RatioService == 0 {
			containsService = []bool{false, false}
		}
	}

	modelTypes := sdk.AvailableWorkerModelType
	if filter.ModelType != "" {
		modelTypes = []string{filter.ModelType}
	}

	args := []interface{}{
		*filter.Since,                      // $1
		*filter.Until,                      // $2
		strings.Join(filter.Statuses, ","), // $3
		containsService[0],                 // $4
		containsService[1],                 // $5
		strings.Join(modelTypes, ","),      // $6
	}

	query := `select distinct workflow_node_run_job.*
	from workflow_node_run_job
	where workflow_node_run_job.queued >= $1
	and workflow_node_run_job.queued <= $2
	and workflow_node_run_job.status = ANY(string_to_array($3, ','))
	AND contains_service IN ($4, $5)
	AND (model_type is NULL OR model_type = '' OR model_type = ANY(string_to_array($6, ',')))
	ORDER BY workflow_node_run_job.queued ASC
	`

	if filter.User != nil && !filter.User.Admin {
		observability.Current(ctx, observability.Tag("isAdmin", false))
		query = `
		SELECT DISTINCT workflow_node_run_job.*
			FROM workflow_node_run_job
			JOIN workflow_node_run ON workflow_node_run.id = workflow_node_run_job.workflow_node_run_id
			JOIN workflow_run ON workflow_run.id = workflow_node_run.workflow_run_id
			JOIN project ON project.id = workflow_run.project_id
		WHERE project.id IN (
			SELECT project_group.project_id
			FROM project_group
			WHERE
				project_group.group_id = ANY(string_to_array($7, ',')::int[])
			OR
				$5 = ANY(string_to_array($7, ',')::int[])
		)
		AND workflow_node_run_job.queued >= $1
		AND workflow_node_run_job.queued <= $2
		AND workflow_node_run_job.status = ANY(string_to_array($3, ','))
		AND contains_service IN ($4, $5)
		AND (model_type is NULL OR model_type = '' OR model_type = ANY(string_to_array($6, ',')))
		ORDER BY workflow_node_run_job.queued ASC
		`

		var groupID string
		for i, g := range filter.User.Groups {
			if i == 0 {
				groupID = fmt.Sprintf("%d", g.ID)
			} else {
				groupID += "," + fmt.Sprintf("%d", g.ID)
			}
		}
		args = append(args, groupID, group.SharedInfraGroup.ID)
	} else {
		observability.Current(ctx, observability.Tag("isAdmin", true))
	}

	if filter.Limit != nil && *filter.Limit > 0 {
		query += `
		LIMIT ` + strconv.Itoa(*filter.Limit)
	}
	isSharedInfraGroup := isSharedInfraGroup(filter.GroupsID)
	sqlJobs := []JobRun{}
	_, next := observability.Span(ctx, "LoadNodeJobRunQueue.select")
	if _, err := db.Select(&sqlJobs, query, args...); err != nil {
		next()
		return nil, sdk.WrapError(err, "workflow.LoadNodeJobRun> Unable to load job runs (Select)")
	}
	next()

	ctx2, next2 := observability.Span(ctx, "LoadNodeJobRunQueue.sqlJobs")

	jobs := make([]sdk.WorkflowNodeJobRun, 0, len(sqlJobs))
	for i := range sqlJobs {
		_, next3 := observability.Span(ctx2, "LoadNodeJobRunQueue.loadHatcheryInfo")
		getHatcheryInfo(store, &sqlJobs[i])
		next3()
		jr, err := sqlJobs[i].WorkflowNodeRunJob()
		if err != nil {
			log.Error("LoadNodeJobRunQueue> WorkflowNodeRunJob error: %v", err)
		}

		var keepJobInQueue bool
		// a shared.infra group can see all jobs
		// a user (not a hatchery or worker) can see all jobs, even if jobs are only RO for him
		if isSharedInfraGroup || filter.Rights == permission.PermissionRead {
			keepJobInQueue = true
		} else {
			// if no shared.infra, we have to filter only executable jobs for worker or hatchery
			for _, g := range jr.ExecGroups {
				if sdk.IsInInt64Array(g.ID, filter.GroupsID) {
					keepJobInQueue = true
					break
				}
			}
		}

		if !keepJobInQueue {
			continue
		}

		jobs = append(jobs, jr)
	}
	next2()

	return jobs, nil
}

// LoadNodeJobRunIDByNodeRunID Load node run job id by node run id
func LoadNodeJobRunIDByNodeRunID(db gorp.SqlExecutor, runNodeID int64) ([]int64, error) {
	query := `SELECT workflow_node_run_job.id FROM workflow_node_run_job WHERE workflow_node_run_id = $1`
	rows, err := db.Query(query, runNodeID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var ids []int64
	for rows.Next() {
		var id int64
		if err := rows.Scan(&id); err != nil {
			return nil, err
		}
		ids = append(ids, id)
	}
	return ids, nil
}

//LoadNodeJobRun load a NodeJobRun given its ID
func LoadNodeJobRun(db gorp.SqlExecutor, store cache.Store, id int64) (*sdk.WorkflowNodeJobRun, error) {
	j := JobRun{}
	query := `select workflow_node_run_job.* from workflow_node_run_job where id = $1`
	if err := db.SelectOne(&j, query, id); err != nil {
		if err == sql.ErrNoRows {
			return nil, sdk.ErrWorkflowNodeRunJobNotFound
		}
		return nil, err
	}
	if store != nil {
		getHatcheryInfo(store, &j)
	}
	jr, err := j.WorkflowNodeRunJob()
	if err != nil {
		return nil, err
	}
	return &jr, nil
}

//LoadAndLockNodeJobRunWait load for update a NodeJobRun given its ID
func LoadAndLockNodeJobRunWait(db gorp.SqlExecutor, store cache.Store, id int64) (*sdk.WorkflowNodeJobRun, error) {
	j := JobRun{}
	query := `select workflow_node_run_job.* from workflow_node_run_job where id = $1 for update`
	if err := db.SelectOne(&j, query, id); err != nil {
		return nil, err
	}
	getHatcheryInfo(store, &j)
	jr, err := j.WorkflowNodeRunJob()
	if err != nil {
		return nil, err
	}
	return &jr, nil
}

//LoadAndLockNodeJobRunNoWait load for update a NodeJobRun given its ID
func LoadAndLockNodeJobRunNoWait(ctx context.Context, db gorp.SqlExecutor, store cache.Store, id int64) (*sdk.WorkflowNodeJobRun, error) {
	var end func()
	_, end = observability.Span(ctx, "workflow.LoadAndLockNodeJobRunNoWait")
	defer end()

	j := JobRun{}
	query := `select workflow_node_run_job.* from workflow_node_run_job where id = $1 for update nowait`
	if err := db.SelectOne(&j, query, id); err != nil {
		return nil, err
	}
	getHatcheryInfo(store, &j)
	jr, err := j.WorkflowNodeRunJob()
	if err != nil {
		return nil, err
	}
	return &jr, nil
}

func insertWorkflowNodeJobRun(db gorp.SqlExecutor, j *sdk.WorkflowNodeJobRun) error {
	dbj := new(JobRun)
	err := dbj.ToJobRun(j)
	if err != nil {
		return err
	}
	if err := db.Insert(dbj); err != nil {
		return err
	}
	j.ID = dbj.ID
	return nil
}

//DeleteNodeJobRuns deletes all workflow_node_run_job for a given workflow_node_run
func DeleteNodeJobRuns(db gorp.SqlExecutor, nodeID int64) error {
	query := `delete from workflow_node_run_job where workflow_node_run_id = $1`
	_, err := db.Exec(query, nodeID)
	return err
}

//UpdateNodeJobRun updates a workflow_node_run_job
func UpdateNodeJobRun(ctx context.Context, db gorp.SqlExecutor, j *sdk.WorkflowNodeJobRun) error {
	var end func()
	_, end = observability.Span(ctx, "workflow.UpdateNodeJobRun")
	defer end()

	dbj := new(JobRun)
	err := dbj.ToJobRun(j)
	if err != nil {
		return err
	}
	if _, err := db.Update(dbj); err != nil {
		return err
	}
	return nil
}

func keyBookJob(id int64) string {
	return cache.Key("book", "job", strconv.FormatInt(id, 10))
}

func getHatcheryInfo(store cache.Store, j *JobRun) {
	h := sdk.Service{}
	if store.Get(keyBookJob(j.ID), &h) {
		j.BookedBy = h
	}
}

// replaceWorkflowJobRunInQueue restart workflow node job
func replaceWorkflowJobRunInQueue(db gorp.SqlExecutor, wNodeJob sdk.WorkflowNodeJobRun) error {
	query := "UPDATE workflow_node_run_job SET status = $1, retry = $2, worker_id = NULL WHERE id = $3"
	if _, err := db.Exec(query, sdk.StatusWaiting.String(), wNodeJob.Retry+1, wNodeJob.ID); err != nil {
		return sdk.WrapError(err, "replaceWorkflowJobRunInQueue> Unable to set workflow_node_run_job id %d with status %s", wNodeJob.ID, sdk.StatusWaiting.String())
	}

	query = "UPDATE worker SET status = $2, action_build_id = NULL where action_build_id = $1"
	if _, err := db.Exec(query, wNodeJob.ID, sdk.StatusDisabled); err != nil {
		return sdk.WrapError(err, "replaceWorkflowJobRunInQueue> Unable to set workers")
	}

	return nil
}
<commit_msg>Update dao_node_run_job.go<commit_after>package workflow

import (
	"context"
	"database/sql"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/group"
	"github.com/ovh/cds/engine/api/observability"
	"github.com/ovh/cds/engine/api/permission"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// loadPrepareGroup returns true if groupsID contains shareInfraGroup
// and list of groups, comma separated
func isSharedInfraGroup(groupsID []int64) bool {
	return sdk.IsInInt64Array(group.SharedInfraGroup.ID, groupsID)
}

// QueueFilter contains all criterias used to fetch queue
type QueueFilter struct {
	ModelType    string
	RatioService *int
	GroupsID     []int64
	User         *sdk.User
	Rights       int
	Since        *time.Time
	Until        *time.Time
	Limit        *int
	Statuses     []string
}

// CountNodeJobRunQueue count all workflow_node_run_job accessible
func CountNodeJobRunQueue(ctx context.Context, db gorp.SqlExecutor, store cache.Store, filter QueueFilter) (sdk.WorkflowNodeJobRunCount, error) {
	c := sdk.WorkflowNodeJobRunCount{}

	queue, err := LoadNodeJobRunQueue(ctx, db, store, filter)
	if err != nil {
		return c, sdk.WrapError(err, "CountNodeJobRunQueue> unable to load queue")
	}

	c.Count = int64(len(queue))
	if filter.Since != nil {
		c.Since = *filter.Since
	}
	if filter.Until != nil {
		c.Until = *filter.Until
	}
	return c, nil
}

// LoadNodeJobRunQueue load all workflow_node_run_job accessible
func LoadNodeJobRunQueue(ctx context.Context, db gorp.SqlExecutor, store cache.Store, filter QueueFilter) ([]sdk.WorkflowNodeJobRun, error) {
	ctx, end := observability.Span(ctx, "LoadNodeJobRunQueue")
	defer end()
	if filter.Since == nil {
		filter.Since = new(time.Time)
	}

	if filter.Until == nil {
		now := time.Now()
		filter.Until = &now
	}

	if len(filter.Statuses) == 0 {
		filter.Statuses = []string{sdk.StatusWaiting.String()}
	}

	containsService := []bool{true, false}
	if filter.RatioService != nil {
		if *filter.RatioService == 100 {
			containsService = []bool{true, true}
		} else if *filter.RatioService == 0 {
			containsService = []bool{false, false}
		}
	}

	modelTypes := sdk.AvailableWorkerModelType
	if filter.ModelType != "" {
		modelTypes = []string{filter.ModelType}
	}

	args := []interface{}{
		*filter.Since,                      // $1
		*filter.Until,                      // $2
		strings.Join(filter.Statuses, ","), // $3
		containsService[0],                 // $4
		containsService[1],                 // $5
		strings.Join(modelTypes, ","),      // $6
	}

	query := `select distinct workflow_node_run_job.*
	from workflow_node_run_job
	where workflow_node_run_job.queued >= $1
	and workflow_node_run_job.queued <= $2
	and workflow_node_run_job.status = ANY(string_to_array($3, ','))
	AND contains_service IN ($4, $5)
	AND (model_type is NULL OR model_type = '' OR model_type = ANY(string_to_array($6, ',')))
	ORDER BY workflow_node_run_job.queued ASC
	`

	if filter.User != nil && !filter.User.Admin {
		observability.Current(ctx, observability.Tag("isAdmin", false))
		query = `
		SELECT DISTINCT workflow_node_run_job.*
			FROM workflow_node_run_job
			JOIN workflow_node_run ON workflow_node_run.id = workflow_node_run_job.workflow_node_run_id
			JOIN workflow_run ON workflow_run.id = workflow_node_run.workflow_run_id
			JOIN project ON project.id = workflow_run.project_id
		WHERE project.id IN (
			SELECT project_group.project_id
			FROM project_group
			WHERE
				project_group.group_id = ANY(string_to_array($7, ',')::int[])
			OR
				$7 = ANY(string_to_array($7, ',')::int[])
		)
		AND workflow_node_run_job.queued >= $1
		AND workflow_node_run_job.queued <= $2
		AND workflow_node_run_job.status = ANY(string_to_array($3, ','))
		AND contains_service IN ($4, $5)
		AND (model_type is NULL OR model_type = '' OR model_type = ANY(string_to_array($6, ',')))
		ORDER BY workflow_node_run_job.queued ASC
		`

		var groupID string
		for i, g := range filter.User.Groups {
			if i == 0 {
				groupID = fmt.Sprintf("%d", g.ID)
			} else {
				groupID += "," + fmt.Sprintf("%d", g.ID)
			}
		}
		args = append(args, groupID, group.SharedInfraGroup.ID)
	} else {
		observability.Current(ctx, observability.Tag("isAdmin", true))
	}

	if filter.Limit != nil && *filter.Limit > 0 {
		query += `
		LIMIT ` + strconv.Itoa(*filter.Limit)
	}
	isSharedInfraGroup := isSharedInfraGroup(filter.GroupsID)
	sqlJobs := []JobRun{}
	_, next := observability.Span(ctx, "LoadNodeJobRunQueue.select")
	if _, err := db.Select(&sqlJobs, query, args...); err != nil {
		next()
		return nil, sdk.WrapError(err, "workflow.LoadNodeJobRun> Unable to load job runs (Select)")
	}
	next()

	ctx2, next2 := observability.Span(ctx, "LoadNodeJobRunQueue.sqlJobs")

	jobs := make([]sdk.WorkflowNodeJobRun, 0, len(sqlJobs))
	for i := range sqlJobs {
		_, next3 := observability.Span(ctx2, "LoadNodeJobRunQueue.loadHatcheryInfo")
		getHatcheryInfo(store, &sqlJobs[i])
		next3()
		jr, err := sqlJobs[i].WorkflowNodeRunJob()
		if err != nil {
			log.Error("LoadNodeJobRunQueue> WorkflowNodeRunJob error: %v", err)
		}

		var keepJobInQueue bool
		// a shared.infra group can see all jobs
		// a user (not a hatchery or worker) can see all jobs, even if jobs are only RO for him
		if isSharedInfraGroup || filter.Rights == permission.PermissionRead {
			keepJobInQueue = true
		} else {
			// if no shared.infra, we have to filter only executable jobs for worker or hatchery
			for _, g := range jr.ExecGroups {
				if sdk.IsInInt64Array(g.ID, filter.GroupsID) {
					keepJobInQueue = true
					break
				}
			}
		}

		if !keepJobInQueue {
			continue
		}

		jobs = append(jobs, jr)
	}
	next2()

	return jobs, nil
}

// LoadNodeJobRunIDByNodeRunID Load node run job id by node run id
func LoadNodeJobRunIDByNodeRunID(db gorp.SqlExecutor, runNodeID int64) ([]int64, error) {
	query := `SELECT workflow_node_run_job.id FROM workflow_node_run_job WHERE workflow_node_run_id = $1`
	rows, err := db.Query(query, runNodeID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var ids []int64
	for rows.Next() {
		var id int64
		if err := rows.Scan(&id); err != nil {
			return nil, err
		}
		ids = append(ids, id)
	}
	return ids, nil
}

//LoadNodeJobRun load a NodeJobRun given its ID
func LoadNodeJobRun(db gorp.SqlExecutor, store cache.Store, id int64) (*sdk.WorkflowNodeJobRun, error) {
	j := JobRun{}
	query := `select workflow_node_run_job.* from workflow_node_run_job where id = $1`
	if err := db.SelectOne(&j, query, id); err != nil {
		if err == sql.ErrNoRows {
			return nil, sdk.ErrWorkflowNodeRunJobNotFound
		}
		return nil, err
	}
	if store != nil {
		getHatcheryInfo(store, &j)
	}
	jr, err := j.WorkflowNodeRunJob()
	if err != nil {
		return nil, err
	}
	return &jr, nil
}

//LoadAndLockNodeJobRunWait load for update a NodeJobRun given its ID
func LoadAndLockNodeJobRunWait(db gorp.SqlExecutor, store cache.Store, id int64) (*sdk.WorkflowNodeJobRun, error) {
	j := JobRun{}
	query := `select workflow_node_run_job.* from workflow_node_run_job where id = $1 for update`
	if err := db.SelectOne(&j, query, id); err != nil {
		return nil, err
	}
	getHatcheryInfo(store, &j)
	jr, err := j.WorkflowNodeRunJob()
	if err != nil {
		return nil, err
	}
	return &jr, nil
}

//LoadAndLockNodeJobRunNoWait load for update a NodeJobRun given its ID
func LoadAndLockNodeJobRunNoWait(ctx context.Context, db gorp.SqlExecutor, store cache.Store, id int64) (*sdk.WorkflowNodeJobRun, error) {
	var end func()
	_, end = observability.Span(ctx, "workflow.LoadAndLockNodeJobRunNoWait")
	defer end()

	j := JobRun{}
	query := `select workflow_node_run_job.* from workflow_node_run_job where id = $1 for update nowait`
	if err := db.SelectOne(&j, query, id); err != nil {
		return nil, err
	}
	getHatcheryInfo(store, &j)
	jr, err := j.WorkflowNodeRunJob()
	if err != nil {
		return nil, err
	}
	return &jr, nil
}

func insertWorkflowNodeJobRun(db gorp.SqlExecutor, j *sdk.WorkflowNodeJobRun) error {
	dbj := new(JobRun)
	err := dbj.ToJobRun(j)
	if err != nil {
		return err
	}
	if err := db.Insert(dbj); err != nil {
		return err
	}
	j.ID = dbj.ID
	return nil
}

//DeleteNodeJobRuns deletes all workflow_node_run_job for a given workflow_node_run
func DeleteNodeJobRuns(db gorp.SqlExecutor, nodeID int64) error {
	query := `delete from workflow_node_run_job where workflow_node_run_id = $1`
	_, err := db.Exec(query, nodeID)
	return err
}

//UpdateNodeJobRun updates a workflow_node_run_job
func UpdateNodeJobRun(ctx context.Context, db gorp.SqlExecutor, j *sdk.WorkflowNodeJobRun) error {
	var end func()
	_, end = observability.Span(ctx, "workflow.UpdateNodeJobRun")
	defer end()

	dbj := new(JobRun)
	err := dbj.ToJobRun(j)
	if err != nil {
		return err
	}
	if _, err := db.Update(dbj); err != nil {
		return err
	}
	return nil
}

func keyBookJob(id int64) string {
	return cache.Key("book", "job", strconv.FormatInt(id, 10))
}

func getHatcheryInfo(store cache.Store, j *JobRun) {
	h := sdk.Service{}
	if store.Get(keyBookJob(j.ID), &h) {
		j.BookedBy = h
	}
}

// replaceWorkflowJobRunInQueue restart workflow node job
func replaceWorkflowJobRunInQueue(db gorp.SqlExecutor, wNodeJob sdk.WorkflowNodeJobRun) error {
	query := "UPDATE workflow_node_run_job SET status = $1, retry = $2, worker_id = NULL WHERE id = $3"
	if _, err := db.Exec(query, sdk.StatusWaiting.String(), wNodeJob.Retry+1, wNodeJob.ID); err != nil {
		return sdk.WrapError(err, "replaceWorkflowJobRunInQueue> Unable to set workflow_node_run_job id %d with status %s", wNodeJob.ID, sdk.StatusWaiting.String())
	}

	query = "UPDATE worker SET status = $2, action_build_id = NULL where action_build_id = $1"
	if _, err := db.Exec(query, wNodeJob.ID, sdk.StatusDisabled); err != nil {
		return sdk.WrapError(err, "replaceWorkflowJobRunInQueue> Unable to set workers")
	}

	return nil
}
<|endoftext|>


<commit_before>// +build spell

package main

import (
	"fmt"
	"regexp"
	"strings"
)

type Spellcheck struct {
	speller Speller
	toks    map[string]struct{}
	check   CheckFunc
	strips  []*regexp.Regexp
}

func NewSpellcheck(ts TokenSet, ignoreFile string) (sc *Spellcheck, err error) {
	igns, err := WithPassIgnores(ignoreFile)
	if err != nil {
		return nil, err
	}

	strips := []*regexp.Regexp{
		regexp.MustCompile("http(s|):[^ ]*"),
		regexp.MustCompile("TODO[ ]*\\([a-z]*"),
	}

	hsp := NewHunSpeller()
	if hsp == nil {
		return nil, fmt.Errorf("bad hunspell")
	}
	asp, err := NewASpeller()
	if err != nil {
		return nil, err
	}

	sc = &Spellcheck{
		strips:  strips,
		toks:    ts,
		speller: NewMultiSpeller(asp, hsp),
	}

	checks := []CheckFunc{
		sc.WithPassTokens(),
		igns,
		WithPassNumbers(),
		sc.WithSpeller(),
	}

	sc.check = func(w string) bool {
		for _, chk := range checks {
			if chk(w) {
				return true
			}
		}
		return false
	}
	return sc, nil
}

func (sc *Spellcheck) Close() { sc.speller.Close() }

func (sc *Spellcheck) WithPassTokens() CheckFunc {
	return func(word string) bool {
		_, ok := sc.toks[word]
		return ok
	}
}

func (sc *Spellcheck) WithSpeller() CheckFunc {
	return func(word string) bool { return sc.speller.Check(word) }
}

func (sc *Spellcheck) Check() CheckPipe {
	return func(lc <-chan *Lexeme, outc chan<- *CheckedLexeme) {
		sc.checkComments(lc, outc)
	}
}

func (sc *Spellcheck) checkComments(lc <-chan *Lexeme, outc chan<- *CheckedLexeme) {
	ch := Filter(lc, CommentFilter)
	for comm := range ch {
		if ct := sc.checkLexeme(comm); ct != nil {
			outc <- ct
		}
	}
}

func (sc *Spellcheck) suggest(word string) string {
	if sc.check(word) {
		return ""
	}
	if sugg := sc.speller.Suggest(word); len(sugg) > 0 {
		return sugg[0]
	}
	return ""
}

func (sc *Spellcheck) tokenize(s string) []string {
	for _, re := range sc.strips {
		s = string(re.ReplaceAll([]byte(s), []byte("")))
	}
	x := []string{
		".", "`", "\"", ",", "!", "?",
		";", ")", "(", "/", ":", "=",
		"*", "-", ">", "]", "[", "_",
		"|", "{", "}", "+", "\t", "' ",
		" '", "&", "<", "'s "}
	for _, v := range x {
		s = strings.Replace(s, v, " ", -1)
	}
	return strings.Fields(s)
}

func (sc *Spellcheck) checkLexeme(ct *Lexeme) (ret *CheckedLexeme) {
	for _, word := range sc.tokenize(ct.lit) {
		if sc.check(word) {
			continue
		}
		if ret == nil {
			ret = &CheckedLexeme{ct, "spell", nil}
		}
		ret.words = append(ret.words, CheckedWord{word, sc.suggest(word)})
	}
	return ret
}
<commit_msg>speller: ignore comment lines that are code<commit_after>// +build spell

package main

import (
	"fmt"
	"regexp"
	"strings"
)

type Spellcheck struct {
	speller Speller
	toks    map[string]struct{}
	check   CheckFunc
	strips  []*regexp.Regexp
}

// isCodeRE matches assignment, function calls, imports, and braces.
var isCodeRE *regexp.Regexp = regexp.MustCompile(
	"(:=|[a-zA-Z1-9]\\(|{|}|[a-z]/[a-z]|\\[)")

func NewSpellcheck(ts TokenSet, ignoreFile string) (sc *Spellcheck, err error) {
	igns, err := WithPassIgnores(ignoreFile)
	if err != nil {
		return nil, err
	}

	strips := []*regexp.Regexp{
		regexp.MustCompile("http(s|):[^ ]*"),
		regexp.MustCompile("TODO[ ]*\\([a-z]*"),
	}

	hsp := NewHunSpeller()
	if hsp == nil {
		return nil, fmt.Errorf("bad hunspell")
	}
	asp, err := NewASpeller()
	if err != nil {
		return nil, err
	}

	sc = &Spellcheck{
		strips:  strips,
		toks:    ts,
		speller: NewMultiSpeller(asp, hsp),
	}

	checks := []CheckFunc{
		sc.WithPassTokens(),
		igns,
		WithPassNumbers(),
		sc.WithSpeller(),
	}

	sc.check = func(w string) bool {
		for _, chk := range checks {
			if chk(w) {
				return true
			}
		}
		return false
	}
	return sc, nil
}

func (sc *Spellcheck) Close() { sc.speller.Close() }

func (sc *Spellcheck) WithPassTokens() CheckFunc {
	return func(word string) bool {
		_, ok := sc.toks[word]
		return ok
	}
}

func (sc *Spellcheck) WithSpeller() CheckFunc {
	return func(word string) bool { return sc.speller.Check(word) }
}

func (sc *Spellcheck) Check() CheckPipe {
	return func(lc <-chan *Lexeme, outc chan<- *CheckedLexeme) {
		sc.checkComments(lc, outc)
	}
}

func (sc *Spellcheck) checkComments(lc <-chan *Lexeme, outc chan<- *CheckedLexeme) {
	ch := Filter(lc, CommentFilter)
	for comm := range ch {
		if isCodeComment(comm.lit) {
			continue
		}
		if ct := sc.checkLexeme(comm); ct != nil {
			outc <- ct
		}
	}
}

// isCodeComment finds comments that contain example code as indented blocks:
// 	abc, def := indented.with(spaceTab)
//	a, b := tabOnly()
//      if xyz != nil { panic(indentWithSpace) }
// Since it's an example, sometimes the identifiers aren't in the
// code, which will cause a spell check false positive.
func isCodeComment(s string) bool {
	hasPfx := false
	for _, pfx := range []string{"//  ", "// \t", "//\t"} {
		if hasPfx = strings.HasPrefix(s, pfx); hasPfx {
			break
		}
	}
	// Check if line seems to be code to avoid false negatives on
	// indented comments like lists.
	return hasPfx && isCodeRE.MatchString(s)
}

func (sc *Spellcheck) suggest(word string) string {
	if sc.check(word) {
		return ""
	}
	if sugg := sc.speller.Suggest(word); len(sugg) > 0 {
		return sugg[0]
	}
	return ""
}

func (sc *Spellcheck) tokenize(s string) []string {
	for _, re := range sc.strips {
		s = string(re.ReplaceAll([]byte(s), []byte("")))
	}
	x := []string{
		".", "`", "\"", ",", "!", "?",
		";", ")", "(", "/", ":", "=",
		"*", "-", ">", "]", "[", "_",
		"|", "{", "}", "+", "\t", "' ",
		" '", "&", "<", "'s "}
	for _, v := range x {
		s = strings.Replace(s, v, " ", -1)
	}
	return strings.Fields(s)
}

func (sc *Spellcheck) checkLexeme(ct *Lexeme) (ret *CheckedLexeme) {
	for _, word := range sc.tokenize(ct.lit) {
		if sc.check(word) {
			continue
		}
		if ret == nil {
			ret = &CheckedLexeme{ct, "spell", nil}
		}
		ret.words = append(ret.words, CheckedWord{word, sc.suggest(word)})
	}
	return ret
}
<|endoftext|>


<commit_before>package server

import (
	"github.com/Sirupsen/logrus"
	"github.com/eternnoir/gbf-bike/bike"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/pkg/errors"
	"net/http"
	"strconv"
	"strings"
	"time"
)

var log = logrus.New().WithFields(logrus.Fields{
	"system": "GBF-Bike",
	"module": "APIServer",
})

var (
	ErrConvertTimeout = errors.New("Can not convert timeout value")
)

var DefaultTimeout = "5"

func NewApi(port string) *ApiServer {
	e := echo.New()
	e.Use(middleware.Logger())
	e.Use(middleware.Recover())
	return &ApiServer{hs: e, revChanList: make(map[chan (*bike.BattleInfo)]bool), port: port}
}

type ApiServer struct {
	hs          *echo.Echo
	port        string
	revChanList map[chan (*bike.BattleInfo)]bool
}

func (api *ApiServer) Start() error {
	e := api.hs
	e.GET("/query", api.query)
	e.Logger.Fatal(e.Start(":" + api.port))
	return nil
}

func (api *ApiServer) NewBattleInfo(battleInfo *bike.BattleInfo) error {
	log.Debugf("%#v", battleInfo)
	for infoCh, alive := range api.revChanList {
		if !alive {
			delete(api.revChanList, infoCh)
			continue
		}
		go func() {
			select {
			case infoCh <- battleInfo:
				log.Debugf("Push battle info to channel %#v", battleInfo)
			}
		}()
	}
	return nil
}

func (api *ApiServer) query(c echo.Context) error {
	// Get team and member from the query string
	level := c.QueryParam("level")
	mobs := c.QueryParam("mobs")
	timeoutStr := c.QueryParam("timeout")
	if timeoutStr == "" {
		timeoutStr = DefaultTimeout
	}
	timeout, err := convertTimeout(timeoutStr)
	if err != nil {
		return ErrConvertTimeout
	}
	recCh := make(chan (*bike.BattleInfo))
	api.revChanList[recCh] = true
	stopCh := make(chan bool)
	go func() {

		time.Sleep(time.Second * time.Duration(timeout))
		stopCh <- true
	}()
	result := api.pushBattleToList(level, mobs, recCh, stopCh)
	close(recCh)
	close(stopCh)
	return c.JSON(http.StatusOK, result)
}

func (api *ApiServer) pushBattleToList(level, mobs string, ch chan (*bike.BattleInfo), stopCh chan bool) []*bike.BattleInfo {
	list := make([]*bike.BattleInfo, 0)
	mobMap := convertStringMap(mobs)
	levelMap := convertStringMap(level)
	for {
		select {
		case <-stopCh:
			api.revChanList[ch] = false
			return list
		default:
		}
		bi := <-ch
		if isWantedMission(levelMap, mobMap, bi) {
			log.Debug("Push BattleInfo to list")
			list = append(list, bi)
		}
	}
	return list
}

func isWantedMission(levels, mobs map[string]struct{}, info *bike.BattleInfo) bool {
	return isWantedLevel(levels, info) && isWantedMob(mobs, info)
}

func isWantedMob(mobs map[string]struct{}, info *bike.BattleInfo) bool {
	if len(mobs) == 0 {
		return true
	}
	_, ok := mobs[info.MobName]
	return ok
}

func isWantedLevel(levels map[string]struct{}, info *bike.BattleInfo) bool {
	if len(levels) == 0 {
		return true
	}
	_, ok := levels[info.Level]
	return ok
}

func convertStringMap(mobstr string) map[string]struct{} {
	if mobstr == "" {
		return make(map[string]struct{})
	}
	moblist := strings.Split(mobstr, ";")
	ret := make(map[string]struct{})
	for _, mobname := range moblist {
		ret[mobname] = struct{}{}
	}
	return ret
}

func convertTimeout(timeout string) (int64, error) {
	i, err := strconv.Atoi(timeout)
	if err != nil {
		return int64(0), err
	}
	return int64(i), nil
}
<commit_msg>Add channel buffer.<commit_after>package server

import (
	"github.com/Sirupsen/logrus"
	"github.com/eternnoir/gbf-bike/bike"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/pkg/errors"
	"net/http"
	"strconv"
	"strings"
	"time"
)

var log = logrus.New().WithFields(logrus.Fields{
	"system": "GBF-Bike",
	"module": "APIServer",
})

var (
	ErrConvertTimeout = errors.New("Can not convert timeout value")
)

var DefaultTimeout = "5"

func NewApi(port string) *ApiServer {
	e := echo.New()
	e.Use(middleware.Logger())
	e.Use(middleware.Recover())
	return &ApiServer{hs: e, revChanList: make(map[chan (*bike.BattleInfo)]bool), port: port}
}

type ApiServer struct {
	hs          *echo.Echo
	port        string
	revChanList map[chan (*bike.BattleInfo)]bool
}

func (api *ApiServer) Start() error {
	e := api.hs
	e.GET("/query", api.query)
	e.Logger.Fatal(e.Start(":" + api.port))
	return nil
}

func (api *ApiServer) NewBattleInfo(battleInfo *bike.BattleInfo) error {
	log.Debugf("%#v", battleInfo)
	for infoCh, alive := range api.revChanList {
		if !alive {
			delete(api.revChanList, infoCh)
			continue
		}
		go func() {
			select {
			case infoCh <- battleInfo:
				log.Debugf("Push battle info to channel %#v", battleInfo)
			}
		}()
	}
	return nil
}

func (api *ApiServer) query(c echo.Context) error {
	// Get team and member from the query string
	level := c.QueryParam("level")
	mobs := c.QueryParam("mobs")
	timeoutStr := c.QueryParam("timeout")
	if timeoutStr == "" {
		timeoutStr = DefaultTimeout
	}
	timeout, err := convertTimeout(timeoutStr)
	if err != nil {
		return ErrConvertTimeout
	}
	recCh := make(chan (*bike.BattleInfo), 50)
	api.revChanList[recCh] = true
	stopCh := make(chan bool)
	go func() {
		time.Sleep(time.Second * time.Duration(timeout))
		stopCh <- true
	}()
	result := api.pushBattleToList(level, mobs, recCh, stopCh)
	close(stopCh)
	go func() {
		time.Sleep(3 * time.Second)
		close(recCh)
	}()
	return c.JSON(http.StatusOK, result)
}

func (api *ApiServer) pushBattleToList(level, mobs string, ch chan (*bike.BattleInfo), stopCh chan bool) []*bike.BattleInfo {
	list := make([]*bike.BattleInfo, 0)
	mobMap := convertStringMap(mobs)
	levelMap := convertStringMap(level)
	for {
		select {
		case <-stopCh:
			api.revChanList[ch] = false
			return list
		default:
		}
		bi := <-ch
		if isWantedMission(levelMap, mobMap, bi) {
			log.Debug("Push BattleInfo to list")
			list = append(list, bi)
		}
	}
	return list
}

func isWantedMission(levels, mobs map[string]struct{}, info *bike.BattleInfo) bool {
	return isWantedLevel(levels, info) && isWantedMob(mobs, info)
}

func isWantedMob(mobs map[string]struct{}, info *bike.BattleInfo) bool {
	if len(mobs) == 0 {
		return true
	}
	_, ok := mobs[info.MobName]
	return ok
}

func isWantedLevel(levels map[string]struct{}, info *bike.BattleInfo) bool {
	if len(levels) == 0 {
		return true
	}
	_, ok := levels[info.Level]
	return ok
}

func convertStringMap(mobstr string) map[string]struct{} {
	if mobstr == "" {
		return make(map[string]struct{})
	}
	moblist := strings.Split(mobstr, ";")
	ret := make(map[string]struct{})
	for _, mobname := range moblist {
		ret[mobname] = struct{}{}
	}
	return ret
}

func convertTimeout(timeout string) (int64, error) {
	i, err := strconv.Atoi(timeout)
	if err != nil {
		return int64(0), err
	}
	return int64(i), nil
}
<|endoftext|>


<commit_before>// Copyright 2016 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package reverseproxy

import (
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/braintree/manners"
	"github.com/nu7hatch/gouuid"
	"github.com/tsuru/planb/log"
)

var (
	emptyResponseBody   = &fixedReadCloser{}
	noRouteResponseBody = &fixedReadCloser{value: noRouteResponseContent}
	noopDirector        = func(*http.Request) {}

	_ ReverseProxy = &NativeReverseProxy{}
)

type NativeReverseProxy struct {
	http.Transport
	ReverseProxyConfig
	server *manners.GracefulServer
	rp     *httputil.ReverseProxy
	dialer *net.Dialer
}

type fixedReadCloser struct {
	value []byte
}

func (r *fixedReadCloser) Read(p []byte) (n int, err error) {
	return copy(p, r.value), io.EOF
}

func (r *fixedReadCloser) Close() error {
	return nil
}

type bufferPool struct {
	syncPool sync.Pool
}

func (p *bufferPool) Get() []byte {
	b := p.syncPool.Get()
	if b == nil {
		return make([]byte, 32*1024)
	}
	return b.([]byte)
}

func (p *bufferPool) Put(b []byte) {
	p.syncPool.Put(b)
}

func (rp *NativeReverseProxy) Initialize(rpConfig ReverseProxyConfig) error {
	rp.ReverseProxyConfig = rpConfig
	rp.server = manners.NewWithServer(&http.Server{Handler: rp})
	rp.dialer = &net.Dialer{
		Timeout:   rp.DialTimeout,
		KeepAlive: 30 * time.Second,
	}
	rp.Transport = http.Transport{
		Dial:                rp.dialer.Dial,
		TLSHandshakeTimeout: rp.DialTimeout,
		MaxIdleConnsPerHost: 100,
	}
	rp.rp = &httputil.ReverseProxy{
		Director:      noopDirector,
		Transport:     rp,
		FlushInterval: rp.FlushInterval,
		BufferPool:    &bufferPool{},
	}
	return nil
}

func (rp *NativeReverseProxy) Listen(listener net.Listener) {
	rp.server.Serve(listener)
}

func (rp *NativeReverseProxy) Stop() {
	rp.server.Close()
}

func (rp *NativeReverseProxy) ridString(req *http.Request) string {
	return rp.RequestIDHeader + ":" + req.Header.Get(rp.RequestIDHeader)
}

func (rp *NativeReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	if req.Host == "__ping__" && req.URL.Path == "/" {
		rw.WriteHeader(http.StatusOK)
		rw.Write(okResponse)
		return
	}
	if rp.RequestIDHeader != "" && req.Header.Get(rp.RequestIDHeader) == "" {
		unparsedID, err := uuid.NewV4()
		if err == nil {
			req.Header.Set(rp.RequestIDHeader, unparsedID.String())
		}
	}
	upgrade := req.Header.Get("Upgrade")
	if upgrade != "" && strings.ToLower(upgrade) == "websocket" {
		reqData, err := rp.serveWebsocket(rw, req)
		if err != nil {
			reqData.logError(req.URL.Path, rp.ridString(req), err)
			http.Error(rw, "", http.StatusBadGateway)
		}
		return
	}
	rp.rp.ServeHTTP(rw, req)
}

func (rp *NativeReverseProxy) serveWebsocket(rw http.ResponseWriter, req *http.Request) (*RequestData, error) {
	reqData, err := rp.Router.ChooseBackend(req.Host)
	if err != nil {
		return reqData, err
	}
	url, err := url.Parse(reqData.Backend)
	if err != nil {
		return reqData, err
	}
	req.Host = url.Host
	dstConn, err := rp.dialer.Dial("tcp", url.Host)
	if err != nil {
		return reqData, err
	}
	defer dstConn.Close()
	hj, ok := rw.(http.Hijacker)
	if !ok {
		return reqData, errors.New("not a hijacker")
	}
	conn, _, err := hj.Hijack()
	if err != nil {
		return reqData, err
	}
	defer conn.Close()
	var clientIP string
	if clientIP, _, err = net.SplitHostPort(req.RemoteAddr); err == nil {
		if prior, ok := req.Header["X-Forwarded-For"]; ok {
			clientIP = strings.Join(prior, ", ") + ", " + clientIP
		}
		req.Header.Set("X-Forwarded-For", clientIP)
	}
	err = req.Write(dstConn)
	if err != nil {
		return reqData, err
	}
	errc := make(chan error, 2)
	cp := func(dst io.Writer, src io.Reader) {
		_, err := io.Copy(dst, src)
		errc <- err
	}
	go cp(dstConn, conn)
	go cp(conn, dstConn)
	<-errc
	return reqData, nil
}

func (rp *NativeReverseProxy) RoundTrip(req *http.Request) (*http.Response, error) {
	req.URL.Scheme = ""
	req.URL.Host = ""
	reqData, err := rp.Router.ChooseBackend(req.Host)
	if err != nil {
		reqData.logError(req.URL.Path, rp.ridString(req), err)
		return rp.roundTripWithData(req, reqData, err), nil
	}
	u, err := url.Parse(reqData.Backend)
	if err == nil {
		req.URL.Host = u.Host
		req.URL.Scheme = u.Scheme
	}
	if req.URL.Host == "" {
		req.URL.Scheme = "http"
		req.URL.Host = reqData.Backend
	}
	return rp.roundTripWithData(req, reqData, nil), nil
}

func (rp *NativeReverseProxy) doResponse(req *http.Request, reqData *RequestData, rsp *http.Response, isDebug bool, isDead bool, backendDuration time.Duration) *http.Response {
	logEntry := func() *log.LogEntry {
		return &log.LogEntry{
			Now:             time.Now(),
			BackendDuration: backendDuration,
			TotalDuration:   time.Since(reqData.StartTime),
			BackendKey:      reqData.BackendKey,
			RemoteAddr:      req.RemoteAddr,
			Method:          req.Method,
			Path:            req.URL.Path,
			Proto:           req.Proto,
			Referer:         req.Referer(),
			UserAgent:       req.UserAgent(),
			RequestIDHeader: rp.RequestIDHeader,
			RequestID:       req.Header.Get(rp.RequestIDHeader),
			StatusCode:      rsp.StatusCode,
			ContentLength:   rsp.ContentLength,
		}
	}
	rsp.Request = req
	rsp.ProtoMajor = req.ProtoMajor
	rsp.ProtoMinor = req.ProtoMinor
	if rsp.Header == nil {
		rsp.Header = http.Header{}
	}
	if isDebug {
		rsp.Header.Set("X-Debug-Backend-Url", reqData.Backend)
		rsp.Header.Set("X-Debug-Backend-Id", strconv.FormatUint(uint64(reqData.BackendIdx), 10))
		rsp.Header.Set("X-Debug-Frontend-Key", reqData.Host)
	}
	err := rp.Router.EndRequest(reqData, isDead, logEntry)
	if err != nil {
		reqData.logError(req.URL.Path, rp.ridString(req), err)
	}
	return rsp
}

func (rp *NativeReverseProxy) roundTripWithData(req *http.Request, reqData *RequestData, err error) (rsp *http.Response) {
	isDebug := req.Header.Get("X-Debug-Router") != ""
	req.Header.Del("X-Debug-Router")
	if err != nil || req.URL.Scheme == "" || req.URL.Host == "" {
		switch err {
		case nil, ErrNoRegisteredBackends:
			rsp = &http.Response{
				StatusCode:    http.StatusBadRequest,
				ContentLength: int64(len(noRouteResponseBody.value)),
				Body:          noRouteResponseBody,
			}
		default:
			rsp = &http.Response{
				StatusCode: http.StatusServiceUnavailable,
				Body:       emptyResponseBody,
			}
		}
		return rp.doResponse(req, reqData, rsp, isDebug, false, 0)
	}
	var timedout int32
	if rp.RequestTimeout > 0 {
		timer := time.AfterFunc(rp.RequestTimeout, func() {
			atomic.AddInt32(&timedout, 1)
			rp.Transport.CancelRequest(req)
		})
		defer timer.Stop()
	}
	host, _, _ := net.SplitHostPort(req.URL.Host)
	if host == "" {
		host = req.URL.Host
	}
	isIP := net.ParseIP(host) != nil
	if !isIP {
		req.Header.Set("X-Host", req.Host)
		req.Header.Set("X-Forwarded-Host", req.Host)
		req.Host = host
	}
	t0 := time.Now().UTC()
	rsp, err = rp.Transport.RoundTrip(req)
	backendDuration := time.Since(t0)
	markAsDead := false
	if err != nil {
		var dialTimeout, requestTimeout bool
		if netErr, ok := err.(net.Error); ok {
			markAsDead = !netErr.Temporary()
			dialTimeout = netErr.Timeout()
		}
		requestTimeout = atomic.LoadInt32(&timedout) == int32(1)
		if requestTimeout {
			markAsDead = false
			err = fmt.Errorf("request timeout after %v: %s", time.Since(reqData.StartTime), err)
		} else if dialTimeout {
			markAsDead = true
			err = fmt.Errorf("dial timeout after %v: %s", time.Since(reqData.StartTime), err)
		} else {
			err = fmt.Errorf("error in backend request: %s", err)
		}
		if markAsDead {
			err = fmt.Errorf("%s *DEAD*", err)
		}
		reqData.logError(req.URL.Path, rp.ridString(req), err)
		rsp = &http.Response{
			StatusCode: http.StatusServiceUnavailable,
			Body:       emptyResponseBody,
		}
	}
	return rp.doResponse(req, reqData, rsp, isDebug, markAsDead, backendDuration)
}
<commit_msg>Isolate manners.GracefulServer for each listener<commit_after>// Copyright 2016 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package reverseproxy

import (
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/braintree/manners"
	"github.com/nu7hatch/gouuid"
	"github.com/tsuru/planb/log"
)

var (
	emptyResponseBody   = &fixedReadCloser{}
	noRouteResponseBody = &fixedReadCloser{value: noRouteResponseContent}
	noopDirector        = func(*http.Request) {}

	_ ReverseProxy = &NativeReverseProxy{}
)

type NativeReverseProxy struct {
	http.Transport
	ReverseProxyConfig
	servers []*manners.GracefulServer
	rp      *httputil.ReverseProxy
	dialer  *net.Dialer
}

type fixedReadCloser struct {
	value []byte
}

func (r *fixedReadCloser) Read(p []byte) (n int, err error) {
	return copy(p, r.value), io.EOF
}

func (r *fixedReadCloser) Close() error {
	return nil
}

type bufferPool struct {
	syncPool sync.Pool
}

func (p *bufferPool) Get() []byte {
	b := p.syncPool.Get()
	if b == nil {
		return make([]byte, 32*1024)
	}
	return b.([]byte)
}

func (p *bufferPool) Put(b []byte) {
	p.syncPool.Put(b)
}

func (rp *NativeReverseProxy) Initialize(rpConfig ReverseProxyConfig) error {
	rp.ReverseProxyConfig = rpConfig
	rp.servers = make([]*manners.GracefulServer, 0)

	rp.dialer = &net.Dialer{
		Timeout:   rp.DialTimeout,
		KeepAlive: 30 * time.Second,
	}
	rp.Transport = http.Transport{
		Dial:                rp.dialer.Dial,
		TLSHandshakeTimeout: rp.DialTimeout,
		MaxIdleConnsPerHost: 100,
	}
	rp.rp = &httputil.ReverseProxy{
		Director:      noopDirector,
		Transport:     rp,
		FlushInterval: rp.FlushInterval,
		BufferPool:    &bufferPool{},
	}
	return nil
}

func (rp *NativeReverseProxy) Listen(listener net.Listener) {
	server := manners.NewWithServer(&http.Server{Handler: rp})
	rp.servers = append(rp.servers, server)
	server.Serve(listener)
}

func (rp *NativeReverseProxy) Stop() {
	for _, server := range rp.servers {
		server.Close()
	}
}

func (rp *NativeReverseProxy) ridString(req *http.Request) string {
	return rp.RequestIDHeader + ":" + req.Header.Get(rp.RequestIDHeader)
}

func (rp *NativeReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	if req.Host == "__ping__" && req.URL.Path == "/" {
		rw.WriteHeader(http.StatusOK)
		rw.Write(okResponse)
		return
	}
	if rp.RequestIDHeader != "" && req.Header.Get(rp.RequestIDHeader) == "" {
		unparsedID, err := uuid.NewV4()
		if err == nil {
			req.Header.Set(rp.RequestIDHeader, unparsedID.String())
		}
	}
	upgrade := req.Header.Get("Upgrade")
	if upgrade != "" && strings.ToLower(upgrade) == "websocket" {
		reqData, err := rp.serveWebsocket(rw, req)
		if err != nil {
			reqData.logError(req.URL.Path, rp.ridString(req), err)
			http.Error(rw, "", http.StatusBadGateway)
		}
		return
	}
	rp.rp.ServeHTTP(rw, req)
}

func (rp *NativeReverseProxy) serveWebsocket(rw http.ResponseWriter, req *http.Request) (*RequestData, error) {
	reqData, err := rp.Router.ChooseBackend(req.Host)
	if err != nil {
		return reqData, err
	}
	url, err := url.Parse(reqData.Backend)
	if err != nil {
		return reqData, err
	}
	req.Host = url.Host
	dstConn, err := rp.dialer.Dial("tcp", url.Host)
	if err != nil {
		return reqData, err
	}
	defer dstConn.Close()
	hj, ok := rw.(http.Hijacker)
	if !ok {
		return reqData, errors.New("not a hijacker")
	}
	conn, _, err := hj.Hijack()
	if err != nil {
		return reqData, err
	}
	defer conn.Close()
	var clientIP string
	if clientIP, _, err = net.SplitHostPort(req.RemoteAddr); err == nil {
		if prior, ok := req.Header["X-Forwarded-For"]; ok {
			clientIP = strings.Join(prior, ", ") + ", " + clientIP
		}
		req.Header.Set("X-Forwarded-For", clientIP)
	}
	err = req.Write(dstConn)
	if err != nil {
		return reqData, err
	}
	errc := make(chan error, 2)
	cp := func(dst io.Writer, src io.Reader) {
		_, err := io.Copy(dst, src)
		errc <- err
	}
	go cp(dstConn, conn)
	go cp(conn, dstConn)
	<-errc
	return reqData, nil
}

func (rp *NativeReverseProxy) RoundTrip(req *http.Request) (*http.Response, error) {
	req.URL.Scheme = ""
	req.URL.Host = ""
	reqData, err := rp.Router.ChooseBackend(req.Host)
	if err != nil {
		reqData.logError(req.URL.Path, rp.ridString(req), err)
		return rp.roundTripWithData(req, reqData, err), nil
	}
	u, err := url.Parse(reqData.Backend)
	if err == nil {
		req.URL.Host = u.Host
		req.URL.Scheme = u.Scheme
	}
	if req.URL.Host == "" {
		req.URL.Scheme = "http"
		req.URL.Host = reqData.Backend
	}
	return rp.roundTripWithData(req, reqData, nil), nil
}

func (rp *NativeReverseProxy) doResponse(req *http.Request, reqData *RequestData, rsp *http.Response, isDebug bool, isDead bool, backendDuration time.Duration) *http.Response {
	logEntry := func() *log.LogEntry {
		return &log.LogEntry{
			Now:             time.Now(),
			BackendDuration: backendDuration,
			TotalDuration:   time.Since(reqData.StartTime),
			BackendKey:      reqData.BackendKey,
			RemoteAddr:      req.RemoteAddr,
			Method:          req.Method,
			Path:            req.URL.Path,
			Proto:           req.Proto,
			Referer:         req.Referer(),
			UserAgent:       req.UserAgent(),
			RequestIDHeader: rp.RequestIDHeader,
			RequestID:       req.Header.Get(rp.RequestIDHeader),
			StatusCode:      rsp.StatusCode,
			ContentLength:   rsp.ContentLength,
		}
	}
	rsp.Request = req
	rsp.ProtoMajor = req.ProtoMajor
	rsp.ProtoMinor = req.ProtoMinor
	if rsp.Header == nil {
		rsp.Header = http.Header{}
	}
	if isDebug {
		rsp.Header.Set("X-Debug-Backend-Url", reqData.Backend)
		rsp.Header.Set("X-Debug-Backend-Id", strconv.FormatUint(uint64(reqData.BackendIdx), 10))
		rsp.Header.Set("X-Debug-Frontend-Key", reqData.Host)
	}
	err := rp.Router.EndRequest(reqData, isDead, logEntry)
	if err != nil {
		reqData.logError(req.URL.Path, rp.ridString(req), err)
	}
	return rsp
}

func (rp *NativeReverseProxy) roundTripWithData(req *http.Request, reqData *RequestData, err error) (rsp *http.Response) {
	isDebug := req.Header.Get("X-Debug-Router") != ""
	req.Header.Del("X-Debug-Router")
	if err != nil || req.URL.Scheme == "" || req.URL.Host == "" {
		switch err {
		case nil, ErrNoRegisteredBackends:
			rsp = &http.Response{
				StatusCode:    http.StatusBadRequest,
				ContentLength: int64(len(noRouteResponseBody.value)),
				Body:          noRouteResponseBody,
			}
		default:
			rsp = &http.Response{
				StatusCode: http.StatusServiceUnavailable,
				Body:       emptyResponseBody,
			}
		}
		return rp.doResponse(req, reqData, rsp, isDebug, false, 0)
	}
	var timedout int32
	if rp.RequestTimeout > 0 {
		timer := time.AfterFunc(rp.RequestTimeout, func() {
			atomic.AddInt32(&timedout, 1)
			rp.Transport.CancelRequest(req)
		})
		defer timer.Stop()
	}
	host, _, _ := net.SplitHostPort(req.URL.Host)
	if host == "" {
		host = req.URL.Host
	}
	isIP := net.ParseIP(host) != nil
	if !isIP {
		req.Header.Set("X-Host", req.Host)
		req.Header.Set("X-Forwarded-Host", req.Host)
		req.Host = host
	}
	t0 := time.Now().UTC()
	rsp, err = rp.Transport.RoundTrip(req)
	backendDuration := time.Since(t0)
	markAsDead := false
	if err != nil {
		var dialTimeout, requestTimeout bool
		if netErr, ok := err.(net.Error); ok {
			markAsDead = !netErr.Temporary()
			dialTimeout = netErr.Timeout()
		}
		requestTimeout = atomic.LoadInt32(&timedout) == int32(1)
		if requestTimeout {
			markAsDead = false
			err = fmt.Errorf("request timeout after %v: %s", time.Since(reqData.StartTime), err)
		} else if dialTimeout {
			markAsDead = true
			err = fmt.Errorf("dial timeout after %v: %s", time.Since(reqData.StartTime), err)
		} else {
			err = fmt.Errorf("error in backend request: %s", err)
		}
		if markAsDead {
			err = fmt.Errorf("%s *DEAD*", err)
		}
		reqData.logError(req.URL.Path, rp.ridString(req), err)
		rsp = &http.Response{
			StatusCode: http.StatusServiceUnavailable,
			Body:       emptyResponseBody,
		}
	}
	return rp.doResponse(req, reqData, rsp, isDebug, markAsDead, backendDuration)
}
<|endoftext|>


<commit_before><commit_msg>Updated the comments in the code<commit_after><|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package nginx

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/tv42/httpunix"
)

// PID defines the location of the pid file used by NGINX
var PID = "/tmp/nginx.pid"

// StatusSocket defines the location of the unix socket used by NGINX for the status server
var StatusSocket = "/tmp/nginx-status-server.sock"

// HealthPath defines the path used to define the health check location in NGINX
var HealthPath = "/healthz"

// HealthCheckTimeout defines the time limit in seconds for a probe to health-check-path to succeed
var HealthCheckTimeout = 10 * time.Second

// StatusPath defines the path used to expose the NGINX status page
// http://nginx.org/en/docs/http/ngx_http_stub_status_module.html
var StatusPath = "/nginx_status"

// StreamSocket defines the location of the unix socket used by NGINX for the NGINX stream configuration socket
var StreamSocket = "/tmp/ingress-stream.sock"

var statusLocation = "nginx-status"

// NewGetStatusRequest creates a new GET request to the internal NGINX status server
func NewGetStatusRequest(path string) (int, []byte, error) {
	url := fmt.Sprintf("http+unix://%v%v", statusLocation, path)

	res, err := buildUnixSocketClient(HealthCheckTimeout).Get(url)
	if err != nil {
		return 0, nil, err
	}
	defer res.Body.Close()

	data, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return 0, nil, err
	}

	return res.StatusCode, data, nil
}

// NewPostStatusRequest creates a new POST request to the internal NGINX status server
func NewPostStatusRequest(path, contentType string, data interface{}) (int, []byte, error) {
	url := fmt.Sprintf("http+unix://%v%v", statusLocation, path)

	buf, err := json.Marshal(data)
	if err != nil {
		return 0, nil, err
	}

	res, err := buildUnixSocketClient(HealthCheckTimeout).Post(url, contentType, bytes.NewReader(buf))
	if err != nil {
		return 0, nil, err
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return 0, nil, err
	}

	return res.StatusCode, body, nil
}

// GetServerBlock takes an nginx.conf file and a host and tries to find the server block for that host
func GetServerBlock(conf string, host string) (string, error) {
	startMsg := fmt.Sprintf("## start server %v\n", host)
	endMsg := fmt.Sprintf("## end server %v", host)

	blockStart := strings.Index(conf, startMsg)
	if blockStart < 0 {
		return "", fmt.Errorf("host %v was not found in the controller's nginx.conf", host)
	}
	blockStart = blockStart + len(startMsg)

	blockEnd := strings.Index(conf, endMsg)
	if blockEnd < 0 {
		return "", fmt.Errorf("the end of the host server block could not be found, but the beginning was")
	}

	return conf[blockStart:blockEnd], nil
}

// ReadNginxConf reads the nginx configuration file into a string
func ReadNginxConf() (string, error) {
	return ReadFileToString("/etc/nginx/nginx.conf")
}

// ReadFileToString reads any file into a string
func ReadFileToString(path string) (string, error) {
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()

	contents, err := ioutil.ReadAll(f)
	if err != nil {
		return "", err
	}
	return string(contents), nil
}

func buildUnixSocketClient(timeout time.Duration) *http.Client {
	u := &httpunix.Transport{
		DialTimeout:           1 * time.Second,
		RequestTimeout:        timeout,
		ResponseHeaderTimeout: timeout,
	}
	u.RegisterLocation(statusLocation, StatusSocket)

	return &http.Client{
		Transport: u,
	}
}
<commit_msg>Refactor http client for unix sockets<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package nginx

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/tv42/httpunix"
)

// PID defines the location of the pid file used by NGINX
var PID = "/tmp/nginx.pid"

// StatusSocket defines the location of the unix socket used by NGINX for the status server
var StatusSocket = "/tmp/nginx-status-server.sock"

// HealthPath defines the path used to define the health check location in NGINX
var HealthPath = "/healthz"

// HealthCheckTimeout defines the time limit in seconds for a probe to health-check-path to succeed
var HealthCheckTimeout = 10 * time.Second

// StatusPath defines the path used to expose the NGINX status page
// http://nginx.org/en/docs/http/ngx_http_stub_status_module.html
var StatusPath = "/nginx_status"

// StreamSocket defines the location of the unix socket used by NGINX for the NGINX stream configuration socket
var StreamSocket = "/tmp/ingress-stream.sock"

var statusLocation = "nginx-status"

var httpClient *http.Client

func init() {
	httpClient = buildUnixSocketClient(HealthCheckTimeout)
}

// NewGetStatusRequest creates a new GET request to the internal NGINX status server
func NewGetStatusRequest(path string) (int, []byte, error) {
	url := fmt.Sprintf("%v://%v%v", httpunix.Scheme, statusLocation, path)

	res, err := httpClient.Get(url)
	if err != nil {
		return 0, nil, err
	}
	defer res.Body.Close()

	data, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return 0, nil, err
	}

	return res.StatusCode, data, nil
}

// NewPostStatusRequest creates a new POST request to the internal NGINX status server
func NewPostStatusRequest(path, contentType string, data interface{}) (int, []byte, error) {
	url := fmt.Sprintf("%v://%v%v", httpunix.Scheme, statusLocation, path)

	buf, err := json.Marshal(data)
	if err != nil {
		return 0, nil, err
	}

	res, err := httpClient.Post(url, contentType, bytes.NewReader(buf))
	if err != nil {
		return 0, nil, err
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return 0, nil, err
	}

	return res.StatusCode, body, nil
}

// GetServerBlock takes an nginx.conf file and a host and tries to find the server block for that host
func GetServerBlock(conf string, host string) (string, error) {
	startMsg := fmt.Sprintf("## start server %v\n", host)
	endMsg := fmt.Sprintf("## end server %v", host)

	blockStart := strings.Index(conf, startMsg)
	if blockStart < 0 {
		return "", fmt.Errorf("host %v was not found in the controller's nginx.conf", host)
	}
	blockStart = blockStart + len(startMsg)

	blockEnd := strings.Index(conf, endMsg)
	if blockEnd < 0 {
		return "", fmt.Errorf("the end of the host server block could not be found, but the beginning was")
	}

	return conf[blockStart:blockEnd], nil
}

// ReadNginxConf reads the nginx configuration file into a string
func ReadNginxConf() (string, error) {
	return readFileToString("/etc/nginx/nginx.conf")
}

// readFileToString reads any file into a string
func readFileToString(path string) (string, error) {
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()

	contents, err := ioutil.ReadAll(f)
	if err != nil {
		return "", err
	}
	return string(contents), nil
}

func buildUnixSocketClient(timeout time.Duration) *http.Client {
	u := &httpunix.Transport{
		DialTimeout:           1 * time.Second,
		RequestTimeout:        timeout,
		ResponseHeaderTimeout: timeout,
	}
	u.RegisterLocation(statusLocation, StatusSocket)

	return &http.Client{
		Transport: u,
	}
}
<|endoftext|>


<commit_before>package parse

import (
	"github.com/jimmyfrasche/etlite/internal/ast"
	"github.com/jimmyfrasche/etlite/internal/parse/internal/fmtname"
	"github.com/jimmyfrasche/etlite/internal/parse/internal/interpolate"
	"github.com/jimmyfrasche/etlite/internal/token"
)

//parseETL is the initial state that dispatches to other productions.
func (p *parser) parseETL(t token.Value) ast.Node {
	if !t.Head(false) {
		panic(p.unexpected(t))
	}
	switch t.Canon {
	case "USE":
		return p.useStmt(t)
	case "ASSERT":
		return p.assertStmt(t)
	case "DISPLAY":
		return p.displayStmt(t)
	case "IMPORT":
		i, _ := p.importStmt(t, false, true, nil)
		return i
	default:
		return p.parseSQL(t, false, true)
	}
}

//USE [DB|DATABASE] "name"
func (p *parser) useStmt(t token.Value) *ast.Use {
	u := &ast.Use{
		Position: t.Position,
	}

	t = p.next()
	if t.AnyLiteral("DATABASE", "DB") {
		t = p.next()
	}

	nm, ok := t.Unescape()
	if !ok {
		panic(p.unexpected(t))
	}
	u.DB = nm

	p.expect(token.Semicolon)

	return u
}

//ASSERT "message", subquery
func (p *parser) assertStmt(t token.Value) *ast.Assert {
	a := &ast.Assert{
		Position: t.Position,
	}

	t = p.expect(token.String)
	a.Message = t

	t = p.expectLit(",")

	t = p.next()
	switch t.Kind {
	default:
		panic(p.expected("@ or subquery", t))
	case token.LParen:
		a.Subquery = p.parseSQL(t, true, false)
		//trim off ()
		a.Subquery.Tokens = a.Subquery.Tokens[1 : len(a.Subquery.Tokens)-1]
	case token.Argument:
		ts, err := interpolate.DesugarAssert(t)
		if err != nil {
			panic(p.mkErr(t, err))
		}
		a.Subquery = &ast.SQL{
			Tokens: ts,
		}
	}

	p.expect(token.Semicolon)

	return a
}

//DISPLAY [TO device] [AS format] [FRAME name]
func (p *parser) displayStmt(t token.Value) *ast.Display {
	d := &ast.Display{
		Position: t.Position,
	}
	if t.Literal("TO") {
		d.Device, t = p.deviceExpr(t)
	}
	d.Frame, t = p.frameExpr(t)
	if t.Literal("AS") {
		d.Format, t = p.formatExpr(p.next())
	}
	if t.Kind != token.Semicolon {
		panic(p.expected(token.Semicolon, t))
	}
	return d
}

//IMPORT [TEMP] [table] [header] [FROM device] [WITH format] [FRAME name] [LIMIT n] [OFFSET n]
func (p *parser) importStmt(t token.Value, subquery, compound bool, sql *ast.SQL) (ast.Node, token.Value) {
	i := &ast.Import{
		Position: t.Position,
		Header:   make([]string, 0, 16),
		Limit:    -1,
		Offset:   -1,
	}
	t = p.next()
	if t.AnyLiteral("TEMP", "TEMPORARY") {
		i.Temporary = true
		t = p.next()
	}

	if !t.AnyLiteral("FROM", "WITH", "FRAME", "LIMIT", "OFFSET", "UNION", "INTERSECT", "EXCEPT") {
		var name []token.Value
		t, name = p.name(t)
		s, err := fmtname.ToString(name)
		if err != nil {
			panic(err)
		}
		i.Name = s
	}

	//slurp header
	if t.Kind == token.LParen {
		t = p.next()
		for {
			f, ok := t.Unescape()
			if !ok {
				panic(p.unexpected(t))
			}
			i.Header = append(i.Header, f)
			t = p.next()
			if t.Kind == token.RParen {
				t = p.next()
				break
			}
			if !t.Literal(",") {
				panic(p.unexpected(t))
			}
		}
	}

	if t.Literal("FROM") {
		i.Device, t = p.deviceExpr(t)
	}

	if t.Literal("WITH") {
		i.Format, t = p.formatExpr(p.next())
	}

	i.Frame, t = p.frameExpr(t)

	if t.Literal("LIMIT") {
		i.Limit, t = p.int(p.next())
	}

	if t.Literal("OFFSET") {
		i.Offset, t = p.int(p.next())
	}

	if t.AnyLiteral("UNION", "INTERSECT", "EXCEPT") {
		if !compound {
			panic(p.unexpected(t))
		}
		//first term in a compound chain is import, lift result into sql
		if sql == nil {
			sql, t = p.liftSQL(t, i)
			return sql, t
		}
		//otherwise we're already in the sql parser
		//and we know we're not done parsing.
		return i, t
	}

	end := token.Semicolon
	if subquery {
		end = token.RParen
	}
	if t.Kind != end {
		panic(p.expected(end, t))
	}
	return i, t
}

//Any random, regular SQL.
func (p *parser) parseSQL(t token.Value, subquery, allowETLsq bool) *ast.SQL {
	sp := newSqlParser(p)
	sp.top(t, subquery, allowETLsq)
	return sp.sql
}

func (p *parser) liftSQL(t token.Value, i *ast.Import) (*ast.SQL, token.Value) {
	sp := newSqlParser(p)
	sp.synth(t, token.Placeholder) //for the import
	sp.sql.Subqueries = append(sp.sql.Subqueries, i)
	t = sp.regular(t, 0, false, true, true)
	return sp.sql, t
}
<commit_msg>fix lack of step in header parser<commit_after>package parse

import (
	"github.com/jimmyfrasche/etlite/internal/ast"
	"github.com/jimmyfrasche/etlite/internal/parse/internal/fmtname"
	"github.com/jimmyfrasche/etlite/internal/parse/internal/interpolate"
	"github.com/jimmyfrasche/etlite/internal/token"
)

//parseETL is the initial state that dispatches to other productions.
func (p *parser) parseETL(t token.Value) ast.Node {
	if !t.Head(false) {
		panic(p.unexpected(t))
	}
	switch t.Canon {
	case "USE":
		return p.useStmt(t)
	case "ASSERT":
		return p.assertStmt(t)
	case "DISPLAY":
		return p.displayStmt(t)
	case "IMPORT":
		i, _ := p.importStmt(t, false, true, nil)
		return i
	default:
		return p.parseSQL(t, false, true)
	}
}

//USE [DB|DATABASE] "name"
func (p *parser) useStmt(t token.Value) *ast.Use {
	u := &ast.Use{
		Position: t.Position,
	}

	t = p.next()
	if t.AnyLiteral("DATABASE", "DB") {
		t = p.next()
	}

	nm, ok := t.Unescape()
	if !ok {
		panic(p.unexpected(t))
	}
	u.DB = nm

	p.expect(token.Semicolon)

	return u
}

//ASSERT "message", subquery
func (p *parser) assertStmt(t token.Value) *ast.Assert {
	a := &ast.Assert{
		Position: t.Position,
	}

	t = p.expect(token.String)
	a.Message = t

	t = p.expectLit(",")

	t = p.next()
	switch t.Kind {
	default:
		panic(p.expected("@ or subquery", t))
	case token.LParen:
		a.Subquery = p.parseSQL(t, true, false)
		//trim off ()
		a.Subquery.Tokens = a.Subquery.Tokens[1 : len(a.Subquery.Tokens)-1]
	case token.Argument:
		ts, err := interpolate.DesugarAssert(t)
		if err != nil {
			panic(p.mkErr(t, err))
		}
		a.Subquery = &ast.SQL{
			Tokens: ts,
		}
	}

	p.expect(token.Semicolon)

	return a
}

//DISPLAY [TO device] [AS format] [FRAME name]
func (p *parser) displayStmt(t token.Value) *ast.Display {
	d := &ast.Display{
		Position: t.Position,
	}
	if t.Literal("TO") {
		d.Device, t = p.deviceExpr(t)
	}
	d.Frame, t = p.frameExpr(t)
	if t.Literal("AS") {
		d.Format, t = p.formatExpr(p.next())
	}
	if t.Kind != token.Semicolon {
		panic(p.expected(token.Semicolon, t))
	}
	return d
}

//IMPORT [TEMP] [table] [header] [FROM device] [WITH format] [FRAME name] [LIMIT n] [OFFSET n]
func (p *parser) importStmt(t token.Value, subquery, compound bool, sql *ast.SQL) (ast.Node, token.Value) {
	i := &ast.Import{
		Position: t.Position,
		Header:   make([]string, 0, 16),
		Limit:    -1,
		Offset:   -1,
	}
	t = p.next()
	if t.AnyLiteral("TEMP", "TEMPORARY") {
		i.Temporary = true
		t = p.next()
	}

	if t.Kind == token.Literal && !t.AnyLiteral("FROM", "WITH", "FRAME", "LIMIT", "OFFSET", "UNION", "INTERSECT", "EXCEPT") {
		var name []token.Value
		t, name = p.name(t)
		s, err := fmtname.ToString(name)
		if err != nil {
			panic(err)
		}
		i.Name = s
	}

	//slurp header
	if t.Kind == token.LParen {
		t = p.next()
		for {
			f, ok := t.Unescape()
			if !ok {
				panic(p.unexpected(t))
			}
			i.Header = append(i.Header, f)

			t = p.next()
			if t.Kind == token.RParen {
				t = p.next()
				break
			}

			if !t.Literal(",") {
				panic(p.unexpected(t))
			}
			t = p.next()
		}
	}

	if t.Literal("FROM") {
		i.Device, t = p.deviceExpr(t)
	}

	if t.Literal("WITH") {
		i.Format, t = p.formatExpr(p.next())
	}

	i.Frame, t = p.frameExpr(t)

	if t.Literal("LIMIT") {
		i.Limit, t = p.int(p.next())
	}

	if t.Literal("OFFSET") {
		i.Offset, t = p.int(p.next())
	}

	if t.AnyLiteral("UNION", "INTERSECT", "EXCEPT") {
		if !compound {
			panic(p.unexpected(t))
		}
		//first term in a compound chain is import, lift result into sql
		if sql == nil {
			sql, t = p.liftSQL(t, i)
			return sql, t
		}
		//otherwise we're already in the sql parser
		//and we know we're not done parsing.
		return i, t
	}

	end := token.Semicolon
	if subquery {
		end = token.RParen
	}
	if t.Kind != end {
		panic(p.expected(end, t))
	}
	return i, t
}

//Any random, regular SQL.
func (p *parser) parseSQL(t token.Value, subquery, allowETLsq bool) *ast.SQL {
	sp := newSqlParser(p)
	sp.top(t, subquery, allowETLsq)
	return sp.sql
}

func (p *parser) liftSQL(t token.Value, i *ast.Import) (*ast.SQL, token.Value) {
	sp := newSqlParser(p)
	sp.synth(t, token.Placeholder) //for the import
	sp.sql.Subqueries = append(sp.sql.Subqueries, i)
	t = sp.regular(t, 0, false, true, true)
	return sp.sql, t
}
<|endoftext|>


<commit_before>package agollo

import (
	"encoding/json"
	"fmt"
	"net/url"
	"strings"
	"time"
)

const appConfigFileName  ="app.properties"

var (
	long_poll_interval = 2 *time.Second //2s
	long_poll_connect_timeout  = 1 * time.Minute //1m

	connect_timeout  = 1 * time.Second //1s
	//notify timeout
	nofity_connect_timeout  = 10 * time.Minute //10m
	//for on error retry
	on_error_retry_interval = 1 * time.Second //1s
	//for typed config cache of parser result, e.g. integer, double, long, etc.
	//max_config_cache_size    = 500             //500 cache key
	//config_cache_expire_time = 1 * time.Minute //1 minute

	//max retries connect apollo
	max_retries=5

	//refresh ip list
	refresh_ip_list_interval=20 *time.Minute //20m

	//appconfig
	appConfig *AppConfig

	//real servers ip
	servers map[string]*serverInfo=make(map[string]*serverInfo,0)

	//next try connect period - 60 second
	next_try_connect_period int64=60

)

type AppConfig struct {
	AppId string `json:"appId"`
	Cluster string `json:"cluster"`
	NamespaceName string `json:"namespaceName"`
	Ip string `json:"ip"`
	NextTryConnTime int64 `json:"-"`
	BackupConfigPath string `json:"backupConfigPath"`
}

func (this *AppConfig) getBackupConfigPath() string{
	return this.BackupConfigPath
}

func (this *AppConfig) getHost() string{
	if strings.HasPrefix(this.Ip,"http"){
		return this.Ip
	}
	return "http://"+this.Ip+"/"
}

//if this connect is fail will set this time
func (this *AppConfig) setNextTryConnTime(nextTryConnectPeriod int64){
	this.NextTryConnTime=time.Now().Unix()+nextTryConnectPeriod
}

//is connect by ip directly
//false : no
//true : yes
func (this *AppConfig) isConnectDirectly() bool{
	if this.NextTryConnTime>=0&&this.NextTryConnTime>time.Now().Unix(){
		return true
	}

	return false
}

func (this *AppConfig) selectHost() string{
	if !this.isConnectDirectly(){
		return this.getHost()
	}

	for host,server:=range servers{
		// if some node has down then select next node
		if server.IsDown{
			continue
		}
		return host
	}

	return ""
}

func setDownNode(host string) {
	if host=="" || appConfig==nil{
		return
	}

	if host==appConfig.getHost(){
		appConfig.setNextTryConnTime(next_try_connect_period)
	}

	for key,server:=range servers{
		if key==host{
			server.IsDown=true
			break
		}
	}
}

type serverInfo struct {
	AppName string `json:"appName"`
	InstanceId string `json:"instanceId"`
	HomepageUrl string `json:"homepageUrl"`
	IsDown bool `json:"-"`

}

func init() {
	//init common
	initCommon()

	//init config
	initFileConfig()
}

func initCommon()  {
	//init server ip list
	go initServerIpList()
}

func initFileConfig()  {
	// default use application.properties
	initConfig(nil)
}

func initConfig(loadAppConfig func()(*AppConfig,error)) {
	var err error
	//init config file
	appConfig,err = getLoadAppConfig(loadAppConfig)

	if err!=nil{
		return
	}

	func(appConfig *AppConfig) {
		apolloConfig:=&ApolloConfig{}
		apolloConfig.AppId=appConfig.AppId
		apolloConfig.Cluster=appConfig.Cluster
		apolloConfig.NamespaceName=appConfig.NamespaceName

		updateApolloConfig(apolloConfig,false)
	}(appConfig)
}

//init config by custom
func InitCustomConfig(loadAppConfig func()(*AppConfig,error)) {

	initConfig(loadAppConfig)

	//init all notification
	initAllNotifications()

}

// set load app config's function
func getLoadAppConfig(loadAppConfig func()(*AppConfig,error)) (*AppConfig,error) {
	if loadAppConfig!=nil{
		return loadAppConfig()
	}
	return loadJsonConfig(appConfigFileName)
}

//set timer for update ip list
//interval : 20m
func initServerIpList() {
	t2 := time.NewTimer(refresh_ip_list_interval)
	for {
		select {
		case <-t2.C:
			syncServerIpList(nil)
			t2.Reset(refresh_ip_list_interval)
		}
	}
}

func syncServerIpListSuccessCallBack(responseBody []byte)(o interface{},err error){
	tmpServerInfo:=make([]*serverInfo,0)

	err= json.Unmarshal(responseBody,&tmpServerInfo)

	if err!=nil{
		logger.Error("Unmarshal json Fail,Error:",err)
		return
	}

	if len(tmpServerInfo)==0 {
		logger.Info("get no real server!")
		return
	}

	for _,server :=range tmpServerInfo {
		if server==nil{
			continue
		}
		servers[server.HomepageUrl]=server
	}
	return
}

//sync ip list from server
//then
//1.update cache
//2.store in disk
func syncServerIpList(newAppConfig *AppConfig) error{
	appConfig:=GetAppConfig(newAppConfig)
	if appConfig==nil{
		panic("can not find apollo config!please confirm!")
	}

	var url string
	if newAppConfig ==nil{
		getServicesConfigUrl(appConfig)
	}else{
		url=newAppConfig.Ip
	}
	logger.Debug("url:",url)

	_,err:=request(url,&ConnectConfig{
	},&CallBack{
		SuccessCallBack:syncServerIpListSuccessCallBack,
	})

	return err
}

func GetAppConfig(newAppConfig *AppConfig)*AppConfig  {
	if newAppConfig !=nil{
		return newAppConfig
	}
	return appConfig
}

func getConfigUrl(config *AppConfig) string{
	return getConfigUrlByHost(config,config.getHost())
}

func getConfigUrlByHost(config *AppConfig,host string) string{
	current:=GetCurrentApolloConfig()
	return fmt.Sprintf("%sconfigs/%s/%s/%s?releaseKey=%s&ip=%s",
		host,
		url.QueryEscape(config.AppId),
		url.QueryEscape(config.Cluster),
		url.QueryEscape(config.NamespaceName),
		url.QueryEscape(current.ReleaseKey),
		getInternal())
}

func getConfigUrlSuffix(config *AppConfig,newConfig *AppConfig) string{
	if newConfig!=nil{
		return ""
	}
	current:=GetCurrentApolloConfig()
	return fmt.Sprintf("configs/%s/%s/%s?releaseKey=%s&ip=%s",
		url.QueryEscape(config.AppId),
		url.QueryEscape(config.Cluster),
		url.QueryEscape(config.NamespaceName),
		url.QueryEscape(current.ReleaseKey),
		getInternal())
}

func getNotifyUrlSuffix(notifications string,config *AppConfig,newConfig *AppConfig) string{
	if newConfig!=nil{
		return ""
	}
	return fmt.Sprintf("notifications/v2?appId=%s&cluster=%s&notifications=%s",
		url.QueryEscape(config.AppId),
		url.QueryEscape(config.Cluster),
		url.QueryEscape(notifications))
}

func getServicesConfigUrl(config *AppConfig) string{
	return fmt.Sprintf("%sservices/config?appId=%s&ip=%s",
		config.getHost(),
		url.QueryEscape(config.AppId),
		getInternal())
}<commit_msg>fix sync ip list<commit_after>package agollo

import (
	"encoding/json"
	"fmt"
	"net/url"
	"strings"
	"time"
)

const appConfigFileName  ="app.properties"

var (
	long_poll_interval = 2 *time.Second //2s
	long_poll_connect_timeout  = 1 * time.Minute //1m

	connect_timeout  = 1 * time.Second //1s
	//notify timeout
	nofity_connect_timeout  = 10 * time.Minute //10m
	//for on error retry
	on_error_retry_interval = 1 * time.Second //1s
	//for typed config cache of parser result, e.g. integer, double, long, etc.
	//max_config_cache_size    = 500             //500 cache key
	//config_cache_expire_time = 1 * time.Minute //1 minute

	//max retries connect apollo
	max_retries=5

	//refresh ip list
	refresh_ip_list_interval=20 *time.Minute //20m

	//appconfig
	appConfig *AppConfig

	//real servers ip
	servers map[string]*serverInfo=make(map[string]*serverInfo,0)

	//next try connect period - 60 second
	next_try_connect_period int64=60

)

type AppConfig struct {
	AppId string `json:"appId"`
	Cluster string `json:"cluster"`
	NamespaceName string `json:"namespaceName"`
	Ip string `json:"ip"`
	NextTryConnTime int64 `json:"-"`
	BackupConfigPath string `json:"backupConfigPath"`
}

func (this *AppConfig) getBackupConfigPath() string{
	return this.BackupConfigPath
}

func (this *AppConfig) getHost() string{
	if strings.HasPrefix(this.Ip,"http"){
		return this.Ip
	}
	return "http://"+this.Ip+"/"
}

//if this connect is fail will set this time
func (this *AppConfig) setNextTryConnTime(nextTryConnectPeriod int64){
	this.NextTryConnTime=time.Now().Unix()+nextTryConnectPeriod
}

//is connect by ip directly
//false : no
//true : yes
func (this *AppConfig) isConnectDirectly() bool{
	if this.NextTryConnTime>=0&&this.NextTryConnTime>time.Now().Unix(){
		return true
	}

	return false
}

func (this *AppConfig) selectHost() string{
	if !this.isConnectDirectly(){
		return this.getHost()
	}

	for host,server:=range servers{
		// if some node has down then select next node
		if server.IsDown{
			continue
		}
		return host
	}

	return ""
}

func setDownNode(host string) {
	if host=="" || appConfig==nil{
		return
	}

	if host==appConfig.getHost(){
		appConfig.setNextTryConnTime(next_try_connect_period)
	}

	for key,server:=range servers{
		if key==host{
			server.IsDown=true
			break
		}
	}
}

type serverInfo struct {
	AppName string `json:"appName"`
	InstanceId string `json:"instanceId"`
	HomepageUrl string `json:"homepageUrl"`
	IsDown bool `json:"-"`

}

func init() {
	//init config
	initFileConfig()

	//init common
	initCommon()
}

func initCommon()  {
	//init server ip list
	go initServerIpList()
}

func initFileConfig()  {
	// default use application.properties
	initConfig(nil)
}

func initConfig(loadAppConfig func()(*AppConfig,error)) {
	var err error
	//init config file
	appConfig,err = getLoadAppConfig(loadAppConfig)

	if err!=nil{
		return
	}

	func(appConfig *AppConfig) {
		apolloConfig:=&ApolloConfig{}
		apolloConfig.AppId=appConfig.AppId
		apolloConfig.Cluster=appConfig.Cluster
		apolloConfig.NamespaceName=appConfig.NamespaceName

		updateApolloConfig(apolloConfig,false)
	}(appConfig)
}

//init config by custom
func InitCustomConfig(loadAppConfig func()(*AppConfig,error)) {

	initConfig(loadAppConfig)

	//init all notification
	initAllNotifications()

}

// set load app config's function
func getLoadAppConfig(loadAppConfig func()(*AppConfig,error)) (*AppConfig,error) {
	if loadAppConfig!=nil{
		return loadAppConfig()
	}
	return loadJsonConfig(appConfigFileName)
}

//set timer for update ip list
//interval : 20m
func initServerIpList() {
	syncServerIpList(nil)

	t2 := time.NewTimer(refresh_ip_list_interval)
	for {
		select {
		case <-t2.C:
			syncServerIpList(nil)
			t2.Reset(refresh_ip_list_interval)
		}
	}
}

func syncServerIpListSuccessCallBack(responseBody []byte)(o interface{},err error){
	logger.Debug("get all server info:",string(responseBody))

	tmpServerInfo:=make([]*serverInfo,0)

	err= json.Unmarshal(responseBody,&tmpServerInfo)

	if err!=nil{
		logger.Error("Unmarshal json Fail,Error:",err)
		return
	}

	if len(tmpServerInfo)==0 {
		logger.Info("get no real server!")
		return
	}

	for _,server :=range tmpServerInfo {
		if server==nil{
			continue
		}
		servers[server.HomepageUrl]=server
	}
	return
}

//sync ip list from server
//then
//1.update cache
//2.store in disk
func syncServerIpList(newAppConfig *AppConfig) error{
	appConfig:=GetAppConfig(newAppConfig)
	if appConfig==nil{
		panic("can not find apollo config!please confirm!")
	}

	_,err:=request(getServicesConfigUrl(appConfig),&ConnectConfig{
	},&CallBack{
		SuccessCallBack:syncServerIpListSuccessCallBack,
	})

	return err
}

func GetAppConfig(newAppConfig *AppConfig)*AppConfig  {
	if newAppConfig !=nil{
		return newAppConfig
	}
	return appConfig
}

func getConfigUrl(config *AppConfig) string{
	return getConfigUrlByHost(config,config.getHost())
}

func getConfigUrlByHost(config *AppConfig,host string) string{
	current:=GetCurrentApolloConfig()
	return fmt.Sprintf("%sconfigs/%s/%s/%s?releaseKey=%s&ip=%s",
		host,
		url.QueryEscape(config.AppId),
		url.QueryEscape(config.Cluster),
		url.QueryEscape(config.NamespaceName),
		url.QueryEscape(current.ReleaseKey),
		getInternal())
}

func getConfigUrlSuffix(config *AppConfig,newConfig *AppConfig) string{
	if newConfig!=nil{
		return ""
	}
	current:=GetCurrentApolloConfig()
	return fmt.Sprintf("configs/%s/%s/%s?releaseKey=%s&ip=%s",
		url.QueryEscape(config.AppId),
		url.QueryEscape(config.Cluster),
		url.QueryEscape(config.NamespaceName),
		url.QueryEscape(current.ReleaseKey),
		getInternal())
}

func getNotifyUrlSuffix(notifications string,config *AppConfig,newConfig *AppConfig) string{
	if newConfig!=nil{
		return ""
	}
	return fmt.Sprintf("notifications/v2?appId=%s&cluster=%s&notifications=%s",
		url.QueryEscape(config.AppId),
		url.QueryEscape(config.Cluster),
		url.QueryEscape(notifications))
}

func getServicesConfigUrl(config *AppConfig) string{
	return fmt.Sprintf("%sservices/config?appId=%s&ip=%s",
		config.getHost(),
		url.QueryEscape(config.AppId),
		getInternal())
}<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package examples_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta1"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/golang/glog"
)

func validateObject(obj interface{}) (errors []error) {
	switch t := obj.(type) {
	case *api.ReplicationController:
		errors = validation.ValidateManifest(&t.DesiredState.PodTemplate.DesiredState.Manifest)
	case *api.ReplicationControllerList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Service:
		errors = validation.ValidateService(t)
	case *api.ServiceList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Pod:
		errors = validation.ValidateManifest(&t.DesiredState.Manifest)
	case *api.PodList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	default:
		return []error{fmt.Errorf("no validation defined for %#v", obj)}
	}
	return errors
}

func walkJSONFiles(inDir string, fn func(name, path string, data []byte)) error {
	err := filepath.Walk(inDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() && path != inDir {
			return filepath.SkipDir
		}
		name := filepath.Base(path)
		ext := filepath.Ext(name)
		if ext != "" {
			name = name[:len(name)-len(ext)]
		}
		if ext != ".json" {
			return nil
		}
		glog.Infof("Testing %s", path)
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		fn(name, path, data)
		return nil
	})
	return err
}

func TestApiExamples(t *testing.T) {
	expected := map[string]interface{}{
		"controller":       &api.ReplicationController{},
		"controller-list":  &api.ReplicationControllerList{},
		"pod":              &api.Pod{},
		"pod-list":         &api.PodList{},
		"service":          &api.Service{},
		"external-service": &api.Service{},
		"service-list":     &api.ServiceList{},
	}

	tested := 0
	err := walkJSONFiles("../api/examples", func(name, path string, data []byte) {
		expectedType, found := expected[name]
		if !found {
			t.Errorf("%s does not have a test case defined", path)
			return
		}
		tested += 1
		if err := runtime.DecodeInto(data, expectedType); err != nil {
			t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
			return
		}
		if errors := validateObject(expectedType); len(errors) > 0 {
			t.Errorf("%s did not validate correctly: %v", path, errors)
		}
	})
	if err != nil {
		t.Errorf("Expected no error, Got %v", err)
	}
	if tested != len(expected) {
		t.Errorf("Expected %d examples, Got %d", len(expected), tested)
	}
}

func TestExamples(t *testing.T) {
	expected := map[string]interface{}{
		"frontend-controller":    &api.ReplicationController{},
		"redis-slave-controller": &api.ReplicationController{},
		"redis-master":           &api.Pod{},
		"frontend-service":       &api.Service{},
		"redis-master-service":   &api.Service{},
		"redis-slave-service":    &api.Service{},
	}

	tested := 0
	err := walkJSONFiles("../examples/guestbook", func(name, path string, data []byte) {
		expectedType, found := expected[name]
		if !found {
			t.Errorf("%s does not have a test case defined", path)
			return
		}
		tested += 1
		if err := runtime.DecodeInto(data, expectedType); err != nil {
			t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
			return
		}
		if errors := validateObject(expectedType); len(errors) > 0 {
			t.Errorf("%s did not validate correctly: %v", path, errors)
		}
	})
	if err != nil {
		t.Errorf("Expected no error, Got %v", err)
	}
	if tested != len(expected) {
		t.Errorf("Expected %d examples, Got %d", len(expected), tested)
	}
}

var jsonRegexp = regexp.MustCompile("(?ms)^```\\w*\\n(\\{.+?\\})\\w*\\n^```")

func TestReadme(t *testing.T) {
	path := "../README.md"
	data, err := ioutil.ReadFile(path)
	if err != nil {
		t.Fatalf("Unable to read file: %v", err)
	}

	match := jsonRegexp.FindStringSubmatch(string(data))
	if match == nil {
		return
	}
	for _, json := range match[1:] {
		expectedType := &api.Pod{}
		if err := runtime.DecodeInto([]byte(json), expectedType); err != nil {
			t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
			return
		}
		if errors := validateObject(expectedType); len(errors) > 0 {
			t.Errorf("%s did not validate correctly: %v", path, errors)
		}
		encoded, err := runtime.Encode(expectedType)
		if err != nil {
			t.Errorf("Could not encode object: %v", err)
			continue
		}
		t.Logf("Found pod %s\n%s", json, encoded)
	}
}
<commit_msg>Rename Codec and ResourceVersioner to add Default in front, to allow for types of those names<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package examples_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta1"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/golang/glog"
)

func validateObject(obj interface{}) (errors []error) {
	switch t := obj.(type) {
	case *api.ReplicationController:
		errors = validation.ValidateManifest(&t.DesiredState.PodTemplate.DesiredState.Manifest)
	case *api.ReplicationControllerList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Service:
		errors = validation.ValidateService(t)
	case *api.ServiceList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Pod:
		errors = validation.ValidateManifest(&t.DesiredState.Manifest)
	case *api.PodList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	default:
		return []error{fmt.Errorf("no validation defined for %#v", obj)}
	}
	return errors
}

func walkJSONFiles(inDir string, fn func(name, path string, data []byte)) error {
	err := filepath.Walk(inDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() && path != inDir {
			return filepath.SkipDir
		}
		name := filepath.Base(path)
		ext := filepath.Ext(name)
		if ext != "" {
			name = name[:len(name)-len(ext)]
		}
		if ext != ".json" {
			return nil
		}
		glog.Infof("Testing %s", path)
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		fn(name, path, data)
		return nil
	})
	return err
}

func TestApiExamples(t *testing.T) {
	expected := map[string]interface{}{
		"controller":       &api.ReplicationController{},
		"controller-list":  &api.ReplicationControllerList{},
		"pod":              &api.Pod{},
		"pod-list":         &api.PodList{},
		"service":          &api.Service{},
		"external-service": &api.Service{},
		"service-list":     &api.ServiceList{},
	}

	tested := 0
	err := walkJSONFiles("../api/examples", func(name, path string, data []byte) {
		expectedType, found := expected[name]
		if !found {
			t.Errorf("%s does not have a test case defined", path)
			return
		}
		tested += 1
		if err := runtime.DefaultCodec.DecodeInto(data, expectedType); err != nil {
			t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
			return
		}
		if errors := validateObject(expectedType); len(errors) > 0 {
			t.Errorf("%s did not validate correctly: %v", path, errors)
		}
	})
	if err != nil {
		t.Errorf("Expected no error, Got %v", err)
	}
	if tested != len(expected) {
		t.Errorf("Expected %d examples, Got %d", len(expected), tested)
	}
}

func TestExamples(t *testing.T) {
	expected := map[string]interface{}{
		"frontend-controller":    &api.ReplicationController{},
		"redis-slave-controller": &api.ReplicationController{},
		"redis-master":           &api.Pod{},
		"frontend-service":       &api.Service{},
		"redis-master-service":   &api.Service{},
		"redis-slave-service":    &api.Service{},
	}

	tested := 0
	err := walkJSONFiles("../examples/guestbook", func(name, path string, data []byte) {
		expectedType, found := expected[name]
		if !found {
			t.Errorf("%s does not have a test case defined", path)
			return
		}
		tested += 1
		if err := runtime.DefaultCodec.DecodeInto(data, expectedType); err != nil {
			t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
			return
		}
		if errors := validateObject(expectedType); len(errors) > 0 {
			t.Errorf("%s did not validate correctly: %v", path, errors)
		}
	})
	if err != nil {
		t.Errorf("Expected no error, Got %v", err)
	}
	if tested != len(expected) {
		t.Errorf("Expected %d examples, Got %d", len(expected), tested)
	}
}

var jsonRegexp = regexp.MustCompile("(?ms)^```\\w*\\n(\\{.+?\\})\\w*\\n^```")

func TestReadme(t *testing.T) {
	path := "../README.md"
	data, err := ioutil.ReadFile(path)
	if err != nil {
		t.Fatalf("Unable to read file: %v", err)
	}

	match := jsonRegexp.FindStringSubmatch(string(data))
	if match == nil {
		return
	}
	for _, json := range match[1:] {
		expectedType := &api.Pod{}
		if err := runtime.DefaultCodec.DecodeInto([]byte(json), expectedType); err != nil {
			t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
			return
		}
		if errors := validateObject(expectedType); len(errors) > 0 {
			t.Errorf("%s did not validate correctly: %v", path, errors)
		}
		encoded, err := runtime.DefaultCodec.Encode(expectedType)
		if err != nil {
			t.Errorf("Could not encode object: %v", err)
			continue
		}
		t.Logf("Found pod %s\n%s", json, encoded)
	}
}
<|endoftext|>


<commit_before>package tq

import (
	"fmt"
	"time"

	"github.com/git-lfs/git-lfs/api"
	"github.com/git-lfs/git-lfs/errors"
)

type Transfer struct {
	Name          string       `json:"name"`
	Oid           string       `json:"oid,omitempty"`
	Size          int64        `json:"size"`
	Authenticated bool         `json:"authenticated,omitempty"`
	Actions       ActionSet    `json:"actions,omitempty"`
	Error         *ObjectError `json:"error,omitempty"`
	Path          string       `json:"path"`
}

type ObjectError struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
}

// NewTransfer creates a new Transfer instance
func NewTransfer(name string, obj *api.ObjectResource, path string) *Transfer {
	t := &Transfer{
		Name:          name,
		Oid:           obj.Oid,
		Size:          obj.Size,
		Authenticated: obj.Authenticated,
		Actions:       make(ActionSet),
		Path:          path,
	}

	if obj.Error != nil {
		t.Error = &ObjectError{
			Code:    obj.Error.Code,
			Message: obj.Error.Message,
		}
	}

	for rel, action := range obj.Actions {
		t.Actions[rel] = &Action{
			Href:      action.Href,
			Header:    action.Header,
			ExpiresAt: action.ExpiresAt,
		}
	}

	return t

}

type Action struct {
	Href      string            `json:"href"`
	Header    map[string]string `json:"header,omitempty"`
	ExpiresAt time.Time         `json:"expires_at,omitempty"`
}

type ActionSet map[string]*Action

const (
	// objectExpirationToTransfer is the duration we expect to have passed
	// from the time that the object's expires_at property is checked to
	// when the transfer is executed.
	objectExpirationToTransfer = 5 * time.Second
)

func (as ActionSet) Get(rel string) (*Action, error) {
	a, ok := as[rel]
	if !ok {
		return nil, &ActionMissingError{Rel: rel}
	}

	if !a.ExpiresAt.IsZero() && a.ExpiresAt.Before(time.Now().Add(objectExpirationToTransfer)) {
		return nil, errors.NewRetriableError(&ActionExpiredErr{Rel: rel, At: a.ExpiresAt})
	}

	return a, nil
}

type ActionExpiredErr struct {
	Rel string
	At  time.Time
}

func (e ActionExpiredErr) Error() string {
	return fmt.Sprintf("tq: action %q expires at %s",
		e.Rel, e.At.In(time.Local).Format(time.RFC822))
}

type ActionMissingError struct {
	Rel string
}

func (e ActionMissingError) Error() string {
	return fmt.Sprintf("tq: unable to find action %q", e.Rel)
}

func IsActionExpiredError(err error) bool {
	if _, ok := err.(*ActionExpiredErr); ok {
		return true
	}
	return false
}

func IsActionMissingError(err error) bool {
	if _, ok := err.(*ActionMissingError); ok {
		return true
	}
	return false
}

func toApiObject(t *Transfer) *api.ObjectResource {
	o := &api.ObjectResource{
		Oid:           t.Oid,
		Size:          t.Size,
		Authenticated: t.Authenticated,
		Actions:       make(map[string]*api.LinkRelation),
	}

	for rel, a := range t.Actions {
		o.Actions[rel] = &api.LinkRelation{
			Href:      a.Href,
			Header:    a.Header,
			ExpiresAt: a.ExpiresAt,
		}
	}

	if t.Error != nil {
		o.Error = &api.ObjectError{
			Code:    t.Error.Code,
			Message: t.Error.Message,
		}
	}

	return o
}
<commit_msg>tq/object: add deprecation note<commit_after>package tq

import (
	"fmt"
	"time"

	"github.com/git-lfs/git-lfs/api"
	"github.com/git-lfs/git-lfs/errors"
)

type Transfer struct {
	Name          string       `json:"name"`
	Oid           string       `json:"oid,omitempty"`
	Size          int64        `json:"size"`
	Authenticated bool         `json:"authenticated,omitempty"`
	Actions       ActionSet    `json:"actions,omitempty"`
	Error         *ObjectError `json:"error,omitempty"`
	Path          string       `json:"path"`
}

type ObjectError struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
}

// NewTransfer creates a new Transfer instance
//
// XXX(taylor): note, this function will be removed from the `tq` package's API
// before landing in master. It is currently used by the smudge operation to
// download a single file and pass _directly_ to the Adapter, whereas it should
// use a transferqueue.
func NewTransfer(name string, obj *api.ObjectResource, path string) *Transfer {
	t := &Transfer{
		Name:          name,
		Oid:           obj.Oid,
		Size:          obj.Size,
		Authenticated: obj.Authenticated,
		Actions:       make(ActionSet),
		Path:          path,
	}

	if obj.Error != nil {
		t.Error = &ObjectError{
			Code:    obj.Error.Code,
			Message: obj.Error.Message,
		}
	}

	for rel, action := range obj.Actions {
		t.Actions[rel] = &Action{
			Href:      action.Href,
			Header:    action.Header,
			ExpiresAt: action.ExpiresAt,
		}
	}

	return t

}

type Action struct {
	Href      string            `json:"href"`
	Header    map[string]string `json:"header,omitempty"`
	ExpiresAt time.Time         `json:"expires_at,omitempty"`
}

type ActionSet map[string]*Action

const (
	// objectExpirationToTransfer is the duration we expect to have passed
	// from the time that the object's expires_at property is checked to
	// when the transfer is executed.
	objectExpirationToTransfer = 5 * time.Second
)

func (as ActionSet) Get(rel string) (*Action, error) {
	a, ok := as[rel]
	if !ok {
		return nil, &ActionMissingError{Rel: rel}
	}

	if !a.ExpiresAt.IsZero() && a.ExpiresAt.Before(time.Now().Add(objectExpirationToTransfer)) {
		return nil, errors.NewRetriableError(&ActionExpiredErr{Rel: rel, At: a.ExpiresAt})
	}

	return a, nil
}

type ActionExpiredErr struct {
	Rel string
	At  time.Time
}

func (e ActionExpiredErr) Error() string {
	return fmt.Sprintf("tq: action %q expires at %s",
		e.Rel, e.At.In(time.Local).Format(time.RFC822))
}

type ActionMissingError struct {
	Rel string
}

func (e ActionMissingError) Error() string {
	return fmt.Sprintf("tq: unable to find action %q", e.Rel)
}

func IsActionExpiredError(err error) bool {
	if _, ok := err.(*ActionExpiredErr); ok {
		return true
	}
	return false
}

func IsActionMissingError(err error) bool {
	if _, ok := err.(*ActionMissingError); ok {
		return true
	}
	return false
}

func toApiObject(t *Transfer) *api.ObjectResource {
	o := &api.ObjectResource{
		Oid:           t.Oid,
		Size:          t.Size,
		Authenticated: t.Authenticated,
		Actions:       make(map[string]*api.LinkRelation),
	}

	for rel, a := range t.Actions {
		o.Actions[rel] = &api.LinkRelation{
			Href:      a.Href,
			Header:    a.Header,
			ExpiresAt: a.ExpiresAt,
		}
	}

	if t.Error != nil {
		o.Error = &api.ObjectError{
			Code:    t.Error.Code,
			Message: t.Error.Message,
		}
	}

	return o
}
<|endoftext|>