<commit_before>package tensor3

import "runtime"

func init() {
	Hints.Threads = runtime.NumCPU() - 1
	Hints.DefaultChunkSize = 10000
}

// parallel optimisation Hints.
var Hints struct {
	Threads          int   // used to keep chunk to core ratio near parity.
	ChunkSizeFixed   bool
	DefaultChunkSize int   // ideally set at run time to number of items that fit into CPU cache.
}

// selects parallel application of functions to Vectors and Matrices types (slices of Vector and Matrix types).
var Parallel bool

// selects parallel application of functions to Matrix components,(its Vector fields).
// only improves performance if using costly functions, non of the built-ins are likely to benefit. YRMV.
var ParallelComponents bool

// use hints to calc a good chunk size
func chunkSize(l int) int {
	if !Hints.ChunkSizeFixed {
		if cs := l / Hints.Threads; cs > Hints.DefaultChunkSize {
			return cs
		}
	}
	return Hints.DefaultChunkSize
}

// return a channel of Vectors that are chunks of the passed Vectors
func vectorsInChunks(vs Vectors, cs int) chan Vectors {
	c := make(chan Vectors, 1)
	lastSplitMax := len(vs)-cs/2
	go func() {
		var bottom int
		for top := cs; top < lastSplitMax; top += cs {
			c <- vs[bottom:top]
			bottom = top
		}
		c <- vs[bottom:]
		close(c)
	}()
	return c
}

// return a channel of Matrices that are chunks of the passed Matrices
func matricesInChunks(ms Matrices, cs int) chan Matrices {
	c := make(chan Matrices)
	lastSplitMax := len(ms)-cs/2
	go func() {
		var bottom int
		for top := cs; top < lastSplitMax; top += cs {
			c <- ms[bottom:top]
			bottom = top
		}
		c <- ms[bottom:]
		close(c)
	}()
	return c
}

// return a channel of VectorRefs that are chunks of the passed VectorRefs
func vectorRefsInChunks(vs VectorRefs, cs int) chan VectorRefs {
	c := make(chan VectorRefs, 1)
	lastSplitMax := len(vs)-cs/2
	go func() {
		var bottom int
		for top := cs; top < lastSplitMax; top += cs {
			c <- vs[bottom:top]
			bottom = top
		}
		c <- vs[bottom:]
		close(c)
	}()
	return c
}

// return a channel of slices from the passed Vectors.
// the inner slices are al the same, provided, Length.
// the start of each slice is spaced by Stride.
// if wrap is true then the last Vector is considered to join to the first.
// notice: wrapped around slices are newly created, modifying their content, unlike non-wrapped, won't change the source Vectors, if consistent behaviour is needed use VectorRefs chunks.
// notice: if the Stride is less than the Length, then the same Vector will appear in consecutive slices.
// notice: can panic if Length larger than chunksize/2 (ie the min. possible size of the last chunk)
func vectorSlicesInChunks(vs Vectors, cs int,length,stride int, wrap bool) chan []Vectors {
	c := make(chan []Vectors, 2)  // 2 so that the next chunk is being calculated in parallel, here unlike other chunking it has a significant cost, although if all cores kept 100% busy, not beneficial.
	go func(){
		// need to special case last chunk; it might have to include the wrap-round's, but don't know its the last until channel closes, so handle previous loop cycle.
		chunkChan :=vectorsInChunks(vs,cs)
		firstChunk := <- chunkChan // keep first chunk for potential wrap-round
		previousChunk := firstChunk
		var i int
		// TODO all array lengths could be precalculate instead of using append
		for chunk:=range chunkChan{
			var vssc []Vectors
			for ;i<len(previousChunk);i+=stride {
				vssc=append(vssc,previousChunk[i:i+length])
			}
			c <- vssc
			i%=len(previousChunk)  // reset start index, allowing for stride continuation across chunks
			previousChunk=chunk
		}
		// now handle the last (previous) chunk
		if wrap {
			var vssc []Vectors
			for ;i<len(previousChunk)-length+1;i+=stride {
				vssc=append(vssc,previousChunk[i:i+length])
			}
			// add the beginning Vectors
			for ;i<len(previousChunk);i+=stride {
				vssc=append(vssc,previousChunk[i:])
				vssc[len(vssc)-1]=append(vssc[len(vssc)-1],firstChunk[:length-len(vssc[len(vssc)-1])]...)
			}
			c <- vssc
		}else{
			// not wrapping so its just shortened
			var vssc []Vectors
			for ;i<len(previousChunk)-length+1;i+=stride {
				vssc=append(vssc,previousChunk[i:i+length])
			}
			c <- vssc
		}
		close(c)
	}()
	return c
}

// TODO VectorRefSlicesInChunks(vs Vectors, cs int,length,stride int, wrap bool) chan []Vectors {

// return a channel of VectorRefs that are chunks of the passed VectorRefs.
// as an optimisation, which some functions might benefit from, the VectorRefs are split so that each chunk contains all/only the VectorRefs within a spacial region, meaning nearby points are MUCH more likely to be in the same chunk.
func vectorRefsInRegionalChunks(vs VectorRefs, centre Vector, cs int) chan VectorRefs {
	// TODO continue to subdivide if exceed chunk size?
	// TODO return, another channel?, boundingbox of chunk?
	cvr := make(chan VectorRefs,8)  // non blocking since a max on 8 VectorRefs from this split function
	// range over a slice of VectorRefs, returned by the Split function, using a function that splits into 8 regions using which side of the origin Vector, by axis alignment, the point is on.
	for _,s:=range func() []VectorRefs {
		return vs.Split(
			func(v *Vector)(i uint){
				i++   // index never zero, since for this all points go somewhere
				if v.x>=centre.x {i++}
				if v.y>=centre.y {i+=2}
				if v.z>=centre.z {i+=4}
				return
			},
		)
	}(){
		cvr <- s
	}
	close(cvr)
	return cvr
}

<commit_msg>chunking refs<commit_after>package tensor3

import "runtime"

func init() {
	Hints.Threads = runtime.NumCPU() - 1
	Hints.DefaultChunkSize = 10000
}

// parallel optimisation Hints.
var Hints struct {
	Threads          int   // used to keep chunk to core ratio near parity.
	ChunkSizeFixed   bool
	DefaultChunkSize int   // ideally set at run time to number of items that fit into CPU cache.
}

// selects parallel application of functions to Vectors and Matrices types (slices of Vector and Matrix types).
var Parallel bool

// selects parallel application of functions to Matrix components,(its Vector fields).
// only improves performance if using costly functions, non of the built-ins are likely to benefit. YRMV.
var ParallelComponents bool

// use hints to calc a good chunk size
func chunkSize(l int) int {
	if !Hints.ChunkSizeFixed {
		if cs := l / Hints.Threads; cs > Hints.DefaultChunkSize {
			return cs
		}
	}
	return Hints.DefaultChunkSize
}

// return a channel of Vectors that are chunks of the passed Vectors
func vectorsInChunks(vs Vectors, cs int) chan Vectors {
	c := make(chan Vectors, 1)
	lastSplitMax := len(vs)-cs/2
	go func() {
		var bottom int
		for top := cs; top < lastSplitMax; top += cs {
			c <- vs[bottom:top]
			bottom = top
		}
		c <- vs[bottom:]
		close(c)
	}()
	return c
}

// return a channel of Matrices that are chunks of the passed Matrices
func matricesInChunks(ms Matrices, cs int) chan Matrices {
	c := make(chan Matrices)
	lastSplitMax := len(ms)-cs/2
	go func() {
		var bottom int
		for top := cs; top < lastSplitMax; top += cs {
			c <- ms[bottom:top]
			bottom = top
		}
		c <- ms[bottom:]
		close(c)
	}()
	return c
}

// return a channel of VectorRefs that are chunks of the passed VectorRefs
func vectorRefsInChunks(vs VectorRefs, cs int) chan VectorRefs {
	c := make(chan VectorRefs, 1)
	lastSplitMax := len(vs)-cs/2
	go func() {
		var bottom int
		for top := cs; top < lastSplitMax; top += cs {
			c <- vs[bottom:top]
			bottom = top
		}
		c <- vs[bottom:]
		close(c)
	}()
	return c
}

// return a channel of slices from the passed Vectors.
// the inner slices are al the same, provided, Length.
// the start of each slice is spaced by Stride.
// if wrap is true then the last Vector is considered to join to the first.
// notice: wrapped around slices are newly created, modifying their content, unlike non-wrapped, won't change the source Vectors, if consistent behaviour is needed use VectorRefs chunks.
// notice: if the Stride is less than the Length, then the same Vector will appear in consecutive slices.
// notice: can panic if Length larger than chunksize/2 (ie the min. possible size of the last chunk)
func vectorSlicesInChunks(vs Vectors, cs int,length,stride int, wrap bool) chan []Vectors {
	c := make(chan []Vectors, 2)  // 2 so that the next chunk is being calculated in parallel, here unlike other chunking it has a significant cost, although if all cores kept 100% busy, not beneficial.
	go func(){
		// need to special case last chunk; it might have to include the wrap-round's, but don't know its the last until channel closes, so handle previous loop cycle.
		chunkChan :=vectorsInChunks(vs,cs)
		firstChunk := <- chunkChan // keep first chunk for potential wrap-round
		previousChunk := firstChunk
		var i int
		// TODO all array lengths could be precalculate instead of using append
		for chunk:=range chunkChan{
			var vssc []Vectors
			for ;i<len(previousChunk);i+=stride {
				vssc=append(vssc,previousChunk[i:i+length])
			}
			c <- vssc
			i%=len(previousChunk)  // reset start index, allowing for stride continuation across chunks
			previousChunk=chunk
		}
		// now handle the last (previous) chunk
		if wrap {
			var vssc []Vectors
			for ;i<len(previousChunk)-length+1;i+=stride {
				vssc=append(vssc,previousChunk[i:i+length])
			}
			// add the beginning Vectors
			for ;i<len(previousChunk);i+=stride {
				vssc=append(vssc,previousChunk[i:])
				vssc[len(vssc)-1]=append(vssc[len(vssc)-1],firstChunk[:length-len(vssc[len(vssc)-1])]...)
			}
			c <- vssc
		}else{
			// not wrapping so its just shortened
			var vssc []Vectors
			for ;i<len(previousChunk)-length+1;i+=stride {
				vssc=append(vssc,previousChunk[i:i+length])
			}
			c <- vssc
		}
		close(c)
	}()
	return c
}

// as vectorSlicesInChunks
func vectorRefSlicesInChunks(vs VectorRefs, cs int,length,stride int, wrap bool) chan []VectorRefs {
	c := make(chan []VectorRefs, 2)  // 2 so that the next chunk is being calculated in parallel, here unlike other chunking it has a significant cost, although if all cores kept 100% busy, not beneficial.
	go func(){
		// need to special case last chunk; it might have to include the wrap-round's, but don't know its the last until channel closes, so handle previous loop cycle.
		chunkChan :=vectorRefsInChunks(vs,cs)
		firstChunk := <- chunkChan // keep first chunk for potential wrap-round
		previousChunk := firstChunk
		var i int
		// TODO all array lengths could be precalculate instead of using append
		for chunk:=range chunkChan{
			var vssc []VectorRefs
			for ;i<len(previousChunk);i+=stride {
				vssc=append(vssc,previousChunk[i:i+length])
			}
			c <- vssc
			i%=len(previousChunk)  // reset start index, allowing for stride continuation across chunks
			previousChunk=chunk
		}
		// now handle the last (previous) chunk
		if wrap {
			var vssc []VectorRefs
			for ;i<len(previousChunk)-length+1;i+=stride {
				vssc=append(vssc,previousChunk[i:i+length])
			}
			// add the beginning Vectors
			for ;i<len(previousChunk);i+=stride {
				vssc=append(vssc,previousChunk[i:])
				vssc[len(vssc)-1]=append(vssc[len(vssc)-1],firstChunk[:length-len(vssc[len(vssc)-1])]...)
			}
			c <- vssc
		}else{
			// not wrapping so its just shortened
			var vssc []VectorRefs
			for ;i<len(previousChunk)-length+1;i+=stride {
				vssc=append(vssc,previousChunk[i:i+length])
			}
			c <- vssc
		}
		close(c)
	}()
	return c
}

// return a channel of VectorRefs that are chunks of the passed VectorRefs.
// as an optimisation, which some functions might benefit from, the VectorRefs are split so that each chunk contains all/only the VectorRefs within a spacial region, meaning nearby points are MUCH more likely to be in the same chunk.
func vectorRefsInRegionalChunks(vs VectorRefs, centre Vector, cs int) chan VectorRefs {
	// TODO continue to subdivide if exceed chunk size?
	// TODO return, another channel?, boundingbox of chunk?
	cvr := make(chan VectorRefs,8)  // non blocking since a max on 8 VectorRefs from this split function
	// range over a slice of VectorRefs, returned by the Split function, using a function that splits into 8 regions using which side of the origin Vector, by axis alignment, the point is on.
	for _,s:=range func() []VectorRefs {
		return vs.Split(
			func(v *Vector)(i uint){
				i++   // index never zero, since for this all points go somewhere
				if v.x>=centre.x {i++}
				if v.y>=centre.y {i+=2}
				if v.z>=centre.z {i+=4}
				return
			},
		)
	}(){
		cvr <- s
	}
	close(cvr)
	return cvr
}

<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// guru: a tool for answering questions about Go source code.
//
//    http://golang.org/s/oracle-design
//    http://golang.org/s/oracle-user-manual
//
// Run with -help flag or help subcommand for usage information.
//
package commands

import (
	"go/build"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"nvim-go/gb"
	"nvim-go/guru"
	"nvim-go/nvim"

	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
	"golang.org/x/tools/cmd/guru/serial"
)

func init() {
	plugin.HandleCommand("GoGuru", &plugin.CommandOptions{NArgs: "*", Complete: "customlist,GuruCompletelist", Eval: "[expand('%:p:h:h:h:h'), expand('%:p')]"}, Guru)
	plugin.HandleFunction("GuruCompletelist", &plugin.FunctionOptions{}, onComplete)
}

var (
	reflectFlag bool
)

type onGuruEval struct {
	Cwd  string `msgpack:",array"`
	File string
}

func Guru(v *vim.Vim, args []string, eval *onGuruEval) error {
	defer gb.WithGoBuildForPath(eval.Cwd)()
	scopeFlag := []byte(eval.File)

	mode := args[0]
	pos, err := nvim.ByteOffset(v)
	if err != nil {
		return nvim.Echomsg(v, "%v", err)
	}

	gopath := os.Getenv("GOPATH")
	cmd := exec.Command("gb", "env", "GB_PROJECT_DIR")
	output, err := cmd.Output()
	if strings.Replace(string(output[:]), "\n", "", -1) != gopath {
		scopeFlag = output
	} else if err != nil {
		nvim.Echomsg(v, "could not find project root directory")
	}

	query := guru.Query{
		Mode:       mode,
		Pos:        eval.File + ":#" + strconv.FormatInt(int64(pos), 10),
		Build:      &build.Default,
		Scope:      strings.Split(string(scopeFlag[:]), ","),
		Reflection: reflectFlag,
	}

	if err := guru.Run(&query); err != nil {
		return nvim.Echomsg(v, "%s", err)
	}

	d := parseSerial(mode, query.Serial())
	if len(d) < 1 {
		return nvim.Echomsg(v, "0")
	}

	return nil
}

func parseSerial(mode string, s *serial.Result) map[string]interface{} {
	data := map[string]interface{}{
		"Mode": s.Mode,
	}

	switch mode {
	case "callees":
		data = map[string]interface{}{
			"Pos":             s.Callees.Pos,
			"Desc":            s.Callees.Desc,
			"Callees.Callees": s.Callees.Callees,
		}
	case "callers":
		data = map[string]interface{}{
			"Callers": s.Callers,
		}
	case "callstack":
		data = map[string]interface{}{
			"Pos":     s.Callstack.Pos,
			"Target":  s.Callstack.Target,
			"Callers": s.Callstack.Callers,
		}
	case "definition":
		data = map[string]interface{}{
			"ObjPos": s.Definition.ObjPos,
			"Desc":   s.Definition.Desc,
		}
	case "describe":
		data = map[string]interface{}{
			"Desc":         s.Describe.Desc,
			"Pos":          s.Describe.Pos,
			"Detail":       s.Describe.Detail,
			"Package":      s.Describe.Package,
			"Type":         s.Describe.Type,
			"Value.Type":   s.Describe.Value.Type,
			"Value.ObjPos": s.Describe.Value.ObjPos,
		}
	case "freevars":
		data = map[string]interface{}{
			"Freevars": s.Freevars,
		}
	case "implements":
		data = map[string]interface{}{
			"T":                       s.Implements.T,
			"AssignableTo":            s.Implements.AssignableTo,
			"AssignableFrom":          s.Implements.AssignableFrom,
			"AssignableFromPtr":       s.Implements.AssignableFromPtr,
			"Method":                  s.Implements.Method,
			"AssignableToMethod":      s.Implements.AssignableToMethod,
			"AssignableFromMethod":    s.Implements.AssignableFromMethod,
			"AssignableFromPtrMethod": s.Implements.AssignableFromPtrMethod,
		}
	case "peers":
		data = map[string]interface{}{
			"Pos":      s.Peers.Pos,
			"Type":     s.Peers.Type,
			"Allocs":   s.Peers.Allocs,
			"Sends":    s.Peers.Sends,
			"Receives": s.Peers.Receives,
			"Closes":   s.Peers.Closes,
		}
	case "pointsto":
		data = map[string]interface{}{
			"PointsTo": s.PointsTo,
		}
	case "referrers":
		data = map[string]interface{}{
			"Pos":    s.Referrers.Pos,
			"ObjPos": s.Referrers.ObjPos,
			"Desc":   s.Referrers.Desc,
			"Refs":   s.Referrers.Refs,
		}
	case "what":
		data = map[string]interface{}{
			"Enclosing":  s.What.Enclosing,
			"Modes":      s.What.Modes,
			"SrcDir":     s.What.SrcDir,
			"ImportPath": s.What.ImportPath,
		}
	case "whicherrs":
		data = map[string]interface{}{
			"ErrPos":    s.WhichErrs.ErrPos,
			"Globals":   s.WhichErrs.Globals,
			"Constants": s.WhichErrs.Constants,
			"Types":     s.WhichErrs.Types,
		}
	}

	return data
}

func onComplete(v *vim.Vim) ([]string, error) {
	return []string{
		"callers",
		"callees",
		"callstack",
		"definition",
		"describe",
		"freevars",
		"implements",
		"peers",
		"pointsto",
		"referrers",
		"what",
		"whicherrs",
	}, nil
}
<commit_msg>Support all guru commands<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// guru: a tool for answering questions about Go source code.
//
//    http://golang.org/s/oracle-design
//    http://golang.org/s/oracle-user-manual
//
// Run with -help flag or help subcommand for usage information.
//
package commands

import (
	"errors"
	"fmt"
	"go/build"
	"strconv"
	"strings"

	"nvim-go/gb"
	"nvim-go/nvim"

	log "github.com/Sirupsen/logrus"
	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
	"golang.org/x/tools/cmd/guru"
	"golang.org/x/tools/cmd/guru/serial"
)

func init() {
	plugin.HandleCommand("GoGuru", &plugin.CommandOptions{NArgs: "*", Complete: "customlist,GuruCompletelist", Eval: "[expand('%:p:h'), expand('%:p')]"}, Guru)
	plugin.HandleFunction("GuruCompletelist", &plugin.FunctionOptions{}, onComplete)

	log.Debugln("Guru Start")
}

var (
	reflectFlag bool
)

type onGuruEval struct {
	Cwd  string `msgpack:",array"`
	File string
}

func Guru(v *vim.Vim, args []string, eval *onGuruEval) error {
	defer gb.WithGoBuildForPath(eval.File)()

	dir := strings.Split(eval.Cwd, "src/")
	scopeFlag := dir[len(dir)-1]

	mode := args[0]

	pos, err := nvim.ByteOffset(v)
	if err != nil {
		return nvim.Echomsg(v, "%v", err)
	}

	query := guru.Query{
		Mode:       mode,
		Pos:        eval.File + ":#" + strconv.FormatInt(int64(pos), 10),
		Build:      &build.Default,
		Scope:      []string{scopeFlag},
		Reflection: reflectFlag,
	}

	if err := guru.Run(&query); err != nil {
		return nvim.Echomsg(v, "%s", err)
	}

	d, err := parseSerial(mode, query.Serial())
	if err != nil {
		return nvim.Echomsg(v, "GoGuru: %v", err)
	}

	return nvim.Loclist(v, d, true)
}

func parseSerial(mode string, s *serial.Result) ([]*nvim.LoclistData, error) {
	var loclist []*nvim.LoclistData

	switch mode {
	case "callees":
		var calleers string
		for _, n := range s.Callees.Callees {
			calleers += calleers + n.Name
		}
		file, line, col := nvim.SplitPos(s.Callees.Pos)
		loclist = append(loclist, &nvim.LoclistData{
			FileName: file,
			LNum:     line,
			Col:      col,
			Text:     s.Callees.Desc + " " + calleers,
		})
	case "callers":
		for _, e := range s.Callers {
			file, line, col := nvim.SplitPos(e.Pos)
			loclist = append(loclist, &nvim.LoclistData{
				FileName: file,
				LNum:     line,
				Col:      col,
				Text:     e.Desc + " " + e.Caller,
			})
		}
	case "callstack":
		if len(s.Callstack.Callers) != 0 {
			for _, n := range s.Callstack.Callers {
				file, line, col := nvim.SplitPos(n.Pos)
				loclist = append(loclist, &nvim.LoclistData{
					FileName: file,
					LNum:     line,
					Col:      col,
					Text:     n.Desc,
				})
			}
		}
	case "definition":
		file, line, col := nvim.SplitPos(s.Definition.ObjPos)
		loclist = append(loclist, &nvim.LoclistData{
			FileName: file,
			LNum:     line,
			Col:      col,
			Text:     s.Definition.Desc,
		})
	case "describe":
		file, line, col := nvim.SplitPos(s.Describe.Value.ObjPos)
		loclist = append(loclist, &nvim.LoclistData{
			FileName: file,
			LNum:     line,
			Col:      col,
			Text:     s.Describe.Value.Type,
		})
	case "freevars":
		for _, e := range s.Freevars {
			file, line, col := nvim.SplitPos(e.Pos)
			log.Debugln(e)
			loclist = append(loclist, &nvim.LoclistData{
				FileName: file,
				LNum:     line,
				Col:      col,
				Text:     e.Type + "\n" + e.Kind + "\n" + e.Ref,
			})
		}
	case "implements":
		file, line, col := nvim.SplitPos(s.Implements.T.Pos)
		loclist = append(loclist, &nvim.LoclistData{
			FileName: file,
			LNum:     line,
			Col:      col,
			Text:     s.Implements.T.Name,
		})
	case "peers":
		for _, e := range s.Peers.Allocs {
			file, line, col := nvim.SplitPos(e)
			loclist = append(loclist, &nvim.LoclistData{
				FileName: file,
				LNum:     line,
				Col:      col,
				Text:     s.Peers.Type + ": Allocs",
			})
		}
		for _, e := range s.Peers.Sends {
			file, line, col := nvim.SplitPos(e)
			loclist = append(loclist, &nvim.LoclistData{
				FileName: file,
				LNum:     line,
				Col:      col,
				Text:     s.Peers.Type + ": Sends",
			})
		}
		for _, e := range s.Peers.Receives {
			file, line, col := nvim.SplitPos(e)
			loclist = append(loclist, &nvim.LoclistData{
				FileName: file,
				LNum:     line,
				Col:      col,
				Text:     s.Peers.Type + ": Receives",
			})
		}
		for _, e := range s.Peers.Closes {
			file, line, col := nvim.SplitPos(e)
			loclist = append(loclist, &nvim.LoclistData{
				FileName: file,
				LNum:     line,
				Col:      col,
				Text:     s.Peers.Type + ": Closes",
			})
		}
	case "pointsto":
		for _, e := range s.PointsTo {
			if e.NamePos != "" {
				file, line, col := nvim.SplitPos(e.NamePos)
				loclist = append(loclist, &nvim.LoclistData{
					FileName: file,
					LNum:     line,
					Col:      col,
					Text:     e.Type,
				})
			} else {
				loclist = append(loclist, &nvim.LoclistData{
					Text: e.Type,
				})
			}
		}
	case "referrers":
		for _, e := range s.Referrers.Refs {
			file, line, col := nvim.SplitPos(e)
			loclist = append(loclist, &nvim.LoclistData{
				FileName: file,
				LNum:     line,
				Col:      col,
				Text:     s.Referrers.Desc,
			})
		}
	case "what":
		log.Debugln("s.What.Enclosing:", s.What.Enclosing)
		log.Debugln("s.What.Modes:", s.What.Modes)
		log.Debugln("s.What.SrcDir:", s.What.SrcDir)
		log.Debugln("s.What.ImportPath:", s.What.ImportPath)
		log.Debugln("s.What.Object:", s.What.Object)
		log.Debugln("s.What.SameIDs:", s.What.SameIDs)
		var modesText string
		for _, mode := range s.What.Modes {
			modesText += mode + " "
		}
		loclist = append(loclist, &nvim.LoclistData{
			Text: "Modes: " + modesText[:len(modesText)-2],
		})
		loclist = append(loclist, &nvim.LoclistData{
			Text: "SrcDir: " + s.What.SrcDir,
		})
		loclist = append(loclist, &nvim.LoclistData{
			Text: "ImportPath: " + s.What.ImportPath,
		})
		loclist = append(loclist, &nvim.LoclistData{
			Text: "Object: " + s.What.Object,
		})
		sameIDsText := "SameIDs: "
		for _, sameid := range s.What.SameIDs {
			sameIDsText += sameid
		}
		loclist = append(loclist, &nvim.LoclistData{
			Text: sameIDsText,
		})
	case "whicherrs":
		// log.Debugln("s.WhichErrs.ErrPos:", s.WhichErrs.ErrPos)
		// log.Debugln("s.WhichErrs.Globals:", s.WhichErrs.Globals)
		// log.Debugln("s.WhichErrs.Constants:", s.WhichErrs.Constants)
		// log.Debugln("s.WhichErrs.Types:", s.WhichErrs.Types)
		for _, e := range s.WhichErrs.Types {
			file, line, col := nvim.SplitPos(e.Position)
			loclist = append(loclist, &nvim.LoclistData{
				FileName: file,
				LNum:     line,
				Col:      col,
				Text:     e.Type,
			})
		}
	}

	if len(loclist) == 0 {
		return loclist, errors.New(fmt.Sprintf("%s not fount", mode))
	}
	return loclist, nil
}

func onComplete(v *vim.Vim) ([]string, error) {
	return []string{
		"callers",
		"callees",
		"callstack",
		"definition",
		"describe",
		"freevars",
		"implements",
		"peers",
		"pointsto",
		"referrers",
		"what",
		"whicherrs",
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Koichi Shiraishi. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package commands

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"path/filepath"
	"strings"

	"nvim-go/config"
	"nvim-go/context"
	"nvim-go/nvim"
	"nvim-go/nvim/quickfix"
	"nvim-go/nvim/terminal"
	"nvim-go/pathutil"

	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
	"golang.org/x/tools/go/ast/astutil"
)

func init() {
	plugin.HandleCommand("Gotest", &plugin.CommandOptions{NArgs: "*", Eval: "expand('%:p:h')"}, cmdTest)
	plugin.HandleCommand("GoTestSwitch", &plugin.CommandOptions{Eval: "[getcwd(), expand('%:p')]"}, cmdTestSwitch)
}

func cmdTest(v *vim.Vim, args []string, dir string) {
	go Test(v, args, dir)
}

var term *terminal.Terminal

// Test run the package test command use compile tool that determined from
// the directory structure.
func Test(v *vim.Vim, args []string, dir string) error {
	ctxt := new(context.Context)
	defer ctxt.Build.SetContext(dir)()

	cmd := []string{ctxt.Build.Tool, "test"}
	args = append(args, config.TestArgs...)
	if len(args) > 0 {
		cmd = append(cmd, args...)
	}

	if ctxt.Build.Tool == "go" {
		cmd = append(cmd, string("./..."))
	}

	if term == nil {
		term = terminal.NewTerminal(v, "__GO_TEST__", cmd, config.TerminalMode)
		term.Dir = pathutil.FindVcsRoot(dir)
	}

	if err := term.Run(cmd); err != nil {
		return err
	}

	return nil
}

var (
	fset       = token.NewFileSet() // *token.FileSet
	parserMode parser.Mode          // uint
	pos        token.Pos

	testPrefix       = "Test"
	testSuffix       = "_test"
	isTest           bool
	funcName         string
	funcNameNoExport string
)

type cmdTestSwitchEval struct {
	Cwd  string `msgpack:",array"`
	File string
}

func cmdTestSwitch(v *vim.Vim, eval cmdTestSwitchEval) {
	go TestSwitch(v, eval)
}

// TestSwitch switch to corresponds current cursor (test)function.
func TestSwitch(v *vim.Vim, eval cmdTestSwitchEval) error {
	// Check the current buffer name whether '*_test.go'.
	fname := eval.File
	exp := filepath.Ext(fname)
	var switchfile string
	if strings.Index(fname, testSuffix) == -1 {
		isTest = false
		switchfile = strings.Replace(fname, exp, testSuffix+exp, 1) // not testfile
	} else {
		isTest = true
		switchfile = strings.Replace(fname, testSuffix+exp, exp, 1) // testfile
	}

	// Check the exists of switch destination file.
	if _, err := os.Stat(switchfile); err != nil {
		return nvim.EchohlErr(v, "GoTestSwitch", "Switch destination file does not exist")
	}

	ctxt := new(context.Context)
	dir, _ := filepath.Split(fname)
	defer ctxt.Build.SetContext(filepath.Dir(dir))()

	var (
		b vim.Buffer
		w vim.Window
	)

	// Gets the current buffer information.
	p := v.NewPipeline()
	p.CurrentBuffer(&b)
	p.CurrentWindow(&w)
	if err := p.Wait(); err != nil {
		return err
	}

	// Get the byte offset of current cursor position from buffer.
	// TODO(zchee): Eval 'line2byte(line('.'))+(col('.')-2)' is faster and safer?
	byteOffset, err := nvim.ByteOffset(v, b, w)
	if err != nil {
		return err
	}
	// Get the 2d byte slice of current buffer.
	var buf [][]byte
	p.BufferLines(b, 0, -1, true, &buf)
	if err := p.Wait(); err != nil {
		return err
	}

	f, err := parse(fname, fset, nvim.ToByteSlice(buf)) // *ast.File
	if err != nil {
		return err
	}
	offset := fset.File(f.Pos()).Pos(byteOffset) // token.Pos

	// Parses the function ast node from the current cursor position.
	qpos, _ := astutil.PathEnclosingInterval(f, offset, offset) // path []ast.Node, exact bool
	for _, q := range qpos {
		switch x := q.(type) {
		case *ast.FuncDecl:
			if x.Name != nil { // *ast.Ident
				// TODO(zchee): Support parses the function struct name.
				// If the function has a struct, gotests will be generated the
				// mixed camel case test function name include struct name for prefix.
				if !isTest {
					funcName = fmt.Sprintf("%s%s", testPrefix, ToPascalCase(x.Name.Name))
				} else {
					funcName = strings.Replace(x.Name.Name, testPrefix, "", 1)
				}
				funcNameNoExport = ToMixedCase(funcName)
			}
		}
	}

	// Get the switch destination file ast node.
	fswitch, err := parse(switchfile, fset, nil) // *ast.File
	if err != nil {
		return err
	}

	// Reset pos value.
	if pos != token.NoPos {
		pos = 0
	}
	// Parses the switch destination file ast node.
	ast.Walk(visitorFunc(parseFunc), fswitch)

	if !pos.IsValid() {
		return nvim.EchohlErr(v, "GoTestSwitch", "Not found the switch destination function")
	}

	// Jump to the corresponds function.
	return quickfix.GotoPos(v, w, fset.Position(pos), eval.Cwd)
}

// Wrapper of the parser.ParseFile()
func parse(filename string, fset *token.FileSet, src interface{}) (*ast.File, error) {
	file, err := parser.ParseFile(fset, filename, src, parserMode)
	if err != nil {
		return nil, err
	}

	return file, err
}

// visitorFunc for ast.Visit type.
type visitorFunc func(n ast.Node) ast.Visitor

// visit for ast.Visit function.
func (f visitorFunc) Visit(n ast.Node) ast.Visitor {
	return f(n)
}

// Core of the parser of the ast node.
func parseFunc(node ast.Node) ast.Visitor {
	switch x := node.(type) {
	case *ast.FuncDecl:
		if x.Name.Name == funcName || x.Name.Name == funcNameNoExport || indexFuncName(x.Name.Name, funcName, funcNameNoExport) { // x.Name.Name: *ast.Ident.string
			pos = x.Name.NamePos
			return nil
		}
	}

	return visitorFunc(parseFunc)
}

func indexFuncName(s string, sep ...string) bool {
	for _, fn := range sep {
		i := strings.Index(fn, s)
		if i > -1 {
			return true
		}
	}

	return false
}
<commit_msg>cmds/test: Re-enable profiling<commit_after>// Copyright 2016 Koichi Shiraishi. All rights reserved.

package commands

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"path/filepath"
	"strings"
	"time"

	"nvim-go/config"
	"nvim-go/context"
	"nvim-go/nvim"
	"nvim-go/nvim/profile"
	"nvim-go/nvim/quickfix"
	"nvim-go/nvim/terminal"
	"nvim-go/pathutil"

	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
	"golang.org/x/tools/go/ast/astutil"
)

func init() {
	plugin.HandleCommand("Gotest", &plugin.CommandOptions{NArgs: "*", Eval: "expand('%:p:h')"}, cmdTest)
	plugin.HandleCommand("GoTestSwitch", &plugin.CommandOptions{Eval: "[getcwd(), expand('%:p')]"}, cmdTestSwitch)
}

func cmdTest(v *vim.Vim, args []string, dir string) {
	go Test(v, args, dir)
}

var term *terminal.Terminal

// Test run the package test command use compile tool that determined from
// the directory structure.
func Test(v *vim.Vim, args []string, dir string) error {
	defer profile.Start(time.Now(), "GoTest")
	ctxt := new(context.Context)
	defer ctxt.Build.SetContext(dir)()

	cmd := []string{ctxt.Build.Tool, "test"}
	args = append(args, config.TestArgs...)
	if len(args) > 0 {
		cmd = append(cmd, args...)
	}

	if ctxt.Build.Tool == "go" {
		cmd = append(cmd, string("./..."))
	}

	if term == nil {
		term = terminal.NewTerminal(v, "__GO_TEST__", cmd, config.TerminalMode)
		term.Dir = pathutil.FindVcsRoot(dir)
	}

	if err := term.Run(cmd); err != nil {
		return err
	}

	return nil
}

var (
	fset       = token.NewFileSet() // *token.FileSet
	parserMode parser.Mode          // uint
	pos        token.Pos

	testPrefix       = "Test"
	testSuffix       = "_test"
	isTest           bool
	funcName         string
	funcNameNoExport string
)

type cmdTestSwitchEval struct {
	Cwd  string `msgpack:",array"`
	File string
}

func cmdTestSwitch(v *vim.Vim, eval cmdTestSwitchEval) {
	go TestSwitch(v, eval)
}

// TestSwitch switch to corresponds current cursor (test)function.
func TestSwitch(v *vim.Vim, eval cmdTestSwitchEval) error {
	// Check the current buffer name whether '*_test.go'.
	fname := eval.File
	exp := filepath.Ext(fname)
	var switchfile string
	if strings.Index(fname, testSuffix) == -1 {
		isTest = false
		switchfile = strings.Replace(fname, exp, testSuffix+exp, 1) // not testfile
	} else {
		isTest = true
		switchfile = strings.Replace(fname, testSuffix+exp, exp, 1) // testfile
	}

	// Check the exists of switch destination file.
	if _, err := os.Stat(switchfile); err != nil {
		return nvim.EchohlErr(v, "GoTestSwitch", "Switch destination file does not exist")
	}

	ctxt := new(context.Context)
	dir, _ := filepath.Split(fname)
	defer ctxt.Build.SetContext(filepath.Dir(dir))()

	var (
		b vim.Buffer
		w vim.Window
	)

	// Gets the current buffer information.
	p := v.NewPipeline()
	p.CurrentBuffer(&b)
	p.CurrentWindow(&w)
	if err := p.Wait(); err != nil {
		return err
	}

	// Get the byte offset of current cursor position from buffer.
	// TODO(zchee): Eval 'line2byte(line('.'))+(col('.')-2)' is faster and safer?
	byteOffset, err := nvim.ByteOffset(v, b, w)
	if err != nil {
		return err
	}
	// Get the 2d byte slice of current buffer.
	var buf [][]byte
	p.BufferLines(b, 0, -1, true, &buf)
	if err := p.Wait(); err != nil {
		return err
	}

	f, err := parse(fname, fset, nvim.ToByteSlice(buf)) // *ast.File
	if err != nil {
		return err
	}
	offset := fset.File(f.Pos()).Pos(byteOffset) // token.Pos

	// Parses the function ast node from the current cursor position.
	qpos, _ := astutil.PathEnclosingInterval(f, offset, offset) // path []ast.Node, exact bool
	for _, q := range qpos {
		switch x := q.(type) {
		case *ast.FuncDecl:
			if x.Name != nil { // *ast.Ident
				// TODO(zchee): Support parses the function struct name.
				// If the function has a struct, gotests will be generated the
				// mixed camel case test function name include struct name for prefix.
				if !isTest {
					funcName = fmt.Sprintf("%s%s", testPrefix, ToPascalCase(x.Name.Name))
				} else {
					funcName = strings.Replace(x.Name.Name, testPrefix, "", 1)
				}
				funcNameNoExport = ToMixedCase(funcName)
			}
		}
	}

	// Get the switch destination file ast node.
	fswitch, err := parse(switchfile, fset, nil) // *ast.File
	if err != nil {
		return err
	}

	// Reset pos value.
	if pos != token.NoPos {
		pos = 0
	}
	// Parses the switch destination file ast node.
	ast.Walk(visitorFunc(parseFunc), fswitch)

	if !pos.IsValid() {
		return nvim.EchohlErr(v, "GoTestSwitch", "Not found the switch destination function")
	}

	// Jump to the corresponds function.
	return quickfix.GotoPos(v, w, fset.Position(pos), eval.Cwd)
}

// Wrapper of the parser.ParseFile()
func parse(filename string, fset *token.FileSet, src interface{}) (*ast.File, error) {
	file, err := parser.ParseFile(fset, filename, src, parserMode)
	if err != nil {
		return nil, err
	}

	return file, err
}

// visitorFunc for ast.Visit type.
type visitorFunc func(n ast.Node) ast.Visitor

// visit for ast.Visit function.
func (f visitorFunc) Visit(n ast.Node) ast.Visitor {
	return f(n)
}

// Core of the parser of the ast node.
func parseFunc(node ast.Node) ast.Visitor {
	switch x := node.(type) {
	case *ast.FuncDecl:
		if x.Name.Name == funcName || x.Name.Name == funcNameNoExport || indexFuncName(x.Name.Name, funcName, funcNameNoExport) { // x.Name.Name: *ast.Ident.string
			pos = x.Name.NamePos
			return nil
		}
	}

	return visitorFunc(parseFunc)
}

func indexFuncName(s string, sep ...string) bool {
	for _, fn := range sep {
		i := strings.Index(fn, s)
		if i > -1 {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>package crawl

import (
	"fmt"
	"testing"
	"time"

	"github.com/arianht/meantweets/database"
	"github.com/arianht/meantweets/util"
	"golang.org/x/net/context"
)

func TestRealCrawl(t *testing.T) {
	// Skip test if credentials are not found.
	_, err := util.GetTwitterAPICredentialsFromFile("credentials.json")
	if err != nil {
		t.Skipf("Error getting Twitter credentials: %v. Skipping crawler integration test.", err)
	}
	ctx := context.Background()

	twitterCrawler, err := NewTwitterCrawler(ctx)
	if err != nil {
		t.Fatal(err)
	}
	celebrities := []string{"Johnny Depp", "Jennifer Lopez", "Justin Bieber", "Taylor Swift"}
	maxTweets := 5
	twitterCrawler.Crawl(celebrities, maxTweets)

	// Sadly, App Engine Datastore takes time to fully write. Without this sleep,
	// the write won't be done in time for the read.
	time.Sleep(2 * time.Second)

	fmt.Println("Printing out results from the Crawler Integration Test")
	for _, celebrity := range celebrities {
		tweets, err := twitterCrawler.Dao.GetCelebrityTweets(celebrity)
		if err != nil {
			t.Fatal(err)
		}
		// Make sure we crawled tweets and wrote the top ones to the database.
		if len(tweets) != maxTweets {
			t.Errorf("Expected %d tweets for celebrity %v in database, but was %v.", maxTweets, celebrity, len(tweets))
		}
		printCelebrityResults(celebrity, tweets)
	}
	fmt.Println("Crawler integration test finished. Manually verify results.")
}

func printCelebrityResults(celebrity string, tweets []database.Tweet) {
	fmt.Printf("================================\n%v mean tweets: \n", celebrity)
	for _, tweet := range tweets {
		fmt.Printf("Score: %v - Link: http://twitter.com/random/status/%v\n", tweet.Score, tweet.Id)
	}
	fmt.Printf("\n")
}
<commit_msg>"Fix" crawler integration test. It's flaky now...<commit_after>package crawl

import (
	"fmt"
	"testing"
	"time"

	"github.com/arianht/meantweets/database"
	"github.com/arianht/meantweets/util"
	"golang.org/x/net/context"
)

func TestRealCrawl(t *testing.T) {
	// Skip test if credentials are not found.
	_, err := util.GetTwitterAPICredentialsFromFile("credentials.json")
	if err != nil {
		t.Skipf("Error getting Twitter credentials: %v. Skipping crawler integration test.", err)
	}
	ctx := context.Background()

	twitterCrawler, err := NewTwitterCrawler(ctx)
	if err != nil {
		t.Fatal(err)
	}
	celebrities := []string{"Johnny Depp", "Jennifer Lopez", "Justin Bieber", "Taylor Swift"}
	maxTweets := 5
	twitterCrawler.Crawl(celebrities, maxTweets)

	// Sadly, App Engine Datastore takes time to fully write. Without this sleep,
	// the write won't be done in time for the read.
	time.Sleep(4 * time.Second)

	for _, celebrity := range celebrities {
		tweets, err := twitterCrawler.Dao.GetCelebrityTweets(celebrity)
		if err != nil {
			t.Fatal(err)
		}
		// Make sure we crawled tweets and wrote the top ones to the database.
		if len(tweets) != maxTweets {
			t.Errorf("Expected %d tweets for celebrity %v in database, but was %v.", maxTweets, celebrity, len(tweets))
		}
	}
}

func printCelebrityResults(celebrity string, tweets []database.Tweet) {
	fmt.Printf("================================\n%v mean tweets: \n", celebrity)
	for _, tweet := range tweets {
		fmt.Printf("Score: %v - Link: http://twitter.com/random/status/%v\n", tweet.Score, tweet.Id)
	}
	fmt.Printf("\n")
}
<|endoftext|>


<commit_before>// Copyright 2021 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aggrmetrics

import (
	"context"
	"sync"
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/tsmon"
	"go.chromium.org/luci/cv/internal/common"
	"go.chromium.org/luci/server/tq"
)

// reportTTL limits for how long the data remains valid.
//
// If tsmon doesn't flush for this much time since the report is prepared,
// the report will be discarded.
//
// It should be longer than a typical tsmon flush interval and should account
// the fact that Driver.MinuteCron() and tsmon flush aren't synchronized.
const reportTTL = 2 * time.Minute

// New creates a new Driver for metrics aggregation.
func New(ctx context.Context, tqd *tq.Dispatcher) *Driver {
	d := &Driver{
		aggregators: []aggregator{
			&runsAggregator{},
		},
		lastFlush: clock.Now(ctx),
	}
	tsmon.RegisterCallbackIn(ctx, d.tsmonCallback)
	return d
}

// Driver takes care of invoking aggregators and correctly working with
// aggregated metrics, specifically resetting them after they are sent to avoid
// tsmon continuously sending of old values long after they were computed.
type Driver struct {
	aggregators []aggregator

	m              sync.Mutex
	nextReports    []reportFunc
	nextExpireTime time.Time
	lastFlush      time.Time
}

// Cron is expected to be called once per minute, e.g., by GAE cron.
//
// Although there can be more than 1 CV process, e.g., 2+ GAE instances,
// it's expected that cron will call at most 1 CV process at a time.
func (d *Driver) Cron(ctx context.Context) error {
	// Use short timeout to make overlap less likely.
	ctx, cancel := clock.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	active, err := activeProjects(ctx)
	if err != nil {
		return err
	}

	startTime := clock.Now(ctx)

	reports := make([]reportFunc, len(d.aggregators))
	errs := errors.NewLazyMultiError(len(d.aggregators))
	var wg sync.WaitGroup
	wg.Add(len(d.aggregators))
	for i, a := range d.aggregators {
		i, a := i, a
		go func() {
			defer wg.Done()
			switch f, err := a.prepare(ctx, active); {
			case err != nil:
				errs.Assign(i, err)
			default:
				reports[i] = f
			}
		}()
	}
	wg.Wait()

	// Save successfully produced reports regardless of errors.
	if reports = removeNils(reports); len(reports) > 0 {
		d.stageReports(ctx, reports, startTime)
	}
	return common.MostSevereError(errs.Get())
}

func (d *Driver) stageReports(ctx context.Context, reports []reportFunc, start time.Time) {
	d.m.Lock()
	defer d.m.Unlock()
	expire := start.Add(reportTTL)
	// Ensure we aren't overwriting newer report, just in case the prior cron
	// invocation somehow got stuck, e.g. due to a buggy aggregator.
	if !d.nextExpireTime.IsZero() {
		lastStagedStart := d.nextExpireTime.Add(-reportTTL)
		if lastStagedStart.After(start) {
			logging.Errorf(ctx, "aggrmetrics.MinuteCron was stuck since %s, newer report %s is already prepared", start, lastStagedStart)
			return
		}
		logging.Warningf(ctx, "aggrmetrics.MinuteCron overwriting unsent report of %s with %s", lastStagedStart, start)
		// Overwriting report means that data points are lost. This is fine once in
		// a while, but bad if this happens all the time on a GAE instance.
		// Detect the latter by checking when the last Flush happened.
		if delay := clock.Since(ctx, d.lastFlush); delay >= reportTTL {
			logging.Errorf(ctx, "aggrmetrics weren't flushed for a long time: %s", delay)
		}
	}
	d.nextReports = reports
	d.nextExpireTime = expire
}

// tsmonCallback resets old data from registered metrics and possibly sets new
// data.
//
// It's called by tsmon flush implementation on all CV processes,
// but the new values should normally be set on just one of them on whichever
// MinuteCron() was called last.
func (d *Driver) tsmonCallback(ctx context.Context) {
	d.m.Lock()
	defer d.m.Unlock()
	// In all cases, reset all metrics.
	d.resetMetrics(ctx)

	now := clock.Now(ctx)
	d.lastFlush = now
	// Decide if a report should be made.
	switch {
	case d.nextExpireTime.IsZero():
		return
	case d.nextExpireTime.Before(now):
		logging.Warningf(ctx, "aggrmetrics dropping expired report of %s", d.nextExpireTime)
	default:
		// Do the reporting.
		for _, f := range d.nextReports {
			f(ctx)
		}
	}
	d.nextExpireTime = time.Time{}
	d.nextReports = nil
}

func (d *Driver) resetMetrics(ctx context.Context) {
	store := tsmon.GetState(ctx).Store()
	for _, a := range d.aggregators {
		for _, m := range a.metrics() {
			store.Reset(ctx, m)
		}
	}
}
<commit_msg>[cv][aggrmetrics] tune error condition.<commit_after>// Copyright 2021 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aggrmetrics

import (
	"context"
	"sync"
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/tsmon"
	"go.chromium.org/luci/cv/internal/common"
	"go.chromium.org/luci/server/tq"
)

const (
	// reportTTL limits for how long the data remains valid.
	//
	// If tsmon doesn't flush for this much time since the report is prepared,
	// the report will be discarded.
	//
	// It should be longer than a typical tsmon flush interval and should account
	// the fact that Driver.MinuteCron() and tsmon flush aren't synchronized.
	reportTTL = 2 * time.Minute

	// noFlushForTooLong defines threshold for emiting error to logs on unusually
	// long durations without a tsmon flush.
	noFlushForTooLong = 4 * time.Minute
)

// New creates a new Driver for metrics aggregation.
func New(ctx context.Context, tqd *tq.Dispatcher) *Driver {
	d := &Driver{
		aggregators: []aggregator{
			&runsAggregator{},
		},
		lastFlush: clock.Now(ctx),
	}
	tsmon.RegisterCallbackIn(ctx, d.tsmonCallback)
	return d
}

// Driver takes care of invoking aggregators and correctly working with
// aggregated metrics, specifically resetting them after they are sent to avoid
// tsmon continuously sending of old values long after they were computed.
type Driver struct {
	aggregators []aggregator

	m              sync.Mutex
	nextReports    []reportFunc
	nextExpireTime time.Time
	lastFlush      time.Time
}

// Cron is expected to be called once per minute, e.g., by GAE cron.
//
// Although there can be more than 1 CV process, e.g., 2+ GAE instances,
// it's expected that cron will call at most 1 CV process at a time.
func (d *Driver) Cron(ctx context.Context) error {
	// Use short timeout to make overlap less likely.
	ctx, cancel := clock.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	active, err := activeProjects(ctx)
	if err != nil {
		return err
	}

	startTime := clock.Now(ctx)

	reports := make([]reportFunc, len(d.aggregators))
	errs := errors.NewLazyMultiError(len(d.aggregators))
	var wg sync.WaitGroup
	wg.Add(len(d.aggregators))
	for i, a := range d.aggregators {
		i, a := i, a
		go func() {
			defer wg.Done()
			switch f, err := a.prepare(ctx, active); {
			case err != nil:
				errs.Assign(i, err)
			default:
				reports[i] = f
			}
		}()
	}
	wg.Wait()

	// Save successfully produced reports regardless of errors.
	if reports = removeNils(reports); len(reports) > 0 {
		d.stageReports(ctx, reports, startTime)
	}
	return common.MostSevereError(errs.Get())
}

func (d *Driver) stageReports(ctx context.Context, reports []reportFunc, start time.Time) {
	d.m.Lock()
	defer d.m.Unlock()
	expire := start.Add(reportTTL)
	// Ensure we aren't overwriting newer report, just in case the prior cron
	// invocation somehow got stuck, e.g. due to a buggy aggregator.
	if !d.nextExpireTime.IsZero() {
		lastStagedStart := d.nextExpireTime.Add(-reportTTL)
		if lastStagedStart.After(start) {
			logging.Errorf(ctx, "aggrmetrics.MinuteCron was stuck since %s, newer report %s is already prepared", start, lastStagedStart)
			return
		}
		logging.Warningf(ctx, "aggrmetrics.MinuteCron overwriting unsent report of %s with %s", lastStagedStart, start)
		// Overwriting report means that data points are lost. This is fine once in
		// a while, but bad if this happens all the time on a GAE instance.
		// Detect the latter by checking when the last Flush happened.
		if delay := clock.Since(ctx, d.lastFlush); delay > noFlushForTooLong {
			logging.Errorf(ctx, "aggrmetrics weren't flushed for a long time: %s", delay)
		}
	}
	d.nextReports = reports
	d.nextExpireTime = expire
}

// tsmonCallback resets old data from registered metrics and possibly sets new
// data.
//
// It's called by tsmon flush implementation on all CV processes,
// but the new values should normally be set on just one of them on whichever
// MinuteCron() was called last.
func (d *Driver) tsmonCallback(ctx context.Context) {
	d.m.Lock()
	defer d.m.Unlock()
	// In all cases, reset all metrics.
	d.resetMetrics(ctx)

	now := clock.Now(ctx)
	d.lastFlush = now
	// Decide if a report should be made.
	switch {
	case d.nextExpireTime.IsZero():
		return
	case d.nextExpireTime.Before(now):
		logging.Warningf(ctx, "aggrmetrics dropping expired report of %s", d.nextExpireTime)
	default:
		// Do the reporting.
		for _, f := range d.nextReports {
			f(ctx)
		}
	}
	d.nextExpireTime = time.Time{}
	d.nextReports = nil
}

func (d *Driver) resetMetrics(ctx context.Context) {
	store := tsmon.GetState(ctx).Store()
	for _, a := range d.aggregators {
		for _, m := range a.metrics() {
			store.Reset(ctx, m)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"k8s.io/test-infra/prow/config"

	s "istio.io/test-infra/sisyphus"
	u "istio.io/test-infra/toolbox/util"
)

var (
	owner                 = flag.String("owner", "istio", "Github owner or org")
	tokenFile             = flag.String("token_file", "", "File containing Github API Access Token")
	op                    = flag.String("op", "", "Operation to be performed")
	repo                  = flag.String("repo", "", "Repository to which op is applied")
	baseBranch            = flag.String("base_branch", "", "Branch to which op is applied")
	refSHA                = flag.String("ref_sha", "", "Reference commit SHA used to update base branch")
	hub                   = flag.String("hub", "", "Hub of the docker images")
	tag                   = flag.String("tag", "", "Tag of the release candidate")
	releaseOrg            = flag.String("rel_org", "istio-releases", "GitHub Release Org")
	gcsPath               = flag.String("gcs_path", "", "The path to the GCS bucket")
	skip                  = flag.String("skip", "", "comma separated list of jobs to skip")
	maxCommitDepth        = flag.Int("max_commit_depth", 200, "Max number of commits before HEAD to check if green")
	maxRunDepth           = flag.Int("max_run_depth", 500, "Max number of runs before the latest one of which results are checked")
	maxConcurrentRequests = flag.Int("max_concurrent_reqs", 50, "Max number of concurrent requests permitted")
	githubClnt            *u.GithubClient
	ghClntRel             *u.GithubClient
)

const (
	masterBranch = "master"
	// Prow
	prowProject   = "istio-testing"
	prowZone      = "us-west1-a"
	gubernatorURL = "https://k8s-gubernator.appspot.com/build/istio-prow"
	gcsBucket     = "istio-prow"
	// release qualification trigger
	relQualificationPRTtilePrefix = "Release Qualification"
	greenBuildVersionFile         = "greenBuild.VERSION"
	dailyRepo                     = "daily-release"
)

func fastForward(repo, baseBranch, refSHA *string) error {
	u.AssertNotEmpty("repo", repo)
	u.AssertNotEmpty("base_branch", baseBranch)
	u.AssertNotEmpty("ref_sha", refSHA)
	isAncestor, err := githubClnt.SHAIsAncestorOfBranch(*repo, masterBranch, *refSHA)
	if err != nil {
		return err
	}
	if !isAncestor {
		glog.Infof("SHA %s is not an ancestor of branch %s, resorts to no-op\n", *refSHA, masterBranch)
		return nil
	}
	return githubClnt.FastForward(*repo, *baseBranch, *refSHA)
}

type task struct {
	job       string
	runNumber int
}

// preprocessProwResults downloads the most recent prow results up to maxRunDepth
// then returns a two-level map job -> sha -> passed (true) or failed (false)
func preprocessProwResults(postSubmitJobs map[string]struct{}) map[string]map[string]bool {
	glog.Infof("Start preprocessing prow results")
	prowAccessor := s.NewProwAccessor(
		prowProject,
		prowZone,
		gubernatorURL,
		gcsBucket,
		u.NewGCSClient(gcsBucket))
	cache := make(map[string]map[string]bool)
	tasksCh := make(chan *task, *maxConcurrentRequests)
	var wg sync.WaitGroup
	mutex := &sync.Mutex{}
	for i := 0; i < *maxConcurrentRequests; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				t, more := <-tasksCh
				if !more {
					break
				}
				result, err := prowAccessor.GetResult(t.job, t.runNumber)
				if err != nil {
					glog.V(1).Infof("failed to get result of %s at run number %d. Skip.", t.job, t.runNumber)
					continue
				}
				mutex.Lock()
				cache[t.job][result.SHA] = result.Passed
				mutex.Unlock()
			}
		}()
	}
	// if postSubmitJobs is empty, the for loop exits immediately
	for job := range postSubmitJobs {
		cache[job] = make(map[string]bool)
		runNumber, err := prowAccessor.GetLatestRun(job)
		if err != nil {
			glog.Fatalf("failed to get latest run number of %s: %v", job, err)
		}
		// download the most recent prow results up to maxRunDepth
		for i := 0; i < *maxRunDepth; i++ {
			tasksCh <- &task{job, runNumber}
			runNumber--
		}
	}
	close(tasksCh)
	wg.Wait()
	return cache
}

func getLatestGreenSHA() (string, error) {
	u.AssertNotEmpty("repo", repo)
	u.AssertNotEmpty("base_branch", baseBranch)
	u.AssertPositive("max_commit_depth", maxCommitDepth)
	u.AssertPositive("max_run_depth", maxRunDepth)
	postSubmitJobs := readPostsubmitListFromProwConfig(*owner, *repo, *baseBranch)
	skipJobs := strings.Split(*skip, ",")
	for _, skipedJob := range skipJobs {
		delete(postSubmitJobs, skipedJob)
	}
	results := preprocessProwResults(postSubmitJobs)
	sha, err := githubClnt.GetHeadCommitSHA(*repo, *baseBranch)
	if err != nil {
		glog.Fatalf("failed to get the head commit sha of %s/%s: %v", *repo, *baseBranch, err)
	}
	for i := 0; i < *maxCommitDepth; i++ {
		glog.Infof("Checking if [%s] passed all checks. %d commits before HEAD", sha, i)
		allChecksPassed := true
		for job := range postSubmitJobs {
			passed, keyExists := results[job][sha]
			if !keyExists {
				glog.V(1).Infof("Results unknown in local cache for [%s] at [%s], treat the test as failed", job, sha)
			}
			if !passed {
				glog.Infof("[%s] failed on [%s]", sha, job)
				allChecksPassed = false
			}
		}
		if allChecksPassed {
			glog.Infof("Found latest green sha [%s] for %s/%s", sha, *repo, *baseBranch)
			return sha, nil
		}
		parentSHA, err := githubClnt.GetParentSHA(*repo, *baseBranch, sha)
		if err != nil {
			glog.Fatalf("failed to find the parent sha of %s in %s/%s", sha, *repo, *baseBranch)
		}
		sha = parentSHA
	}
	return "", fmt.Errorf("exceeded max commit depth")
}

// DailyReleaseQualification triggers test jobs buy creating a PR that generates
// a GitHub notification. It blocks until PR status is known and returns nonzero
// value if failure. Links to test logs will also be logged to console.
func DailyReleaseQualification(baseBranch *string) error {
	u.AssertNotEmpty("hub", hub) // TODO (chx) default value of hub
	u.AssertNotEmpty("tag", tag)
	u.AssertNotEmpty("gcs_path", gcsPath)
	var dstBranch string
	// we could have made baseBranch have a default value, but that breaks all the places
	// where baseBranch must be passed in cmdline and a default value is not acceptable
	// therefore, if a branch is not passed in use masterBranch as the default destination
	if baseBranch != nil && len(*baseBranch) != 0 {
		dstBranch = *baseBranch
	} else {
		dstBranch = masterBranch
	}
	glog.Infof("Creating PR to trigger release qualifications on %s branch\n", dstBranch)
	prTitle := fmt.Sprintf("%s - %s", relQualificationPRTtilePrefix, *tag)
	prBody := "This is a generated PR that triggers release qualification tests, and will be automatically merged " +
		"if all tests pass. In case some test fails, you can manually rerun the failing tests using /test. Force " +
		"merging this PR will suppress the test failures and let the release pipeline continue."
	timestamp := fmt.Sprintf("%v", time.Now().UnixNano())
	srcBranch := "relQual_" + timestamp
	edit := func() error {
		if err := u.UpdateKeyValueInFile(greenBuildVersionFile, "HUB", *hub); err != nil {
			return err
		}
		if err := u.UpdateKeyValueInFile(greenBuildVersionFile, "TAG", *tag); err != nil {
			return err
		}
		if err := u.UpdateKeyValueInFile(greenBuildVersionFile, "TIME", timestamp); err != nil {
			return err
		}
		if err := u.UpdateKeyValueInFile(greenBuildVersionFile, "ISTIO_REL_URL",
			fmt.Sprintf("https://storage.googleapis.com/%s", *gcsPath)); err != nil {
			return err
		}
		return nil
	}
	pr, err := ghClntRel.CreatePRUpdateRepo(srcBranch, dstBranch, dailyRepo, prTitle, prBody, edit)
	if err != nil {
		return err
	}
	defer func() {
		glog.Infof("Close the PR and delete its branch\n")
		if e := ghClntRel.ClosePRDeleteBranch(dailyRepo, pr); e != nil {
			glog.Infof("Error in ClosePRDeleteBranch: %v\n", e)
		}
	}()

	verbose := true
	ci := u.NewCIState()
	retryDelay := 5 * time.Minute
	maxWait := 20 * time.Hour
	totalRetries := int(maxWait / retryDelay)
	glog.Infof("Waiting for all jobs starting. Results Polling starts in %v.\n", retryDelay)
	time.Sleep(retryDelay)

	err = u.Poll(retryDelay, totalRetries, func() (bool, error) {
		pr, err = ghClntRel.GetPR(dailyRepo, *pr.Number)
		if err != nil {
			return true, err
		}
		if *pr.Merged {
			// PR is apparently closed manually. Exit the loop.
			glog.Infof("pr was manually merged.\n")
			return true, nil
		}
		if *pr.State == "closed" {
			// PR is apparently closed manually. Exit the loop.
			return false, fmt.Errorf("pr close was manually closed")
		}

		status, errPoll := ghClntRel.GetPRTestResults(dailyRepo, pr, verbose)
		verbose = false
		if errPoll != nil {
			return false, errPoll
		}
		exitPolling := false
		switch status {
		case ci.Success:
			exitPolling = true
			glog.Infof("Auto merging this PR to update daily release\n")
			errPoll = ghClntRel.MergePR(dailyRepo, pr)
			*pr.Merged = true

		case ci.Pending:
			glog.Infof("Results still pending. Will check again in %v.\n", retryDelay)
		case ci.Error:
		case ci.Failure:
			// Go back to sleep until timeout, so that release engineer can potentially suppress test failure or retest
			// in github directly.
		}
		return exitPolling, errPoll
	})
	// Fail to poll or merge
	if err != nil {
		return err
	}
	// In case the PR is merged manually or automatically
	if *pr.Merged {
		return nil
	}
	return fmt.Errorf("release qualification failed")
}

func traverseJobTree(job config.Postsubmit, postsubmitJobs map[string]struct{}, targetBranch string) {
	if job.Brancher.RunsAgainstBranch(targetBranch) {
		postsubmitJobs[job.Name] = struct{}{}
		if len(job.RunAfterSuccess) > 0 {
			for _, childJob := range job.RunAfterSuccess {
				traverseJobTree(childJob, postsubmitJobs, targetBranch)
			}
		}
	}
}

func readPostsubmitListFromProwConfig(org, repo, branch string) map[string]struct{} {
	postsubmitJobs := make(map[string]struct{})
	repoRootDir, err := u.Shell("git rev-parse --show-toplevel")
	repoRootDir = strings.TrimSuffix(repoRootDir, "\n")
	if err != nil {
		glog.Fatalf("cannot find repo root directory path")
	}
	prowConfigYaml := filepath.Join(repoRootDir, "prow/config.yaml")
	config, err := config.Load(prowConfigYaml, "")
	if err != nil {
		glog.Fatalf("could not read configs: %v", err)
	}

	for _, job := range config.Postsubmits[fmt.Sprintf("%s/%s", org, repo)] {
		traverseJobTree(job, postsubmitJobs, branch)
	}
	return postsubmitJobs
}

func init() {
	flag.Parse()
	u.AssertNotEmpty("token_file", tokenFile)
	token, err := u.GetAPITokenFromFile(*tokenFile)
	if err != nil {
		glog.Fatalf("Error accessing user supplied token_file: %v\n", err)
	}
	githubClnt = u.NewGithubClient(*owner, token)
	// a new github client is created for istio-releases org
	ghClntRel = u.NewGithubClient(*releaseOrg, token)
}

func main() {
	switch *op {
	case "fastForward":
		if err := fastForward(repo, baseBranch, refSHA); err != nil {
			glog.Infof("Error during fastForward: %v\n", err)
		}
	case "dailyRelQual":
		if err := DailyReleaseQualification(baseBranch); err != nil {
			glog.Infof("Error during DailyReleaseQualification: %v\n", err)
			os.Exit(1)
		}
	case "getLatestGreenSHA":
		latestGreenSHA, err := getLatestGreenSHA()
		if err != nil {
			glog.Infof("Error during getLatestGreenSHA: %v\n", err)
			os.Exit(1)
		}
		fmt.Printf("%s", latestGreenSHA)
	default:
		glog.Infof("Unsupported operation: %s\n", *op)
	}
}
<commit_msg>Remove merging logic in githubctl as it is already covered by merge bot. (#927)<commit_after>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"k8s.io/test-infra/prow/config"

	s "istio.io/test-infra/sisyphus"
	u "istio.io/test-infra/toolbox/util"
)

var (
	owner                 = flag.String("owner", "istio", "Github owner or org")
	tokenFile             = flag.String("token_file", "", "File containing Github API Access Token")
	op                    = flag.String("op", "", "Operation to be performed")
	repo                  = flag.String("repo", "", "Repository to which op is applied")
	baseBranch            = flag.String("base_branch", "", "Branch to which op is applied")
	refSHA                = flag.String("ref_sha", "", "Reference commit SHA used to update base branch")
	hub                   = flag.String("hub", "", "Hub of the docker images")
	tag                   = flag.String("tag", "", "Tag of the release candidate")
	releaseOrg            = flag.String("rel_org", "istio-releases", "GitHub Release Org")
	gcsPath               = flag.String("gcs_path", "", "The path to the GCS bucket")
	skip                  = flag.String("skip", "", "comma separated list of jobs to skip")
	maxCommitDepth        = flag.Int("max_commit_depth", 200, "Max number of commits before HEAD to check if green")
	maxRunDepth           = flag.Int("max_run_depth", 500, "Max number of runs before the latest one of which results are checked")
	maxConcurrentRequests = flag.Int("max_concurrent_reqs", 50, "Max number of concurrent requests permitted")
	githubClnt            *u.GithubClient
	ghClntRel             *u.GithubClient
)

const (
	masterBranch = "master"
	// Prow
	prowProject   = "istio-testing"
	prowZone      = "us-west1-a"
	gubernatorURL = "https://k8s-gubernator.appspot.com/build/istio-prow"
	gcsBucket     = "istio-prow"
	// release qualification trigger
	relQualificationPRTtilePrefix = "Release Qualification"
	greenBuildVersionFile         = "greenBuild.VERSION"
	dailyRepo                     = "daily-release"
)

func fastForward(repo, baseBranch, refSHA *string) error {
	u.AssertNotEmpty("repo", repo)
	u.AssertNotEmpty("base_branch", baseBranch)
	u.AssertNotEmpty("ref_sha", refSHA)
	isAncestor, err := githubClnt.SHAIsAncestorOfBranch(*repo, masterBranch, *refSHA)
	if err != nil {
		return err
	}
	if !isAncestor {
		glog.Infof("SHA %s is not an ancestor of branch %s, resorts to no-op\n", *refSHA, masterBranch)
		return nil
	}
	return githubClnt.FastForward(*repo, *baseBranch, *refSHA)
}

type task struct {
	job       string
	runNumber int
}

// preprocessProwResults downloads the most recent prow results up to maxRunDepth
// then returns a two-level map job -> sha -> passed (true) or failed (false)
func preprocessProwResults(postSubmitJobs map[string]struct{}) map[string]map[string]bool {
	glog.Infof("Start preprocessing prow results")
	prowAccessor := s.NewProwAccessor(
		prowProject,
		prowZone,
		gubernatorURL,
		gcsBucket,
		u.NewGCSClient(gcsBucket))
	cache := make(map[string]map[string]bool)
	tasksCh := make(chan *task, *maxConcurrentRequests)
	var wg sync.WaitGroup
	mutex := &sync.Mutex{}
	for i := 0; i < *maxConcurrentRequests; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				t, more := <-tasksCh
				if !more {
					break
				}
				result, err := prowAccessor.GetResult(t.job, t.runNumber)
				if err != nil {
					glog.V(1).Infof("failed to get result of %s at run number %d. Skip.", t.job, t.runNumber)
					continue
				}
				mutex.Lock()
				cache[t.job][result.SHA] = result.Passed
				mutex.Unlock()
			}
		}()
	}
	// if postSubmitJobs is empty, the for loop exits immediately
	for job := range postSubmitJobs {
		cache[job] = make(map[string]bool)
		runNumber, err := prowAccessor.GetLatestRun(job)
		if err != nil {
			glog.Fatalf("failed to get latest run number of %s: %v", job, err)
		}
		// download the most recent prow results up to maxRunDepth
		for i := 0; i < *maxRunDepth; i++ {
			tasksCh <- &task{job, runNumber}
			runNumber--
		}
	}
	close(tasksCh)
	wg.Wait()
	return cache
}

func getLatestGreenSHA() (string, error) {
	u.AssertNotEmpty("repo", repo)
	u.AssertNotEmpty("base_branch", baseBranch)
	u.AssertPositive("max_commit_depth", maxCommitDepth)
	u.AssertPositive("max_run_depth", maxRunDepth)
	postSubmitJobs := readPostsubmitListFromProwConfig(*owner, *repo, *baseBranch)
	skipJobs := strings.Split(*skip, ",")
	for _, skipedJob := range skipJobs {
		delete(postSubmitJobs, skipedJob)
	}
	results := preprocessProwResults(postSubmitJobs)
	sha, err := githubClnt.GetHeadCommitSHA(*repo, *baseBranch)
	if err != nil {
		glog.Fatalf("failed to get the head commit sha of %s/%s: %v", *repo, *baseBranch, err)
	}
	for i := 0; i < *maxCommitDepth; i++ {
		glog.Infof("Checking if [%s] passed all checks. %d commits before HEAD", sha, i)
		allChecksPassed := true
		for job := range postSubmitJobs {
			passed, keyExists := results[job][sha]
			if !keyExists {
				glog.V(1).Infof("Results unknown in local cache for [%s] at [%s], treat the test as failed", job, sha)
			}
			if !passed {
				glog.Infof("[%s] failed on [%s]", sha, job)
				allChecksPassed = false
			}
		}
		if allChecksPassed {
			glog.Infof("Found latest green sha [%s] for %s/%s", sha, *repo, *baseBranch)
			return sha, nil
		}
		parentSHA, err := githubClnt.GetParentSHA(*repo, *baseBranch, sha)
		if err != nil {
			glog.Fatalf("failed to find the parent sha of %s in %s/%s", sha, *repo, *baseBranch)
		}
		sha = parentSHA
	}
	return "", fmt.Errorf("exceeded max commit depth")
}

// DailyReleaseQualification triggers test jobs buy creating a PR that generates
// a GitHub notification. It blocks until PR status is known and returns nonzero
// value if failure. Links to test logs will also be logged to console.
func DailyReleaseQualification(baseBranch *string) error {
	u.AssertNotEmpty("hub", hub) // TODO (chx) default value of hub
	u.AssertNotEmpty("tag", tag)
	u.AssertNotEmpty("gcs_path", gcsPath)
	var dstBranch string
	// we could have made baseBranch have a default value, but that breaks all the places
	// where baseBranch must be passed in cmdline and a default value is not acceptable
	// therefore, if a branch is not passed in use masterBranch as the default destination
	if baseBranch != nil && len(*baseBranch) != 0 {
		dstBranch = *baseBranch
	} else {
		dstBranch = masterBranch
	}
	glog.Infof("Creating PR to trigger release qualifications on %s branch\n", dstBranch)
	prTitle := fmt.Sprintf("%s - %s", relQualificationPRTtilePrefix, *tag)
	prBody := "This is a generated PR that triggers release qualification tests, and will be automatically merged " +
		"if all tests pass. In case some test fails, you can manually rerun the failing tests using /test. Force " +
		"merging this PR will suppress the test failures and let the release pipeline continue."
	timestamp := fmt.Sprintf("%v", time.Now().UnixNano())
	srcBranch := "relQual_" + timestamp
	edit := func() error {
		if err := u.UpdateKeyValueInFile(greenBuildVersionFile, "HUB", *hub); err != nil {
			return err
		}
		if err := u.UpdateKeyValueInFile(greenBuildVersionFile, "TAG", *tag); err != nil {
			return err
		}
		if err := u.UpdateKeyValueInFile(greenBuildVersionFile, "TIME", timestamp); err != nil {
			return err
		}
		if err := u.UpdateKeyValueInFile(greenBuildVersionFile, "ISTIO_REL_URL",
			fmt.Sprintf("https://storage.googleapis.com/%s", *gcsPath)); err != nil {
			return err
		}
		return nil
	}
	pr, err := ghClntRel.CreatePRUpdateRepo(srcBranch, dstBranch, dailyRepo, prTitle, prBody, edit)
	if err != nil {
		return err
	}
	defer func() {
		glog.Infof("Close the PR and delete its branch\n")
		if e := ghClntRel.ClosePRDeleteBranch(dailyRepo, pr); e != nil {
			glog.Infof("Error in ClosePRDeleteBranch: %v\n", e)
		}
	}()

	retryDelay := 5 * time.Minute
	maxWait := 20 * time.Hour
	totalRetries := int(maxWait / retryDelay)
	glog.Infof("Waiting for all jobs starting. Results Polling starts in %v.\n", retryDelay)
	time.Sleep(retryDelay)

	err = u.Poll(retryDelay, totalRetries, func() (bool, error) {
		pr, err = ghClntRel.GetPR(dailyRepo, *pr.Number)
		if err != nil {
			return true, err
		}
		if *pr.Merged {
			// PR is already merged. Either all tests have passed or it is manually merged. Exit the loop.
			glog.Infof("pr was has been merged.\n")
			return true, nil
		}
		if *pr.State == "closed" {
			// PR is closed and not merged, which is a signal the qualification is abandoned.
			glog.Infof("pr has been closed.\n")
			return true, nil
		}
		return false, nil
	})
	// Fail to poll
	if err != nil {
		return err
	}
	// In case the PR is merged manually or automatically
	if *pr.Merged {
		return nil
	}
	return fmt.Errorf("release qualification failed")
}

func traverseJobTree(job config.Postsubmit, postsubmitJobs map[string]struct{}, targetBranch string) {
	if job.Brancher.RunsAgainstBranch(targetBranch) {
		postsubmitJobs[job.Name] = struct{}{}
		if len(job.RunAfterSuccess) > 0 {
			for _, childJob := range job.RunAfterSuccess {
				traverseJobTree(childJob, postsubmitJobs, targetBranch)
			}
		}
	}
}

func readPostsubmitListFromProwConfig(org, repo, branch string) map[string]struct{} {
	postsubmitJobs := make(map[string]struct{})
	repoRootDir, err := u.Shell("git rev-parse --show-toplevel")
	repoRootDir = strings.TrimSuffix(repoRootDir, "\n")
	if err != nil {
		glog.Fatalf("cannot find repo root directory path")
	}
	prowConfigYaml := filepath.Join(repoRootDir, "prow/config.yaml")
	config, err := config.Load(prowConfigYaml, "")
	if err != nil {
		glog.Fatalf("could not read configs: %v", err)
	}

	for _, job := range config.Postsubmits[fmt.Sprintf("%s/%s", org, repo)] {
		traverseJobTree(job, postsubmitJobs, branch)
	}
	return postsubmitJobs
}

func init() {
	flag.Parse()
	u.AssertNotEmpty("token_file", tokenFile)
	token, err := u.GetAPITokenFromFile(*tokenFile)
	if err != nil {
		glog.Fatalf("Error accessing user supplied token_file: %v\n", err)
	}
	githubClnt = u.NewGithubClient(*owner, token)
	// a new github client is created for istio-releases org
	ghClntRel = u.NewGithubClient(*releaseOrg, token)
}

func main() {
	switch *op {
	case "fastForward":
		if err := fastForward(repo, baseBranch, refSHA); err != nil {
			glog.Infof("Error during fastForward: %v\n", err)
		}
	case "dailyRelQual":
		if err := DailyReleaseQualification(baseBranch); err != nil {
			glog.Infof("Error during DailyReleaseQualification: %v\n", err)
			os.Exit(1)
		}
	case "getLatestGreenSHA":
		latestGreenSHA, err := getLatestGreenSHA()
		if err != nil {
			glog.Infof("Error during getLatestGreenSHA: %v\n", err)
			os.Exit(1)
		}
		fmt.Printf("%s", latestGreenSHA)
	default:
		glog.Infof("Unsupported operation: %s\n", *op)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"encoding/json"
	"io/ioutil"
	"errors"
	"golang.org/x/crypto/bcrypt"
	"os"
	"github.com/roypur/goapi/src"
)

type MACAddress [6]byte

// A MagicPacket is constituted of 6 bytes of 0xFF followed by
// 16 groups of the destination MAC address.

type MagicPacket struct {
    header  [6]byte
    payload [16]MACAddress
}

var args map[int]string;

func main(){

    args = make(map[int]string);

    for i := 0;  i < 7; i++{
        if(len(os.Args) > i){
            args[i] = os.Args[i];
        }else{
            args[i] = "";
        }
    }

    setPath();

    getConf();

    if((args[1] == "delete") || (args[1] == "del")){
        if(args[2] == "user"){
            modUser(args[1]);
        }else if(args[2] == "host"){
            modHost(args[1]);
        }else{
            showHelp();
        }
    }else if(((args[1] == "add") || (args[1] == "edit")) && ((len(args[3]) > 0) && (len(args[4]) > 0))){
        if(args[2] == "user"){
            modUser(args[1]);
        }else if(args[2] == "host"){
            modHost(args[1]);
        }else{
            showHelp();
        }
    }else if(args[1] == "server"){
        goapi.Listen(httpHandler, config["listen"]);
    }else{
        showHelp();
    }
}

func httpHandler(req goapi.Request){

        var status string;

        var user string = req.Header["Goapi-User"]

        var pass string = req.Header["Goapi-Pass"]

        var host string = req.Header["Goapi-Host"]

        var list bool = req.Header["Goapi-List"] == "true"

        if(isUser(user, pass) == nil){
            if(list){
                //var jsonList string;

                var list []string = []string{}

                for k,_ := range getHosts(){
                    list = append(list,k)
                }

                b,err := json.Marshal(list)

                if(err==nil){
                    status = string(b)
                }else{
                    status = "no-hosts-found"
                }
            }else{
                if(sendPacket(host) == nil){
                    status = "ok";
                }else{
                    status = "fail";
                }
            }
        }else{
            status = "denied";
        }
        req.Write(goapi.Ok)
        req.Write(req.Resp)
        req.Write("\r\n\r\n")
        req.Write(status)
        req.Close()

}

func showHelp(){
    fmt.Println("\n");
    fmt.Println("# # # # # # # # # # # # # # # # # # # # # #");
    fmt.Println("#                                         #");
    fmt.Println("#  Usage:                                 #");
    fmt.Println("#                                         #");
    fmt.Println("#  dir <conf-dir> (optional)              #");
    fmt.Println("#                                         #");
    fmt.Println("#  add|edit|delete|server                 #");
    fmt.Println("#                                         #");
    fmt.Println("#  user|host <username>|<computer-name>   #");
    fmt.Println("#                                         #");
    fmt.Println("#  <password>|<mac-address>               #");
    fmt.Println("#                                         #");
    fmt.Println("# # # # # # # # # # # # # # # # # # # # # #");
    fmt.Println("\n");
}

func getHost(host string)(string,error){

    hosts := getHosts()

    if(len(hosts[host]) > 0){
        return hosts[host], nil;
    }
    return "", errors.New("host not found" + host);
}

func getHosts()(map[string]string){

    file, err := ioutil.ReadFile(configPath + "/hosts.json");

    hosts := make(map[string]string);

    if(err == nil){
        err = json.Unmarshal(file, &hosts);

        if(err == nil){
            return hosts;
        }
    }
    return nil;
}

func isUser(user string, pass string)(error){

    file, err := ioutil.ReadFile(configPath + "/users.json");
    users := make(map[string]string);

    if(err == nil){
        err = json.Unmarshal(file, &users);
        if((err == nil)  && (len(users[user]) > 0)){

            err = bcrypt.CompareHashAndPassword([]byte(users[user]), []byte(pass));

            if(err == nil){
                return nil;
            }
        }
    }
    return errors.New("user not found" + user);

}
<commit_msg>use ssl<commit_after>package main

import (
	"fmt"
	"encoding/json"
	"io/ioutil"
	"errors"
	"golang.org/x/crypto/bcrypt"
	"os"
	"github.com/roypur/goapi/src"
)

type MACAddress [6]byte

// A MagicPacket is constituted of 6 bytes of 0xFF followed by
// 16 groups of the destination MAC address.

type MagicPacket struct {
    header  [6]byte
    payload [16]MACAddress
}

var args map[int]string;

func main(){

    args = make(map[int]string);

    for i := 0;  i < 7; i++{
        if(len(os.Args) > i){
            args[i] = os.Args[i];
        }else{
            args[i] = "";
        }
    }

    setPath();

    getConf();

    if((args[1] == "delete") || (args[1] == "del")){
        if(args[2] == "user"){
            modUser(args[1]);
        }else if(args[2] == "host"){
            modHost(args[1]);
        }else{
            showHelp();
        }
    }else if(((args[1] == "add") || (args[1] == "edit")) && ((len(args[3]) > 0) && (len(args[4]) > 0))){
        if(args[2] == "user"){
            modUser(args[1]);
        }else if(args[2] == "host"){
            modHost(args[1]);
        }else{
            showHelp();
        }
    }else if(args[1] == "server"){
        goapi.Listen(httpHandler, config["listen"], config["cert"], config["key"]);
    }else{
        showHelp();
    }
}

func httpHandler(req goapi.Request){

        var status string;

        var user string = req.Header["Goapi-User"]

        var pass string = req.Header["Goapi-Pass"]

        var host string = req.Header["Goapi-Host"]

        var list bool = req.Header["Goapi-List"] == "true"

        if(isUser(user, pass) == nil){
            if(list){
                //var jsonList string;

                var list []string = []string{}

                for k,_ := range getHosts(){
                    list = append(list,k)
                }

                b,err := json.Marshal(list)

                if(err==nil){
                    status = string(b)
                }else{
                    status = "no-hosts-found"
                }
            }else{
                if(sendPacket(host) == nil){
                    status = "ok";
                }else{
                    status = "fail";
                }
            }
        }else{
            status = "denied";
        }
        req.Write(goapi.Ok)
        req.Write(req.Resp)
        req.Write("\r\n\r\n")
        req.Write(status)
        req.Close()

}

func showHelp(){
    fmt.Println("\n");
    fmt.Println("# # # # # # # # # # # # # # # # # # # # # #");
    fmt.Println("#                                         #");
    fmt.Println("#  Usage:                                 #");
    fmt.Println("#                                         #");
    fmt.Println("#  dir <conf-dir> (optional)              #");
    fmt.Println("#                                         #");
    fmt.Println("#  add|edit|delete|server                 #");
    fmt.Println("#                                         #");
    fmt.Println("#  user|host <username>|<computer-name>   #");
    fmt.Println("#                                         #");
    fmt.Println("#  <password>|<mac-address>               #");
    fmt.Println("#                                         #");
    fmt.Println("# # # # # # # # # # # # # # # # # # # # # #");
    fmt.Println("\n");
}

func getHost(host string)(string,error){

    hosts := getHosts()

    if(len(hosts[host]) > 0){
        return hosts[host], nil;
    }
    return "", errors.New("host not found" + host);
}

func getHosts()(map[string]string){

    file, err := ioutil.ReadFile(configPath + "/hosts.json");

    hosts := make(map[string]string);

    if(err == nil){
        err = json.Unmarshal(file, &hosts);

        if(err == nil){
            return hosts;
        }
    }
    return nil;
}

func isUser(user string, pass string)(error){

    file, err := ioutil.ReadFile(configPath + "/users.json");
    users := make(map[string]string);

    if(err == nil){
        err = json.Unmarshal(file, &users);
        if((err == nil)  && (len(users[user]) > 0)){

            err = bcrypt.CompareHashAndPassword([]byte(users[user]), []byte(pass));

            if(err == nil){
                return nil;
            }
        }
    }
    return errors.New("user not found" + user);

}
<|endoftext|>


<commit_before>package falcore

import (
	"net/http"
	"io"
	"fmt"
)

// Implements the RequestFilter using a http.Handleer to produce the response
// This will always return a response due to the requirements of the http.HandleFunc
// interface.
type HandlerFilter struct {
	handler http.Handler
}

func NewHandlerFilter(handler http.Handler) (*HandlerFilter) {
	return &HandlerFilter{ handler: handler}
}

func (h *HandlerFilter) FilterRequest(req *Request) *http.Response {
	rw, respc := newPopulateResponseWriter()
	// this must be done concurrently so that the HandlerFunc can write the response
	// while falcore is copying it to the socket
	go func() {
		h.handler.ServeHTTP(rw, req.HttpRequest)
		rw.finish()
	}()
	return <-respc
}

// copied from net/http/filetransport.go
func newPopulateResponseWriter() (*populateResponse, <-chan *http.Response) {
	pr, pw := io.Pipe()
	rw := &populateResponse{
		ch: make(chan *http.Response),
		pw: pw,
		res: &http.Response{
			Proto:      "HTTP/1.0",
			ProtoMajor: 1,
			Header:     make(http.Header),
			Close:      true,
			Body:       pr,
		},
	}
	return rw, rw.ch
}

// populateResponse is a ResponseWriter that populates the *Response
// in res, and writes its body to a pipe connected to the response
// body. Once writes begin or finish() is called, the response is sent
// on ch.
type populateResponse struct {
	res          *http.Response
	ch           chan *http.Response
	wroteHeader  bool
	hasContent   bool
	sentResponse bool
	pw           *io.PipeWriter
}

func (pr *populateResponse) finish() {
	if !pr.wroteHeader {
		pr.WriteHeader(500)
	}
	if !pr.sentResponse {
		pr.sendResponse()
	}
	pr.pw.Close()
}

func (pr *populateResponse) sendResponse() {
	if pr.sentResponse {
		return
	}
	pr.sentResponse = true

	if pr.hasContent {
		pr.res.ContentLength = -1
	}
	pr.ch <- pr.res
}

func (pr *populateResponse) Header() http.Header {
	return pr.res.Header
}

func (pr *populateResponse) WriteHeader(code int) {
	if pr.wroteHeader {
		return
	}
	pr.wroteHeader = true

	pr.res.StatusCode = code
	pr.res.Status = fmt.Sprintf("%d %s", code, http.StatusText(code))
}

func (pr *populateResponse) Write(p []byte) (n int, err error) {
	if !pr.wroteHeader {
		pr.WriteHeader(http.StatusOK)
	}
	pr.hasContent = true
	if !pr.sentResponse {
		pr.sendResponse()
	}
	return pr.pw.Write(p)
}

<commit_msg>fixed documentation typo<commit_after>package falcore

import (
	"net/http"
	"io"
	"fmt"
)

// Implements a RequestFilter using a http.Handler to produce the response
// This will always return a response due to the requirements of the http.Handler
// interface so it should be placed at the end of the Upstream pipeline.
type HandlerFilter struct {
	handler http.Handler
}

func NewHandlerFilter(handler http.Handler) (*HandlerFilter) {
	return &HandlerFilter{ handler: handler}
}

func (h *HandlerFilter) FilterRequest(req *Request) *http.Response {
	rw, respc := newPopulateResponseWriter()
	// this must be done concurrently so that the HandlerFunc can write the response
	// while falcore is copying it to the socket
	go func() {
		h.handler.ServeHTTP(rw, req.HttpRequest)
		rw.finish()
	}()
	return <-respc
}

// copied from net/http/filetransport.go
func newPopulateResponseWriter() (*populateResponse, <-chan *http.Response) {
	pr, pw := io.Pipe()
	rw := &populateResponse{
		ch: make(chan *http.Response),
		pw: pw,
		res: &http.Response{
			Proto:      "HTTP/1.0",
			ProtoMajor: 1,
			Header:     make(http.Header),
			Close:      true,
			Body:       pr,
		},
	}
	return rw, rw.ch
}

// populateResponse is a ResponseWriter that populates the *Response
// in res, and writes its body to a pipe connected to the response
// body. Once writes begin or finish() is called, the response is sent
// on ch.
type populateResponse struct {
	res          *http.Response
	ch           chan *http.Response
	wroteHeader  bool
	hasContent   bool
	sentResponse bool
	pw           *io.PipeWriter
}

func (pr *populateResponse) finish() {
	if !pr.wroteHeader {
		pr.WriteHeader(500)
	}
	if !pr.sentResponse {
		pr.sendResponse()
	}
	pr.pw.Close()
}

func (pr *populateResponse) sendResponse() {
	if pr.sentResponse {
		return
	}
	pr.sentResponse = true

	if pr.hasContent {
		pr.res.ContentLength = -1
	}
	pr.ch <- pr.res
}

func (pr *populateResponse) Header() http.Header {
	return pr.res.Header
}

func (pr *populateResponse) WriteHeader(code int) {
	if pr.wroteHeader {
		return
	}
	pr.wroteHeader = true

	pr.res.StatusCode = code
	pr.res.Status = fmt.Sprintf("%d %s", code, http.StatusText(code))
}

func (pr *populateResponse) Write(p []byte) (n int, err error) {
	if !pr.wroteHeader {
		pr.WriteHeader(http.StatusOK)
	}
	pr.hasContent = true
	if !pr.sentResponse {
		pr.sendResponse()
	}
	return pr.pw.Write(p)
}

<|endoftext|>


<commit_before>package fire

import (
	"fmt"
	"time"

	"github.com/labstack/echo"
	"github.com/labstack/echo/engine"
	"github.com/labstack/echo/engine/standard"
	"github.com/labstack/echo/middleware"
	"gopkg.in/mgo.v2"
)

// An Application provides an out-of-the-box configuration of components to
// get started with building JSON APIs.
type Application struct {
	set       *Set
	router    *echo.Echo
	bodyLimit string

	enableMethodOverriding bool
	disableCompression     bool
	disableRecovery        bool
	disableCommonSecurity  bool
	enableDevMode          bool
}

// New creates and returns a new Application.
func New(mongoURI, prefix string) *Application {
	// create router
	router := echo.New()

	// connect to database
	sess, err := mgo.Dial(mongoURI)
	if err != nil {
		panic(err)
	}

	set := NewSet(sess, router, prefix)

	return &Application{
		set:       set,
		router:    router,
		bodyLimit: "4K",
	}
}

// Mount will add controllers to the set and register them on the router.
//
// Note: Each controller should only be mounted once.
func (a *Application) Mount(controllers ...*Controller) {
	a.set.Mount(controllers...)
}

// Router will return the internally used echo instance.
func (a *Application) Router() *echo.Echo {
	return a.router
}

// EnableCORS will enable CORS with a general configuration.
//
// Note: You can always add your own CORS middleware to the router.
func (a *Application) EnableCORS(origins ...string) {
	a.router.Use(middleware.CORSWithConfig(middleware.CORSConfig{
		AllowOrigins: origins,
		// TODO: Allow "Accept, Cache-Control"?
		AllowHeaders: []string{echo.HeaderOrigin, echo.HeaderAuthorization,
			echo.HeaderContentType, echo.HeaderXHTTPMethodOverride},
	}))
}

// EnableSecurity will enable further security measures for your application.
func (a *Application) EnableSecurity() {
	a.router.Use()
}

// EnableMethodOverriding will enable the usage of the X-HTTP-Method-Override
// header to set a request method when using the POST method.
//
// Note: This method must be called before calling Run or Start.
func (a *Application) EnableMethodOverriding() {
	a.enableMethodOverriding = true
}

// SetBodyLimit can be used to override the default body limit of 4K with a new
// value in the form of 4K, 2M, 1G or 1P.
//
// Note: This method must be called before calling Run or Start.
func (a *Application) SetBodyLimit(size string) {
	a.bodyLimit = size
}

// DisableCompression will turn of gzip compression.
//
// Note: This method must be called before calling Run or Start.
func (a *Application) DisableCompression() {
	a.disableCompression = true
}

// DisableRecovery will disable the automatic recover mechanism.
//
// Note: This method must be called before calling Run or Start.
func (a *Application) DisableRecovery() {
	a.disableRecovery = true
}

// DisableCommonSecurity will disable common security features including:
// protection against cross-site scripting attacks by setting the
// `X-XSS-Protection` header, protection against overriding Content-Type
// header by setting the `X-Content-Type-Options` header and protection against
// clickjacking by setting the `X-Frame-Options` header.
//
// Note: This method must be called before calling Run or Start.
func (a *Application) DisableCommonSecurity() {
	a.disableCommonSecurity = true
}

// EnableDevMode will enable the development mode that prints all registered
// handlers on boot and all incoming requests.
func (a *Application) EnableDevMode() {
	a.enableDevMode = true
}

// Run will run the application using the passed server.
func (a *Application) Run(server engine.Server) {
	// set body limit
	a.router.Use(middleware.BodyLimit(a.bodyLimit))

	// enable method overriding
	if a.enableMethodOverriding {
		a.router.Pre(middleware.MethodOverride())
	}

	// enable gzip compression
	if !a.disableCompression {
		a.router.Use(middleware.Gzip())
	}

	// enable automatic recovery
	if !a.disableRecovery {
		a.router.Use(middleware.Recover())
	}

	// enable common security
	if !a.disableCommonSecurity {
		a.router.Use(middleware.Secure())
	}

	// enable dev mode
	if a.enableDevMode {
		a.printInfo()
		a.router.Use(a.logger)
	}

	a.router.Run(server)
}

// Start will run the application on the specified address.
func (a *Application) Start(addr string) {
	a.Run(standard.New(addr))
}

func (a *Application) printInfo() {
	fmt.Println("==> Fire application starting...")
	fmt.Println("==> Registered routes:")

	for _, route := range a.router.Routes() {
		fmt.Printf("%6s  %-30s  %s\n", route.Method, route.Path, route.Handler)
	}

	fmt.Println("==> Ready to go!")
}

func (a *Application) logger(next echo.HandlerFunc) echo.HandlerFunc {
	return func(c echo.Context) (err error) {
		req := c.Request()
		res := c.Response()

		start := time.Now()
		if err = next(c); err != nil {
			c.Error(err)
		}

		duration := time.Since(start).String()

		path := req.URL().Path()
		if path == "" {
			path = "/"
		}

		fmt.Printf("%6s  %-30s  %d  %s\n", req.Method(), path, res.Status(), duration)

		return
	}
}
<commit_msg>finish cors support<commit_after>package fire

import (
	"fmt"
	"time"

	"github.com/labstack/echo"
	"github.com/labstack/echo/engine"
	"github.com/labstack/echo/engine/standard"
	"github.com/labstack/echo/middleware"
	"gopkg.in/mgo.v2"
)

// An Application provides an out-of-the-box configuration of components to
// get started with building JSON APIs.
type Application struct {
	set    *Set
	router *echo.Echo

	bodyLimit      string
	allowedOrigins []string
	allowedHeaders []string

	disableCORS            bool
	disableCompression     bool
	disableRecovery        bool
	disableCommonSecurity  bool
	enableMethodOverriding bool
	enableDevMode          bool
}

// New creates and returns a new Application.
func New(mongoURI, prefix string) *Application {
	// create router
	router := echo.New()

	// connect to database
	sess, err := mgo.Dial(mongoURI)
	if err != nil {
		panic(err)
	}

	// create controller set
	set := NewSet(sess, router, prefix)

	return &Application{
		set:            set,
		router:         router,
		bodyLimit:      "4K",
		allowedOrigins: []string{"*"},
		allowedHeaders: []string{
			echo.HeaderOrigin,
			echo.HeaderContentType,
			echo.HeaderAuthorization,
		},
	}
}

// Mount will add controllers to the set and register them on the router.
//
// Note: Each controller should only be mounted once.
func (a *Application) Mount(controllers ...*Controller) {
	a.set.Mount(controllers...)
}

// Router will return the internally used echo instance.
func (a *Application) Router() *echo.Echo {
	return a.router
}

// EnableSecurity will enable further security measures for your application.
func (a *Application) EnableSecurity() {
	a.router.Use()
}

// EnableMethodOverriding will enable the usage of the X-HTTP-Method-Override
// header to set a request method when using the POST method.
//
// Note: This method must be called before calling Run or Start.
func (a *Application) EnableMethodOverriding() {
	a.enableMethodOverriding = true
}

// SetBodyLimit can be used to override the default body limit of 4K with a new
// value in the form of 4K, 2M, 1G or 1P.
//
// Note: This method must be called before calling Run or Start.
func (a *Application) SetBodyLimit(size string) {
	a.bodyLimit = size
}

// SetAllowedOrigins will replace the default allowed origin set `*`.
func (a *Application) SetAllowedOrigins(origins ...string) {
	a.allowedOrigins = origins
}

// AddAllowedHeaders will allow additional headers.
func (a *Application) AddAllowedHeaders(headers ...string) {
	a.allowedHeaders = append(a.allowedHeaders, headers)
}

// DisableCORS will turn off CORS support.
//
// Note: This method must be called before calling Run or Start.
func (a *Application) DisableCORS(origins ...string) {
	a.disableCORS = true
}

// DisableCompression will turn of gzip compression.
//
// Note: This method must be called before calling Run or Start.
func (a *Application) DisableCompression() {
	a.disableCompression = true
}

// DisableRecovery will disable the automatic recover mechanism.
//
// Note: This method must be called before calling Run or Start.
func (a *Application) DisableRecovery() {
	a.disableRecovery = true
}

// DisableCommonSecurity will disable common security features including:
// protection against cross-site scripting attacks by setting the
// `X-XSS-Protection` header, protection against overriding Content-Type
// header by setting the `X-Content-Type-Options` header and protection against
// clickjacking by setting the `X-Frame-Options` header.
//
// Note: This method must be called before calling Run or Start.
func (a *Application) DisableCommonSecurity() {
	a.disableCommonSecurity = true
}

// EnableDevMode will enable the development mode that prints all registered
// handlers on boot and all incoming requests.
func (a *Application) EnableDevMode() {
	a.enableDevMode = true
}

// Run will run the application using the passed server.
func (a *Application) Run(server engine.Server) {
	// set body limit
	a.router.Use(middleware.BodyLimit(a.bodyLimit))

	// enable cors
	if !a.disableCORS {
		allowedHeaders := a.allowedHeaders

		// add method override header if enabled
		if a.enableMethodOverriding {
			allowedHeaders = append(allowedHeaders, echo.HeaderXHTTPMethodOverride)
		}

		// add cors middleware
		a.router.Use(middleware.CORSWithConfig(middleware.CORSConfig{
			AllowOrigins: a.allowedOrigins,
			AllowMethods: []string{echo.GET, echo.POST, echo.PATCH, echo.DELETE},
			AllowHeaders: allowedHeaders,
			MaxAge:       60,
		}))
	}

	// enable gzip compression
	if !a.disableCompression {
		a.router.Use(middleware.Gzip())
	}

	// enable automatic recovery
	if !a.disableRecovery {
		a.router.Use(middleware.Recover())
	}

	// enable common security
	if !a.disableCommonSecurity {
		a.router.Use(middleware.Secure())
	}

	// enable method overriding
	if a.enableMethodOverriding {
		a.router.Pre(middleware.MethodOverride())
	}

	// enable dev mode
	if a.enableDevMode {
		a.printInfo()
		a.router.Use(a.logger)
	}

	a.router.Run(server)
}

// Start will run the application on the specified address.
func (a *Application) Start(addr string) {
	a.Run(standard.New(addr))
}

func (a *Application) printInfo() {
	fmt.Println("==> Fire application starting...")
	fmt.Println("==> Registered routes:")

	for _, route := range a.router.Routes() {
		fmt.Printf("%6s  %-30s  %s\n", route.Method, route.Path, route.Handler)
	}

	fmt.Println("==> Ready to go!")
}

func (a *Application) logger(next echo.HandlerFunc) echo.HandlerFunc {
	return func(c echo.Context) (err error) {
		req := c.Request()
		res := c.Response()

		start := time.Now()
		if err = next(c); err != nil {
			c.Error(err)
		}

		duration := time.Since(start).String()

		path := req.URL().Path()
		if path == "" {
			path = "/"
		}

		fmt.Printf("%6s  %-30s  %d  %s\n", req.Method(), path, res.Status(), duration)

		return
	}
}
<|endoftext|>