<commit_before>// Package route finds the shortest route between two points along a geometrical
// network (e.g., a road network). For now, all network links are assumed to be
// bi-directional (e.g., all roads are two-way).
package route

import (
	"fmt"
	"math"

	"github.com/ctessum/geom"
	"github.com/ctessum/geom/index/rtree"
	"github.com/ctessum/geom/op"
	"gonum.org/v1/gonum/graph"
	"gonum.org/v1/gonum/graph/iterator"
	"gonum.org/v1/gonum/graph/path"
)

// A Network is a holder for network data (e.g., a road network)
type Network struct {
	nodes, edges   *rtree.Rtree
	neighbors      map[int64]map[int64]*edge
	nodeMap        map[int64]*node
	maxID          int
	freeMap        map[int]struct{}
	minimizeOption MinimizeOption
	minimumSpeed   float64 // The minimum speed traveled on any link in the network.
}

// NewNetwork initializes a new Network where m determines how to choose
// the shortest route (either by Distance or Time).
func NewNetwork(m MinimizeOption) *Network {
	return &Network{
		neighbors:      make(map[int64]map[int64]*edge),
		nodeMap:        make(map[int64]*node),
		maxID:          0,
		nodes:          rtree.NewTree(25, 50),
		edges:          rtree.NewTree(25, 50),
		minimizeOption: m,
		minimumSpeed:   math.Inf(1),
	}
}

// Has returns whether the node exists within the graph.
// It is not intended for direct use in this package.
func (net Network) Has(n int64) bool {
	_, ok := net.nodeMap[n]
	return ok
}

// Nodes returns all the nodes in the graph.
// It is not intended for direct use in this package.
func (net Network) Nodes() graph.Nodes {
	nodes := make([]graph.Node, len(net.nodeMap))
	i := 0
	for _, n := range net.nodeMap {
		nodes[i] = n
		i++
	}
	return iterator.NewOrderedNodes(nodes)
}

// From returns all nodes that can be reached directly
// from the given node.
// It is not intended for direct use in this package.
func (net Network) From(n int64) graph.Nodes {
	if !net.Has(n) {
		return nil
	}
	neighbors := make([]graph.Node, len(net.neighbors[n]))
	i := 0
	for id := range net.neighbors[n] {
		neighbors[i] = net.nodeMap[id]
		i++
	}
	return iterator.NewOrderedNodes(neighbors)
}

// HasEdge returns whether an edge exists between
// nodes x and y without considering direction.
// It is not intended for direct use in this package.
func (net Network) HasEdge(x, y int64) bool {
	_, ok := net.neighbors[x][y]
	return ok
}

// Edge returns the edge from u to v if such an edge
// exists and nil otherwise. The node v must be directly
// reachable from u as defined by the From method.
// It is not intended for direct use in this package.
func (net Network) Edge(u, v int64) graph.Edge {
	// We don't need to check if neigh exists because
	// it's implicit in the neighbors access.
	if !net.Has(u) {
		return nil
	}
	return net.neighbors[u][v]
}

func (net Network) Node(id int64) graph.Node { return net.nodeMap[id] }

// The math package only provides explicitly sized max
// values. This ensures we get the max for the actual
// type int.
const maxInt int = int(^uint(0) >> 1)

func (net *Network) newNodeID() int {
	if net.maxID != maxInt {
		net.maxID++
		return net.maxID
	}
	// I cannot foresee this ever happening, but just in case, we check.
	if len(net.nodeMap) == maxInt {
		panic("cannot allocate node: graph too large")
	}
	// Should not happen.
	panic("cannot allocate node id: no free id found")
}

// Check if there is already a node at this location, and if there
// is return that one, otherwise create a new node.
func (net *Network) newNode(p geom.Point) *node {
	nearest := net.nodes.NearestNeighbor(p)
	if nearest != nil && op.PointEquals(p, nearest.(*node).Point) {
		return nearest.(*node)
	}
	return &node{
		Point: p,
		id:    net.newNodeID(),
	}
}

func (net *Network) addNode(n graph.Node) {
	if _, exists := net.nodeMap[n.ID()]; exists {
		panic(fmt.Sprintf("route: node ID collision: %d", n.ID()))
	}
	net.nodeMap[n.ID()] = n.(*node)
	net.neighbors[n.ID()] = make(map[int64]*edge)
	net.nodes.Insert(n.(*node))
}

// AddLink adds a new link l (which is a line string) to the Network, where
// speed is the speed traveled along the link and should have units that
// are compatible with the units of l (for instance, if l is in units of
// meters, and speed is in units of m/s, the time results will be in units
// of seconds).
func (net *Network) AddLink(l geom.LineString, speed float64) {
	from := net.newNode(l[0])
	to := net.newNode(l[len(l)-1])

	length := op.Length(l)
	e := &edge{
		LineString: l,
		start:      from,
		end:        to,
		length:     length,
		speed:      speed,
		time:       length / speed,
	}
	if e.speed < net.minimumSpeed {
		net.minimumSpeed = e.speed
	}
	fid := from.ID()
	tid := to.ID()
	if fid == tid {
		panic("concrete: adding self edge")
	}
	if !net.Has(from.ID()) {
		net.addNode(from)
	}
	if !net.Has(to.ID()) {
		net.addNode(to)
	}
	net.edges.Insert(e)
	net.neighbors[fid][tid] = e
	net.neighbors[tid][fid] = e
}

// Weight returns the weight associated with this edge.
// It is not intended for direct use in this package.
func (net *Network) Weight(e graph.Edge) float64 {
	if n, ok := net.neighbors[e.From().ID()]; ok {
		if we, ok := n[e.To().ID()]; ok {
			switch net.minimizeOption {
			// If we're optimizing by time, return use the minimum speed to
			// calculate the time to ensure the heuristic is less than the actual
			// value
			case Time:
				return we.time
			case Distance:
				// If we're optimizing by distance, just return the distance.
				return we.length
			default:
				panic(fmt.Errorf("Invalid MinimizeOption %v", net.minimizeOption))
			}
		}
	}
	panic("route: attempting to find an edge that is not in the graph")
}

type edge struct {
	geom.LineString
	start, end          *node
	length, speed, time float64
}

// From gives the beginning point of this edge.
func (e edge) From() graph.Node {
	return e.start
}

// To gives the final point of this edge.
func (e edge) To() graph.Node {
	return e.end
}

type node struct {
	geom.Point
	id int
}

func (n node) ID() int64 {
	return int64(n.id)
}

// MinimizeOption specifies how the shortest route should be chosen.
type MinimizeOption float64

const (
	// Distance specifies that we are looking to travel the minimum distance.
	Distance MinimizeOption = iota
	// Time specifies that we are looking to travel the minimum time.
	Time
)

// ShortestRoute calculates the shortest route along the network between the
// from and to points. It returns the route ("path"), the distance traveled
// along the route ("distance"), the time it would take travel along the route
// ("time"; this does not count time spent getting to and from the network),
// the distance traveled from the starting
// point to get to the nearest node (e.g., intersection) along the route
// ("startDistance") and the distance traveled to the ending point from
// the nearest node along the route ("endDistance"). This function does
// not change the Network, so multiple function calls can be run concurrently.
func (net Network) ShortestRoute(from, to geom.Point) (
	route geom.MultiLineString, distance, time, startDistance, endDistance float64) {
	startNode := net.nodes.NearestNeighbor(from).(*node)
	endNode := net.nodes.NearestNeighbor(to).(*node)
	startDistance = op.Distance(from, startNode.Point)
	endDistance = op.Distance(to, endNode.Point)
	shortest, _ := path.AStar(startNode, endNode, net, net.costHeuristic)
	nodes, _ := shortest.To(endNode.ID())
	for i := 0; i < len(nodes)-1; i++ {
		e, ok := net.neighbors[nodes[i].ID()][nodes[i+1].ID()]
		if !ok {
			panic("route: missing edge; this shouldn't happen")
		}
		route = append(route, e.LineString)
		distance += e.length
		time += e.time
	}
	return
}

// costHeuristic provides a cost estimate this is guaranteed to be equal
// to or less than the actual cost.
func (net *Network) costHeuristic(x, y graph.Node) float64 {
	distance := op.Distance(x.(*node).Point, y.(*node).Point)
	switch net.minimizeOption {
	// If we're optimizing by time, return use the minimum speed to
	// calculate the time to ensure the heuristic is less than the actual
	// value
	case Time:
		return distance / net.minimumSpeed
	case Distance:
		// If we're optimizing by distance, just return the distance.
		return distance
	default:
		panic(fmt.Errorf("Invalid MinimizeOption %v", net.minimizeOption))
	}
}

// HasEdgeBetween returns whether an edge exists between nodes x and y without
// considering direction.
func (net Network) HasEdgeBetween(xid, yid int64) bool {
	if _, ok := net.neighbors[xid][yid]; ok {
		return true
	}
	_, ok := net.neighbors[yid][xid]
	return ok
}
<commit_msg>route: Make sure rtree is initialized before querying<commit_after>// Package route finds the shortest route between two points along a geometrical
// network (e.g., a road network). For now, all network links are assumed to be
// bi-directional (e.g., all roads are two-way).
package route

import (
	"fmt"
	"math"

	"github.com/ctessum/geom"
	"github.com/ctessum/geom/index/rtree"
	"github.com/ctessum/geom/op"
	"gonum.org/v1/gonum/graph"
	"gonum.org/v1/gonum/graph/iterator"
	"gonum.org/v1/gonum/graph/path"
)

// A Network is a holder for network data (e.g., a road network)
type Network struct {
	nodes, edges   *rtree.Rtree
	neighbors      map[int64]map[int64]*edge
	nodeMap        map[int64]*node
	maxID          int
	freeMap        map[int]struct{}
	minimizeOption MinimizeOption
	minimumSpeed   float64 // The minimum speed traveled on any link in the network.
}

// NewNetwork initializes a new Network where m determines how to choose
// the shortest route (either by Distance or Time).
func NewNetwork(m MinimizeOption) *Network {
	return &Network{
		neighbors:      make(map[int64]map[int64]*edge),
		nodeMap:        make(map[int64]*node),
		maxID:          0,
		nodes:          rtree.NewTree(25, 50),
		edges:          rtree.NewTree(25, 50),
		minimizeOption: m,
		minimumSpeed:   math.Inf(1),
	}
}

// Has returns whether the node exists within the graph.
// It is not intended for direct use in this package.
func (net Network) Has(n int64) bool {
	_, ok := net.nodeMap[n]
	return ok
}

// Nodes returns all the nodes in the graph.
// It is not intended for direct use in this package.
func (net Network) Nodes() graph.Nodes {
	nodes := make([]graph.Node, len(net.nodeMap))
	i := 0
	for _, n := range net.nodeMap {
		nodes[i] = n
		i++
	}
	return iterator.NewOrderedNodes(nodes)
}

// From returns all nodes that can be reached directly
// from the given node.
// It is not intended for direct use in this package.
func (net Network) From(n int64) graph.Nodes {
	if !net.Has(n) {
		return nil
	}
	neighbors := make([]graph.Node, len(net.neighbors[n]))
	i := 0
	for id := range net.neighbors[n] {
		neighbors[i] = net.nodeMap[id]
		i++
	}
	return iterator.NewOrderedNodes(neighbors)
}

// HasEdge returns whether an edge exists between
// nodes x and y without considering direction.
// It is not intended for direct use in this package.
func (net Network) HasEdge(x, y int64) bool {
	_, ok := net.neighbors[x][y]
	return ok
}

// Edge returns the edge from u to v if such an edge
// exists and nil otherwise. The node v must be directly
// reachable from u as defined by the From method.
// It is not intended for direct use in this package.
func (net Network) Edge(u, v int64) graph.Edge {
	// We don't need to check if neigh exists because
	// it's implicit in the neighbors access.
	if !net.Has(u) {
		return nil
	}
	return net.neighbors[u][v]
}

func (net Network) Node(id int64) graph.Node { return net.nodeMap[id] }

// The math package only provides explicitly sized max
// values. This ensures we get the max for the actual
// type int.
const maxInt int = int(^uint(0) >> 1)

func (net *Network) newNodeID() int {
	if net.maxID != maxInt {
		net.maxID++
		return net.maxID
	}
	// I cannot foresee this ever happening, but just in case, we check.
	if len(net.nodeMap) == maxInt {
		panic("cannot allocate node: graph too large")
	}
	// Should not happen.
	panic("cannot allocate node id: no free id found")
}

// Check if there is already a node at this location, and if there
// is return that one, otherwise create a new node.
func (net *Network) newNode(p geom.Point) *node {
	if net.nodes.Size() != 0 {
		nearest := net.nodes.NearestNeighbor(p)
		if nearest != nil && op.PointEquals(p, nearest.(*node).Point) {
			return nearest.(*node)
		}
	}
	return &node{
		Point: p,
		id:    net.newNodeID(),
	}
}

func (net *Network) addNode(n graph.Node) {
	if _, exists := net.nodeMap[n.ID()]; exists {
		panic(fmt.Sprintf("route: node ID collision: %d", n.ID()))
	}
	net.nodeMap[n.ID()] = n.(*node)
	net.neighbors[n.ID()] = make(map[int64]*edge)
	net.nodes.Insert(n.(*node))
}

// AddLink adds a new link l (which is a line string) to the Network, where
// speed is the speed traveled along the link and should have units that
// are compatible with the units of l (for instance, if l is in units of
// meters, and speed is in units of m/s, the time results will be in units
// of seconds).
func (net *Network) AddLink(l geom.LineString, speed float64) {
	from := net.newNode(l[0])
	to := net.newNode(l[len(l)-1])

	length := op.Length(l)
	e := &edge{
		LineString: l,
		start:      from,
		end:        to,
		length:     length,
		speed:      speed,
		time:       length / speed,
	}
	if e.speed < net.minimumSpeed {
		net.minimumSpeed = e.speed
	}
	fid := from.ID()
	tid := to.ID()
	if fid == tid {
		panic("concrete: adding self edge")
	}
	if !net.Has(from.ID()) {
		net.addNode(from)
	}
	if !net.Has(to.ID()) {
		net.addNode(to)
	}
	net.edges.Insert(e)
	net.neighbors[fid][tid] = e
	net.neighbors[tid][fid] = e
}

// Weight returns the weight associated with this edge.
// It is not intended for direct use in this package.
func (net *Network) Weight(e graph.Edge) float64 {
	if n, ok := net.neighbors[e.From().ID()]; ok {
		if we, ok := n[e.To().ID()]; ok {
			switch net.minimizeOption {
			// If we're optimizing by time, return use the minimum speed to
			// calculate the time to ensure the heuristic is less than the actual
			// value
			case Time:
				return we.time
			case Distance:
				// If we're optimizing by distance, just return the distance.
				return we.length
			default:
				panic(fmt.Errorf("Invalid MinimizeOption %v", net.minimizeOption))
			}
		}
	}
	panic("route: attempting to find an edge that is not in the graph")
}

type edge struct {
	geom.LineString
	start, end          *node
	length, speed, time float64
}

// From gives the beginning point of this edge.
func (e edge) From() graph.Node {
	return e.start
}

// To gives the final point of this edge.
func (e edge) To() graph.Node {
	return e.end
}

type node struct {
	geom.Point
	id int
}

func (n node) ID() int64 {
	return int64(n.id)
}

// MinimizeOption specifies how the shortest route should be chosen.
type MinimizeOption float64

const (
	// Distance specifies that we are looking to travel the minimum distance.
	Distance MinimizeOption = iota
	// Time specifies that we are looking to travel the minimum time.
	Time
)

// ShortestRoute calculates the shortest route along the network between the
// from and to points. It returns the route ("path"), the distance traveled
// along the route ("distance"), the time it would take travel along the route
// ("time"; this does not count time spent getting to and from the network),
// the distance traveled from the starting
// point to get to the nearest node (e.g., intersection) along the route
// ("startDistance") and the distance traveled to the ending point from
// the nearest node along the route ("endDistance"). This function does
// not change the Network, so multiple function calls can be run concurrently.
func (net Network) ShortestRoute(from, to geom.Point) (
	route geom.MultiLineString, distance, time, startDistance, endDistance float64) {
	startNode := net.nodes.NearestNeighbor(from).(*node)
	endNode := net.nodes.NearestNeighbor(to).(*node)
	startDistance = op.Distance(from, startNode.Point)
	endDistance = op.Distance(to, endNode.Point)
	shortest, _ := path.AStar(startNode, endNode, net, net.costHeuristic)
	nodes, _ := shortest.To(endNode.ID())
	for i := 0; i < len(nodes)-1; i++ {
		e, ok := net.neighbors[nodes[i].ID()][nodes[i+1].ID()]
		if !ok {
			panic("route: missing edge; this shouldn't happen")
		}
		route = append(route, e.LineString)
		distance += e.length
		time += e.time
	}
	return
}

// costHeuristic provides a cost estimate this is guaranteed to be equal
// to or less than the actual cost.
func (net *Network) costHeuristic(x, y graph.Node) float64 {
	distance := op.Distance(x.(*node).Point, y.(*node).Point)
	switch net.minimizeOption {
	// If we're optimizing by time, return use the minimum speed to
	// calculate the time to ensure the heuristic is less than the actual
	// value
	case Time:
		return distance / net.minimumSpeed
	case Distance:
		// If we're optimizing by distance, just return the distance.
		return distance
	default:
		panic(fmt.Errorf("Invalid MinimizeOption %v", net.minimizeOption))
	}
}

// HasEdgeBetween returns whether an edge exists between nodes x and y without
// considering direction.
func (net Network) HasEdgeBetween(xid, yid int64) bool {
	if _, ok := net.neighbors[xid][yid]; ok {
		return true
	}
	_, ok := net.neighbors[yid][xid]
	return ok
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/cloudfoundry/gorouter"
)

var configFile string

func init() {
	flag.StringVar(&configFile, "c", "", "Configuration File")

	flag.Parse()
}

func main() {
	c := router.DefaultConfig()
	if configFile != "" {
		c = router.InitConfigFromFile(configFile)
	}

	router.SetupLoggerFromConfig(c)

	router.NewRouter(c).Run()
}
<commit_msg>fix main<commit_after>package main

import (
	"flag"

	"github.com/cloudfoundry/gorouter"
	"github.com/cloudfoundry/gorouter/config"
	"github.com/cloudfoundry/gorouter/log"
)

var configFile string

func init() {
	flag.StringVar(&configFile, "c", "", "Configuration File")

	flag.Parse()
}

func main() {
	c := config.DefaultConfig()
	if configFile != "" {
		c = config.InitConfigFromFile(configFile)
	}

	log.SetupLoggerFromConfig(c)

	router.NewRouter(c).Run()
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"os/exec"
	"path/filepath"
)

type LocalContainerCreator struct {
	Root string
}

func NewLocalContainerCreator() *LocalContainerCreator {
	r := &LocalContainerCreator{
		Root: filepath.Join(os.TempDir(), "pixelpixel"),
	}
	return r
}

func (lcc *LocalContainerCreator) CreateContainer(fs *tar.Reader, envInjections []string) (Container, error) {
	dir := filepath.Join(lcc.Root, GenerateAlnumString(32))
	err := os.MkdirAll(dir, os.FileMode(0755))
	if err != nil {
		return nil, err
	}

	ctr := &localContainer{
		Root:        dir,
		LogBuffer:   &bytes.Buffer{},
		Terminating: make(chan bool),
	}

	purge := true
	defer func() {
		if purge {
			close(ctr.Terminating)
			ctr.Terminated = true
			os.RemoveAll(dir)
		}
	}()

	err = ctr.extractFileSystem(fs)
	if err != nil {
		return nil, err
	}

	err = ctr.compile()
	if err != nil {
		return ctr, nil
	}

	purge = false
	ctr.Cmd = exec.Command(filepath.Join(dir, "pixel"))
	ctr.Cmd.Dir = dir
	ctr.Cmd.Stdout = ctr.LogBuffer
	ctr.Cmd.Stderr = ctr.LogBuffer
	ctr.Port = <-portGenerator
	ctr.Cmd.Env = stringList(os.Environ(), envInjections, fmt.Sprintf("PORT=%d", ctr.Port))
	err = ctr.Cmd.Start()
	if err != nil {
		return ctr, err
	}

	go func() {
		ctr.Cmd.Wait()
		close(ctr.Terminating)
		ctr.Terminated = true
	}()

	return ctr, nil
}

type localContainer struct {
	Root        string
	Cmd         *exec.Cmd
	Terminating chan bool
	LogBuffer   *bytes.Buffer
	Port        int
	Terminated  bool
}

func (lc *localContainer) IsRunning() bool {
	return !lc.Terminated
}

func (lc *localContainer) Address() net.Addr {
	addr, _ := net.ResolveTCPAddr("tcp4", fmt.Sprintf("localhost:%d", lc.Port))
	return addr
}

func (lc *localContainer) Logs() string {
	return lc.LogBuffer.String()
}

func (lc *localContainer) SoftKill() {
	if !lc.IsRunning() || lc.Cmd == nil || lc.Cmd.Process == nil {
		return
	}
	lc.Cmd.Process.Signal(os.Interrupt)
}

func (lc *localContainer) HardKill() {
	if !lc.IsRunning() || lc.Cmd == nil || lc.Cmd.Process == nil {
		return
	}
	lc.Cmd.Process.Signal(os.Kill)
}

func (lc *localContainer) Wait() {
	select {
	case <-lc.Terminating:
	}
}

func (lc *localContainer) Cleanup() {
	lc.Wait()
	os.RemoveAll(lc.Root)
}

func (lc *localContainer) compile() error {
	cmd := exec.Command("go", "get", "-d")
	cmd.Dir = lc.Root
	cmd.Stdout = lc.LogBuffer
	cmd.Stderr = lc.LogBuffer
	if err := cmd.Run(); err != nil {
		return err
	}

	files, err := filepath.Glob(filepath.Join(lc.Root, "*.go"))
	if err != nil {
		return err
	}

	// add extension .exe to compiled program in windows environment
	ext := ""
	if ( runtime.GOOS == "windows" ) {
		ext = ".exe"
	}

	cmd = exec.Command("go", stringList("build", "-o", "pixel", files)...)
	cmd.Dir = lc.Root
	cmd.Stdout = lc.LogBuffer
	cmd.Stderr = lc.LogBuffer
	return cmd.Run()
}

func (lc *localContainer) extractFileSystem(fs *tar.Reader) error {
	hdr, err := fs.Next()
	for err != io.EOF {
		if err != nil {
			return err
		}
		file := filepath.Join(lc.Root, hdr.Name)
		switch hdr.Typeflag {
		case tar.TypeDir:
			err := os.MkdirAll(file, os.FileMode(0755))
			if err != nil {
				return err
			}
		case tar.TypeReg:
			err := lc.writeFile(file, fs)
			if err != nil {
				return err
			}
		default:
			log.Printf("Encountered unknown file type 0x%02x, skipping", hdr.Typeflag)
		}

		hdr, err = fs.Next()
	}
	return nil
}

func (lc *localContainer) writeFile(file string, ff io.Reader) error {
	f, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY, os.FileMode(0644))
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = io.Copy(f, ff)
	return err
}

// Stolen from $GOROOT/src/cmd/go/main.go
// stringList's arguments should be a sequence of string or []string values.
// stringList flattens them into a single []string.
func stringList(args ...interface{}) []string {
	var x []string
	for _, arg := range args {
		switch arg := arg.(type) {
		case []string:
			x = append(x, arg...)
		case string:
			x = append(x, arg)
		default:
			panic("stringList: invalid argument")
		}
	}
	return x
}

var (
	portGenerator <-chan int
)

func init() {
	c := make(chan int)
	portGenerator = c
	go func() {
		for {
			for i := 49000; i < 65535; i++ {
				c <- i
			}
		}
	}()
}
<commit_msg>added extension .exe to pixelpixel tmp file in windows environment<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"os/exec"
	"path/filepath"
)

type LocalContainerCreator struct {
	Root string
}

func NewLocalContainerCreator() *LocalContainerCreator {
	r := &LocalContainerCreator{
		Root: filepath.Join(os.TempDir(), "pixelpixel"),
	}
	return r
}

func (lcc *LocalContainerCreator) CreateContainer(fs *tar.Reader, envInjections []string) (Container, error) {
	dir := filepath.Join(lcc.Root, GenerateAlnumString(32))
	err := os.MkdirAll(dir, os.FileMode(0755))
	if err != nil {
		return nil, err
	}

	ctr := &localContainer{
		Root:        dir,
		LogBuffer:   &bytes.Buffer{},
		Terminating: make(chan bool),
	}

	purge := true
	defer func() {
		if purge {
			close(ctr.Terminating)
			ctr.Terminated = true
			os.RemoveAll(dir)
		}
	}()

	err = ctr.extractFileSystem(fs)
	if err != nil {
		return nil, err
	}

	err = ctr.compile()
	if err != nil {
		return ctr, nil
	}

	purge = false
	ctr.Cmd = exec.Command(filepath.Join(dir, "pixel"))
	ctr.Cmd.Dir = dir
	ctr.Cmd.Stdout = ctr.LogBuffer
	ctr.Cmd.Stderr = ctr.LogBuffer
	ctr.Port = <-portGenerator
	ctr.Cmd.Env = stringList(os.Environ(), envInjections, fmt.Sprintf("PORT=%d", ctr.Port))
	err = ctr.Cmd.Start()
	if err != nil {
		return ctr, err
	}

	go func() {
		ctr.Cmd.Wait()
		close(ctr.Terminating)
		ctr.Terminated = true
	}()

	return ctr, nil
}

type localContainer struct {
	Root        string
	Cmd         *exec.Cmd
	Terminating chan bool
	LogBuffer   *bytes.Buffer
	Port        int
	Terminated  bool
}

func (lc *localContainer) IsRunning() bool {
	return !lc.Terminated
}

func (lc *localContainer) Address() net.Addr {
	addr, _ := net.ResolveTCPAddr("tcp4", fmt.Sprintf("localhost:%d", lc.Port))
	return addr
}

func (lc *localContainer) Logs() string {
	return lc.LogBuffer.String()
}

func (lc *localContainer) SoftKill() {
	if !lc.IsRunning() || lc.Cmd == nil || lc.Cmd.Process == nil {
		return
	}
	lc.Cmd.Process.Signal(os.Interrupt)
}

func (lc *localContainer) HardKill() {
	if !lc.IsRunning() || lc.Cmd == nil || lc.Cmd.Process == nil {
		return
	}
	lc.Cmd.Process.Signal(os.Kill)
}

func (lc *localContainer) Wait() {
	select {
	case <-lc.Terminating:
	}
}

func (lc *localContainer) Cleanup() {
	lc.Wait()
	os.RemoveAll(lc.Root)
}

func (lc *localContainer) compile() error {
	cmd := exec.Command("go", "get", "-d")
	cmd.Dir = lc.Root
	cmd.Stdout = lc.LogBuffer
	cmd.Stderr = lc.LogBuffer
	if err := cmd.Run(); err != nil {
		return err
	}

	files, err := filepath.Glob(filepath.Join(lc.Root, "*.go"))
	if err != nil {
		return err
	}

	// add extension .exe to compiled program in windows environment
	ext := ""
	if ( runtime.GOOS == "windows" ) {
		ext = ".exe"
	}

	cmd = exec.Command("go", stringList("build", "-o", "pixel" + ext, files)...)
	cmd.Dir = lc.Root
	cmd.Stdout = lc.LogBuffer
	cmd.Stderr = lc.LogBuffer
	return cmd.Run()
}

func (lc *localContainer) extractFileSystem(fs *tar.Reader) error {
	hdr, err := fs.Next()
	for err != io.EOF {
		if err != nil {
			return err
		}
		file := filepath.Join(lc.Root, hdr.Name)
		switch hdr.Typeflag {
		case tar.TypeDir:
			err := os.MkdirAll(file, os.FileMode(0755))
			if err != nil {
				return err
			}
		case tar.TypeReg:
			err := lc.writeFile(file, fs)
			if err != nil {
				return err
			}
		default:
			log.Printf("Encountered unknown file type 0x%02x, skipping", hdr.Typeflag)
		}

		hdr, err = fs.Next()
	}
	return nil
}

func (lc *localContainer) writeFile(file string, ff io.Reader) error {
	f, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY, os.FileMode(0644))
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = io.Copy(f, ff)
	return err
}

// Stolen from $GOROOT/src/cmd/go/main.go
// stringList's arguments should be a sequence of string or []string values.
// stringList flattens them into a single []string.
func stringList(args ...interface{}) []string {
	var x []string
	for _, arg := range args {
		switch arg := arg.(type) {
		case []string:
			x = append(x, arg...)
		case string:
			x = append(x, arg)
		default:
			panic("stringList: invalid argument")
		}
	}
	return x
}

var (
	portGenerator <-chan int
)

func init() {
	c := make(chan int)
	portGenerator = c
	go func() {
		for {
			for i := 49000; i < 65535; i++ {
				c <- i
			}
		}
	}()
}
<|endoftext|>


<commit_before>//Package logging Functions to set unique configuration for use with the logrus logger
package logging

import (
	"fmt"
	"os"
	"os/user"
	"runtime"
	"regexp"
	"strconv"

	log "github.com/sirupsen/logrus"
	msgs "github.com/crunchydata/postgres-operator/apiservermsgs"
)

func SetParameters() LogValues {
	var logval LogValues

	logval.version = msgs.PGO_VERSION
	//set defaults of unknown for each of the remaining log values
	//logval.host, logval.user = "Unknown", "Unknown"
	logval.user = "Unknown"

	//set username value
	username, oku := getUsername()
	if oku {
		logval.user = username
	}
	return logval
}

//LogValues holds the standard log value types
type LogValues struct {
	user string
	version  string
	//host string
}

// formatter adds default fields to each log entry.
type formatter struct {
	fields log.Fields
	lf     log.Formatter
}

// Format satisfies the logrus.Formatter interface.
func (f *formatter) Format(e *log.Entry) ([]byte, error) {
	for k, v := range f.fields {
			e.Data[k] = v
	}
	return f.lf.Format(e)
}

//GetUsername gets the current OS username
func getUsername() (string, bool) {
	user, err := user.Current()
	if err != nil {
		CrunchyLogger(LogValues{"UnknownUser", "UnknownVersion"})
		log.Infof("Problem gathering username, will provide User ID instead. Message: %v", err)
		//if username not found, return UID
		return strconv.Itoa(os.Getuid()), true
	}
	return user.Username, true
}

//CrunchyLogger adds the customized logging fields to the logrus instance context
func CrunchyLogger(logDetails LogValues) {
	//Sets calling method as a field
	log.SetReportCaller(true)

	crunchyTextFormatter := &log.TextFormatter{
		CallerPrettyfier: func(f *runtime.Frame) (string, string) {
			filename := f.File
			function := f.Function
			re1 := regexp.MustCompile(`postgres-operator/(.*go)`)
			result1 := re1.FindStringSubmatch(f.File)
			if len(result1) > 1 {
				filename = result1[1]
			}

			re2 := regexp.MustCompile(`postgres-operator/(.*)`)
			result2 := re2.FindStringSubmatch(f.Function)
			if len(result2) > 1 {
				function = result2[1]
			}
			return fmt.Sprintf("%s()", function), fmt.Sprintf("%s:%d", filename, f.Line)
		},
		//ForceColors: true,
		FullTimestamp: true,
		//DisableTruncation: true,
	}

	crunchyTextFormatter.TimestampFormat = "2006-01-02 15:04:05 -0700"

	log.SetFormatter(&formatter{
		fields: log.Fields{
			"user": logDetails.user,
			"version":  logDetails.version,
		},
		lf: crunchyTextFormatter,
	})

	// Output to stdout instead of the default stderr
	// Can be any io.Writer, see below for File example
	log.SetOutput(os.Stdout)

	// Only log the debug severity or above.
	log.SetLevel(log.DebugLevel)
}<commit_msg>Removed logging user tag (#904)<commit_after>//Package logging Functions to set unique configuration for use with the logrus logger
package logging

import (
	"fmt"
	"os"
	"runtime"
	"regexp"

	log "github.com/sirupsen/logrus"
	msgs "github.com/crunchydata/postgres-operator/apiservermsgs"
)

func SetParameters() LogValues {
	var logval LogValues

	logval.version = msgs.PGO_VERSION

	return logval
}

//LogValues holds the standard log value types
type LogValues struct {
	version  string
}

// formatter adds default fields to each log entry.
type formatter struct {
	fields log.Fields
	lf     log.Formatter
}

// Format satisfies the logrus.Formatter interface.
func (f *formatter) Format(e *log.Entry) ([]byte, error) {
	for k, v := range f.fields {
			e.Data[k] = v
	}
	return f.lf.Format(e)
}

//CrunchyLogger adds the customized logging fields to the logrus instance context
func CrunchyLogger(logDetails LogValues) {
	//Sets calling method as a field
	log.SetReportCaller(true)

	crunchyTextFormatter := &log.TextFormatter{
		CallerPrettyfier: func(f *runtime.Frame) (string, string) {
			filename := f.File
			function := f.Function
			re1 := regexp.MustCompile(`postgres-operator/(.*go)`)
			result1 := re1.FindStringSubmatch(f.File)
			if len(result1) > 1 {
				filename = result1[1]
			}

			re2 := regexp.MustCompile(`postgres-operator/(.*)`)
			result2 := re2.FindStringSubmatch(f.Function)
			if len(result2) > 1 {
				function = result2[1]
			}
			return fmt.Sprintf("%s()", function), fmt.Sprintf("%s:%d", filename, f.Line)
		},
		//ForceColors: true,
		FullTimestamp: true,
		//DisableTruncation: true,
	}

	crunchyTextFormatter.TimestampFormat = "2006-01-02 15:04:05 -0700"

	log.SetFormatter(&formatter{
		fields: log.Fields{
			"version":  logDetails.version,
		},
		lf: crunchyTextFormatter,
	})

	// Output to stdout instead of the default stderr
	// Can be any io.Writer, see below for File example
	log.SetOutput(os.Stdout)

	// Only log the debug severity or above.
	log.SetLevel(log.DebugLevel)
}<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logs

import (
	"os"
	"time"

	"github.com/go-logr/logr"
	"github.com/go-logr/zapr"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"

	"k8s.io/component-base/config"
	"k8s.io/component-base/logs/registry"
)

var (
	// timeNow stubbed out for testing
	timeNow = time.Now
)

// NewJSONLogger creates a new json logr.Logger and its associated
// flush function. The separate error stream is optional and may be nil.
// The encoder config is also optional.
func NewJSONLogger(infoStream, errorStream zapcore.WriteSyncer, encoderConfig *zapcore.EncoderConfig) (logr.Logger, func()) {
	if encoderConfig == nil {
		encoderConfig = &zapcore.EncoderConfig{
			MessageKey:     "msg",
			CallerKey:      "caller",
			TimeKey:        "ts",
			EncodeTime:     epochMillisTimeEncoder,
			EncodeDuration: zapcore.StringDurationEncoder,
			EncodeCaller:   zapcore.ShortCallerEncoder,
		}
	}

	encoder := zapcore.NewJSONEncoder(*encoderConfig)
	var core zapcore.Core
	if errorStream == nil {
		core = zapcore.NewCore(encoder, infoStream, zapcore.Level(-127))
	} else {
		highPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
			return lvl >= zapcore.ErrorLevel
		})
		lowPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
			return lvl < zapcore.ErrorLevel
		})
		core = zapcore.NewTee(
			zapcore.NewCore(encoder, errorStream, highPriority),
			zapcore.NewCore(encoder, infoStream, lowPriority),
		)
	}
	l := zap.New(core, zap.WithCaller(true))
	return zapr.NewLoggerWithOptions(l, zapr.LogInfoLevel("v"), zapr.ErrorKey("err")), func() {
		l.Sync()
	}
}

func epochMillisTimeEncoder(_ time.Time, enc zapcore.PrimitiveArrayEncoder) {
	nanos := timeNow().UnixNano()
	millis := float64(nanos) / float64(time.Millisecond)
	enc.AppendFloat64(millis)
}

// Factory produces JSON logger instances.
type Factory struct{}

var _ registry.LogFormatFactory = Factory{}

func (f Factory) Create(options config.FormatOptions) (logr.Logger, func()) {
	stderr := zapcore.Lock(os.Stderr)
	if options.JSON.SplitStream {
		stdout := zapcore.Lock(os.Stdout)
		size := options.JSON.InfoBufferSize.Value()
		if size > 0 {
			// Prevent integer overflow.
			if size > 2*1024*1024*1024 {
				size = 2 * 1024 * 1024 * 1024
			}
			stdout = &zapcore.BufferedWriteSyncer{
				WS:   stdout,
				Size: int(size),
			}
		}
		// stdout for info messages, stderr for errors.
		return NewJSONLogger(stdout, stderr, nil)
	}
	// Write info messages and errors to stderr to prevent mixing with normal program output.
	return NewJSONLogger(stderr, nil, nil)
}
<commit_msg>json: never call fsync for stdout or stderr<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logs

import (
	"io"
	"os"
	"time"

	"github.com/go-logr/logr"
	"github.com/go-logr/zapr"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"

	"k8s.io/component-base/config"
	"k8s.io/component-base/logs/registry"
)

var (
	// timeNow stubbed out for testing
	timeNow = time.Now
)

// NewJSONLogger creates a new json logr.Logger and its associated
// flush function. The separate error stream is optional and may be nil.
// The encoder config is also optional.
func NewJSONLogger(infoStream, errorStream zapcore.WriteSyncer, encoderConfig *zapcore.EncoderConfig) (logr.Logger, func()) {
	if encoderConfig == nil {
		encoderConfig = &zapcore.EncoderConfig{
			MessageKey:     "msg",
			CallerKey:      "caller",
			TimeKey:        "ts",
			EncodeTime:     epochMillisTimeEncoder,
			EncodeDuration: zapcore.StringDurationEncoder,
			EncodeCaller:   zapcore.ShortCallerEncoder,
		}
	}

	encoder := zapcore.NewJSONEncoder(*encoderConfig)
	var core zapcore.Core
	if errorStream == nil {
		core = zapcore.NewCore(encoder, infoStream, zapcore.Level(-127))
	} else {
		highPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
			return lvl >= zapcore.ErrorLevel
		})
		lowPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
			return lvl < zapcore.ErrorLevel
		})
		core = zapcore.NewTee(
			zapcore.NewCore(encoder, errorStream, highPriority),
			zapcore.NewCore(encoder, infoStream, lowPriority),
		)
	}
	l := zap.New(core, zap.WithCaller(true))
	return zapr.NewLoggerWithOptions(l, zapr.LogInfoLevel("v"), zapr.ErrorKey("err")), func() {
		l.Sync()
	}
}

func epochMillisTimeEncoder(_ time.Time, enc zapcore.PrimitiveArrayEncoder) {
	nanos := timeNow().UnixNano()
	millis := float64(nanos) / float64(time.Millisecond)
	enc.AppendFloat64(millis)
}

// Factory produces JSON logger instances.
type Factory struct{}

var _ registry.LogFormatFactory = Factory{}

func (f Factory) Create(options config.FormatOptions) (logr.Logger, func()) {
	// We intentionally avoid all os.File.Sync calls. Output is unbuffered,
	// therefore we don't need to flush, and calling the underlying fsync
	// would just slow down writing.
	//
	// The assumption is that logging only needs to ensure that data gets
	// written to the output stream before the process terminates, but
	// doesn't need to worry about data not being written because of a
	// system crash or powerloss.
	stderr := zapcore.Lock(AddNopSync(os.Stderr))
	if options.JSON.SplitStream {
		stdout := zapcore.Lock(AddNopSync(os.Stdout))
		size := options.JSON.InfoBufferSize.Value()
		if size > 0 {
			// Prevent integer overflow.
			if size > 2*1024*1024*1024 {
				size = 2 * 1024 * 1024 * 1024
			}
			stdout = &zapcore.BufferedWriteSyncer{
				WS:   stdout,
				Size: int(size),
			}
		}
		// stdout for info messages, stderr for errors.
		return NewJSONLogger(stdout, stderr, nil)
	}
	// Write info messages and errors to stderr to prevent mixing with normal program output.
	return NewJSONLogger(stderr, nil, nil)
}

// AddNoSync adds a NOP Sync implementation.
func AddNopSync(writer io.Writer) zapcore.WriteSyncer {
	return nopSync{Writer: writer}
}

type nopSync struct {
	io.Writer
}

func (f nopSync) Sync() error {
	return nil
}
<|endoftext|>


<commit_before>package main

// ./airline_arrival_delay_mapper < ~/tmp/clean/On_Time_On_Time_Performance_1990_1_head.csv |./airline_arrival_perf_reducer

import(
	"bufio"
	"fmt"
	"os"
  "strconv"
)

var arrDelays map[string]int64 = make(map[string]int64)
var arrCounts map[string]int64 = make(map[string]int64)

func airline_arrival_perf_reducer_main() {
  scanner := bufio.NewScanner(os.Stdin)
  scanner.Split(bufio.ScanWords)
  for scanner.Scan() {
    airline := scanner.Text()
    checkBool(scanner.Scan())
    delay_str := scanner.Text()
    if delay_str == "_" {
      // "_" is added by cleanup script for records where it is not available
      continue
    }
    delay, err := strconv.Atoi(delay_str)
    check(err, delay_str)
    arrCounts[airline]++
    arrDelays[airline] += int64(delay)
  }
  for airline,delay := range arrDelays {
    fmt.Println(airline, "\t", float64(delay)/float64(arrCounts[airline]));
  }
}
<commit_msg>tolerate int parsing errors in arrival delay<commit_after>package main

// ./airline_arrival_delay_mapper < ~/tmp/clean/On_Time_On_Time_Performance_1990_1_head.csv |./airline_arrival_perf_reducer

import(
	"bufio"
	"fmt"
	"os"
  "strconv"
)

var arrDelays map[string]int64 = make(map[string]int64)
var arrCounts map[string]int64 = make(map[string]int64)

func airline_arrival_perf_reducer_main() {
  delay_parse_errors := 0
  scanner := bufio.NewScanner(os.Stdin)
  scanner.Split(bufio.ScanWords)
  for scanner.Scan() {
    airline := scanner.Text()
    checkBool(scanner.Scan())
    delay_str := scanner.Text()
    if delay_str == "_" {
      // "_" is added by cleanup script for records where it is not available
      continue
    }
    delay, err := strconv.Atoi(delay_str)
    if err != nil {
      delay_parse_errors++
      continue
    }
    arrCounts[airline]++
    arrDelays[airline] += int64(delay)
  }
  for airline,delay := range arrDelays {
    fmt.Println(airline, "\t", float64(delay)/float64(arrCounts[airline]));
  }
  fmt.Println("Number of records with bad arrival delay:", delay_parse_errors)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"k8s.io/test-infra/testgrid/config"
	"path"
	"strconv"
	"strings"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	prowConfig "k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/pod-utils/downwardapi"
	prowGCS "k8s.io/test-infra/prow/pod-utils/gcs"
)

const testgridCreateTestGroupAnnotation = "testgrid-create-test-group"
const testgridDashboardsAnnotation = "testgrid-dashboards"
const testgridTabNameAnnotation = "testgrid-tab-name"
const testgridEmailAnnotation = "testgrid-alert-email"
const testgridNumColumnsRecentAnnotation = "testgrid-num-columns-recent"
const testgridAlertStaleResultsHoursAnnotation = "testgrid-alert-stale-results-hours"
const testgridNumFailuresToAlertAnnotation = "testgrid-num-failures-to-alert"
const descriptionAnnotation = "description"
const minPresubmitNumColumnsRecent = 20

// Talk to @michelle192837 if you're thinking about adding more of these!

func applySingleProwjobAnnotations(c *Config, pc *prowConfig.Config, j prowConfig.JobBase, jobType prowapi.ProwJobType, reconcile bool) error {
	tabName := j.Name
	testGroupName := j.Name
	description := j.Name

	mustMakeGroup := j.Annotations[testgridCreateTestGroupAnnotation] == "true"
	mustNotMakeGroup := j.Annotations[testgridCreateTestGroupAnnotation] == "false"
	dashboards, addToDashboards := j.Annotations[testgridDashboardsAnnotation]
	mightMakeGroup := (mustMakeGroup || addToDashboards || jobType != prowapi.PresubmitJob) && !mustNotMakeGroup
	var testGroup *config.TestGroup

	if mightMakeGroup {
		if testGroup = c.config.FindTestGroup(testGroupName); testGroup != nil {
			if mustMakeGroup {
				return fmt.Errorf("test group %q already exists", testGroupName)
			}
		} else {
			var prefix string
			if j.DecorationConfig != nil && j.DecorationConfig.GCSConfiguration != nil {
				prefix = path.Join(j.DecorationConfig.GCSConfiguration.Bucket, j.DecorationConfig.GCSConfiguration.PathPrefix)
			} else if pc.Plank.DefaultDecorationConfig != nil && pc.Plank.DefaultDecorationConfig.GCSConfiguration != nil {
				prefix = path.Join(pc.Plank.DefaultDecorationConfig.GCSConfiguration.Bucket, pc.Plank.DefaultDecorationConfig.GCSConfiguration.PathPrefix)
			} else {
				return fmt.Errorf("job %s: couldn't figure out a default decoration config", j.Name)
			}

			testGroup = &config.TestGroup{
				Name:      testGroupName,
				GcsPrefix: path.Join(prefix, prowGCS.RootForSpec(&downwardapi.JobSpec{Job: j.Name, Type: jobType})),
			}
			if reconcile {
				ReconcileTestGroup(testGroup, c.defaultConfig.DefaultTestGroup)
			}
			c.config.TestGroups = append(c.config.TestGroups, testGroup)
		}
	} else {
		testGroup = c.config.FindTestGroup(testGroupName)
	}

	if testGroup == nil {
		for _, a := range []string{testgridNumColumnsRecentAnnotation, testgridAlertStaleResultsHoursAnnotation,
			testgridNumFailuresToAlertAnnotation, testgridTabNameAnnotation, testgridEmailAnnotation} {
			_, ok := j.Annotations[a]
			if ok {
				return fmt.Errorf("no testgroup exists for job %q, but annotation %q implies one should exist", j.Name, a)
			}
		}
		// exit early: with no test group, there's nothing else for us to usefully do with the job.
		return nil
	}

	if ncr, ok := j.Annotations[testgridNumColumnsRecentAnnotation]; ok {
		ncrInt, err := strconv.ParseInt(ncr, 10, 32)
		if err != nil {
			return fmt.Errorf("%s value %q is not a valid integer", testgridNumColumnsRecentAnnotation, ncr)
		}
		testGroup.NumColumnsRecent = int32(ncrInt)
	} else if jobType == prowapi.PresubmitJob && testGroup.NumColumnsRecent < minPresubmitNumColumnsRecent {
		testGroup.NumColumnsRecent = minPresubmitNumColumnsRecent
	}

	if srh, ok := j.Annotations[testgridAlertStaleResultsHoursAnnotation]; ok {
		srhInt, err := strconv.ParseInt(srh, 10, 32)
		if err != nil {
			return fmt.Errorf("%s value %q is not a valid integer", testgridAlertStaleResultsHoursAnnotation, srh)
		}
		testGroup.AlertStaleResultsHours = int32(srhInt)
	}

	if nfta, ok := j.Annotations[testgridNumFailuresToAlertAnnotation]; ok {
		nftaInt, err := strconv.ParseInt(nfta, 10, 32)
		if err != nil {
			return fmt.Errorf("%s value %q is not a valid integer", testgridNumFailuresToAlertAnnotation, nfta)
		}
		testGroup.NumFailuresToAlert = int32(nftaInt)
	}

	if tn, ok := j.Annotations[testgridTabNameAnnotation]; ok {
		tabName = tn
	}
	if d := j.Annotations[descriptionAnnotation]; d != "" {
		description = d
	}

	if addToDashboards {
		firstDashboard := true
		for _, dashboardName := range strings.Split(dashboards, ",") {
			dashboardName = strings.TrimSpace(dashboardName)
			d := c.config.FindDashboard(dashboardName)
			if d == nil {
				return fmt.Errorf("couldn't find dashboard %q for job %q", dashboardName, j.Name)
			}
			dt := &config.DashboardTab{
				Name:          tabName,
				TestGroupName: testGroupName,
				Description:   description,
			}
			if firstDashboard {
				firstDashboard = false
				if emails, ok := j.Annotations[testgridEmailAnnotation]; ok {
					dt.AlertOptions = &config.DashboardTabAlertOptions{AlertMailToAddresses: emails}
				}
			}
			if reconcile {
				ReconcileDashboardTab(dt, c.defaultConfig.DefaultDashboardTab)
			}
			d.DashboardTab = append(d.DashboardTab, dt)
		}
	}

	return nil
}

func applyProwjobAnnotations(c *Config, prowConfigAgent *prowConfig.Agent, reconcileDefaults bool) error {
	pc := prowConfigAgent.Config()
	if pc == nil {
		return nil
	}
	jobs := prowConfigAgent.Config().JobConfig
	for _, j := range jobs.AllPeriodics() {
		if err := applySingleProwjobAnnotations(c, pc, j.JobBase, prowapi.PeriodicJob, reconcileDefaults); err != nil {
			return err
		}
	}
	for _, j := range jobs.AllPostsubmits(nil) {
		if err := applySingleProwjobAnnotations(c, pc, j.JobBase, prowapi.PostsubmitJob, reconcileDefaults); err != nil {
			return err
		}
	}
	for _, j := range jobs.AllPresubmits(nil) {
		if err := applySingleProwjobAnnotations(c, pc, j.JobBase, prowapi.PresubmitJob, reconcileDefaults); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Generate a relevant URL template for annotated jobs.<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"k8s.io/test-infra/testgrid/config"
	"path"
	"strconv"
	"strings"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	prowConfig "k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/pod-utils/downwardapi"
	prowGCS "k8s.io/test-infra/prow/pod-utils/gcs"
)

const testgridCreateTestGroupAnnotation = "testgrid-create-test-group"
const testgridDashboardsAnnotation = "testgrid-dashboards"
const testgridTabNameAnnotation = "testgrid-tab-name"
const testgridEmailAnnotation = "testgrid-alert-email"
const testgridNumColumnsRecentAnnotation = "testgrid-num-columns-recent"
const testgridAlertStaleResultsHoursAnnotation = "testgrid-alert-stale-results-hours"
const testgridNumFailuresToAlertAnnotation = "testgrid-num-failures-to-alert"
const descriptionAnnotation = "description"
const minPresubmitNumColumnsRecent = 20

// Talk to @michelle192837 if you're thinking about adding more of these!

func applySingleProwjobAnnotations(c *Config, pc *prowConfig.Config, j prowConfig.JobBase, jobType prowapi.ProwJobType, repo string, reconcile bool) error {
	tabName := j.Name
	testGroupName := j.Name
	description := j.Name

	mustMakeGroup := j.Annotations[testgridCreateTestGroupAnnotation] == "true"
	mustNotMakeGroup := j.Annotations[testgridCreateTestGroupAnnotation] == "false"
	dashboards, addToDashboards := j.Annotations[testgridDashboardsAnnotation]
	mightMakeGroup := (mustMakeGroup || addToDashboards || jobType != prowapi.PresubmitJob) && !mustNotMakeGroup
	var testGroup *config.TestGroup

	if mightMakeGroup {
		if testGroup = c.config.FindTestGroup(testGroupName); testGroup != nil {
			if mustMakeGroup {
				return fmt.Errorf("test group %q already exists", testGroupName)
			}
		} else {
			var prefix string
			if j.DecorationConfig != nil && j.DecorationConfig.GCSConfiguration != nil {
				prefix = path.Join(j.DecorationConfig.GCSConfiguration.Bucket, j.DecorationConfig.GCSConfiguration.PathPrefix)
			} else if pc.Plank.DefaultDecorationConfig != nil && pc.Plank.DefaultDecorationConfig.GCSConfiguration != nil {
				prefix = path.Join(pc.Plank.DefaultDecorationConfig.GCSConfiguration.Bucket, pc.Plank.DefaultDecorationConfig.GCSConfiguration.PathPrefix)
			} else {
				return fmt.Errorf("job %s: couldn't figure out a default decoration config", j.Name)
			}

			testGroup = &config.TestGroup{
				Name:      testGroupName,
				GcsPrefix: path.Join(prefix, prowGCS.RootForSpec(&downwardapi.JobSpec{Job: j.Name, Type: jobType})),
			}
			if reconcile {
				ReconcileTestGroup(testGroup, c.defaultConfig.DefaultTestGroup)
			}
			c.config.TestGroups = append(c.config.TestGroups, testGroup)
		}
	} else {
		testGroup = c.config.FindTestGroup(testGroupName)
	}

	if testGroup == nil {
		for _, a := range []string{testgridNumColumnsRecentAnnotation, testgridAlertStaleResultsHoursAnnotation,
			testgridNumFailuresToAlertAnnotation, testgridTabNameAnnotation, testgridEmailAnnotation} {
			_, ok := j.Annotations[a]
			if ok {
				return fmt.Errorf("no testgroup exists for job %q, but annotation %q implies one should exist", j.Name, a)
			}
		}
		// exit early: with no test group, there's nothing else for us to usefully do with the job.
		return nil
	}

	if ncr, ok := j.Annotations[testgridNumColumnsRecentAnnotation]; ok {
		ncrInt, err := strconv.ParseInt(ncr, 10, 32)
		if err != nil {
			return fmt.Errorf("%s value %q is not a valid integer", testgridNumColumnsRecentAnnotation, ncr)
		}
		testGroup.NumColumnsRecent = int32(ncrInt)
	} else if jobType == prowapi.PresubmitJob && testGroup.NumColumnsRecent < minPresubmitNumColumnsRecent {
		testGroup.NumColumnsRecent = minPresubmitNumColumnsRecent
	}

	if srh, ok := j.Annotations[testgridAlertStaleResultsHoursAnnotation]; ok {
		srhInt, err := strconv.ParseInt(srh, 10, 32)
		if err != nil {
			return fmt.Errorf("%s value %q is not a valid integer", testgridAlertStaleResultsHoursAnnotation, srh)
		}
		testGroup.AlertStaleResultsHours = int32(srhInt)
	}

	if nfta, ok := j.Annotations[testgridNumFailuresToAlertAnnotation]; ok {
		nftaInt, err := strconv.ParseInt(nfta, 10, 32)
		if err != nil {
			return fmt.Errorf("%s value %q is not a valid integer", testgridNumFailuresToAlertAnnotation, nfta)
		}
		testGroup.NumFailuresToAlert = int32(nftaInt)
	}

	if tn, ok := j.Annotations[testgridTabNameAnnotation]; ok {
		tabName = tn
	}
	if d := j.Annotations[descriptionAnnotation]; d != "" {
		description = d
	}

	if addToDashboards {
		firstDashboard := true
		for _, dashboardName := range strings.Split(dashboards, ",") {
			dashboardName = strings.TrimSpace(dashboardName)
			d := c.config.FindDashboard(dashboardName)
			if d == nil {
				return fmt.Errorf("couldn't find dashboard %q for job %q", dashboardName, j.Name)
			}
			if repo == "" {
				if len(j.ExtraRefs) > 0 {
					repo = fmt.Sprintf("%s/%s", j.ExtraRefs[0].Org, j.ExtraRefs[0].Repo)
				}
			}
			var linkTemplate *config.LinkTemplate
			if repo != "" {
				linkTemplate = &config.LinkTemplate{
					Url: fmt.Sprintf("https://github.com/%s/compare/<start-custom-0>...<end-custom-0>", repo),
				}
			}
			dt := &config.DashboardTab{
				Name:                  tabName,
				TestGroupName:         testGroupName,
				Description:           description,
				CodeSearchUrlTemplate: linkTemplate,
			}
			if firstDashboard {
				firstDashboard = false
				if emails, ok := j.Annotations[testgridEmailAnnotation]; ok {
					dt.AlertOptions = &config.DashboardTabAlertOptions{AlertMailToAddresses: emails}
				}
			}
			if reconcile {
				ReconcileDashboardTab(dt, c.defaultConfig.DefaultDashboardTab)
			}
			d.DashboardTab = append(d.DashboardTab, dt)
		}
	}

	return nil
}

func applyProwjobAnnotations(c *Config, prowConfigAgent *prowConfig.Agent, reconcileDefaults bool) error {
	pc := prowConfigAgent.Config()
	if pc == nil {
		return nil
	}
	jobs := prowConfigAgent.Config().JobConfig
	for _, j := range jobs.AllPeriodics() {
		if err := applySingleProwjobAnnotations(c, pc, j.JobBase, prowapi.PeriodicJob, "", reconcileDefaults); err != nil {
			return err
		}
	}

	for repo, js := range jobs.Postsubmits {
		for _, j := range js {
			if err := applySingleProwjobAnnotations(c, pc, j.JobBase, prowapi.PostsubmitJob, repo, reconcileDefaults); err != nil {
				return err
			}
		}
	}

	for repo, js := range jobs.Presubmits {
		for _, j := range js {
			if err := applySingleProwjobAnnotations(c, pc, j.JobBase, prowapi.PresubmitJob, repo, reconcileDefaults); err != nil {
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// deck: make slide decks
package main

import (
	"bufio"
	"encoding/xml"
	"fmt"
	"os"
	"strings"

	"github.com/ajstarks/openvg"
)

// Deck defines the structure of a presentation deck
// The size of the canvas, and series of slides
type Deck struct {
	Canvas canvas  `xml:"canvas"`
	Slide  []slide `xml:"slide"`
}

type canvas struct {
	Width  int `xml:"width,attr"`
	Height int `xml:"height,attr"`
}

type list struct {
	Xp    float64  `xml:"xp,attr"`
	Yp    float64  `xml:"yp,attr"`
	Sp    float64  `xml:"sp,attr"`
	Type  string   `xml:"type,attr"`
	Align string   `xml:"align,attr"`
	Color string   `xml:"color,attr"`
	Li    []string `xml:"li"`
}

type text struct {
	Xp    float64 `xml:"xp,attr"`
	Yp    float64 `xml:"yp,attr"`
	Sp    float64 `xml:"sp,attr"`
	Type  string  `xml:"type,attr"`
	Align string  `xml:"align,attr"`
	Color string  `xml:"color,attr"`
	Tdata string  `xml:",chardata"`
}

type image struct {
	Xp     float64 `xml:"xp,attr"`
	Yp     float64 `xml:"yp,attr"`
	Width  int     `xml:"width,attr"`
	Height int     `xml:"height,attr"`
	Name   string  `xml:"name,attr"`
}

type slide struct {
	Bg    string  `xml:"bg,attr"`
	Fg    string  `xml:"fg,attr"`
	List  []list  `xml:"list"`
	Text  []text  `xml:"text"`
	Image []image `xml:"image"`
}

// dodeck sets up the graphics environment and kicks off the interaction
func dodeck(filename string) {
	w, h := openvg.Init()
	interact(filename, w, h)
	openvg.Finish()
}

// readdeck reads the deck description file
func readdeck(filename string) (Deck, error) {
	var d Deck
	r, err := os.Open(filename)
	if err != nil {
		return d, err
	}
	defer r.Close()

	err = xml.NewDecoder(r).Decode(&d)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return d, err
	}
	return d, err
}

// interact controls the display of the deck
func interact(filename string, w, h int) {
	openvg.SaveTerm()
	defer openvg.RestoreTerm()
	var d Deck
	var err error
	d, err = readdeck(filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return
	}
	if d.Canvas.Width == 0 {
		d.Canvas.Width = w
	}
	if d.Canvas.Height == 0 {
		d.Canvas.Height = h
	}
	openvg.RawTerm()
	r := bufio.NewReader(os.Stdin)
	firstslide := 0
	lastslide := len(d.Slide) - 1
	n := firstslide

	for cmd := byte('0'); cmd != 'q'; cmd = readcmd(r) {
		switch cmd {
		case 'r':
			d, err = readdeck(filename)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%v\n", err)
				return
			}
		case '0', '1':
			n = firstslide
			showslide(d, n)

		case '*':
			n = lastslide
			showslide(d, n)

		case '+', 'n', '\n', ' ':
			n++
			if n > lastslide {
				n = firstslide
			}
			showslide(d, n)

		case '-', 'p', 8, 127:
			n--
			if n < firstslide {
				n = lastslide
			}
			showslide(d, n)

		case '/':
			openvg.RestoreTerm()
			searchterm, serr := r.ReadString('\n')
			openvg.RawTerm()
			if serr != nil {
				continue
			}
			if len(searchterm) > 2 {
				ns := searchdeck(d, searchterm[0:len(searchterm)-1])
				if ns >= 0 {
					showslide(d, ns)
				}
			}
		}
	}
}

// dimen computes the coordinates and size of an object
func dimen(c canvas, xp, yp, sp float64) (x, y float64, s int) {
	x = (xp / 100) * float64(c.Width)
	y = (yp / 100) * float64(c.Height)
	s = int((sp / 100) * float64(c.Width))
	return
}

//showtext displays text
func showtext(x, y float64, s, align, font string, fontsize int) {
	switch align {
	case "center", "middle", "mid":
		openvg.TextMid(x, y, s, font, fontsize)
	case "right", "end":
		openvg.TextEnd(x, y, s, font, fontsize)
	default:
		openvg.Text(x, y, s, font, fontsize)
	}
}

// showlide displays slides
func showslide(d Deck, n int) {
	if n < 0 || n > len(d.Slide)-1 {
		return
	}
	s := d.Slide[n]
	openvg.Start(d.Canvas.Width, d.Canvas.Height)
	if s.Bg == "" {
		s.Bg = "white"
	}
	if s.Fg == "" {
		s.Fg = "black"
	}
	openvg.BackgroundColor(s.Bg)
	openvg.FillColor(s.Fg)

	var x, y float64
	var fontsize int

	// every image in the slide
	for _, im := range s.Image {
		x = (im.Xp / 100) * float64(d.Canvas.Width)
		y = (im.Yp / 100) * float64(d.Canvas.Height)
		openvg.Image(x-float64(im.Width/2), y-float64(im.Height/2), im.Width, im.Height, im.Name)
	}
	// every list in the slide
	var offset float64
	for _, l := range s.List {
		x, y, fontsize = dimen(d.Canvas, l.Xp, l.Yp, l.Sp)
		if l.Type == "bullet" {
			offset = 1.2 * float64(fontsize)
		} else {
			offset = 0
		}
		if l.Color != "" {
			openvg.FillColor(l.Color)
		} else {
			openvg.FillColor(s.Fg)
		}
		// every list item
		for _, li := range l.Li {
			if l.Type == "bullet" {
				boffset := float64(fontsize) / 2
				openvg.Circle(x, y+boffset, boffset)
			}
			showtext(x+offset, y, li, l.Align, "sans", fontsize)
			y -= float64(fontsize) * 2.0
		}
	}
	openvg.FillColor(s.Fg)

	// every text in the slide
	var font string
	for _, t := range s.Text {
		x, y, fontsize = dimen(d.Canvas, t.Xp, t.Yp, t.Sp)
		td := strings.Split(t.Tdata, "\n")
		if t.Type == "code" {
			tdepth := (float64(fontsize) * 1.8) * float64(len(td)-1)
			font = "mono"
			openvg.FillColor("rgb(240,240,240)")
			openvg.Rect(x-20, y-tdepth+40, float64(d.Canvas.Width)-20-x, tdepth)
		} else {
			font = "sans"
		}
		if t.Color != "" {
			openvg.FillColor(t.Color)
		} else {
			openvg.FillColor(s.Fg)
		}
		// every text line
		for _, txt := range td {
			showtext(x, y, txt, t.Align, font, fontsize)
			y -= float64(fontsize) * 1.8
		}
	}
	openvg.FillColor(s.Fg)
	openvg.End()
}

// searchdeck searches the deck for the specified text, returning the slide number if found
func searchdeck(d Deck, s string) int {
	// for every slide...
	for i := 0; i < len(d.Slide); i++ {
		// search lists
		for _, l := range d.Slide[i].List {
			for _, ll := range l.Li {
				if strings.Contains(ll, s) {
					return i
				}
			}
		}
		// search text
		for _, t := range d.Slide[i].Text {
			if strings.Contains(t.Tdata, s) {
				return i
			}
		}
	}
	return -1
}

// dumpdeck shows the decoded description
func dumpdeck(d Deck) {
	fmt.Printf("Canvas = %v\n", d.Canvas)
	for i, s := range d.Slide {
		fmt.Printf("Slide #%d = %v %v\n", i, s.Bg, s.Fg)
		for j, l := range s.List {
			fmt.Printf("\tList #%d = %#v\n", j, l)
		}
		for k, t := range s.Text {
			fmt.Printf("\tText #%d = %#v\n", k, t)
		}
		for m, im := range s.Image {
			fmt.Printf("\tImage #%d = %#v\n", m, im)
		}
	}
}

// readcmd reads interaction commands
func readcmd(r *bufio.Reader) byte {
	s, err := r.ReadByte()
	if err != nil {
		return 'e'
	}
	return s
}

// for every file, make a deck
func main() {
	if len(os.Args) > 1 {
		for _, f := range os.Args[1:] {
			dodeck(f)
		}
	}
}
<commit_msg>fix "r" command when no canvas specified, cleanups<commit_after>// deck: make slide decks
package main

import (
	"bufio"
	"encoding/xml"
	"fmt"
	"os"
	"strings"

	"github.com/ajstarks/openvg"
)

// Deck defines the structure of a presentation deck
// The size of the canvas, and series of slides
type Deck struct {
	Canvas canvas  `xml:"canvas"`
	Slide  []slide `xml:"slide"`
}

type canvas struct {
	Width  int `xml:"width,attr"`
	Height int `xml:"height,attr"`
}

type list struct {
	Xp    float64  `xml:"xp,attr"`
	Yp    float64  `xml:"yp,attr"`
	Sp    float64  `xml:"sp,attr"`
	Type  string   `xml:"type,attr"`
	Align string   `xml:"align,attr"`
	Color string   `xml:"color,attr"`
	Li    []string `xml:"li"`
}

type text struct {
	Xp    float64 `xml:"xp,attr"`
	Yp    float64 `xml:"yp,attr"`
	Sp    float64 `xml:"sp,attr"`
	Type  string  `xml:"type,attr"`
	Align string  `xml:"align,attr"`
	Color string  `xml:"color,attr"`
	Tdata string  `xml:",chardata"`
}

type image struct {
	Xp     float64 `xml:"xp,attr"`
	Yp     float64 `xml:"yp,attr"`
	Width  int     `xml:"width,attr"`
	Height int     `xml:"height,attr"`
	Name   string  `xml:"name,attr"`
}

type slide struct {
	Bg    string  `xml:"bg,attr"`
	Fg    string  `xml:"fg,attr"`
	List  []list  `xml:"list"`
	Text  []text  `xml:"text"`
	Image []image `xml:"image"`
}

// dodeck sets up the graphics environment and kicks off the interaction
func dodeck(filename string) {
	w, h := openvg.Init()
	interact(filename, w, h)
	openvg.Finish()
}

// readdeck reads the deck description file
func readdeck(filename string, w, h int) (Deck, error) {
	var d Deck
	r, err := os.Open(filename)
	if err != nil {
		return d, err
	}

	err = xml.NewDecoder(r).Decode(&d)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
	}
	if d.Canvas.Width == 0 {
		d.Canvas.Width = w
	}
	if d.Canvas.Height == 0 {
		d.Canvas.Height = h
	}
	r.Close()
	return d, err
}

// interact controls the display of the deck
func interact(filename string, w, h int) {
	openvg.SaveTerm()
	defer openvg.RestoreTerm()
	var d Deck
	var err error
	d, err = readdeck(filename, w, h)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return
	}
	openvg.RawTerm()
	r := bufio.NewReader(os.Stdin)
	firstslide := 0
	lastslide := len(d.Slide) - 1
	n := firstslide

	for cmd := byte('0'); cmd != 'q'; cmd = readcmd(r) {
		switch cmd {
		case 'r':
			d, err = readdeck(filename, w, h)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%v\n", err)
				return
			}
		case '0', '1':
			n = firstslide
			showslide(d, n)

		case '*':
			n = lastslide
			showslide(d, n)

		case '+', 'n', '\n', ' ':
			n++
			if n > lastslide {
				n = firstslide
			}
			showslide(d, n)

		case '-', 'p', 8, 127:
			n--
			if n < firstslide {
				n = lastslide
			}
			showslide(d, n)

		case '/':
			openvg.RestoreTerm()
			searchterm, serr := r.ReadString('\n')
			openvg.RawTerm()
			if serr != nil {
				continue
			}
			if len(searchterm) > 2 {
				ns := searchdeck(d, searchterm[0:len(searchterm)-1])
				if ns >= 0 {
					showslide(d, ns)
				}
			}
		}
	}
}

// dimen computes the coordinates and size of an object
func dimen(c canvas, xp, yp, sp float64) (x, y float64, s int) {
	x = (xp / 100) * float64(c.Width)
	y = (yp / 100) * float64(c.Height)
	s = int((sp / 100) * float64(c.Width))
	return
}

//showtext displays text
func showtext(x, y float64, s, align, font string, fontsize int) {
	switch align {
	case "center", "middle", "mid":
		openvg.TextMid(x, y, s, font, fontsize)
	case "right", "end":
		openvg.TextEnd(x, y, s, font, fontsize)
	default:
		openvg.Text(x, y, s, font, fontsize)
	}
}

// showlide displays slides
func showslide(d Deck, n int) {
	if n < 0 || n > len(d.Slide)-1 {
		return
	}
	s := d.Slide[n]
	openvg.Start(d.Canvas.Width, d.Canvas.Height)
	if s.Bg == "" {
		s.Bg = "white"
	}
	if s.Fg == "" {
		s.Fg = "black"
	}
	openvg.BackgroundColor(s.Bg)
	openvg.FillColor(s.Fg)

	var x, y float64
	var fontsize int

	// every image in the slide
	for _, im := range s.Image {
		x = (im.Xp / 100) * float64(d.Canvas.Width)
		y = (im.Yp / 100) * float64(d.Canvas.Height)
		openvg.Image(x-float64(im.Width/2), y-float64(im.Height/2), im.Width, im.Height, im.Name)
	}
	// every list in the slide
	var offset float64
	const blinespacing = 2.0
	for _, l := range s.List {
		x, y, fontsize = dimen(d.Canvas, l.Xp, l.Yp, l.Sp)
		fs := float64(fontsize)
		if l.Type == "bullet" {
			offset = 1.2 * float64(fontsize)
		} else {
			offset = 0
		}
		if l.Color != "" {
			openvg.FillColor(l.Color)
		} else {
			openvg.FillColor(s.Fg)
		}
		// every list item
		for _, li := range l.Li {
			if l.Type == "bullet" {
				boffset := fs / 2
				openvg.Circle(x, y+boffset, boffset)
			}
			showtext(x+offset, y, li, l.Align, "sans", fontsize)
			y -= fs * blinespacing
		}
	}
	openvg.FillColor(s.Fg)

	// every text in the slide
	var font string
	const linespacing = 1.8
	for _, t := range s.Text {
		x, y, fontsize = dimen(d.Canvas, t.Xp, t.Yp, t.Sp)
		fs := float64(fontsize)
		td := strings.Split(t.Tdata, "\n")
		if t.Type == "code" {
			tdepth := ((fs*linespacing)*float64(len(td))) + fs
			font = "mono"
			openvg.FillColor("rgb(240,240,240)")
			openvg.Rect(x-20, y-tdepth+(fs*linespacing), float64(d.Canvas.Width)-20-x, tdepth)
		} else {
			font = "sans"
		}
		if t.Color != "" {
			openvg.FillColor(t.Color)
		} else {
			openvg.FillColor(s.Fg)
		}
		// every text line
		for _, txt := range td {
			showtext(x, y, txt, t.Align, font, fontsize)
			y -= (fs * linespacing)
		}
	}
	openvg.FillColor(s.Fg)
	openvg.End()
	//openvg.SaveEnd(fmt.Sprintf("slide%0d.raw", n))
}

// searchdeck searches the deck for the specified text, returning the slide number if found
func searchdeck(d Deck, s string) int {
	// for every slide...
	for i := 0; i < len(d.Slide); i++ {
		// search lists
		for _, l := range d.Slide[i].List {
			for _, ll := range l.Li {
				if strings.Contains(ll, s) {
					return i
				}
			}
		}
		// search text
		for _, t := range d.Slide[i].Text {
			if strings.Contains(t.Tdata, s) {
				return i
			}
		}
	}
	return -1
}

// dumpdeck shows the decoded description
func dumpdeck(d Deck) {
	fmt.Printf("Canvas = %v\n", d.Canvas)
	for i, s := range d.Slide {
		fmt.Printf("Slide #%d = %v %v\n", i, s.Bg, s.Fg)
		for j, l := range s.List {
			fmt.Printf("\tList #%d = %#v\n", j, l)
		}
		for k, t := range s.Text {
			fmt.Printf("\tText #%d = %#v\n", k, t)
		}
		for m, im := range s.Image {
			fmt.Printf("\tImage #%d = %#v\n", m, im)
		}
	}
}

// readcmd reads interaction commands
func readcmd(r *bufio.Reader) byte {
	s, err := r.ReadByte()
	if err != nil {
		return 'e'
	}
	return s
}

// for every file, make a deck
func main() {
	if len(os.Args) > 1 {
		for _, f := range os.Args[1:] {
			dodeck(f)
		}
	}
}
<|endoftext|>


<commit_before>package dirContent

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

// Root directory global variable
var ROOT_DIR string = "E:\\Pete\\Documents"

// Directory content struct
type Content struct {
	Name string `json:"name"`
	Path string `json:"path"`
	Size int64  `json:"size"`
	Type string `json:"type"`
}

// Path struct for unmarshalling JSON array of paths.
type Path struct {
	Path string
}

// Get directory Content and return a JSON-encoded string
func GetDirectoryContentInJSON(dir string) ([]Content, int) {

	fullDir := filepath.Join(ROOT_DIR, dir)

	// Retrieve all the files in the input directory
	files, err := ioutil.ReadDir(fullDir)
	if err != nil {
		return nil, 404 // Status 404 is path not available
	}

	// Create a Content struct for each file and append it to a Content list.
	var listOfContent []Content
	var ext string
	var name string
	var fullPath string

	for _, file := range files {
		// Determine whether it is a file or a folder
		if file.IsDir() {
			ext = "folder"
			name = file.Name()
			fullPath = filepath.Join(fullDir, file.Name())
		} else {
			ext = filepath.Ext(file.Name())
			name = strings.TrimSuffix(file.Name(), ext)
			fullPath = filepath.Join(fullDir, file.Name())
		}
		c := Content{
			Name: name,
			Path: fullPath,
			Size: file.Size(),
			Type: ext}

		listOfContent = append(listOfContent, c)
	}

	return listOfContent, 0
}

// Delete user-chosen content from given directory.
func deleteSelectedContentFromDir(jsonSelected []byte) bool {
	var paths []Path
	jsonErr := json.Unmarshal(jsonSelected, &paths) //TODO: Use Decoder
	if jsonErr != nil {
		log.Fatal(jsonErr)
	}

	// Run through the paths and delete them.
	for n := range paths {
		err := os.Remove(paths[n].Path)
		if err != nil {
			log.Fatal(err)
			return false
		}
	}
	return true
}
<commit_msg>fullPath in the dir / file check updated<commit_after>package dirContent

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

// Root directory global variable
var ROOT_DIR string = "E:\\Pete\\Documents"

// Directory content struct
type Content struct {
	Name string `json:"name"`
	Path string `json:"path"`
	Size int64  `json:"size"`
	Type string `json:"type"`
}

// Path struct for unmarshalling JSON array of paths.
type Path struct {
	Path string
}

// Get directory Content and return a JSON-encoded string
func GetDirectoryContentInJSON(dir string) ([]Content, int) {

	fullDir := filepath.Join(ROOT_DIR, dir)

	// Retrieve all the files in the input directory
	files, err := ioutil.ReadDir(fullDir)
	if err != nil {
		return nil, 404 // Status 404 is path not available
	}

	// Create a Content struct for each file and append it to a Content list.
	var listOfContent []Content
	var ext string
	var name string
	var fullPath string

	for _, file := range files {
		// Determine whether it is a file or a folder
		if file.IsDir() {
			ext = "folder"
			name = file.Name()
		} else {
			ext = filepath.Ext(file.Name())
			name = strings.TrimSuffix(file.Name(), ext)
		}
		fullPath = filepath.Join(fullDir, file.Name())
		c := Content{
			Name: name,
			Path: fullPath,
			Size: file.Size(),
			Type: ext}

		listOfContent = append(listOfContent, c)
	}

	return listOfContent, 0
}

// Delete user-chosen content from given directory.
func deleteSelectedContentFromDir(jsonSelected []byte) bool {
	var paths []Path
	jsonErr := json.Unmarshal(jsonSelected, &paths) //TODO: Use Decoder
	if jsonErr != nil {
		log.Fatal(jsonErr)
	}

	// Run through the paths and delete them.
	for n := range paths {
		err := os.Remove(paths[n].Path)
		if err != nil {
			log.Fatal(err)
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>// Copyright 2016, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package vtgate

import (
	"errors"
	"fmt"

	"github.com/youtube/vitess/go/sqltypes"
	"github.com/youtube/vitess/go/vt/vtgate/planbuilder"

	querypb "github.com/youtube/vitess/go/vt/proto/query"
)

func (rtr *Router) execPlan(vcursor *requestContext, plan interface{}) (*sqltypes.Result, error) {
	route, ok := plan.(*planbuilder.Route)
	if !ok {
		return nil, errors.New("no joins yet")
	}
	return rtr.execRoute(vcursor, route)
}

func (rtr *Router) execJoin(vcursor *requestContext, join *planbuilder.Join) (*sqltypes.Result, error) {
	lresult, err := rtr.execPlan(vcursor, join.Left)
	if err != nil {
		return nil, err
	}
	saved := copyBindVars(vcursor.JoinVars)
	defer func() { vcursor.JoinVars = saved }()
	result := &sqltypes.Result{}
	if len(lresult.Rows) == 0 {
		// We still need field info from the RHS.
		// TODO(sougou): Change this to use an impossible query.
		for k := range join.Vars {
			vcursor.JoinVars[k] = nil
		}
		rresult, err := rtr.execPlan(vcursor, join.Right)
		if err != nil {
			return nil, err
		}
		result.Fields = joinFields(lresult.Fields, rresult.Fields, join.Cols)
		return result, nil
	}
	for _, lrow := range lresult.Rows {
		for k, col := range join.Vars {
			vcursor.JoinVars[k] = lrow[col]
		}
		rresult, err := rtr.execPlan(vcursor, join.Right)
		if err != nil {
			return nil, err
		}
		result.Fields = joinFields(lresult.Fields, rresult.Fields, join.Cols)
		if len(rresult.Rows) == 0 {
			return result, nil
		}
		if join.IsLeft && len(rresult.Rows) == 0 {
			rresult.Rows = [][]sqltypes.Value{make([]sqltypes.Value, len(rresult.Fields))}
		}
		for _, rrow := range rresult.Rows {
			result.Rows = append(result.Rows, joinRows(lrow, rrow, join.Cols))
		}
		result.RowsAffected += uint64(len(rresult.Rows))
	}
	return result, nil
}

func copyBindVars(bindVars map[string]interface{}) map[string]interface{} {
	out := make(map[string]interface{})
	for k, v := range bindVars {
		out[k] = v
	}
	return out
}

func joinFields(lfields, rfields []*querypb.Field, cols []int) []*querypb.Field {
	if lfields == nil || rfields == nil {
		return nil
	}
	fields := make([]*querypb.Field, len(cols))
	for i, index := range cols {
		if index < 0 {
			fields[i] = lfields[-index-1]
			continue
		}
		fields[i] = rfields[index-1]
	}
	return fields
}

func joinRows(lrow, rrow []sqltypes.Value, cols []int) []sqltypes.Value {
	row := make([]sqltypes.Value, len(cols))
	for i, index := range cols {
		if index < 0 {
			row[i] = lrow[-index-1]
			continue
		}
		row[i] = rrow[index-1]
	}
	return row
}

func (rtr *Router) execRoute(vcursor *requestContext, route *planbuilder.Route) (*sqltypes.Result, error) {
	var err error
	var params *scatterParams
	switch route.PlanID {
	case planbuilder.SelectUnsharded:
		params, err = rtr.paramsUnsharded2(vcursor, route)
	case planbuilder.SelectEqual, planbuilder.SelectEqualUnique:
		params, err = rtr.paramsSelectEqual2(vcursor, route)
	case planbuilder.SelectIN:
		params, err = rtr.paramsSelectIN2(vcursor, route)
	case planbuilder.SelectScatter:
		params, err = rtr.paramsSelectScatter2(vcursor, route)
	default:
		// TODO(sougou): improve error.
		return nil, fmt.Errorf("unsupported query plan: %v", route)
	}
	if err != nil {
		return nil, err
	}
	saved := copyBindVars(vcursor.bindVars)
	defer func() { vcursor.bindVars = saved }()
	for k := range route.UseVars {
		vcursor.bindVars[k] = vcursor.JoinVars[k]
	}
	return rtr.scatterConn.ExecuteMulti(
		vcursor.ctx,
		params.query,
		params.ks,
		params.shardVars,
		vcursor.tabletType,
		NewSafeSession(vcursor.session),
		vcursor.notInTransaction,
	)
}

func (rtr *Router) streamExecPlan(vcursor *requestContext, plan interface{}, sendReply func(*sqltypes.Result) error) error {
	route, ok := plan.(*planbuilder.Route)
	if !ok {
		return errors.New("no joins yet")
	}
	return rtr.streamExecRoute(vcursor, route, sendReply)
}

func (rtr *Router) streamExecJoin(vcursor *requestContext, join *planbuilder.Join, sendReply func(*sqltypes.Result) error) error {
	fieldSent := false
	var rhsLen int
	err := rtr.streamExecPlan(vcursor, join.Left, func(lresult *sqltypes.Result) error {
		saved := copyBindVars(vcursor.JoinVars)
		defer func() { vcursor.JoinVars = saved }()
		for _, lrow := range lresult.Rows {
			for k, col := range join.Vars {
				vcursor.JoinVars[k] = lrow[col]
			}
			rowSent := false
			err := rtr.streamExecPlan(vcursor, join.Right, func(rresult *sqltypes.Result) error {
				result := &sqltypes.Result{}
				if !fieldSent {
					result.Fields = joinFields(lresult.Fields, rresult.Fields, join.Cols)
					fieldSent = true
					rhsLen = len(rresult.Fields)
				}
				if len(result.Rows) != 0 {
					rowSent = true
				}
				for _, rrow := range rresult.Rows {
					result.Rows = append(result.Rows, joinRows(lrow, rrow, join.Cols))
				}
				return sendReply(result)
			})
			if err != nil {
				return err
			}
			if join.IsLeft && !rowSent {
				result := &sqltypes.Result{}
				result.Rows = [][]sqltypes.Value{joinRows(
					lrow,
					make([]sqltypes.Value, rhsLen),
					join.Cols,
				)}
				err := sendReply(result)
				if err != nil {
					return err
				}
			}
		}
		if !fieldSent {
			for k := range join.Vars {
				vcursor.JoinVars[k] = nil
			}
			return rtr.streamExecPlan(vcursor, join.Right, func(rresult *sqltypes.Result) error {
				result := &sqltypes.Result{}
				result.Fields = joinFields(lresult.Fields, rresult.Fields, join.Cols)
				fieldSent = true
				rhsLen = len(rresult.Fields)
				return sendReply(result)
			})
		}
		return nil
	})
	return err
}

func (rtr *Router) streamExecRoute(vcursor *requestContext, route *planbuilder.Route, sendReply func(*sqltypes.Result) error) error {
	var err error
	var params *scatterParams
	switch route.PlanID {
	case planbuilder.SelectUnsharded:
		params, err = rtr.paramsUnsharded2(vcursor, route)
	case planbuilder.SelectEqual, planbuilder.SelectEqualUnique:
		params, err = rtr.paramsSelectEqual2(vcursor, route)
	case planbuilder.SelectIN:
		params, err = rtr.paramsSelectIN2(vcursor, route)
	case planbuilder.SelectScatter:
		params, err = rtr.paramsSelectScatter2(vcursor, route)
	default:
		// TODO(sougou): improve error.
		return fmt.Errorf("unsupported query plan: %v", route)
	}
	if err != nil {
		return err
	}
	saved := copyBindVars(vcursor.bindVars)
	defer func() { vcursor.bindVars = saved }()
	for k := range route.UseVars {
		vcursor.bindVars[k] = vcursor.JoinVars[k]
	}
	return rtr.scatterConn.StreamExecuteMulti(
		vcursor.ctx,
		params.query,
		params.ks,
		params.shardVars,
		vcursor.tabletType,
		sendReply,
	)
}

func (rtr *Router) paramsUnsharded2(vcursor *requestContext, route *planbuilder.Route) (*scatterParams, error) {
	ks, _, allShards, err := getKeyspaceShards(vcursor.ctx, rtr.serv, rtr.cell, route.Keyspace.Name, vcursor.tabletType)
	if err != nil {
		return nil, fmt.Errorf("paramsUnsharded: %v", err)
	}
	if len(allShards) != 1 {
		return nil, fmt.Errorf("unsharded keyspace %s has multiple shards", ks)
	}
	return newScatterParams(vcursor.sql, ks, vcursor.bindVars, []string{allShards[0].Name}), nil
}

func (rtr *Router) paramsSelectEqual2(vcursor *requestContext, route *planbuilder.Route) (*scatterParams, error) {
	keys, err := rtr.resolveKeys([]interface{}{route.Values}, vcursor.bindVars)
	if err != nil {
		return nil, fmt.Errorf("paramsSelectEqual: %v", err)
	}
	ks, routing, err := rtr.resolveShards2(vcursor, keys, route)
	if err != nil {
		return nil, fmt.Errorf("paramsSelectEqual: %v", err)
	}
	return newScatterParams(route.Query, ks, vcursor.bindVars, routing.Shards()), nil
}

func (rtr *Router) paramsSelectIN2(vcursor *requestContext, route *planbuilder.Route) (*scatterParams, error) {
	keys, err := rtr.resolveKeys(route.Values.([]interface{}), vcursor.bindVars)
	if err != nil {
		return nil, fmt.Errorf("paramsSelectIN: %v", err)
	}
	ks, routing, err := rtr.resolveShards2(vcursor, keys, route)
	if err != nil {
		return nil, fmt.Errorf("paramsSelectEqual: %v", err)
	}
	return &scatterParams{
		query:     route.Query,
		ks:        ks,
		shardVars: routing.ShardVars(vcursor.bindVars),
	}, nil
}

func (rtr *Router) paramsSelectScatter2(vcursor *requestContext, route *planbuilder.Route) (*scatterParams, error) {
	ks, _, allShards, err := getKeyspaceShards(vcursor.ctx, rtr.serv, rtr.cell, route.Keyspace.Name, vcursor.tabletType)
	if err != nil {
		return nil, fmt.Errorf("paramsSelectScatter: %v", err)
	}
	var shards []string
	for _, shard := range allShards {
		shards = append(shards, shard.Name)
	}
	return newScatterParams(route.Query, ks, vcursor.bindVars, shards), nil
}

func (rtr *Router) resolveShards2(vcursor *requestContext, vindexKeys []interface{}, route *planbuilder.Route) (newKeyspace string, routing routingMap, err error) {
	newKeyspace, _, allShards, err := getKeyspaceShards(vcursor.ctx, rtr.serv, rtr.cell, route.Keyspace.Name, vcursor.tabletType)
	if err != nil {
		return "", nil, err
	}
	routing = make(routingMap)
	switch mapper := route.Vindex.(type) {
	case planbuilder.Unique:
		ksids, err := mapper.Map(vcursor, vindexKeys)
		if err != nil {
			return "", nil, err
		}
		for i, ksid := range ksids {
			if len(ksid) == 0 {
				continue
			}
			shard, err := getShardForKeyspaceID(allShards, ksid)
			if err != nil {
				return "", nil, err
			}
			routing.Add(shard, vindexKeys[i])
		}
	case planbuilder.NonUnique:
		ksidss, err := mapper.Map(vcursor, vindexKeys)
		if err != nil {
			return "", nil, err
		}
		for i, ksids := range ksidss {
			for _, ksid := range ksids {
				shard, err := getShardForKeyspaceID(allShards, ksid)
				if err != nil {
					return "", nil, err
				}
				routing.Add(shard, vindexKeys[i])
			}
		}
	default:
		panic("unexpected")
	}
	return newKeyspace, routing, nil
}
<commit_msg>v3: enable join execution<commit_after>// Copyright 2016, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package vtgate

import (
	"fmt"

	"github.com/youtube/vitess/go/sqltypes"
	"github.com/youtube/vitess/go/vt/vtgate/planbuilder"

	querypb "github.com/youtube/vitess/go/vt/proto/query"
)

func (rtr *Router) execPlan(vcursor *requestContext, plan interface{}) (*sqltypes.Result, error) {
	switch plan := plan.(type) {
	case *planbuilder.Join:
		return rtr.execJoin(vcursor, plan)
	case *planbuilder.Route:
		return rtr.execRoute(vcursor, plan)
	}
	panic("unreachable")
}

func (rtr *Router) execJoin(vcursor *requestContext, join *planbuilder.Join) (*sqltypes.Result, error) {
	lresult, err := rtr.execPlan(vcursor, join.Left)
	if err != nil {
		return nil, err
	}
	saved := copyBindVars(vcursor.JoinVars)
	defer func() { vcursor.JoinVars = saved }()
	result := &sqltypes.Result{}
	if len(lresult.Rows) == 0 {
		// We still need field info from the RHS.
		// TODO(sougou): Change this to use an impossible query.
		for k := range join.Vars {
			vcursor.JoinVars[k] = nil
		}
		rresult, err := rtr.execPlan(vcursor, join.Right)
		if err != nil {
			return nil, err
		}
		result.Fields = joinFields(lresult.Fields, rresult.Fields, join.Cols)
		return result, nil
	}
	for _, lrow := range lresult.Rows {
		for k, col := range join.Vars {
			vcursor.JoinVars[k] = lrow[col]
		}
		rresult, err := rtr.execPlan(vcursor, join.Right)
		if err != nil {
			return nil, err
		}
		result.Fields = joinFields(lresult.Fields, rresult.Fields, join.Cols)
		if join.IsLeft && len(rresult.Rows) == 0 {
			rresult.Rows = [][]sqltypes.Value{make([]sqltypes.Value, len(rresult.Fields))}
		}
		for _, rrow := range rresult.Rows {
			result.Rows = append(result.Rows, joinRows(lrow, rrow, join.Cols))
		}
		result.RowsAffected += uint64(len(rresult.Rows))
	}
	return result, nil
}

func copyBindVars(bindVars map[string]interface{}) map[string]interface{} {
	out := make(map[string]interface{})
	for k, v := range bindVars {
		out[k] = v
	}
	return out
}

func joinFields(lfields, rfields []*querypb.Field, cols []int) []*querypb.Field {
	if lfields == nil || rfields == nil {
		return nil
	}
	fields := make([]*querypb.Field, len(cols))
	for i, index := range cols {
		if index < 0 {
			fields[i] = lfields[-index-1]
			continue
		}
		fields[i] = rfields[index-1]
	}
	return fields
}

func joinRows(lrow, rrow []sqltypes.Value, cols []int) []sqltypes.Value {
	row := make([]sqltypes.Value, len(cols))
	for i, index := range cols {
		if index < 0 {
			row[i] = lrow[-index-1]
			continue
		}
		row[i] = rrow[index-1]
	}
	return row
}

func (rtr *Router) execRoute(vcursor *requestContext, route *planbuilder.Route) (*sqltypes.Result, error) {
	var err error
	var params *scatterParams
	switch route.PlanID {
	case planbuilder.SelectUnsharded:
		params, err = rtr.paramsUnsharded2(vcursor, route)
	case planbuilder.SelectEqual, planbuilder.SelectEqualUnique:
		params, err = rtr.paramsSelectEqual2(vcursor, route)
	case planbuilder.SelectIN:
		params, err = rtr.paramsSelectIN2(vcursor, route)
	case planbuilder.SelectScatter:
		params, err = rtr.paramsSelectScatter2(vcursor, route)
	default:
		// TODO(sougou): improve error.
		return nil, fmt.Errorf("unsupported query plan: %v", route)
	}
	if err != nil {
		return nil, err
	}
	saved := copyBindVars(vcursor.bindVars)
	defer func() { vcursor.bindVars = saved }()
	for k := range route.UseVars {
		vcursor.bindVars[k] = vcursor.JoinVars[k]
	}
	return rtr.scatterConn.ExecuteMulti(
		vcursor.ctx,
		params.query,
		params.ks,
		params.shardVars,
		vcursor.tabletType,
		NewSafeSession(vcursor.session),
		vcursor.notInTransaction,
	)
}

func (rtr *Router) streamExecPlan(vcursor *requestContext, plan interface{}, sendReply func(*sqltypes.Result) error) error {
	switch plan := plan.(type) {
	case *planbuilder.Join:
		return rtr.streamExecPlan(vcursor, plan, sendReply)
	case *planbuilder.Route:
		return rtr.streamExecRoute(vcursor, plan, sendReply)
	}
	panic("unreachable")
}

func (rtr *Router) streamExecJoin(vcursor *requestContext, join *planbuilder.Join, sendReply func(*sqltypes.Result) error) error {
	fieldSent := false
	var rhsLen int
	err := rtr.streamExecPlan(vcursor, join.Left, func(lresult *sqltypes.Result) error {
		saved := copyBindVars(vcursor.JoinVars)
		defer func() { vcursor.JoinVars = saved }()
		for _, lrow := range lresult.Rows {
			for k, col := range join.Vars {
				vcursor.JoinVars[k] = lrow[col]
			}
			rowSent := false
			err := rtr.streamExecPlan(vcursor, join.Right, func(rresult *sqltypes.Result) error {
				result := &sqltypes.Result{}
				if !fieldSent {
					result.Fields = joinFields(lresult.Fields, rresult.Fields, join.Cols)
					fieldSent = true
					rhsLen = len(rresult.Fields)
				}
				if len(result.Rows) != 0 {
					rowSent = true
				}
				for _, rrow := range rresult.Rows {
					result.Rows = append(result.Rows, joinRows(lrow, rrow, join.Cols))
				}
				return sendReply(result)
			})
			if err != nil {
				return err
			}
			if join.IsLeft && !rowSent {
				result := &sqltypes.Result{}
				result.Rows = [][]sqltypes.Value{joinRows(
					lrow,
					make([]sqltypes.Value, rhsLen),
					join.Cols,
				)}
				err := sendReply(result)
				if err != nil {
					return err
				}
			}
		}
		if !fieldSent {
			for k := range join.Vars {
				vcursor.JoinVars[k] = nil
			}
			return rtr.streamExecPlan(vcursor, join.Right, func(rresult *sqltypes.Result) error {
				result := &sqltypes.Result{}
				result.Fields = joinFields(lresult.Fields, rresult.Fields, join.Cols)
				fieldSent = true
				rhsLen = len(rresult.Fields)
				return sendReply(result)
			})
		}
		return nil
	})
	return err
}

func (rtr *Router) streamExecRoute(vcursor *requestContext, route *planbuilder.Route, sendReply func(*sqltypes.Result) error) error {
	var err error
	var params *scatterParams
	switch route.PlanID {
	case planbuilder.SelectUnsharded:
		params, err = rtr.paramsUnsharded2(vcursor, route)
	case planbuilder.SelectEqual, planbuilder.SelectEqualUnique:
		params, err = rtr.paramsSelectEqual2(vcursor, route)
	case planbuilder.SelectIN:
		params, err = rtr.paramsSelectIN2(vcursor, route)
	case planbuilder.SelectScatter:
		params, err = rtr.paramsSelectScatter2(vcursor, route)
	default:
		// TODO(sougou): improve error.
		return fmt.Errorf("unsupported query plan: %v", route)
	}
	if err != nil {
		return err
	}
	saved := copyBindVars(vcursor.bindVars)
	defer func() { vcursor.bindVars = saved }()
	for k := range route.UseVars {
		vcursor.bindVars[k] = vcursor.JoinVars[k]
	}
	return rtr.scatterConn.StreamExecuteMulti(
		vcursor.ctx,
		params.query,
		params.ks,
		params.shardVars,
		vcursor.tabletType,
		sendReply,
	)
}

func (rtr *Router) paramsUnsharded2(vcursor *requestContext, route *planbuilder.Route) (*scatterParams, error) {
	ks, _, allShards, err := getKeyspaceShards(vcursor.ctx, rtr.serv, rtr.cell, route.Keyspace.Name, vcursor.tabletType)
	if err != nil {
		return nil, fmt.Errorf("paramsUnsharded: %v", err)
	}
	if len(allShards) != 1 {
		return nil, fmt.Errorf("unsharded keyspace %s has multiple shards", ks)
	}
	return newScatterParams(vcursor.sql, ks, vcursor.bindVars, []string{allShards[0].Name}), nil
}

func (rtr *Router) paramsSelectEqual2(vcursor *requestContext, route *planbuilder.Route) (*scatterParams, error) {
	keys, err := rtr.resolveKeys([]interface{}{route.Values}, vcursor.bindVars)
	if err != nil {
		return nil, fmt.Errorf("paramsSelectEqual: %v", err)
	}
	ks, routing, err := rtr.resolveShards2(vcursor, keys, route)
	if err != nil {
		return nil, fmt.Errorf("paramsSelectEqual: %v", err)
	}
	return newScatterParams(route.Query, ks, vcursor.bindVars, routing.Shards()), nil
}

func (rtr *Router) paramsSelectIN2(vcursor *requestContext, route *planbuilder.Route) (*scatterParams, error) {
	keys, err := rtr.resolveKeys(route.Values.([]interface{}), vcursor.bindVars)
	if err != nil {
		return nil, fmt.Errorf("paramsSelectIN: %v", err)
	}
	ks, routing, err := rtr.resolveShards2(vcursor, keys, route)
	if err != nil {
		return nil, fmt.Errorf("paramsSelectEqual: %v", err)
	}
	return &scatterParams{
		query:     route.Query,
		ks:        ks,
		shardVars: routing.ShardVars(vcursor.bindVars),
	}, nil
}

func (rtr *Router) paramsSelectScatter2(vcursor *requestContext, route *planbuilder.Route) (*scatterParams, error) {
	ks, _, allShards, err := getKeyspaceShards(vcursor.ctx, rtr.serv, rtr.cell, route.Keyspace.Name, vcursor.tabletType)
	if err != nil {
		return nil, fmt.Errorf("paramsSelectScatter: %v", err)
	}
	var shards []string
	for _, shard := range allShards {
		shards = append(shards, shard.Name)
	}
	return newScatterParams(route.Query, ks, vcursor.bindVars, shards), nil
}

func (rtr *Router) resolveShards2(vcursor *requestContext, vindexKeys []interface{}, route *planbuilder.Route) (newKeyspace string, routing routingMap, err error) {
	newKeyspace, _, allShards, err := getKeyspaceShards(vcursor.ctx, rtr.serv, rtr.cell, route.Keyspace.Name, vcursor.tabletType)
	if err != nil {
		return "", nil, err
	}
	routing = make(routingMap)
	switch mapper := route.Vindex.(type) {
	case planbuilder.Unique:
		ksids, err := mapper.Map(vcursor, vindexKeys)
		if err != nil {
			return "", nil, err
		}
		for i, ksid := range ksids {
			if len(ksid) == 0 {
				continue
			}
			shard, err := getShardForKeyspaceID(allShards, ksid)
			if err != nil {
				return "", nil, err
			}
			routing.Add(shard, vindexKeys[i])
		}
	case planbuilder.NonUnique:
		ksidss, err := mapper.Map(vcursor, vindexKeys)
		if err != nil {
			return "", nil, err
		}
		for i, ksids := range ksidss {
			for _, ksid := range ksids {
				shard, err := getShardForKeyspaceID(allShards, ksid)
				if err != nil {
					return "", nil, err
				}
				routing.Add(shard, vindexKeys[i])
			}
		}
	default:
		panic("unexpected")
	}
	return newKeyspace, routing, nil
}
<|endoftext|>