<commit_before>/*
 * Copyright (c) 2015 Alex Yatskov <alex@foosoft.net>
 * Author: Alex Yatskov <alex@foosoft.net>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of
 * this software and associated documentation files (the "Software"), to deal in
 * the Software without restriction, including without limitation the rights to
 * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 * the Software, and to permit persons to whom the Software is furnished to do so,
 * subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

package main

import (
	"fmt"
	"log"
)

type task struct {
	Deps   []string
	Links  [][]string
	Cmds   [][]string
	Envs   [][]string
	WTrue  [][]string
	WFalse [][]string
}

func (t *task) deps(conf *config) []string {
	deps := t.Deps

	if conf.flags&flagNoCmds == 0 {
		for _, currCmd := range t.Cmds {
			deps = append(deps, findCmdDeps(currCmd, conf)...)
		}
	}

	return deps
}

func (t *task) process(conf *config) error {
	for _, currTask := range t.deps(conf) {
		if err := processTask(currTask, conf); err != nil {
			return err
		}
	}

	for _, currEnv := range t.Envs {
		if err := processEnv(currEnv, conf); err != nil {
			return err
		}
	}

	if conf.flags&flagNoCmds == 0 {
	CmdLoop:
		for {
			for _, currCnd := range t.WTrue {
				if err := processCmd(currCnd, false, conf); err != nil {
					break CmdLoop
				}
			}

			for _, currCnd := range t.WFalse {
				if err := processCmd(currCnd, false, conf); err == nil {
					break CmdLoop
				}
			}

			for _, currCmd := range t.Cmds {
				if err := processCmd(currCmd, true, conf); err != nil {
					return err
				}
			}

			break
		}
	}

	if conf.flags&flagNoLinks == 0 {
		for _, currLink := range t.Links {
			if err := processLink(currLink, conf); err != nil {
				return err
			}
		}
	}

	return nil
}

func processTask(taskName string, conf *config) error {
	for _, tn := range makeVariantNames(taskName, conf.variant) {
		t, ok := conf.Tasks[tn]
		if !ok {
			continue
		}

		if conf.handled[tn] {
			if conf.flags&flagVerbose != 0 {
				log.Printf("skipping processed task: %s", tn)
			}

			return nil
		}

		if conf.flags&flagVerbose != 0 {
			log.Printf("processing task: %s", tn)
		}

		conf.handled[tn] = true
		return t.process(conf)
	}

	return fmt.Errorf("task or variant not found: %s", taskName)
}
<commit_msg>More renaming of conditionls<commit_after>/*
 * Copyright (c) 2015 Alex Yatskov <alex@foosoft.net>
 * Author: Alex Yatskov <alex@foosoft.net>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of
 * this software and associated documentation files (the "Software"), to deal in
 * the Software without restriction, including without limitation the rights to
 * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 * the Software, and to permit persons to whom the Software is furnished to do so,
 * subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

package main

import (
	"fmt"
	"log"
)

type task struct {
	Deps    []string
	Links   [][]string
	Cmds    [][]string
	Envs    [][]string
	Accepts [][]string
	Rejects [][]string
}

func (t *task) deps(conf *config) []string {
	deps := t.Deps

	if conf.flags&flagNoCmds == 0 {
		for _, currCmd := range t.Cmds {
			deps = append(deps, findCmdDeps(currCmd, conf)...)
		}
	}

	return deps
}

func (t *task) process(conf *config) error {
	for _, currTask := range t.deps(conf) {
		if err := processTask(currTask, conf); err != nil {
			return err
		}
	}

	for _, currEnv := range t.Envs {
		if err := processEnv(currEnv, conf); err != nil {
			return err
		}
	}

	if conf.flags&flagNoCmds == 0 {
	CmdLoop:
		for {
			for _, currCnd := range t.Accepts {
				if err := processCmd(currCnd, false, conf); err != nil {
					break CmdLoop
				}
			}

			for _, currCnd := range t.Rejects {
				if err := processCmd(currCnd, false, conf); err == nil {
					break CmdLoop
				}
			}

			for _, currCmd := range t.Cmds {
				if err := processCmd(currCmd, true, conf); err != nil {
					return err
				}
			}

			break
		}
	}

	if conf.flags&flagNoLinks == 0 {
		for _, currLink := range t.Links {
			if err := processLink(currLink, conf); err != nil {
				return err
			}
		}
	}

	return nil
}

func processTask(taskName string, conf *config) error {
	for _, tn := range makeVariantNames(taskName, conf.variant) {
		t, ok := conf.Tasks[tn]
		if !ok {
			continue
		}

		if conf.handled[tn] {
			if conf.flags&flagVerbose != 0 {
				log.Printf("skipping processed task: %s", tn)
			}

			return nil
		}

		if conf.flags&flagVerbose != 0 {
			log.Printf("processing task: %s", tn)
		}

		conf.handled[tn] = true
		return t.process(conf)
	}

	return fmt.Errorf("task or variant not found: %s", taskName)
}
<|endoftext|>


<commit_before>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"strings"
	"testing"

	"k8s.io/minikube/test/integration/util"
)

func TestFunctional(t *testing.T) {
	minikubeRunner := util.MinikubeRunner{
		BinaryPath: *binaryPath,
		Args:       *args,
		T:          t}
	minikubeRunner.EnsureRunning()

	// This one is not parallel, and ensures the cluster comes up
	// before we run any other tests.
	t.Run("Status", testClusterStatus)

	t.Run("DNS", testClusterDNS)
	t.Run("Logs", testClusterLogs)
	t.Run("Addons", testAddons)
	t.Run("Dashboard", testDashboard)
	t.Run("ServicesList", testServicesList)
	t.Run("Provisioning", testProvisioning)

	if !strings.Contains(*args, "--vm-driver=none") {
		t.Run("EnvVars", testClusterEnv)
		t.Run("SSH", testClusterSSH)
		// t.Run("Mounting", testMounting)
	}
}
<commit_msg>Cache busybox in integration test<commit_after>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"strings"
	"testing"

	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/machine"
	"k8s.io/minikube/test/integration/util"
)

func TestFunctional(t *testing.T) {
	minikubeRunner := util.MinikubeRunner{
		BinaryPath: *binaryPath,
		Args:       *args,
		T:          t}
	minikubeRunner.EnsureRunning()
	integrationTestImages := []string{"busybox:glibc"}
	if err := machine.CacheImages(integrationTestImages, constants.ImageCacheDir); err != nil {
		t.Fatalf("caching images: %s", err)
	}
	if err := machine.LoadFromCacheBlocking(&minikubeRunner, constants.ImageCacheDir); err != nil {
		t.Fatalf("loading images: %s", err)
	}
	// This one is not parallel, and ensures the cluster comes up
	// before we run any other tests.
	t.Run("Status", testClusterStatus)

	t.Run("DNS", testClusterDNS)
	t.Run("Logs", testClusterLogs)
	t.Run("Addons", testAddons)
	t.Run("Dashboard", testDashboard)
	t.Run("ServicesList", testServicesList)
	t.Run("Provisioning", testProvisioning)

	if !strings.Contains(*args, "--vm-driver=none") {
		t.Run("EnvVars", testClusterEnv)
		t.Run("SSH", testClusterSSH)
		// t.Run("Mounting", testMounting)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/nlopes/slack"
)

// Input channel
type CommandChannel struct {
	Channel *slack.Channel
	Event   *slack.MessageEvent
	UserID  string
}

// Output channel
type ReplyChannel struct {
	Channel *slack.Channel
	Message *slack.Msg
}

var (
	api *slack.Client
	//	userMessages      Messages
	botID             string
	botCommandChannel chan *CommandChannel
	botReplyChannel   chan ReplyChannel
)

func handleBotCommands(c chan ReplyChannel) {
	var rc ReplyChannel

	for {
		botChannel := <-botCommandChannel

		reply := &slack.Msg{
			Text: "Hello",
			User: "Concierge",
		}
		rc.Channel = botChannel.Channel
		rc.Message = reply
		c <- rc
		fmt.Printf("Pushed to channel\n")

		//reply := &slack.Msg{}

		//commandArray := strings.Fields(botChannel.Event.Text)
		// switch commandArray[1] {

	}
}

func handleBotReply(rtm *slack.RTM) {
	for {
		ac := <-botReplyChannel
		rtm.SendMessage(rtm.NewOutgoingMessage(ac.Message.Text, ac.Channel.ID))
	}
}

func main() {
	if len(os.Args) != 3 || os.Args[1] != "slack" {
		fmt.Fprintf(os.Stderr, "usage: concierge slack bot-token\n")
		os.Exit(1)
	}
	api := slack.New(os.Args[2])
	rtm := api.NewRTM()

	botCommandChannel = make(chan *CommandChannel)
	botReplyChannel = make(chan ReplyChannel)

	//userMessages = make(Messages, 0)

	go rtm.ManageConnection()
	go handleBotCommands(botReplyChannel)
	go handleBotReply(rtm)

Loop:
	for {
		select {
		case msg := <-rtm.IncomingEvents:
			switch ev := msg.Data.(type) {

			case *slack.ConnectedEvent:
				botID = ev.Info.User.ID

			case *slack.MessageEvent:
				fmt.Printf("Got message: %s\n", msg.Data)
				channelInfo, err := api.GetChannelInfo(ev.Channel)
				if err != nil {
					log.Fatalln(err)
				}

				command := &CommandChannel{
					Channel: channelInfo,
					Event:   ev,
					UserID:  ev.User,
				}

				fmt.Printf("Type: %s\n", ev.Type)
				fmt.Printf("Text: %s\n", ev.Text)
				fmt.Printf("botID: %s\n", botID)

				if ev.Type == "message" && strings.HasPrefix(ev.Text, "<@"+botID+">") {
					fmt.Printf("Valid message\n")
					botCommandChannel <- command
				}

			case *slack.RTMError:
				fmt.Printf("Error: %s\n", ev.Error())

			case *slack.InvalidAuthEvent:
				fmt.Printf("Invalid credentials")
				break Loop

			default:
				// Ignore other events..
				// fmt.Printf("Unexpected: %v\n", msg.Data)
			}
		}
	}

}
<commit_msg>better naming structs<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/nlopes/slack"
)

// Input channel
type InputPacket struct {
	Channel *slack.Channel
	Event   *slack.MessageEvent
	UserID  string
}

// Output channel
type ReplyChannel struct {
	Channel *slack.Channel
	Message *slack.Msg
}

var (
	api *slack.Client
	//	userMessages      Messages
	botID           string
	botInputChannel chan *InputPacket
	botReplyChannel chan ReplyChannel
)

func handleBotCommands(c chan ReplyChannel) {
	var rc ReplyChannel

	for {
		botChannel := <-botInputChannel

		reply := &slack.Msg{
			Text: "Hello",
			User: "Concierge",
		}
		rc.Channel = botChannel.Channel
		rc.Message = reply
		c <- rc

		//reply := &slack.Msg{}

		//commandArray := strings.Fields(botChannel.Event.Text)
		// switch commandArray[1] {

	}
}

func handleBotReply(rtm *slack.RTM) {
	for {
		ac := <-botReplyChannel
		rtm.SendMessage(rtm.NewOutgoingMessage(ac.Message.Text, ac.Channel.ID))
	}
}

func main() {
	if len(os.Args) != 3 || os.Args[1] != "slack" {
		fmt.Fprintf(os.Stderr, "usage: concierge slack bot-token\n")
		os.Exit(1)
	}
	api := slack.New(os.Args[2])
	rtm := api.NewRTM()

	botInputChannel = make(chan *InputPacket)
	botReplyChannel = make(chan ReplyChannel)

	go rtm.ManageConnection()
	go handleBotCommands(botReplyChannel)
	go handleBotReply(rtm)

Loop:
	for {
		select {
		case msg := <-rtm.IncomingEvents:
			switch ev := msg.Data.(type) {

			case *slack.ConnectedEvent:
				botID = ev.Info.User.ID

			case *slack.MessageEvent:
				fmt.Printf("Got message: %s\n", msg.Data)
				channelInfo, err := api.GetChannelInfo(ev.Channel)
				if err != nil {
					log.Fatalln(err)
				}

				command := &InputPacket{
					Channel: channelInfo,
					Event:   ev,
					UserID:  ev.User,
				}

				fmt.Printf("Type: %s, Text: %s, botID: %s\n", ev.Type, ev.Text, botID)

				if ev.Type == "message" && strings.HasPrefix(ev.Text, "<@"+botID+">") {
					fmt.Printf("Valid message\n")
					botInputChannel <- command
				}

			case *slack.RTMError:
				fmt.Printf("Error: %s\n", ev.Error())

			case *slack.InvalidAuthEvent:
				fmt.Printf("Invalid credentials")
				break Loop

			default:
				// Ignore other events..
				// fmt.Printf("Unexpected: %v\n", msg.Data)
			}
		}
	}

}
<|endoftext|>


<commit_before>package conf

import (
	"fmt"
	"io"
	"io/ioutil"
	"regexp"
	"runtime"
	"strings"
	"text/template"

	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/tsaf/conf/parse"
	"github.com/StackExchange/tsaf/expr"
	eparse "github.com/StackExchange/tsaf/expr/parse"
)

type Conf struct {
	Vars
	Name        string // Config file name
	WebDir      string // Static content web directory: web
	TsdbHost    string // OpenTSDB relay and query destination: ny-devtsdb04:4242
	RelayListen string // OpenTSDB relay listen address: :4242
	HttpListen  string // Web server listen address: :80
	SmtpHost    string // SMTP address: ny-mail:25
	Templates   map[string]*Template
	Alerts      map[string]*Alert

	tree *parse.Tree
	node parse.Node
}

// at marks the state to be on node n, for error reporting.
func (c *Conf) at(node parse.Node) {
	c.node = node
}

func (c *Conf) error(err error) {
	c.errorf(err.Error())
}

// errorf formats the error and terminates processing.
func (c *Conf) errorf(format string, args ...interface{}) {
	if c.node == nil {
		format = fmt.Sprintf("conf: %s: %s", c.Name, format)
	} else {
		location, context := c.tree.ErrorContext(c.node)
		format = fmt.Sprintf("conf: %s: at <%s>: %s", location, context, format)
	}
	panic(fmt.Errorf(format, args...))
}

// errRecover is the handler that turns panics into returns from the top
// level of Parse.
func errRecover(errp *error) {
	e := recover()
	if e != nil {
		switch err := e.(type) {
		case runtime.Error:
			panic(e)
		case error:
			*errp = err
		default:
			panic(e)
		}
	}
}

type Alert struct {
	Vars
	*Template  `json:"-"`
	Name       string
	Owner      string     `json:",omitempty"`
	Crit       *expr.Expr `json:",omitempty"`
	Warn       *expr.Expr `json:",omitempty"`
	Overriders []*Alert   `json:"-"`
	Overrides  *Alert     `json:",omitempty"`

	crit, warn string
	template   string
	override   string
}

type Template struct {
	Vars
	Name          string
	Body, Subject *template.Template

	body, subject string
}

type context struct {
	Alert   *Alert
	Tags    opentsdb.TagSet
	Context opentsdb.Context
}

// E executes the given expression and returns a value with corresponding tags
// to the context's tags. If no such result is found, the first result with nil
// tags is returned. If no such result is found, nil is returned.
func (c *context) E(v string) expr.Value {
	e, err := expr.New(v)
	if err != nil {
		return nil
	}
	res, err := e.Execute(c.Context, nil)
	if err != nil {
		return nil
	}
	for _, r := range res {
		if r.Group.Equal(c.Tags) {
			return r.Value
		}
	}
	for _, r := range res {
		if r.Group == nil {
			return r.Value
		}
	}
	return nil
}

func (a *Alert) data(group opentsdb.TagSet, c opentsdb.Context) interface{} {
	return &context{
		a,
		group,
		c,
	}
}

func (a *Alert) ExecuteBody(w io.Writer, group opentsdb.TagSet, c opentsdb.Context) error {
	return a.Template.Body.Execute(w, a.data(group, c))
}

func (a *Alert) ExecuteSubject(w io.Writer, group opentsdb.TagSet, c opentsdb.Context) error {
	return a.Template.Subject.Execute(w, a.data(group, c))
}

type Vars map[string]string

func ParseFile(fname string) (*Conf, error) {
	f, err := ioutil.ReadFile(fname)
	if err != nil {
		return nil, err
	}
	return New(fname, string(f))
}

func New(name, text string) (c *Conf, err error) {
	defer errRecover(&err)
	c = &Conf{
		Name:        name,
		HttpListen:  ":8070",
		RelayListen: ":4242",
		WebDir:      "web",
		Vars:        make(map[string]string),
		Templates:   make(map[string]*Template),
		Alerts:      make(map[string]*Alert),
	}
	c.tree, err = parse.Parse(name, text)
	if err != nil {
		c.error(err)
	}
	for _, n := range c.tree.Root.Nodes {
		c.at(n)
		switch n := n.(type) {
		case *parse.PairNode:
			c.loadGlobal(n)
		case *parse.SectionNode:
			c.loadSection(n)
		default:
			c.errorf("unexpected parse node %s", n)
		}
	}
	if c.TsdbHost == "" {
		c.at(nil)
		c.errorf("tsdbHost required")
	}
	return
}

func (c *Conf) loadGlobal(p *parse.PairNode) {
	v := p.Val.Text
	switch k := p.Key.Text; k {
	case "tsdbHost":
		c.TsdbHost = c.expand(v, nil)
	case "httpListen":
		c.HttpListen = c.expand(v, nil)
	case "relayListen":
		c.RelayListen = c.expand(v, nil)
	case "webDir":
		c.WebDir = c.expand(v, nil)
	case "smtpHost":
		c.SmtpHost = c.expand(v, nil)
	default:
		if !strings.HasPrefix(k, "$") {
			c.errorf("unknown key %s", k)
		}
		c.Vars[k] = c.expand(v, nil)
	}
}

func (c *Conf) loadSection(s *parse.SectionNode) {
	switch s.SectionType.Text {
	case "template":
		c.loadTemplate(s)
	case "alert":
		c.loadAlert(s)
	default:
		c.errorf("unknown section type: %s", s.SectionType.Text)
	}
}

func (c *Conf) loadTemplate(s *parse.SectionNode) {
	name := s.Name.Text
	if _, ok := c.Templates[name]; ok {
		c.errorf("duplicate template name: %s", name)
	}
	t := Template{
		Vars: make(map[string]string),
		Name: name,
	}
	V := func(v string) string {
		return c.expand(v, t.Vars)
	}
	master := template.New(name).Funcs(template.FuncMap{
		"V": V,
	})
	for _, p := range s.Nodes {
		c.at(p)
		v := p.Val.Text
		switch k := p.Key.Text; k {
		case "body":
			t.body = v
			tmpl := master.New(k)
			_, err := tmpl.Parse(t.body)
			if err != nil {
				c.error(err)
			}
			t.Body = tmpl
		case "subject":
			t.subject = v
			tmpl := master.New(k)
			_, err := tmpl.Parse(t.subject)
			if err != nil {
				c.error(err)
			}
			t.Subject = tmpl
		default:
			if !strings.HasPrefix(k, "$") {
				c.errorf("unknown key %s", k)
			}
			t.Vars[k] = v
			t.Vars[k[1:]] = t.Vars[k]
		}
	}
	c.at(s)
	if t.Body == nil && t.Subject == nil {
		c.errorf("neither body or subject specified")
	}
	c.Templates[name] = &t
}

func (c *Conf) loadAlert(s *parse.SectionNode) {
	name := s.Name.Text
	if _, ok := c.Alerts[name]; ok {
		c.errorf("duplicate template name: %s", name)
	}
	a := Alert{
		Vars: make(map[string]string),
		Name: name,
	}
	for _, p := range s.Nodes {
		c.at(p)
		v := p.Val.Text
		switch k := p.Key.Text; k {
		case "owner":
			if c.SmtpHost == "" {
				c.errorf("no smtpHost specified, can't specify owner")
			}
			a.Owner = c.expand(v, a.Vars)
		case "template":
			a.template = c.expand(v, a.Vars)
			t, ok := c.Templates[a.template]
			if !ok {
				c.errorf("unknown template %s", a.template)
			}
			a.Template = t
		case "override":
			a.override = c.expand(v, a.Vars)
			o, ok := c.Alerts[a.override]
			if !ok {
				c.errorf("unknown alert %s", a.override)
			}
			a.Overrides = o
			o.Overriders = append(o.Overriders, &a)
		case "crit":
			a.crit = c.expand(v, a.Vars)
			crit, err := expr.New(a.crit)
			if err != nil {
				c.error(err)
			}
			if crit.Root.Return() != eparse.TYPE_NUMBER {
				c.errorf("crit must return a number")
			}
			a.Crit = crit
		case "warn":
			a.warn = c.expand(v, a.Vars)
			warn, err := expr.New(a.warn)
			if err != nil {
				c.error(err)
			}
			if warn.Root.Return() != eparse.TYPE_NUMBER {
				c.errorf("warn must return a number")
			}
			a.Warn = warn
		default:
			if !strings.HasPrefix(k, "$") {
				c.errorf("unknown key %s", k)
			}
			a.Vars[k] = c.expand(v, a.Vars)
			a.Vars[k[1:]] = a.Vars[k]
		}
	}
	c.at(s)
	if a.Crit == nil && a.Warn == nil {
		c.errorf("neither crit or warn specified")
	}
	c.Alerts[name] = &a
}

var exRE = regexp.MustCompile(`\$\w+`)

func (c *Conf) expand(v string, vars map[string]string) string {
	v = exRE.ReplaceAllStringFunc(v, func(s string) string {
		if vars != nil {
			if n, ok := vars[s]; ok {
				return c.expand(n, vars)
			}
		}
		n, ok := c.Vars[s]
		if !ok {
			c.errorf("unknown variable %s", s)
		}
		return c.expand(n, nil)
	})
	return v
}
<commit_msg>Allow nil template body or subject<commit_after>package conf

import (
	"fmt"
	"io"
	"io/ioutil"
	"regexp"
	"runtime"
	"strings"
	"text/template"

	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/tsaf/conf/parse"
	"github.com/StackExchange/tsaf/expr"
	eparse "github.com/StackExchange/tsaf/expr/parse"
)

type Conf struct {
	Vars
	Name        string // Config file name
	WebDir      string // Static content web directory: web
	TsdbHost    string // OpenTSDB relay and query destination: ny-devtsdb04:4242
	RelayListen string // OpenTSDB relay listen address: :4242
	HttpListen  string // Web server listen address: :80
	SmtpHost    string // SMTP address: ny-mail:25
	Templates   map[string]*Template
	Alerts      map[string]*Alert

	tree *parse.Tree
	node parse.Node
}

// at marks the state to be on node n, for error reporting.
func (c *Conf) at(node parse.Node) {
	c.node = node
}

func (c *Conf) error(err error) {
	c.errorf(err.Error())
}

// errorf formats the error and terminates processing.
func (c *Conf) errorf(format string, args ...interface{}) {
	if c.node == nil {
		format = fmt.Sprintf("conf: %s: %s", c.Name, format)
	} else {
		location, context := c.tree.ErrorContext(c.node)
		format = fmt.Sprintf("conf: %s: at <%s>: %s", location, context, format)
	}
	panic(fmt.Errorf(format, args...))
}

// errRecover is the handler that turns panics into returns from the top
// level of Parse.
func errRecover(errp *error) {
	e := recover()
	if e != nil {
		switch err := e.(type) {
		case runtime.Error:
			panic(e)
		case error:
			*errp = err
		default:
			panic(e)
		}
	}
}

type Alert struct {
	Vars
	*Template  `json:"-"`
	Name       string
	Owner      string     `json:",omitempty"`
	Crit       *expr.Expr `json:",omitempty"`
	Warn       *expr.Expr `json:",omitempty"`
	Overriders []*Alert   `json:"-"`
	Overrides  *Alert     `json:",omitempty"`

	crit, warn string
	template   string
	override   string
}

type Template struct {
	Vars
	Name          string
	Body, Subject *template.Template

	body, subject string
}

type context struct {
	Alert   *Alert
	Tags    opentsdb.TagSet
	Context opentsdb.Context
}

// E executes the given expression and returns a value with corresponding tags
// to the context's tags. If no such result is found, the first result with nil
// tags is returned. If no such result is found, nil is returned.
func (c *context) E(v string) expr.Value {
	e, err := expr.New(v)
	if err != nil {
		return nil
	}
	res, err := e.Execute(c.Context, nil)
	if err != nil {
		return nil
	}
	for _, r := range res {
		if r.Group.Equal(c.Tags) {
			return r.Value
		}
	}
	for _, r := range res {
		if r.Group == nil {
			return r.Value
		}
	}
	return nil
}

func (a *Alert) data(group opentsdb.TagSet, c opentsdb.Context) interface{} {
	return &context{
		a,
		group,
		c,
	}
}

func (a *Alert) ExecuteBody(w io.Writer, group opentsdb.TagSet, c opentsdb.Context) error {
	if a.Template.Body == nil {
		return nil
	}
	return a.Template.Body.Execute(w, a.data(group, c))
}

func (a *Alert) ExecuteSubject(w io.Writer, group opentsdb.TagSet, c opentsdb.Context) error {
	if a.Template.Subject == nil {
		return nil
	}
	return a.Template.Subject.Execute(w, a.data(group, c))
}

type Vars map[string]string

func ParseFile(fname string) (*Conf, error) {
	f, err := ioutil.ReadFile(fname)
	if err != nil {
		return nil, err
	}
	return New(fname, string(f))
}

func New(name, text string) (c *Conf, err error) {
	defer errRecover(&err)
	c = &Conf{
		Name:        name,
		HttpListen:  ":8070",
		RelayListen: ":4242",
		WebDir:      "web",
		Vars:        make(map[string]string),
		Templates:   make(map[string]*Template),
		Alerts:      make(map[string]*Alert),
	}
	c.tree, err = parse.Parse(name, text)
	if err != nil {
		c.error(err)
	}
	for _, n := range c.tree.Root.Nodes {
		c.at(n)
		switch n := n.(type) {
		case *parse.PairNode:
			c.loadGlobal(n)
		case *parse.SectionNode:
			c.loadSection(n)
		default:
			c.errorf("unexpected parse node %s", n)
		}
	}
	if c.TsdbHost == "" {
		c.at(nil)
		c.errorf("tsdbHost required")
	}
	return
}

func (c *Conf) loadGlobal(p *parse.PairNode) {
	v := p.Val.Text
	switch k := p.Key.Text; k {
	case "tsdbHost":
		c.TsdbHost = c.expand(v, nil)
	case "httpListen":
		c.HttpListen = c.expand(v, nil)
	case "relayListen":
		c.RelayListen = c.expand(v, nil)
	case "webDir":
		c.WebDir = c.expand(v, nil)
	case "smtpHost":
		c.SmtpHost = c.expand(v, nil)
	default:
		if !strings.HasPrefix(k, "$") {
			c.errorf("unknown key %s", k)
		}
		c.Vars[k] = c.expand(v, nil)
	}
}

func (c *Conf) loadSection(s *parse.SectionNode) {
	switch s.SectionType.Text {
	case "template":
		c.loadTemplate(s)
	case "alert":
		c.loadAlert(s)
	default:
		c.errorf("unknown section type: %s", s.SectionType.Text)
	}
}

func (c *Conf) loadTemplate(s *parse.SectionNode) {
	name := s.Name.Text
	if _, ok := c.Templates[name]; ok {
		c.errorf("duplicate template name: %s", name)
	}
	t := Template{
		Vars: make(map[string]string),
		Name: name,
	}
	V := func(v string) string {
		return c.expand(v, t.Vars)
	}
	master := template.New(name).Funcs(template.FuncMap{
		"V": V,
	})
	for _, p := range s.Nodes {
		c.at(p)
		v := p.Val.Text
		switch k := p.Key.Text; k {
		case "body":
			t.body = v
			tmpl := master.New(k)
			_, err := tmpl.Parse(t.body)
			if err != nil {
				c.error(err)
			}
			t.Body = tmpl
		case "subject":
			t.subject = v
			tmpl := master.New(k)
			_, err := tmpl.Parse(t.subject)
			if err != nil {
				c.error(err)
			}
			t.Subject = tmpl
		default:
			if !strings.HasPrefix(k, "$") {
				c.errorf("unknown key %s", k)
			}
			t.Vars[k] = v
			t.Vars[k[1:]] = t.Vars[k]
		}
	}
	c.at(s)
	if t.Body == nil && t.Subject == nil {
		c.errorf("neither body or subject specified")
	}
	c.Templates[name] = &t
}

func (c *Conf) loadAlert(s *parse.SectionNode) {
	name := s.Name.Text
	if _, ok := c.Alerts[name]; ok {
		c.errorf("duplicate template name: %s", name)
	}
	a := Alert{
		Vars: make(map[string]string),
		Name: name,
	}
	for _, p := range s.Nodes {
		c.at(p)
		v := p.Val.Text
		switch k := p.Key.Text; k {
		case "owner":
			if c.SmtpHost == "" {
				c.errorf("no smtpHost specified, can't specify owner")
			}
			a.Owner = c.expand(v, a.Vars)
		case "template":
			a.template = c.expand(v, a.Vars)
			t, ok := c.Templates[a.template]
			if !ok {
				c.errorf("unknown template %s", a.template)
			}
			a.Template = t
		case "override":
			a.override = c.expand(v, a.Vars)
			o, ok := c.Alerts[a.override]
			if !ok {
				c.errorf("unknown alert %s", a.override)
			}
			a.Overrides = o
			o.Overriders = append(o.Overriders, &a)
		case "crit":
			a.crit = c.expand(v, a.Vars)
			crit, err := expr.New(a.crit)
			if err != nil {
				c.error(err)
			}
			if crit.Root.Return() != eparse.TYPE_NUMBER {
				c.errorf("crit must return a number")
			}
			a.Crit = crit
		case "warn":
			a.warn = c.expand(v, a.Vars)
			warn, err := expr.New(a.warn)
			if err != nil {
				c.error(err)
			}
			if warn.Root.Return() != eparse.TYPE_NUMBER {
				c.errorf("warn must return a number")
			}
			a.Warn = warn
		default:
			if !strings.HasPrefix(k, "$") {
				c.errorf("unknown key %s", k)
			}
			a.Vars[k] = c.expand(v, a.Vars)
			a.Vars[k[1:]] = a.Vars[k]
		}
	}
	c.at(s)
	if a.Crit == nil && a.Warn == nil {
		c.errorf("neither crit or warn specified")
	}
	c.Alerts[name] = &a
}

var exRE = regexp.MustCompile(`\$\w+`)

func (c *Conf) expand(v string, vars map[string]string) string {
	v = exRE.ReplaceAllStringFunc(v, func(s string) string {
		if vars != nil {
			if n, ok := vars[s]; ok {
				return c.expand(n, vars)
			}
		}
		n, ok := c.Vars[s]
		if !ok {
			c.errorf("unknown variable %s", s)
		}
		return c.expand(n, nil)
	})
	return v
}
<|endoftext|>


<commit_before>/* This is a FLEXible file which can be used by both client and daemon.
 * Teehee.
 */
package shared

var Version = "2.5"
var UserAgent = "LXD " + Version

/*
 * Please increment the api compat number every time you change the API.
 *
 * Version 1.0: ping
 */
var APIVersion = "1.0"
<commit_msg>Release LXD 2.6<commit_after>/* This is a FLEXible file which can be used by both client and daemon.
 * Teehee.
 */
package shared

var Version = "2.6"
var UserAgent = "LXD " + Version

/*
 * Please increment the api compat number every time you change the API.
 *
 * Version 1.0: ping
 */
var APIVersion = "1.0"
<|endoftext|>


<commit_before><commit_msg>converger: Add new methods to the API<commit_after><|endoftext|>


<commit_before>package providers

import (
	"fmt"
	"os"

	"golang.org/x/net/context"

	log "github.com/Sirupsen/logrus"
	"github.com/camptocamp/conplicity/handler"
	"github.com/camptocamp/conplicity/volume"
	docker "github.com/docker/engine-api/client"
	"github.com/docker/engine-api/types"
)

// A Provider is an interface for providers
type Provider interface {
	GetName() string
	GetPrepareCommand(*types.MountPoint) []string
	GetHandler() *handler.Conplicity
	GetVolume() *volume.Volume
	GetBackupDir() string
	SetVolumeBackupDir()
}

// BaseProvider is a struct implementing the Provider interface
type BaseProvider struct {
	handler   *handler.Conplicity
	vol       *volume.Volume
	backupDir string
}

// GetProvider detects which provider suits the passed volume and returns it
func GetProvider(c *handler.Conplicity, vol *volume.Volume) Provider {
	v := vol
	log.WithFields(log.Fields{
		"volume": v.Name,
	}).Info("Detecting provider")
	p := &BaseProvider{
		handler: c,
		vol:     v,
	}
	if f, err := os.Stat(v.Mountpoint + "/PG_VERSION"); err == nil && f.Mode().IsRegular() {
		log.WithFields(log.Fields{
			"volume": v.Name,
		}).Debug("PG_VERSION file found, this should be a PostgreSQL datadir")
		return &PostgreSQLProvider{
			BaseProvider: p,
		}
	} else if f, err := os.Stat(v.Mountpoint + "/mysql"); err == nil && f.Mode().IsDir() {
		log.WithFields(log.Fields{
			"volume": v.Name,
		}).Debug("mysql directory found, this should be MySQL datadir")
		return &MySQLProvider{
			BaseProvider: p,
		}
	} else if f, err := os.Stat(v.Mountpoint + "/DB_CONFIG"); err == nil && f.Mode().IsRegular() {
		log.WithFields(log.Fields{
			"volume": v.Name,
		}).Debug("DB_CONFIG file found, this should be and OpenLDAP datadir")
		return &OpenLDAPProvider{
			BaseProvider: p,
		}
	}

	return &DefaultProvider{
		BaseProvider: p,
	}
}

// PrepareBackup sets up the data before backup
func PrepareBackup(p Provider) (err error) {
	p.SetVolumeBackupDir()

	c := p.GetHandler()
	vol := p.GetVolume()
	containers, err := c.ContainerList(context.Background(), types.ContainerListOptions{})
	if err != nil {
		return fmt.Errorf("failed to list containers: %v", err)
	}

	// Work around https://github.com/docker/engine-api/issues/303
	client, err := docker.NewClient(c.Config.Docker.Endpoint, "", nil, nil)
	if err != nil {
		return fmt.Errorf("failed to create new Docker client: %v", err)
	}

	for _, container := range containers {
		container, err := client.ContainerInspect(context.Background(), container.ID)
		if err != nil {
			return fmt.Errorf("failed to inspect container %v: %v", container.ID, err)
		}
		for _, mount := range container.Mounts {
			if mount.Name == vol.Name {
				log.WithFields(log.Fields{
					"volume":    vol.Name,
					"container": container.ID,
				}).Debug("Container found using volume")

				cmd := p.GetPrepareCommand(&mount)
				if cmd != nil {
					exec, err := client.ContainerExecCreate(context.Background(), container.ID, types.ExecConfig{
						Cmd: p.GetPrepareCommand(&mount),
					},
					)
					if err != nil {
						return fmt.Errorf("failed to create exec: %v", err)
					}

					err = client.ContainerExecStart(context.Background(), exec.ID, types.ExecStartCheck{})
					if err != nil {
						return fmt.Errorf("failed to start exec: %v", err)
					}
				} else {
					log.WithFields(log.Fields{
						"volume":    vol.Name,
						"container": container.ID,
					}).Info("No prepare command to execute in container")
				}
			}
		}
	}
	return
}

// GetHandler returns the handler associated with the provider
func (p *BaseProvider) GetHandler() *handler.Conplicity {
	return p.handler
}

// GetVolume returns the volume associated with the provider
func (p *BaseProvider) GetVolume() *volume.Volume {
	return p.vol
}

// GetBackupDir returns the backup directory used by the provider
func (p *BaseProvider) GetBackupDir() string {
	return p.backupDir
}

// SetVolumeBackupDir sets the backup dir for the volume
func (p *BaseProvider) SetVolumeBackupDir() {
	p.vol.BackupDir = p.GetBackupDir()
}
<commit_msg>Check exit code in PrepareBackup()<commit_after>package providers

import (
	"fmt"
	"os"

	"golang.org/x/net/context"

	log "github.com/Sirupsen/logrus"
	"github.com/camptocamp/conplicity/handler"
	"github.com/camptocamp/conplicity/volume"
	docker "github.com/docker/engine-api/client"
	"github.com/docker/engine-api/types"
)

// A Provider is an interface for providers
type Provider interface {
	GetName() string
	GetPrepareCommand(*types.MountPoint) []string
	GetHandler() *handler.Conplicity
	GetVolume() *volume.Volume
	GetBackupDir() string
	SetVolumeBackupDir()
}

// BaseProvider is a struct implementing the Provider interface
type BaseProvider struct {
	handler   *handler.Conplicity
	vol       *volume.Volume
	backupDir string
}

// GetProvider detects which provider suits the passed volume and returns it
func GetProvider(c *handler.Conplicity, vol *volume.Volume) Provider {
	v := vol
	log.WithFields(log.Fields{
		"volume": v.Name,
	}).Info("Detecting provider")
	p := &BaseProvider{
		handler: c,
		vol:     v,
	}
	if f, err := os.Stat(v.Mountpoint + "/PG_VERSION"); err == nil && f.Mode().IsRegular() {
		log.WithFields(log.Fields{
			"volume": v.Name,
		}).Debug("PG_VERSION file found, this should be a PostgreSQL datadir")
		return &PostgreSQLProvider{
			BaseProvider: p,
		}
	} else if f, err := os.Stat(v.Mountpoint + "/mysql"); err == nil && f.Mode().IsDir() {
		log.WithFields(log.Fields{
			"volume": v.Name,
		}).Debug("mysql directory found, this should be MySQL datadir")
		return &MySQLProvider{
			BaseProvider: p,
		}
	} else if f, err := os.Stat(v.Mountpoint + "/DB_CONFIG"); err == nil && f.Mode().IsRegular() {
		log.WithFields(log.Fields{
			"volume": v.Name,
		}).Debug("DB_CONFIG file found, this should be and OpenLDAP datadir")
		return &OpenLDAPProvider{
			BaseProvider: p,
		}
	}

	return &DefaultProvider{
		BaseProvider: p,
	}
}

// PrepareBackup sets up the data before backup
func PrepareBackup(p Provider) (err error) {
	p.SetVolumeBackupDir()

	c := p.GetHandler()
	vol := p.GetVolume()
	containers, err := c.ContainerList(context.Background(), types.ContainerListOptions{})
	if err != nil {
		return fmt.Errorf("failed to list containers: %v", err)
	}

	// Work around https://github.com/docker/engine-api/issues/303
	client, err := docker.NewClient(c.Config.Docker.Endpoint, "", nil, nil)
	if err != nil {
		return fmt.Errorf("failed to create new Docker client: %v", err)
	}

	for _, container := range containers {
		container, err := client.ContainerInspect(context.Background(), container.ID)
		if err != nil {
			return fmt.Errorf("failed to inspect container %v: %v", container.ID, err)
		}
		for _, mount := range container.Mounts {
			if mount.Name == vol.Name {
				log.WithFields(log.Fields{
					"volume":    vol.Name,
					"container": container.ID,
				}).Debug("Container found using volume")

				cmd := p.GetPrepareCommand(&mount)
				if cmd != nil {
					exec, err := client.ContainerExecCreate(context.Background(), container.ID, types.ExecConfig{
						Cmd: p.GetPrepareCommand(&mount),
					},
					)
					if err != nil {
						return fmt.Errorf("failed to create exec: %v", err)
					}

					err = client.ContainerExecStart(context.Background(), exec.ID, types.ExecStartCheck{})
					if err != nil {
						return fmt.Errorf("failed to start exec: %v", err)
					}

					inspect, err := client.ContainerExecInspect(context.Background(), exec.ID)
					if err != nil {
						return fmt.Errorf("failed to check prepare command exit code: %v", err)
					}
					if c := inspect.ExitCode; c != 0 {
						return fmt.Errorf("prepare command exited with code %v", c)
					}
				} else {
					log.WithFields(log.Fields{
						"volume":    vol.Name,
						"container": container.ID,
					}).Info("No prepare command to execute in container")
				}
			}
		}
	}
	return
}

// GetHandler returns the handler associated with the provider
func (p *BaseProvider) GetHandler() *handler.Conplicity {
	return p.handler
}

// GetVolume returns the volume associated with the provider
func (p *BaseProvider) GetVolume() *volume.Volume {
	return p.vol
}

// GetBackupDir returns the backup directory used by the provider
func (p *BaseProvider) GetBackupDir() string {
	return p.backupDir
}

// SetVolumeBackupDir sets the backup dir for the volume
func (p *BaseProvider) SetVolumeBackupDir() {
	p.vol.BackupDir = p.GetBackupDir()
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/pkg/flagutil"
	"k8s.io/test-infra/prow/bugzilla"
	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/config/secret"
	prowflagutil "k8s.io/test-infra/prow/flagutil"
	"k8s.io/test-infra/prow/git/v2"
	"k8s.io/test-infra/prow/githubeventserver"
	"k8s.io/test-infra/prow/hook"
	"k8s.io/test-infra/prow/interrupts"
	"k8s.io/test-infra/prow/logrusutil"
	"k8s.io/test-infra/prow/metrics"
	"k8s.io/test-infra/prow/pjutil"
	pluginhelp "k8s.io/test-infra/prow/pluginhelp/hook"
	"k8s.io/test-infra/prow/plugins"
	bzplugin "k8s.io/test-infra/prow/plugins/bugzilla"
	"k8s.io/test-infra/prow/repoowners"
	"k8s.io/test-infra/prow/slack"
)

type options struct {
	port int

	configPath    string
	jobConfigPath string
	pluginConfig  string

	dryRun                 bool
	gracePeriod            time.Duration
	kubernetes             prowflagutil.KubernetesOptions
	github                 prowflagutil.GitHubOptions
	bugzilla               prowflagutil.BugzillaOptions
	instrumentationOptions prowflagutil.InstrumentationOptions

	webhookSecretFile string
	slackTokenFile    string
}

func (o *options) Validate() error {
	for _, group := range []flagutil.OptionGroup{&o.kubernetes, &o.github, &o.bugzilla} {
		if err := group.Validate(o.dryRun); err != nil {
			return err
		}
	}

	return nil
}

func gatherOptions(fs *flag.FlagSet, args ...string) options {
	var o options
	fs.IntVar(&o.port, "port", 8888, "Port to listen on.")

	fs.StringVar(&o.configPath, "config-path", "", "Path to config.yaml.")
	fs.StringVar(&o.jobConfigPath, "job-config-path", "", "Path to prow job configs.")
	fs.StringVar(&o.pluginConfig, "plugin-config", "/etc/plugins/plugins.yaml", "Path to plugin config file.")

	fs.BoolVar(&o.dryRun, "dry-run", true, "Dry run for testing. Uses API tokens but does not mutate.")
	fs.DurationVar(&o.gracePeriod, "grace-period", 180*time.Second, "On shutdown, try to handle remaining events for the specified duration. ")
	for _, group := range []flagutil.OptionGroup{&o.kubernetes, &o.github, &o.bugzilla, &o.instrumentationOptions} {
		group.AddFlags(fs)
	}

	fs.StringVar(&o.webhookSecretFile, "hmac-secret-file", "/etc/webhook/hmac", "Path to the file containing the GitHub HMAC secret.")
	fs.StringVar(&o.slackTokenFile, "slack-token-file", "", "Path to the file containing the Slack token to use.")
	fs.Parse(args)
	return o
}

func main() {
	logrusutil.ComponentInit()

	o := gatherOptions(flag.NewFlagSet(os.Args[0], flag.ExitOnError), os.Args[1:]...)
	if err := o.Validate(); err != nil {
		logrus.WithError(err).Fatal("Invalid options")
	}

	configAgent := &config.Agent{}
	if err := configAgent.Start(o.configPath, o.jobConfigPath); err != nil {
		logrus.WithError(err).Fatal("Error starting config agent.")
	}

	var tokens []string

	// Append the path of hmac and github secrets.
	tokens = append(tokens, o.github.TokenPath)
	tokens = append(tokens, o.webhookSecretFile)

	// This is necessary since slack token is optional.
	if o.slackTokenFile != "" {
		tokens = append(tokens, o.slackTokenFile)
	}

	if o.bugzilla.ApiKeyPath != "" {
		tokens = append(tokens, o.bugzilla.ApiKeyPath)
	}

	secretAgent := &secret.Agent{}
	if err := secretAgent.Start(tokens); err != nil {
		logrus.WithError(err).Fatal("Error starting secrets agent.")
	}

	pluginAgent := &plugins.ConfigAgent{}
	if err := pluginAgent.Start(o.pluginConfig, true); err != nil {
		logrus.WithError(err).Fatal("Error starting plugins.")
	}

	githubClient, err := o.github.GitHubClient(secretAgent, o.dryRun)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting GitHub client.")
	}
	gitClient, err := o.github.GitClient(secretAgent, o.dryRun)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting Git client.")
	}

	var bugzillaClient bugzilla.Client
	if orgs, repos := pluginAgent.Config().EnabledReposForPlugin(bzplugin.PluginName); orgs != nil || repos != nil {
		client, err := o.bugzilla.BugzillaClient(secretAgent)
		if err != nil {
			logrus.WithError(err).Fatal("Error getting Bugzilla client.")
		}
		bugzillaClient = client
	}

	infrastructureClient, err := o.kubernetes.InfrastructureClusterClient(o.dryRun)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting Kubernetes client for infrastructure cluster.")
	}

	buildClusterCoreV1Clients, err := o.kubernetes.BuildClusterCoreV1Clients(o.dryRun)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting Kubernetes clients for build cluster.")
	}

	prowJobClient, err := o.kubernetes.ProwJobClient(configAgent.Config().ProwJobNamespace, o.dryRun)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting ProwJob client for infrastructure cluster.")
	}

	var slackClient *slack.Client
	if !o.dryRun && string(secretAgent.GetSecret(o.slackTokenFile)) != "" {
		logrus.Info("Using real slack client.")
		slackClient = slack.NewClient(secretAgent.GetTokenGenerator(o.slackTokenFile))
	}
	if slackClient == nil {
		logrus.Info("Using fake slack client.")
		slackClient = slack.NewFakeClient()
	}

	mdYAMLEnabled := func(org, repo string) bool {
		return pluginAgent.Config().MDYAMLEnabled(org, repo)
	}
	skipCollaborators := func(org, repo string) bool {
		return pluginAgent.Config().SkipCollaborators(org, repo)
	}
	ownersDirBlacklist := func() config.OwnersDirBlacklist {
		return configAgent.Config().OwnersDirBlacklist
	}
	ownersClient := repoowners.NewClient(git.ClientFactoryFrom(gitClient), githubClient, mdYAMLEnabled, skipCollaborators, ownersDirBlacklist)

	clientAgent := &plugins.ClientAgent{
		GitHubClient:              githubClient,
		ProwJobClient:             prowJobClient,
		KubernetesClient:          infrastructureClient,
		BuildClusterCoreV1Clients: buildClusterCoreV1Clients,
		GitClient:                 git.ClientFactoryFrom(gitClient),
		SlackClient:               slackClient,
		OwnersClient:              ownersClient,
		BugzillaClient:            bugzillaClient,
	}

	promMetrics := githubeventserver.NewMetrics()

	defer interrupts.WaitForGracefulShutdown()

	// Expose prometheus metrics
	metrics.ExposeMetrics("hook", configAgent.Config().PushGateway, o.instrumentationOptions.MetricsPort)
	pjutil.ServePProf(o.instrumentationOptions.PProfPort)

	server := &hook.Server{
		ClientAgent:    clientAgent,
		ConfigAgent:    configAgent,
		Plugins:        pluginAgent,
		Metrics:        promMetrics,
		TokenGenerator: secretAgent.GetTokenGenerator(o.webhookSecretFile),
	}
	interrupts.OnInterrupt(func() {
		server.GracefulShutdown()
		if err := gitClient.Clean(); err != nil {
			logrus.WithError(err).Error("Could not clean up git client cache.")
		}
	})

	health := pjutil.NewHealth()

	// TODO remove this health endpoint when the migration to health endpoint is done
	// Return 200 on / for health checks.
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {})

	// For /hook, handle a webhook normally.
	http.Handle("/hook", server)
	// Serve plugin help information from /plugin-help.
	http.Handle("/plugin-help", pluginhelp.NewHelpAgent(pluginAgent, githubClient))

	httpServer := &http.Server{Addr: ":" + strconv.Itoa(o.port)}

	health.ServeReady()

	interrupts.ListenAndServe(httpServer, o.gracePeriod)
}
<commit_msg>hook: ensure bugzilla client is non-nil<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/pkg/flagutil"
	"k8s.io/test-infra/prow/bugzilla"
	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/config/secret"
	prowflagutil "k8s.io/test-infra/prow/flagutil"
	"k8s.io/test-infra/prow/git/v2"
	"k8s.io/test-infra/prow/githubeventserver"
	"k8s.io/test-infra/prow/hook"
	"k8s.io/test-infra/prow/interrupts"
	"k8s.io/test-infra/prow/logrusutil"
	"k8s.io/test-infra/prow/metrics"
	"k8s.io/test-infra/prow/pjutil"
	pluginhelp "k8s.io/test-infra/prow/pluginhelp/hook"
	"k8s.io/test-infra/prow/plugins"
	bzplugin "k8s.io/test-infra/prow/plugins/bugzilla"
	"k8s.io/test-infra/prow/repoowners"
	"k8s.io/test-infra/prow/slack"
)

type options struct {
	port int

	configPath    string
	jobConfigPath string
	pluginConfig  string

	dryRun                 bool
	gracePeriod            time.Duration
	kubernetes             prowflagutil.KubernetesOptions
	github                 prowflagutil.GitHubOptions
	bugzilla               prowflagutil.BugzillaOptions
	instrumentationOptions prowflagutil.InstrumentationOptions

	webhookSecretFile string
	slackTokenFile    string
}

func (o *options) Validate() error {
	for _, group := range []flagutil.OptionGroup{&o.kubernetes, &o.github, &o.bugzilla} {
		if err := group.Validate(o.dryRun); err != nil {
			return err
		}
	}

	return nil
}

func gatherOptions(fs *flag.FlagSet, args ...string) options {
	var o options
	fs.IntVar(&o.port, "port", 8888, "Port to listen on.")

	fs.StringVar(&o.configPath, "config-path", "", "Path to config.yaml.")
	fs.StringVar(&o.jobConfigPath, "job-config-path", "", "Path to prow job configs.")
	fs.StringVar(&o.pluginConfig, "plugin-config", "/etc/plugins/plugins.yaml", "Path to plugin config file.")

	fs.BoolVar(&o.dryRun, "dry-run", true, "Dry run for testing. Uses API tokens but does not mutate.")
	fs.DurationVar(&o.gracePeriod, "grace-period", 180*time.Second, "On shutdown, try to handle remaining events for the specified duration. ")
	for _, group := range []flagutil.OptionGroup{&o.kubernetes, &o.github, &o.bugzilla, &o.instrumentationOptions} {
		group.AddFlags(fs)
	}

	fs.StringVar(&o.webhookSecretFile, "hmac-secret-file", "/etc/webhook/hmac", "Path to the file containing the GitHub HMAC secret.")
	fs.StringVar(&o.slackTokenFile, "slack-token-file", "", "Path to the file containing the Slack token to use.")
	fs.Parse(args)
	return o
}

func main() {
	logrusutil.ComponentInit()

	o := gatherOptions(flag.NewFlagSet(os.Args[0], flag.ExitOnError), os.Args[1:]...)
	if err := o.Validate(); err != nil {
		logrus.WithError(err).Fatal("Invalid options")
	}

	configAgent := &config.Agent{}
	if err := configAgent.Start(o.configPath, o.jobConfigPath); err != nil {
		logrus.WithError(err).Fatal("Error starting config agent.")
	}

	var tokens []string

	// Append the path of hmac and github secrets.
	tokens = append(tokens, o.github.TokenPath)
	tokens = append(tokens, o.webhookSecretFile)

	// This is necessary since slack token is optional.
	if o.slackTokenFile != "" {
		tokens = append(tokens, o.slackTokenFile)
	}

	if o.bugzilla.ApiKeyPath != "" {
		tokens = append(tokens, o.bugzilla.ApiKeyPath)
	}

	secretAgent := &secret.Agent{}
	if err := secretAgent.Start(tokens); err != nil {
		logrus.WithError(err).Fatal("Error starting secrets agent.")
	}

	pluginAgent := &plugins.ConfigAgent{}
	if err := pluginAgent.Start(o.pluginConfig, true); err != nil {
		logrus.WithError(err).Fatal("Error starting plugins.")
	}

	githubClient, err := o.github.GitHubClient(secretAgent, o.dryRun)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting GitHub client.")
	}
	gitClient, err := o.github.GitClient(secretAgent, o.dryRun)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting Git client.")
	}

	var bugzillaClient bugzilla.Client
	if orgs, repos := pluginAgent.Config().EnabledReposForPlugin(bzplugin.PluginName); orgs != nil || repos != nil {
		client, err := o.bugzilla.BugzillaClient(secretAgent)
		if err != nil {
			logrus.WithError(err).Fatal("Error getting Bugzilla client.")
		}
		bugzillaClient = client
	} else {
		// we want something non-nil here with good no-op behavior,
		// so the test fake is a cheap way to do that
		bugzillaClient = &bugzilla.Fake{}
	}

	infrastructureClient, err := o.kubernetes.InfrastructureClusterClient(o.dryRun)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting Kubernetes client for infrastructure cluster.")
	}

	buildClusterCoreV1Clients, err := o.kubernetes.BuildClusterCoreV1Clients(o.dryRun)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting Kubernetes clients for build cluster.")
	}

	prowJobClient, err := o.kubernetes.ProwJobClient(configAgent.Config().ProwJobNamespace, o.dryRun)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting ProwJob client for infrastructure cluster.")
	}

	var slackClient *slack.Client
	if !o.dryRun && string(secretAgent.GetSecret(o.slackTokenFile)) != "" {
		logrus.Info("Using real slack client.")
		slackClient = slack.NewClient(secretAgent.GetTokenGenerator(o.slackTokenFile))
	}
	if slackClient == nil {
		logrus.Info("Using fake slack client.")
		slackClient = slack.NewFakeClient()
	}

	mdYAMLEnabled := func(org, repo string) bool {
		return pluginAgent.Config().MDYAMLEnabled(org, repo)
	}
	skipCollaborators := func(org, repo string) bool {
		return pluginAgent.Config().SkipCollaborators(org, repo)
	}
	ownersDirBlacklist := func() config.OwnersDirBlacklist {
		return configAgent.Config().OwnersDirBlacklist
	}
	ownersClient := repoowners.NewClient(git.ClientFactoryFrom(gitClient), githubClient, mdYAMLEnabled, skipCollaborators, ownersDirBlacklist)

	clientAgent := &plugins.ClientAgent{
		GitHubClient:              githubClient,
		ProwJobClient:             prowJobClient,
		KubernetesClient:          infrastructureClient,
		BuildClusterCoreV1Clients: buildClusterCoreV1Clients,
		GitClient:                 git.ClientFactoryFrom(gitClient),
		SlackClient:               slackClient,
		OwnersClient:              ownersClient,
		BugzillaClient:            bugzillaClient,
	}

	promMetrics := githubeventserver.NewMetrics()

	defer interrupts.WaitForGracefulShutdown()

	// Expose prometheus metrics
	metrics.ExposeMetrics("hook", configAgent.Config().PushGateway, o.instrumentationOptions.MetricsPort)
	pjutil.ServePProf(o.instrumentationOptions.PProfPort)

	server := &hook.Server{
		ClientAgent:    clientAgent,
		ConfigAgent:    configAgent,
		Plugins:        pluginAgent,
		Metrics:        promMetrics,
		TokenGenerator: secretAgent.GetTokenGenerator(o.webhookSecretFile),
	}
	interrupts.OnInterrupt(func() {
		server.GracefulShutdown()
		if err := gitClient.Clean(); err != nil {
			logrus.WithError(err).Error("Could not clean up git client cache.")
		}
	})

	health := pjutil.NewHealth()

	// TODO remove this health endpoint when the migration to health endpoint is done
	// Return 200 on / for health checks.
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {})

	// For /hook, handle a webhook normally.
	http.Handle("/hook", server)
	// Serve plugin help information from /plugin-help.
	http.Handle("/plugin-help", pluginhelp.NewHelpAgent(pluginAgent, githubClient))

	httpServer := &http.Server{Addr: ":" + strconv.Itoa(o.port)}

	health.ServeReady()

	interrupts.ListenAndServe(httpServer, o.gracePeriod)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/sirupsen/logrus"
	"sigs.k8s.io/yaml"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/config/secret"
	prowflagutil "k8s.io/test-infra/prow/flagutil"
	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/pjutil"
)

type options struct {
	jobName       string
	configPath    string
	jobConfigPath string

	baseRef    string
	baseSha    string
	pullNumber int
	pullSha    string
	pullAuthor string
	org        string
	repo       string

	local bool

	github       prowflagutil.GitHubOptions
	githubClient githubClient
	pullRequest  *github.PullRequest
}

func (o *options) getPullRequest() (*github.PullRequest, error) {
	if o.pullRequest != nil {
		return o.pullRequest, nil
	}
	pr, err := o.githubClient.GetPullRequest(o.org, o.repo, o.pullNumber)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch PullRequest from GitHub: %v", err)
	}
	o.pullRequest = pr
	return pr, nil
}

func (o *options) defaultPR(pjs *prowapi.ProwJobSpec) error {
	if pjs.Refs.Pulls[0].Number == 0 {
		fmt.Fprint(os.Stderr, "PR Number: ")
		var pullNumber int
		fmt.Scanln(&pullNumber)
		pjs.Refs.Pulls[0].Number = pullNumber
		o.pullNumber = pullNumber
	}
	if pjs.Refs.Pulls[0].Author == "" {
		pr, err := o.getPullRequest()
		if err != nil {
			return err
		}
		pjs.Refs.Pulls[0].Author = pr.User.Login
	}
	if pjs.Refs.Pulls[0].SHA == "" {
		pr, err := o.getPullRequest()
		if err != nil {
			return err
		}
		pjs.Refs.Pulls[0].SHA = pr.Head.SHA
	}
	return nil
}

func (o *options) defaultBaseRef(pjs *prowapi.ProwJobSpec) error {
	if pjs.Refs == nil {
		return nil
	}
	if pjs.Refs.BaseRef == "" {
		if o.pullNumber != 0 {
			pr, err := o.getPullRequest()
			if err != nil {
				return err
			}
			pjs.Refs.BaseRef = pr.Base.Ref
		} else {
			fmt.Fprint(os.Stderr, "Base ref (e.g. master): ")
			fmt.Scanln(&pjs.Refs.BaseRef)
		}
	}
	if pjs.Refs.BaseSHA == "" {
		if o.pullNumber != 0 {
			pr, err := o.getPullRequest()
			if err != nil {
				return err
			}
			pjs.Refs.BaseSHA = pr.Base.SHA
		} else {
			baseSHA, err := o.githubClient.GetRef(o.org, o.repo, fmt.Sprintf("heads/%s", pjs.Refs.BaseRef))
			if err != nil {
				logrus.Fatalf("failed to get base sha: %v", err)
				return err
			}
			pjs.Refs.BaseSHA = baseSHA
		}
	}
	return nil
}

type githubClient interface {
	GetPullRequest(org, repo string, number int) (*github.PullRequest, error)
	GetRef(org, repo, ref string) (string, error)
}

func (o *options) Validate() error {
	if o.jobName == "" {
		return errors.New("required flag --job was unset")
	}

	if o.configPath == "" {
		return errors.New("required flag --config-path was unset")
	}

	if err := o.github.Validate(false); err != nil {
		return err
	}

	return nil
}

func gatherOptions() options {
	var o options
	fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	fs.StringVar(&o.jobName, "job", "", "Job to run.")
	fs.BoolVar(&o.local, "local", false, "Print help for running locally")
	fs.StringVar(&o.configPath, "config-path", "", "Path to config.yaml.")
	fs.StringVar(&o.jobConfigPath, "job-config-path", "", "Path to prow job configs.")
	fs.StringVar(&o.baseRef, "base-ref", "", "Git base ref under test")
	fs.StringVar(&o.baseSha, "base-sha", "", "Git base SHA under test")
	fs.IntVar(&o.pullNumber, "pull-number", 0, "Git pull number under test")
	fs.StringVar(&o.pullSha, "pull-sha", "", "Git pull SHA under test")
	fs.StringVar(&o.pullAuthor, "pull-author", "", "Git pull author under test")
	o.github.AddFlagsWithoutDefaultGitHubTokenPath(fs)
	fs.Parse(os.Args[1:])
	return o
}

func main() {
	o := gatherOptions()
	if err := o.Validate(); err != nil {
		logrus.WithError(err).Fatalf("Bad flags")
	}

	conf, err := config.Load(o.configPath, o.jobConfigPath)
	if err != nil {
		logrus.WithError(err).Fatal("Error loading config")
	}

	var secretAgent *secret.Agent
	if o.github.TokenPath != "" {
		secretAgent = &secret.Agent{}
		if err := secretAgent.Start([]string{o.github.TokenPath}); err != nil {
			logrus.WithError(err).Fatal("Failed to start secret agent")
		}
	}
	o.githubClient, err = o.github.GitHubClient(secretAgent, false)
	if err != nil {
		logrus.WithError(err).Fatal("Failed to get GitHub client")
	}

	var pjs prowapi.ProwJobSpec
	var labels map[string]string
	var annotations map[string]string
	var found bool
	var needsBaseRef bool
	var needsPR bool
	for fullRepoName, ps := range conf.Presubmits {
		org, repo, err := splitRepoName(fullRepoName)
		if err != nil {
			logrus.WithError(err).Warnf("Invalid repo name %s.", fullRepoName)
			continue
		}
		for _, p := range ps {
			if p.Name == o.jobName {
				pjs = pjutil.PresubmitSpec(p, prowapi.Refs{
					Org:     org,
					Repo:    repo,
					BaseRef: o.baseRef,
					BaseSHA: o.baseSha,
					Pulls: []prowapi.Pull{{
						Author: o.pullAuthor,
						Number: o.pullNumber,
						SHA:    o.pullSha,
					}},
				})
				labels = p.Labels
				annotations = p.Annotations
				found = true
				needsBaseRef = true
				needsPR = true
			}
		}
	}
	for fullRepoName, ps := range conf.Postsubmits {
		org, repo, err := splitRepoName(fullRepoName)
		if err != nil {
			logrus.WithError(err).Warnf("Invalid repo name %s.", fullRepoName)
			continue
		}
		for _, p := range ps {
			if p.Name == o.jobName {
				pjs = pjutil.PostsubmitSpec(p, prowapi.Refs{
					Org:     org,
					Repo:    repo,
					BaseRef: o.baseRef,
					BaseSHA: o.baseSha,
				})
				labels = p.Labels
				annotations = p.Annotations
				found = true
				needsBaseRef = true
			}
		}
	}
	for _, p := range conf.Periodics {
		if p.Name == o.jobName {
			pjs = pjutil.PeriodicSpec(p)
			labels = p.Labels
			annotations = p.Annotations
			found = true
		}
	}
	if !found {
		logrus.Fatalf("Job %s not found.", o.jobName)
	}
	if pjs.Refs != nil {
		o.org = pjs.Refs.Org
		o.repo = pjs.Refs.Repo
	}
	if needsPR {
		if err := o.defaultPR(&pjs); err != nil {
			logrus.WithError(err).Fatal("Failed to default PR")
		}
	}
	if needsBaseRef {
		if err := o.defaultBaseRef(&pjs); err != nil {
			logrus.WithError(err).Fatal("Failed to default base ref")
		}
	}
	pj := pjutil.NewProwJob(pjs, labels, annotations)
	b, err := yaml.Marshal(&pj)
	if err != nil {
		logrus.WithError(err).Fatal("Error marshalling YAML.")
	}
	fmt.Print(string(b))
	if o.local {
		logrus.Info("Use 'bazel run //prow/cmd/phaino' to run this job locally in docker")
	}
}

func splitRepoName(repo string) (string, string, error) {
	s := strings.SplitN(repo, "/", 2)
	if len(s) != 2 {
		return "", "", fmt.Errorf("repo %s cannot be split into org/repo", repo)
	}
	return s[0], s[1], nil
}
<commit_msg>mkpj: simplify defaulting<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/sirupsen/logrus"
	"sigs.k8s.io/yaml"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/config/secret"
	prowflagutil "k8s.io/test-infra/prow/flagutil"
	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/pjutil"
)

type options struct {
	jobName       string
	configPath    string
	jobConfigPath string

	baseRef    string
	baseSha    string
	pullNumber int
	pullSha    string
	pullAuthor string
	org        string
	repo       string

	local bool

	github       prowflagutil.GitHubOptions
	githubClient githubClient
	pullRequest  *github.PullRequest
}

func (o *options) getPullRequest() (*github.PullRequest, error) {
	if o.pullRequest != nil {
		return o.pullRequest, nil
	}
	pr, err := o.githubClient.GetPullRequest(o.org, o.repo, o.pullNumber)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch PullRequest from GitHub: %v", err)
	}
	o.pullRequest = pr
	return pr, nil
}

func (o *options) defaultPR(pjs *prowapi.ProwJobSpec) error {
	if pjs.Refs.Pulls[0].Number == 0 {
		fmt.Fprint(os.Stderr, "PR Number: ")
		var pullNumber int
		fmt.Scanln(&pullNumber)
		pjs.Refs.Pulls[0].Number = pullNumber
		o.pullNumber = pullNumber
	}
	if pjs.Refs.Pulls[0].Author == "" {
		pr, err := o.getPullRequest()
		if err != nil {
			return err
		}
		pjs.Refs.Pulls[0].Author = pr.User.Login
	}
	if pjs.Refs.Pulls[0].SHA == "" {
		pr, err := o.getPullRequest()
		if err != nil {
			return err
		}
		pjs.Refs.Pulls[0].SHA = pr.Head.SHA
	}
	return nil
}

func (o *options) defaultBaseRef(pjs *prowapi.ProwJobSpec) error {
	if pjs.Refs.BaseRef == "" {
		if o.pullNumber != 0 {
			pr, err := o.getPullRequest()
			if err != nil {
				return err
			}
			pjs.Refs.BaseRef = pr.Base.Ref
		} else {
			fmt.Fprint(os.Stderr, "Base ref (e.g. master): ")
			fmt.Scanln(&pjs.Refs.BaseRef)
		}
	}
	if pjs.Refs.BaseSHA == "" {
		if o.pullNumber != 0 {
			pr, err := o.getPullRequest()
			if err != nil {
				return err
			}
			pjs.Refs.BaseSHA = pr.Base.SHA
		} else {
			baseSHA, err := o.githubClient.GetRef(o.org, o.repo, fmt.Sprintf("heads/%s", pjs.Refs.BaseRef))
			if err != nil {
				logrus.Fatalf("failed to get base sha: %v", err)
				return err
			}
			pjs.Refs.BaseSHA = baseSHA
		}
	}
	return nil
}

type githubClient interface {
	GetPullRequest(org, repo string, number int) (*github.PullRequest, error)
	GetRef(org, repo, ref string) (string, error)
}

func (o *options) Validate() error {
	if o.jobName == "" {
		return errors.New("required flag --job was unset")
	}

	if o.configPath == "" {
		return errors.New("required flag --config-path was unset")
	}

	if err := o.github.Validate(false); err != nil {
		return err
	}

	return nil
}

func gatherOptions() options {
	var o options
	fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	fs.StringVar(&o.jobName, "job", "", "Job to run.")
	fs.BoolVar(&o.local, "local", false, "Print help for running locally")
	fs.StringVar(&o.configPath, "config-path", "", "Path to config.yaml.")
	fs.StringVar(&o.jobConfigPath, "job-config-path", "", "Path to prow job configs.")
	fs.StringVar(&o.baseRef, "base-ref", "", "Git base ref under test")
	fs.StringVar(&o.baseSha, "base-sha", "", "Git base SHA under test")
	fs.IntVar(&o.pullNumber, "pull-number", 0, "Git pull number under test")
	fs.StringVar(&o.pullSha, "pull-sha", "", "Git pull SHA under test")
	fs.StringVar(&o.pullAuthor, "pull-author", "", "Git pull author under test")
	o.github.AddFlagsWithoutDefaultGitHubTokenPath(fs)
	fs.Parse(os.Args[1:])
	return o
}

func main() {
	o := gatherOptions()
	if err := o.Validate(); err != nil {
		logrus.WithError(err).Fatalf("Bad flags")
	}

	conf, err := config.Load(o.configPath, o.jobConfigPath)
	if err != nil {
		logrus.WithError(err).Fatal("Error loading config")
	}

	var secretAgent *secret.Agent
	if o.github.TokenPath != "" {
		secretAgent = &secret.Agent{}
		if err := secretAgent.Start([]string{o.github.TokenPath}); err != nil {
			logrus.WithError(err).Fatal("Failed to start secret agent")
		}
	}
	o.githubClient, err = o.github.GitHubClient(secretAgent, false)
	if err != nil {
		logrus.WithError(err).Fatal("Failed to get GitHub client")
	}

	var pjs prowapi.ProwJobSpec
	var labels map[string]string
	var annotations map[string]string
	var found bool
	for fullRepoName, ps := range conf.Presubmits {
		org, repo, err := splitRepoName(fullRepoName)
		if err != nil {
			logrus.WithError(err).Warnf("Invalid repo name %s.", fullRepoName)
			continue
		}
		for _, p := range ps {
			if p.Name == o.jobName {
				pjs = pjutil.PresubmitSpec(p, prowapi.Refs{
					Org:     org,
					Repo:    repo,
					BaseRef: o.baseRef,
					BaseSHA: o.baseSha,
					Pulls: []prowapi.Pull{{
						Author: o.pullAuthor,
						Number: o.pullNumber,
						SHA:    o.pullSha,
					}},
				})
				labels = p.Labels
				annotations = p.Annotations
				found = true
			}
		}
	}
	for fullRepoName, ps := range conf.Postsubmits {
		org, repo, err := splitRepoName(fullRepoName)
		if err != nil {
			logrus.WithError(err).Warnf("Invalid repo name %s.", fullRepoName)
			continue
		}
		for _, p := range ps {
			if p.Name == o.jobName {
				pjs = pjutil.PostsubmitSpec(p, prowapi.Refs{
					Org:     org,
					Repo:    repo,
					BaseRef: o.baseRef,
					BaseSHA: o.baseSha,
				})
				labels = p.Labels
				annotations = p.Annotations
				found = true
			}
		}
	}
	for _, p := range conf.Periodics {
		if p.Name == o.jobName {
			pjs = pjutil.PeriodicSpec(p)
			labels = p.Labels
			annotations = p.Annotations
			found = true
		}
	}
	if !found {
		logrus.Fatalf("Job %s not found.", o.jobName)
	}
	if pjs.Refs != nil {
		o.org = pjs.Refs.Org
		o.repo = pjs.Refs.Repo
		if len(pjs.Refs.Pulls) != 0 {
			if err := o.defaultPR(&pjs); err != nil {
				logrus.WithError(err).Fatal("Failed to default PR")
			}
		}
		if err := o.defaultBaseRef(&pjs); err != nil {
			logrus.WithError(err).Fatal("Failed to default base ref")
		}
	}
	pj := pjutil.NewProwJob(pjs, labels, annotations)
	b, err := yaml.Marshal(&pj)
	if err != nil {
		logrus.WithError(err).Fatal("Error marshalling YAML.")
	}
	fmt.Print(string(b))
	if o.local {
		logrus.Info("Use 'bazel run //prow/cmd/phaino' to run this job locally in docker")
	}
}

func splitRepoName(repo string) (string, string, error) {
	s := strings.SplitN(repo, "/", 2)
	if len(s) != 2 {
		return "", "", fmt.Errorf("repo %s cannot be split into org/repo", repo)
	}
	return s[0], s[1], nil
}
<|endoftext|>


<commit_before>package proxystorage

import (
	"context"
	"fmt"
	"net/url"
	"reflect"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/jacksontj/promxy/config"
	"github.com/jacksontj/promxy/promclient"
	"github.com/jacksontj/promxy/proxyquerier"
	"github.com/jacksontj/promxy/servergroup"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/pkg/timestamp"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/storage"
	"github.com/prometheus/prometheus/storage/remote"
	"github.com/sirupsen/logrus"
)

type proxyStorageState struct {
	serverGroups []*servergroup.ServerGroup
	cfg          *proxyconfig.PromxyConfig
	appender     storage.Appender
}

func (p *proxyStorageState) Ready() {
	for _, sg := range p.serverGroups {
		<-sg.Ready
	}
}

func (p *proxyStorageState) Cancel() {
	if p.serverGroups != nil {
		for _, sg := range p.serverGroups {
			sg.Cancel()
		}
	}
}

func NewProxyStorage() (*ProxyStorage, error) {
	return &ProxyStorage{}, nil
}

// TODO: rename?
type ProxyStorage struct {
	state atomic.Value
}

func (p *ProxyStorage) GetState() *proxyStorageState {
	tmp := p.state.Load()
	if sg, ok := tmp.(*proxyStorageState); ok {
		return sg
	} else {
		return nil
	}
}

func (p *ProxyStorage) ApplyConfig(c *proxyconfig.Config) error {
	failed := false

	newState := &proxyStorageState{
		serverGroups: make([]*servergroup.ServerGroup, len(c.ServerGroups)),
		cfg:          &c.PromxyConfig,
	}
	for i, sgCfg := range c.ServerGroups {
		tmp := servergroup.New()
		if err := tmp.ApplyConfig(sgCfg); err != nil {
			failed = true
			logrus.Errorf("Error applying config to server group: %s", err)
		}
		newState.serverGroups[i] = tmp
	}

	if failed {
		for _, sg := range newState.serverGroups {
			sg.Cancel()
		}
		return fmt.Errorf("Error Applying Config to one or more server group(s)")
	}

	// Check for remote_write (for appender)
	if c.PromConfig.RemoteWriteConfigs != nil {
		remote := remote.NewStorage(nil, func() (int64, error) { return 0, nil }, 1*time.Second)
		if err := remote.ApplyConfig(&c.PromConfig); err != nil {
			return err
		}
		newState.appender = remote
	} else {
		newState.appender = &appenderStub{}

	}

	newState.Ready()         // Wait for the newstate to be ready
	oldState := p.GetState() // Fetch the old state
	p.state.Store(newState)  // Store the new state
	if oldState != nil {
		oldState.Cancel() // Cancel the old one
	}

	return nil
}

func (p *ProxyStorage) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) {
	state := p.GetState()
	return &proxyquerier.ProxyQuerier{
		ctx,
		timestamp.Time(mint),
		timestamp.Time(maxt),
		state.serverGroups,

		state.cfg,
	}, nil
}

func (p *ProxyStorage) StartTime() (int64, error) {
	return 0, nil
}

func (p *ProxyStorage) Appender() (storage.Appender, error) {
	state := p.GetState()
	return state.appender, nil
}

// TODO: actually close things?
func (p *ProxyStorage) Close() error { return nil }

// This replaces promql Nodes with more efficient-to-fetch ones. This works by taking lower-layer
// chunks of the query, farming them out to prometheus hosts, then stitching the results back together.
// An example would be a sum, we can sum multiple sums and come up with the same result -- so we do.
// There are a few ground rules for this:
//      - Children cannot be AggregateExpr: aggregates have their own combining logic, so its not safe to send a subquery with additional aggregations
//      - offsets within the subtree must match: if they don't then we'll get mismatched data, so we wait until we are far enough down the tree that they converge
//      - Don't reduce accuracy/granularity: the intention of this is to get the correct data faster, meaning correctness overrules speed.
func (p *ProxyStorage) NodeReplacer(ctx context.Context, s *promql.EvalStmt, node promql.Node) (promql.Node, error) {

	isAgg := func(node promql.Node) bool {
		_, ok := node.(*promql.AggregateExpr)
		return ok
	}

	// If there is a child that is an aggregator we cannot do anything (as they have their own
	// rules around combining). We'll skip this node and let a lower layer take this on
	aggFinder := &BooleanFinder{Func: isAgg}
	offsetFinder := &OffsetFinder{}

	visitor := &MultiVisitor{[]promql.Visitor{aggFinder, offsetFinder}}

	if _, err := promql.Walk(ctx, visitor, s, node, nil, nil); err != nil {
		return nil, err
	}

	if aggFinder.Found > 0 {
		// If there was a single agg and that was us, then we're okay
		if !(isAgg(node) && aggFinder.Found == 1) {
			return nil, nil
		}
	}

	// If the tree below us is not all the same offset, then we can't do anything below -- we'll need
	// to wait until further in execution where they all match
	var offset time.Duration

	// If we couldn't find an offset, then something is wrong-- lets skip
	// Also if there was an error, skip
	if !offsetFinder.Found || offsetFinder.Error != nil {
		return nil, nil
	}
	offset = offsetFinder.Offset

	// Function to recursivelt remove offset. This is needed as we're using
	// the node API to String() the query to downstreams. Promql's iterators require
	// that the time be the absolute time, whereas the API returns them based on the
	// range you ask for (with the offset being implicit)
	// TODO: rename
	removeOffset := func() error {
		_, err := promql.Walk(ctx, &OffsetRemover{}, s, node, nil, nil)
		return err
	}

	state := p.GetState()
	serverGroups := servergroup.ServerGroups(state.serverGroups)
	switch n := node.(type) {
	// Some AggregateExprs can be composed (meaning they are "reentrant". If the aggregation op
	// is reentrant/composable then we'll do so, otherwise we let it fall through to normal query mechanisms
	case *promql.AggregateExpr:
		logrus.Debugf("AggregateExpr %v", n)

		var result model.Value
		var err error

		// Not all Aggregation functions are composable, so we'll do what we can
		switch n.Op.String() {
		// All "reentrant" cases (meaning they can be done repeatedly and the outcome doesn't change)
		case "sum", "min", "max", "topk", "bottomk":
			var urlBase string
			values := url.Values{}
			removeOffset()
			values.Add("query", n.String())

			if s.Interval > 0 {
				values.Add("start", model.Time(timestamp.FromTime(s.Start.Add(-offset-promql.LookbackDelta))).String())
				values.Add("end", model.Time(timestamp.FromTime(s.End.Add(-offset))).String())
				values.Add("step", strconv.FormatFloat(s.Interval.Seconds(), 'f', -1, 64))
				urlBase = "/api/v1/query_range"
			} else {
				values.Add("time", model.Time(timestamp.FromTime(s.Start.Add(-offset))).String())
				urlBase = "/api/v1/query"
			}

			result, err = serverGroups.GetData(ctx, urlBase, values)
			if err != nil {
				return nil, err
			}

		// Convert avg into sum() / count()
		case "avg":
			// Replace with sum() / count()
			return &promql.BinaryExpr{
				Op: 24, // Divide  TODO
				LHS: &promql.AggregateExpr{
					Op:       41, // sum() TODO
					Expr:     n.Expr,
					Param:    n.Param,
					Grouping: n.Grouping,
					Without:  n.Without,
				},

				RHS: &promql.AggregateExpr{
					Op:       40, // count() TODO
					Expr:     n.Expr,
					Param:    n.Param,
					Grouping: n.Grouping,
					Without:  n.Without,
				},
				VectorMatching: &promql.VectorMatching{Card: promql.CardOneToOne},
			}, nil

		// For count we simply need to change this to a sum over the data we get back
		case "count":
			var urlBase string
			values := url.Values{}
			removeOffset()
			values.Add("query", n.String())

			if s.Interval > 0 {
				values.Add("start", model.Time(timestamp.FromTime(s.Start.Add(-offset-promql.LookbackDelta))).String())
				values.Add("end", model.Time(timestamp.FromTime(s.End.Add(-offset))).String())
				values.Add("step", strconv.FormatFloat(s.Interval.Seconds(), 'f', -1, 64))
				urlBase = "/api/v1/query_range"
			} else {
				values.Add("time", model.Time(timestamp.FromTime(s.Start.Add(-offset))).String())
				urlBase = "/api/v1/query"
			}

			result, err = serverGroups.GetData(ctx, urlBase, values)
			if err != nil {
				return nil, err
			}
			// TODO: have a reverse method in promql/lex.go
			n.Op = 41 // SUM

		case "stddev": // TODO: something?
		case "stdvar": // TODO: something?

			// Do nothing, we want to allow the VectorSelector to fall through to do a query_range.
			// Unless we find another way to decompose the query, this is all we can do
		case "count_values":
			// DO NOTHING

		case "quantile": // TODO: something?

		}

		if result != nil {
			iterators := promclient.IteratorsForValue(result)

			series := make([]storage.Series, len(iterators))
			for i, iterator := range iterators {
				series[i] = &proxyquerier.Series{iterator}
			}

			ret := &promql.VectorSelector{Offset: offset}
			ret.SetSeries(series)
			n.Expr = ret
			return n, nil
		}

	// Call is for things such as rate() etc. This can be sent directly to the
	// prometheus node to answer
	case *promql.Call:
		logrus.Debugf("call %v %v", n, n.Type())
		var urlBase string
		values := url.Values{}
		removeOffset()
		values.Add("query", n.String())

		if s.Interval > 0 {
			values.Add("start", model.Time(timestamp.FromTime(s.Start.Add(-offset-promql.LookbackDelta))).String())
			values.Add("end", model.Time(timestamp.FromTime(s.End.Add(-offset))).String())
			values.Add("step", strconv.FormatFloat(s.Interval.Seconds(), 'f', -1, 64))
			urlBase = "/api/v1/query_range"
		} else {
			values.Add("time", model.Time(timestamp.FromTime(s.Start.Add(-offset))).String())
			urlBase = "/api/v1/query"
		}

		result, err := serverGroups.GetData(ctx, urlBase, values)
		if err != nil {
			return nil, err
		}
		iterators := promclient.IteratorsForValue(result)
		series := make([]storage.Series, len(iterators))
		for i, iterator := range iterators {
			series[i] = &proxyquerier.Series{iterator}
		}

		ret := &promql.VectorSelector{Offset: offset}
		ret.SetSeries(series)
		return ret, nil

		// If we are simply fetching a Vector then we can fetch the data using the same step that
		// the query came in as (reducing the amount of data we need to fetch)
	// If we are simply fetching data, we skip here to let it fall through to the normal
	// storage API
	case *promql.VectorSelector:
		// Do Nothing
		return nil, nil

	// If we hit this someone is asking for a matrix directly, if so then we don't
	// have anyway to ask for less-- since this is exactly what they are asking for
	case *promql.MatrixSelector:
		// DO NOTHING

	default:
		logrus.Debugf("default %v %s", n, reflect.TypeOf(n))

	}
	return nil, nil

}
<commit_msg>Use ApplyConfig where possible (instead of always creating new)<commit_after>package proxystorage

import (
	"context"
	"fmt"
	"net/url"
	"reflect"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/jacksontj/promxy/config"
	"github.com/jacksontj/promxy/promclient"
	"github.com/jacksontj/promxy/proxyquerier"
	"github.com/jacksontj/promxy/servergroup"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/pkg/timestamp"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/storage"
	"github.com/prometheus/prometheus/storage/remote"
	"github.com/sirupsen/logrus"
)

type proxyStorageState struct {
	serverGroups   []*servergroup.ServerGroup
	cfg            *proxyconfig.PromxyConfig
	appender       storage.Appender
	appenderCloser func() error
}

func (p *proxyStorageState) Ready() {
	for _, sg := range p.serverGroups {
		<-sg.Ready
	}
}

func (p *proxyStorageState) Cancel(n *proxyStorageState) {
	if p.serverGroups != nil {
		for _, sg := range p.serverGroups {
			sg.Cancel()
		}
	}
	// We call close if the new one is nil, or if the appanders don't match
	if n == nil || p.appender != n.appender {
		if p.appenderCloser != nil {
			p.appenderCloser()
		}
	}
}

func NewProxyStorage() (*ProxyStorage, error) {
	return &ProxyStorage{}, nil
}

// TODO: rename?
type ProxyStorage struct {
	state atomic.Value
}

func (p *ProxyStorage) GetState() *proxyStorageState {
	tmp := p.state.Load()
	if sg, ok := tmp.(*proxyStorageState); ok {
		return sg
	} else {
		return &proxyStorageState{}
	}
}

func (p *ProxyStorage) ApplyConfig(c *proxyconfig.Config) error {
	oldState := p.GetState() // Fetch the old state

	failed := false

	newState := &proxyStorageState{
		serverGroups: make([]*servergroup.ServerGroup, len(c.ServerGroups)),
		cfg:          &c.PromxyConfig,
	}
	for i, sgCfg := range c.ServerGroups {
		tmp := servergroup.New()
		if err := tmp.ApplyConfig(sgCfg); err != nil {
			failed = true
			logrus.Errorf("Error applying config to server group: %s", err)
		}
		newState.serverGroups[i] = tmp
	}

	if failed {
		newState.Cancel(nil)
		return fmt.Errorf("Error Applying Config to one or more server group(s)")
	}

	// Check for remote_write (for appender)
	if c.PromConfig.RemoteWriteConfigs != nil {
		switch oldAppender := oldState.appender.(type) {
		// If the old one was a remote storage, we just need to apply config
		case *remote.Storage:
			if err := oldAppender.ApplyConfig(&c.PromConfig); err != nil {
				return err
			}
			newState.appender = oldState.appender
			newState.appenderCloser = oldState.appenderCloser
		// if it was an appenderstub we just need to replace
		default:
			remote := remote.NewStorage(nil, func() (int64, error) { return 0, nil }, 1*time.Second)
			if err := remote.ApplyConfig(&c.PromConfig); err != nil {
				return err
			}
			newState.appender = remote
			newState.appenderCloser = remote.Close
		}
	} else {
		newState.appender = &appenderStub{}
	}

	newState.Ready()        // Wait for the newstate to be ready
	p.state.Store(newState) // Store the new state
	if oldState != nil {
		oldState.Cancel(newState) // Cancel the old one
	}

	return nil
}

func (p *ProxyStorage) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) {
	state := p.GetState()
	return &proxyquerier.ProxyQuerier{
		ctx,
		timestamp.Time(mint),
		timestamp.Time(maxt),
		state.serverGroups,

		state.cfg,
	}, nil
}

func (p *ProxyStorage) StartTime() (int64, error) {
	return 0, nil
}

func (p *ProxyStorage) Appender() (storage.Appender, error) {
	state := p.GetState()
	return state.appender, nil
}

// TODO: actually close things?
func (p *ProxyStorage) Close() error { return nil }

// This replaces promql Nodes with more efficient-to-fetch ones. This works by taking lower-layer
// chunks of the query, farming them out to prometheus hosts, then stitching the results back together.
// An example would be a sum, we can sum multiple sums and come up with the same result -- so we do.
// There are a few ground rules for this:
//      - Children cannot be AggregateExpr: aggregates have their own combining logic, so its not safe to send a subquery with additional aggregations
//      - offsets within the subtree must match: if they don't then we'll get mismatched data, so we wait until we are far enough down the tree that they converge
//      - Don't reduce accuracy/granularity: the intention of this is to get the correct data faster, meaning correctness overrules speed.
func (p *ProxyStorage) NodeReplacer(ctx context.Context, s *promql.EvalStmt, node promql.Node) (promql.Node, error) {

	isAgg := func(node promql.Node) bool {
		_, ok := node.(*promql.AggregateExpr)
		return ok
	}

	// If there is a child that is an aggregator we cannot do anything (as they have their own
	// rules around combining). We'll skip this node and let a lower layer take this on
	aggFinder := &BooleanFinder{Func: isAgg}
	offsetFinder := &OffsetFinder{}

	visitor := &MultiVisitor{[]promql.Visitor{aggFinder, offsetFinder}}

	if _, err := promql.Walk(ctx, visitor, s, node, nil, nil); err != nil {
		return nil, err
	}

	if aggFinder.Found > 0 {
		// If there was a single agg and that was us, then we're okay
		if !(isAgg(node) && aggFinder.Found == 1) {
			return nil, nil
		}
	}

	// If the tree below us is not all the same offset, then we can't do anything below -- we'll need
	// to wait until further in execution where they all match
	var offset time.Duration

	// If we couldn't find an offset, then something is wrong-- lets skip
	// Also if there was an error, skip
	if !offsetFinder.Found || offsetFinder.Error != nil {
		return nil, nil
	}
	offset = offsetFinder.Offset

	// Function to recursivelt remove offset. This is needed as we're using
	// the node API to String() the query to downstreams. Promql's iterators require
	// that the time be the absolute time, whereas the API returns them based on the
	// range you ask for (with the offset being implicit)
	// TODO: rename
	removeOffset := func() error {
		_, err := promql.Walk(ctx, &OffsetRemover{}, s, node, nil, nil)
		return err
	}

	state := p.GetState()
	serverGroups := servergroup.ServerGroups(state.serverGroups)
	switch n := node.(type) {
	// Some AggregateExprs can be composed (meaning they are "reentrant". If the aggregation op
	// is reentrant/composable then we'll do so, otherwise we let it fall through to normal query mechanisms
	case *promql.AggregateExpr:
		logrus.Debugf("AggregateExpr %v", n)

		var result model.Value
		var err error

		// Not all Aggregation functions are composable, so we'll do what we can
		switch n.Op.String() {
		// All "reentrant" cases (meaning they can be done repeatedly and the outcome doesn't change)
		case "sum", "min", "max", "topk", "bottomk":
			var urlBase string
			values := url.Values{}
			removeOffset()
			values.Add("query", n.String())

			if s.Interval > 0 {
				values.Add("start", model.Time(timestamp.FromTime(s.Start.Add(-offset-promql.LookbackDelta))).String())
				values.Add("end", model.Time(timestamp.FromTime(s.End.Add(-offset))).String())
				values.Add("step", strconv.FormatFloat(s.Interval.Seconds(), 'f', -1, 64))
				urlBase = "/api/v1/query_range"
			} else {
				values.Add("time", model.Time(timestamp.FromTime(s.Start.Add(-offset))).String())
				urlBase = "/api/v1/query"
			}

			result, err = serverGroups.GetData(ctx, urlBase, values)
			if err != nil {
				return nil, err
			}

		// Convert avg into sum() / count()
		case "avg":
			// Replace with sum() / count()
			return &promql.BinaryExpr{
				Op: 24, // Divide  TODO
				LHS: &promql.AggregateExpr{
					Op:       41, // sum() TODO
					Expr:     n.Expr,
					Param:    n.Param,
					Grouping: n.Grouping,
					Without:  n.Without,
				},

				RHS: &promql.AggregateExpr{
					Op:       40, // count() TODO
					Expr:     n.Expr,
					Param:    n.Param,
					Grouping: n.Grouping,
					Without:  n.Without,
				},
				VectorMatching: &promql.VectorMatching{Card: promql.CardOneToOne},
			}, nil

		// For count we simply need to change this to a sum over the data we get back
		case "count":
			var urlBase string
			values := url.Values{}
			removeOffset()
			values.Add("query", n.String())

			if s.Interval > 0 {
				values.Add("start", model.Time(timestamp.FromTime(s.Start.Add(-offset-promql.LookbackDelta))).String())
				values.Add("end", model.Time(timestamp.FromTime(s.End.Add(-offset))).String())
				values.Add("step", strconv.FormatFloat(s.Interval.Seconds(), 'f', -1, 64))
				urlBase = "/api/v1/query_range"
			} else {
				values.Add("time", model.Time(timestamp.FromTime(s.Start.Add(-offset))).String())
				urlBase = "/api/v1/query"
			}

			result, err = serverGroups.GetData(ctx, urlBase, values)
			if err != nil {
				return nil, err
			}
			// TODO: have a reverse method in promql/lex.go
			n.Op = 41 // SUM

		case "stddev": // TODO: something?
		case "stdvar": // TODO: something?

			// Do nothing, we want to allow the VectorSelector to fall through to do a query_range.
			// Unless we find another way to decompose the query, this is all we can do
		case "count_values":
			// DO NOTHING

		case "quantile": // TODO: something?

		}

		if result != nil {
			iterators := promclient.IteratorsForValue(result)

			series := make([]storage.Series, len(iterators))
			for i, iterator := range iterators {
				series[i] = &proxyquerier.Series{iterator}
			}

			ret := &promql.VectorSelector{Offset: offset}
			ret.SetSeries(series)
			n.Expr = ret
			return n, nil
		}

	// Call is for things such as rate() etc. This can be sent directly to the
	// prometheus node to answer
	case *promql.Call:
		logrus.Debugf("call %v %v", n, n.Type())
		var urlBase string
		values := url.Values{}
		removeOffset()
		values.Add("query", n.String())

		if s.Interval > 0 {
			values.Add("start", model.Time(timestamp.FromTime(s.Start.Add(-offset-promql.LookbackDelta))).String())
			values.Add("end", model.Time(timestamp.FromTime(s.End.Add(-offset))).String())
			values.Add("step", strconv.FormatFloat(s.Interval.Seconds(), 'f', -1, 64))
			urlBase = "/api/v1/query_range"
		} else {
			values.Add("time", model.Time(timestamp.FromTime(s.Start.Add(-offset))).String())
			urlBase = "/api/v1/query"
		}

		result, err := serverGroups.GetData(ctx, urlBase, values)
		if err != nil {
			return nil, err
		}
		iterators := promclient.IteratorsForValue(result)
		series := make([]storage.Series, len(iterators))
		for i, iterator := range iterators {
			series[i] = &proxyquerier.Series{iterator}
		}

		ret := &promql.VectorSelector{Offset: offset}
		ret.SetSeries(series)
		return ret, nil

		// If we are simply fetching a Vector then we can fetch the data using the same step that
		// the query came in as (reducing the amount of data we need to fetch)
	// If we are simply fetching data, we skip here to let it fall through to the normal
	// storage API
	case *promql.VectorSelector:
		// Do Nothing
		return nil, nil

	// If we hit this someone is asking for a matrix directly, if so then we don't
	// have anyway to ask for less-- since this is exactly what they are asking for
	case *promql.MatrixSelector:
		// DO NOTHING

	default:
		logrus.Debugf("default %v %s", n, reflect.TypeOf(n))

	}
	return nil, nil

}
<|endoftext|>